chore: Add year to output folders, copy query templates, use archive for previous years

SEPTREFHUM.py

@@ -30,6 +30,7 @@ import pytz
 import netCDF4 as nc
 import configparser
 import logging
+import shutil
 
 # Paths config
 # Create a .env file from dotenv-sample
@@ -40,14 +41,14 @@ config.read("SEPTREFHUM.cfg")
 
 # Paths to weather data
 model_id = os.getenv("MODEL_ID")
+home_dir = os.getenv("HOME_DIR")
 infile_path  = os.getenv("WEATHER_DATA_DIR")
 infile_archive_path  = os.getenv("ARCHIVE_WEATHER_DATA_DIR")
 # Used for iterating the weather data files
 filename_pattern = os.getenv("FILENAME_PATTERN")
 # Date format of weather data filenames
 filename_dateformat = os.getenv("FILENAME_DATEFORMAT")
-# Path to store generated GeoTIFF files
-outfile_path = os.getenv("DATA_DIR")
+
 # Names of weather parameters in NetCDF files
 RR = os.getenv("RR")
 UM = os.getenv("UM")
@@ -60,9 +61,17 @@ if len(sys.argv) > 1:
 else:
     year = today.year
 
-# Where to store intermediary calculations
+weather_data_path = weather_data_dir
+if year != today.year:
+    weather_data_path = f"{archive_weather_data_dir}{year}/"
+
+# Path to store intermediary calculations
 tmpfile_path = f"tmp/{year}/"
 
+# Paths to store generated result files
+outfile_path = f"os.getenv('DATA_DIR'){year}/"
+mapfile_outdir = f"os.getenv('MAPFILE_DIR'){year}/"
+
 TEMPERATURE_THRESHOLD = 8.0
 
 DEBUG = False if os.getenv("DEBUG") is None or os.getenv("DEBUG").lower() == "false" else True
@@ -87,10 +96,10 @@ if last_wh_date is not None:
     logging.info(f"Last date of WH calculations is {last_wh_date}. Start date = {start_date}")
 
 weatherdata_files = glob.glob(f"{infile_path}{filename_pattern}")
-if DEBUG:
-    logging.info(f"{infile_path}{filename_pattern}")
-    logging.info("What are the weatherdata files?")
-    logging.info(weatherdata_files)
+logging.debug(f"{infile_path}{filename_pattern}")
+logging.debug("What are the weatherdata files?")
+logging.debug(weatherdata_files)
+
 for file_path in sorted(weatherdata_files):
     # TODO: When filename/pattern is configurable: make the string search adaptable
     file_name = os.path.basename(file_path)
@@ -98,7 +107,7 @@ for file_path in sorted(weatherdata_files):
     try:
         wh_sum_date = local_timezone.localize(datetime.strptime(file_name, filename_dateformat))
     except ValueError as e:
-        logging.info(e)
+        logging.error(e)
         continue
 
     # Only process files from the three last days (if this is not a work from scratch)
@@ -247,18 +256,22 @@ template = env.get_template("mapfile/template.j2")
 output = template.render({
      "model_id":model_id,
      "timestep_dates": timestep_dates,
-     "mapserver_data_dir": os.getenv("MAPSERVER_DATA_DIR"),
-     "mapserver_mapfile_dir": os.getenv("MAPSERVER_MAPFILE_DIR"),
+     "mapserver_data_dir": f"{os.getenv('MAPSERVER_DATA_DIR')}{year}/",
+     "mapserver_mapfile_dir": f"{os.getenv('MAPSERVER_MAPFILE_DIR')}{year}/",
      "mapserver_log_file": os.getenv("MAPSERVER_LOG_FILE"),
      "mapserver_image_path": os.getenv("MAPSERVER_IMAGE_PATH"),
      "mapserver_extent": os.getenv("MAPSERVER_EXTENT"),
      "languages": languages,
      "language_codes": language_codes
 })
-mapfile_outdir = os.getenv("MAPFILE_DIR")
-with open(f"{mapfile_outdir}/{model_id}.map", 'w') as f:
+
+with open(f"{mapfile_outdir}{model_id}.map", 'w') as f:
         f.write(output)
 
+query_template = os.path.join(home_dir, "mapfile/query_template.xml")
+query_template_whs = os.path.join(home_dir, "mapfile/query_template_WHS.xml")
+shutil.copy(query_template, mapfile_outdir)
+shutil.copy(query_template_whs, mapfile_outdir)
 
 # Remove all temporary/intermediary files
 subprocess.run(f"rm {tmpfile_path}wh_3daysum*.nc", shell=True)