log: Add log config, format code, remove prints

ADASMELIAE.py

@@ -23,10 +23,14 @@
 # * For Python: See requirements.txt
 
 
-import os, sys, subprocess, glob
+import os
+import sys
+import subprocess
+import glob
 from dotenv import load_dotenv
-from datetime import datetime, timezone, timedelta
+from datetime import datetime, timedelta
 from jinja2 import Environment, FileSystemLoader
+import logging
 import pytz
 import netCDF4 as nc
 import configparser
@@ -38,9 +42,16 @@ load_dotenv()
 config = configparser.ConfigParser()
 config.read("ADASMELIAE.cfg")
 
+DEBUG = False if os.getenv("DEBUG") is None or os.getenv("DEBUG").lower() == "false" else True
+
+logging.basicConfig(
+    level=logging.DEBUG if DEBUG else logging.INFO,
+    format="%(asctime)s - %(levelname).4s - (%(filename)s:%(lineno)d) - %(message)s",
+)
+
 # Path to weather data
 model_id = os.getenv("MODEL_ID")
-infile_path  = os.getenv("WEATHER_DATA_DIR")
+infile_path = os.getenv("WEATHER_DATA_DIR")
 # Used for iterating the weather data files
 filename_pattern = os.getenv("FILENAME_PATTERN")
 # Date format of weather data filenames
@@ -54,29 +65,27 @@ RR = os.getenv("RR")
 UM = os.getenv("UM")
 local_timezone = pytz.timezone(os.getenv("LOCAL_TIMEZONE"))
 
-DEBUG = False if os.getenv("DEBUG") is None or os.getenv("DEBUG").lower() == "false" else True
-
-
 # Iterate the set of hourly weather data files
 # 1. When's the latest wh_[DATE].nc file? - set earliest weather data file: start_date = [DATE]-2 days
 # --> If no wh_DATE.nc file exists, set start_date = None
 start_date = None
 last_wh_date = None
-#"""
-for wh_file in glob.glob(f"{tmpfile_path}wh_2[0-9][0-9][0-9]-[01][0-9]-[0123][0-9].nc"):          
+
+logging.info("Start running model")
+logging.info(f"Read files from path pattern: {tmpfile_path}wh_2[0-9][0-9][0-9]-[01][0-9]-[0123][0-9].nc")
+for wh_file in glob.glob(f"{tmpfile_path}wh_2[0-9][0-9][0-9]-[01][0-9]-[0123][0-9].nc"):
     current_wh_file_date = local_timezone.localize(datetime.strptime(f"{wh_file[7:17]}", "%Y-%m-%d"))
     if last_wh_date is None or last_wh_date < current_wh_file_date:
-       last_wh_date = current_wh_file_date
+        last_wh_date = current_wh_file_date
 if last_wh_date is not None:
     start_date = last_wh_date - timedelta(days=2)
-    print(f"Last date of WH calculations is {last_wh_date}. Start date = {start_date}")
+    logging.info(f"Last date of WH calculations is {last_wh_date}. Start date = {start_date}")
 
 
 weatherdata_files = glob.glob(f"{infile_path}{filename_pattern}")
-if DEBUG:
-    print(f"{infile_path}{filename_pattern}")
-    print("What are the weatherdata files?")
-    print(weatherdata_files)
+logging.debug(f"{infile_path}{filename_pattern}")
+logging.debug("What are the weatherdata files?")
+logging.debug(weatherdata_files)
 for file_path in sorted(weatherdata_files):
     # TODO: When filename/pattern is configurable: make the string search adaptable
     file_name = os.path.basename(file_path)
@@ -84,18 +93,18 @@ for file_path in sorted(weatherdata_files):
     try:
         wh_sum_date = local_timezone.localize(datetime.strptime(file_name, filename_dateformat))
     except ValueError as e:
-        print(e)
+        logging.info(e)
         continue
 
     # Only process files from the three last days (if this is not a work from scratch)
     if start_date is not None and wh_sum_date < start_date:
-         continue
-    
+        continue
+
     # Check that the file has at least 23 timesteps
     with nc.Dataset(file_path, 'r') as weatherdata_file:
         file_timesteps = len(weatherdata_file.variables["time"])
-        if  file_timesteps < 23:
-            print(f"{file_path} has {file_timesteps} timesteps. Skipping it.")
+        if file_timesteps < 23:
+            logging.info(f"{file_path} has {file_timesteps} timesteps. Skipping it.")
             continue
 
     # Produce daily files with WH_SUM, which is the number of "Wet hours" (WH) for a given day
@@ -120,7 +129,7 @@ for timestep in timesteps:
         if timestep - previous_timestep != 86400.0:
             timestep_str = datetime.fromtimestamp(timestep).astimezone(local_timezone).strftime("%Y-%m-%d")
             previous_timestep_str = datetime.fromtimestamp(previous_timestep).astimezone(local_timezone).strftime("%Y-%m-%d")
-            print(f"ERROR: Missing weather data between {previous_timestep_str} and {timestep_str}. Exiting.", file=sys.stderr)
+            logging.error(f"Missing weather data between {previous_timestep_str} and {timestep_str}. Exiting.", file=sys.stderr)
             exit(1)
     previous_timestep = timestep
 wh_daysum.close()
@@ -157,7 +166,7 @@ subprocess.run(f'cdo -s -chname,WH_DAYSUM,WHS -mergetime {tmpfile_path}wh_3daysu
 # Using netCDF4 to accomplish this
 
 wh_3daysum = nc.Dataset(f'{tmpfile_path}wh_3daysum_tmp_merged.nc', 'r')
-time_bnds = wh_3daysum.variables["time_bnds"][:] 
+time_bnds = wh_3daysum.variables["time_bnds"][:]
 # Assuming that wrong time bounds only exist at the end of the time series, this works
 number_of_timesteps_to_remove = 0
 for time_bnd in time_bnds:
@@ -179,7 +188,7 @@ subprocess.run(f'cdo -s -aexpr,"WARNING_STATUS = WHS <= 0 ? 0 : -1; WARNING_STAT
 # Env variable MASK_FILE must be set
 if os.getenv("MASK_FILE") is not None:
     mask_file = os.getenv("MASK_FILE")
-    print(f"Applying mask file {mask_file} to result.nc")
+    logging.info(f"Applying mask file {mask_file} to result.nc")
     subprocess.run(f'cdo -maskregion,{mask_file} {tmpfile_path}result_unmasked.nc {tmpfile_path}result.nc', shell=True)
 else:
     os.rename(f"{tmpfile_path}result_unmasked.nc", f"{tmpfile_path}result.nc")
@@ -200,7 +209,7 @@ for timestep in timesteps:
     # If start_date is set, that means that we should only generate result files from that date on 
     # -- IF GeoTIFF fiels already exists before that.
     if start_date is not None and timestep_date < start_date:
-         if os.path.isfile(f'{outfile_path}result_WARNING_STATUS_{file_date}.tif') and os.path.isfile(f'{outfile_path}result_{file_date}.tif'):
+        if os.path.isfile(f'{outfile_path}result_WARNING_STATUS_{file_date}.tif') and os.path.isfile(f'{outfile_path}result_{file_date}.tif'):
             continue
 
     # Create NetCDF result file
@@ -225,8 +234,8 @@ for timestep in timesteps:
 languages = []
 language_codes = config["i18n"]["languages"].split(",");
 for language_code in language_codes:
-     language = {"language_code": language_code}
-     if ("i18n.%s" % language_code) in config:
+    language = {"language_code": language_code}
+    if ("i18n.%s" % language_code) in config:
         for keyword in config["i18n.%s" % language_code]:
             language[keyword] = config["i18n.%s" % language_code][keyword]
         languages.append(language)
@@ -247,7 +256,7 @@ output = template.render({
 })
 mapfile_outdir = os.getenv("MAPFILE_DIR")
 with open(f"{mapfile_outdir}/{model_id}.map", 'w') as f:
-        f.write(output)
+    f.write(output)
 
 
 # Remove all temporary/intermediary files