feat: Add param year, improve logging and doc

NAERSTADMO.py

@@ -20,6 +20,7 @@
 # Author: Brita Linnestad <brita.linnestad@nibio.no>
 
 import os
+import sys
 import subprocess,glob
 from dotenv import load_dotenv
 from datetime import datetime, timedelta
@@ -31,34 +32,61 @@ import configparser
 import netCDF4 as nc
 import numpy as np
 
-logging.basicConfig(level=logging.INFO)
-
 load_dotenv()
+
+DEBUG = (
+    False
+    if os.getenv("DEBUG") is None or os.getenv("DEBUG").lower() == "false"
+    else True
+)
+logging.basicConfig(
+    level=logging.DEBUG if DEBUG else logging.INFO,
+    format="%(asctime)s - %(levelname).4s - (%(filename)s:%(lineno)d) - %(message)s",
+)
+
 # Get language stuff
 config = configparser.ConfigParser()
 config.read("NAERSTADMO.cfg")
 
+local_timezone = pytz.timezone(os.getenv("LOCAL_TIMEZONE"))
+today = datetime.now(local_timezone)
+if len(sys.argv) > 1:
+    year = int(sys.argv[1])
+else:
+    year = today.year
+
+# Don't run if before start_date
+recurring_start_date = os.getenv("RECURRING_START_DATE")
+start_date = datetime.strptime(f"{year}-{recurring_start_date}", "%Y-%m-%d")
+if datetime.now() <= start_date:
+    logging.error(f"Today is before the configured start date of {start_date}. Exiting.")
+    exit(0)
+
+
+infile_path = f"{os.getenv("WEATHER_DATA_DIR")}{year}/"
 
-infile_path = os.getenv("WEATHER_DATA_DIR")
-outfile_path = os.getenv("DATA_DIR")
-outtmp_path = "out/"
-tmpfile_path = "tmp/"
+outtmp_path = f"out/{year}/"
+os.makedirs(outtmp_path, exist_ok=True)
+tmpfile_path = f"tmp/{year}/"
+os.makedirs(tmpfile_path, exist_ok=True)
+outfile_path = f"{os.getenv("DATA_DIR")}{year}/"
+os.makedirs(outfile_path, exist_ok=True)
+mapfile_outdir = f"{os.getenv("MAPFILE_DIR")}{year}/"
+os.makedirs(mapfile_outdir, exist_ok=True)
 
 bg_filename = f"{tmpfile_path}background_data.nc"
 tmp_filename = f"{tmpfile_path}background_data_tmp.nc"
 prepareWHS = f"{tmpfile_path}prepare_WHS.nc"
 
 utc_offset = "+02:00"
-local_timezone = pytz.timezone("Europe/Oslo")
 
 filename = f"{tmpfile_path}weather_data.nc"
 
-
 def create_dataset():
     # Find the latest file from previous run to create a start date
     last_final_date = None
     list_of_files = glob.glob(
-        f"{outtmp_path}final_2[0-9][0-9][0-9]-[01][0-9]-[0123][0-9].nc", recursive=True
+        f"{outtmp_path}final_{year}-[01][0-9]-[0123][0-9].nc", recursive=True
     )
 
     if list_of_files:
@@ -70,16 +98,16 @@ def create_dataset():
         last_final_date = file_date
 
     if last_final_date is None or last_final_date < file_date:
-        start_date = datetime.strptime(os.getenv("START_DATE"), "%Y-%m-%d")
+        start_date = datetime.strptime(start_date, "%Y-%m-%d")
     if last_final_date is not None:
         start_date = datetime.strptime(last_final_date, "%Y-%m-%d") - timedelta(days=4)
-    print(
+    logging.info(
         f"Last date of final calculations is {last_final_date}. Start date = {start_date}"
     )
 
     # Find the set of data to merge and use as input file based on start date
     list_weatherdata_files = glob.glob(
-        f"{infile_path}/met_1_0km_nordic-2[0-9][0-9][0-9]-[01][0-9]-[0123][0-9].nc"
+        f"{infile_path}/met_1_0km_nordic-{year}-[01][0-9]-[0123][0-9].nc"
     )
 
     for file in list_weatherdata_files:
@@ -87,6 +115,7 @@ def create_dataset():
         file_date = file_name[
             file_name.index("nordic-") + 7 : file_name.index("nordic-") + 17
         ]
+        logging.info(f"Work on date {file_date}")
 
         end_date = None
         end_date = start_date + timedelta(days=5)
@@ -94,7 +123,7 @@ def create_dataset():
         if file_date >= start_date.strftime(
             "%Y-%m-%d"
         ) and file_date <= end_date.strftime("%Y-%m-%d"):
-            if os.path.exists(f"{tmpfile_path}weather_data.nc") != True:
+            if not os.path.exists(f"{tmpfile_path}weather_data.nc"):
                 subprocess.run(f"cp {file} {tmpfile_path}weather_data.nc", shell=True)
             else:
                 subprocess.run(
@@ -109,7 +138,7 @@ def create_dataset():
 
     # Ensure that model is not run if weather data is not available
     if not os.path.exists(f"{tmpfile_path}weather_data.nc"):
-        print(f"{tmpfile_path}weather_data.nc does not exist. Exit.")
+        logging.error(f"{tmpfile_path}weather_data.nc does not exist. Exit.")
         return
 
     subprocess.run(f"rm {outtmp_path}final_*", shell=True)
@@ -146,7 +175,7 @@ def create_warning_status(start_date):
         # Env variable MASK_FILE must be set
         if os.getenv("MASK_FILE") is not None:
             mask_file = os.getenv("MASK_FILE")
-            print(f"Applying mask file {mask_file} to result.nc")
+            logging.info(f"Applying mask file {mask_file} to result.nc")
             subprocess.run(
                 f"cdo -maskregion,{mask_file} {tmpfile_path}result_unmasked.nc {tmpfile_path}result_{file_date}.nc",
                 shell=True,
@@ -226,7 +255,6 @@ def create_warning_status(start_date):
             "language_codes": language_codes,
         }
     )
-    mapfile_outdir = os.getenv("MAPFILE_DIR")
     with open(f"{mapfile_outdir}/NAERSTADMO.map", "w") as f:
         f.write(output)
 
@@ -520,18 +548,11 @@ def prepare_WHS():
 
 def run_cdo(cdo_command):
     try:
-        print(cdo_command)
         subprocess.run(cdo_command, check=True)
         logging.info(f"CDO command {cdo_command[1]} executed successfully.")
     except subprocess.CalledProcessError as e:
         logging.error(f"CDO command {cdo_command[1]} failed:", e)
         quit()
 
-# Don't run if before start_date
-start_date = datetime.strptime(os.getenv("START_DATE"), "%Y-%m-%d")
-if datetime.now() <= start_date:
-    print(f"Today is before the configured start date of {start_date}. Exiting.")
-    exit(0)
-
 # Run model
 create_dataset()

README.md

@@ -24,12 +24,14 @@ The model assumes weather data files named `met_1_0km_nordic-[YYYY-MM-DD].nc` wi
 It is required that you have set the following environment variables:
 
 ```bash
-# Path to this code (HOME_DIR + NAERSTADMO)
-HOME_DIR=/foobar/gridmodels/
+# Path to this code
+HOME_DIR=/foobar/gridmodels/NAERSTADMO/
 # Path to the weather data files. Expecting hourly values in files named met_1_0km_nordic-[YYYY-MM-DD].nc
 WEATHER_DATA_DIR=/foobar/met_1_0km_nordic/2024/
-# Start date for the model 
-START_DATE=2024-05-15
+# Start date for the model (MM-DD)
+RECURRING_START_DATE=05-15
+# Local time zone
+LOCAL_TIMEZONE=Europe/Oslo
 # Use this file to crop the output of the grid
 MASK_FILE=Norge_landomrader.csv
 # Where the GeoTIFF files will be output
@@ -46,21 +48,24 @@ MAPSERVER_LOG_FILE=/foobar2/mapserver/log/NAERSTADMO.log
 MAPSERVER_IMAGE_PATH=/foobar2/mapserver/tmp/
 # Extent of map  (written to mapfile)
 MAPSERVER_EXTENT="-1.5831861262936526 52.4465003983706595 39.2608060398730458 71.7683216082912736"
+# Whether or not to debug log. Default value is False
+DEBUG=False
 ```
 
 ...this is the contents of the `env-sample` file
 
 ```bash
-$ ./run_PSILARTEMP.sh
+$ ./run_NAERSTADMO.sh
 ```
 This creates a Python virtualenv, installs all the Python dependencies, runs the model and stores output in a log file. 
 
-Alternatively, primarily for development purposes, you can run the Python script PSILARTEMP directly:
+The script can be run for a specific year like this:
 
 ```bash
-$ ./PSILARTEMP.py
+$ ./run_NAERSTADMO.sh 2024
 ```
 
+
 #### Viewing the result of the model
 
 **TODO** Add more details

env-sample

 # Use this example to create your own .env file
-# Path to this code (HOME_DIR + NAERSTADMO)
-HOME_DIR=/foobar/gridmodels/
+# Path to this code
+HOME_DIR=/foobar/gridmodels/NAERSTADMO/
 # Path to the weather data files. Expecting hourly values in files named met_1_0km_nordic-[YYYY-MM-DD].nc
 WEATHER_DATA_DIR=/foobar/met_1_0km_nordic/2024/
-# Start date for the model 
-START_DATE=2024-05-15
+# Start date for the model (MM-DD)
+RECURRING_START_DATE=05-15
+# Local time zone
+LOCAL_TIMEZONE=Europe/Oslo
 # Use this file to crop the output of the grid
 MASK_FILE=Norge_landomrader.csv
 # Where the GeoTIFF files will be output
@@ -21,3 +23,5 @@ MAPSERVER_LOG_FILE=/foobar2/mapserver/log/NAERSTADMO.log
 MAPSERVER_IMAGE_PATH=/foobar2/mapserver/tmp/
 # Extent of map  (written to mapfile)
 MAPSERVER_EXTENT="-1.5831861262936526 52.4465003983706595 39.2608060398730458 71.7683216082912736"
+# Whether or not to debug log. Default value is False
+DEBUG=False
\ No newline at end of file

run_NAERSTADMO.sh

@@ -18,9 +18,25 @@
 # Configures environment and logging before running the model
 # @author: Tor-Einar Skog <tor-einar.skog@nibio.no>
 
+validate_year() {
+    if [[ $1 =~ ^[0-9]{4}$ ]]; then
+        return 0
+    else
+        return 1
+    fi
+}
 
+# Check if the year parameter is passed and validate it
+if [ -n "$1" ]; then
+    if validate_year "$1"; then
+        year=$1
+    else
+        echo "Invalid year: $1. Please provide a valid 4-digit year."
+        exit 1
+    fi
+fi
 
-# First: Test that we have CDO and GDAL installed
+# Test that we have CDO and GDAL installed
 if ! command -v cdo &> /dev/null
 then
     echo "ERROR: CDO could not be found. Exiting."
@@ -44,11 +60,10 @@ then
 fi
 
 # Paths to scripts and requirements
-APP_PATH=${HOME_DIR}NAERSTADMO/
-LOG_FILE=${APP_PATH}log/NAERSTADMO.log
-REQUIREMENTS=${APP_PATH}requirements.txt
+LOG_FILE=${HOME_DIR}log/NAERSTADMO.log
+REQUIREMENTS=${HOME_DIR}requirements.txt
 
-cd $APP_PATH
+cd $HOME_DIR
 
 # Create and activate the virtual environment
 python3 -m venv .venv
@@ -56,10 +71,14 @@ python3 -m venv .venv
 python3 -m pip install -q --upgrade pip
 pip install -q -r $REQUIREMENTS
 
-# Run the model
-echo "==== `date`: Running model" &>> "$LOG_FILE"
-python3 $APP_PATH/NAERSTADMO.py &>> "$LOG_FILE"
-echo "==== `date`: DONE running model" &>> "$LOG_FILE"
+if [ -z "${year}" ]; then
+    echo "==== `date`: Run model for current year" >> "$LOG_FILE" 2>&1
+    python3 ${HOME_DIR}NAERSTADMO.py >> "$LOG_FILE" 2>&1
+else
+    echo "==== `date`: Run model for $year" >> "$LOG_FILE" 2>&1
+    python3 ${HOME_DIR}NAERSTADMO.py "$year" >> "$LOG_FILE" 2>&1
+fi
+echo "==== `date`: DONE running model" >> "$LOG_FILE" 2>&1
 
 # Deactivate the virtual environment
 deactivate