feat: Improve end date logic, provide year as param to script

ADASMELIAE.py

@@ -128,7 +128,8 @@ def remove_temporary_files():
 
 # Remove previously calculated results
 def remove_old_results(tif_dir, mapfile_dir):
-    logging.info(f"Remove previously calculated results from {tif_dir} and {mapfile_dir}")
+    logging.info(f"Remove previously calculated results from {tif_dir}")
+    logging.info(f"Remove previously calculated results from {mapfile_dir}")
     if glob.glob(f"{tif_dir}*.tif"):
         run_command(command=f"rm {tif_dir}*.tif")
     if glob.glob(f"rm {mapfile_dir}*.map"):
@@ -158,10 +159,12 @@ def split_by_date(input_file, prefix):
 if __name__ == "__main__":
     start_time = time.time()  # For logging execution time
     today = datetime.now().date()
+    cron_run = True
 
     # Check if a year argument is provided
     if len(sys.argv) > 1:
         year = int(sys.argv[1])
+        cron_run = False
     else:
         today = datetime.now().date()
         year = today.year
@@ -180,11 +183,16 @@ if __name__ == "__main__":
     model_start_date = datetime.strptime(f"{year}-{start_MM_DD}", "%Y-%m-%d").date()
     model_end_date = datetime.strptime(f"{year}-{end_MM_DD}", "%Y-%m-%d").date()
 
+    # Ensure that model is not run after model period (for cron runs)
+    if cron_run and today > model_end_date:
+        logging.error(f"Cron run and today is after model period end {model_end_date}. Quit.")
+        sys.exit()
+
     start_date = model_start_date
-    if year == today.year: # Run for current year
+    if year == today.year and today + timedelta(days=2) <= model_end_date:
         end_date = today + timedelta(days=2)
-    else: # Run for previous year
-        end_date = model_end_date + timedelta(days=2)
+    else:
+        end_date = model_end_date
 
     logging.info(
         f"Model start date {model_start_date}. Model end date {model_end_date}."
@@ -194,11 +202,6 @@ if __name__ == "__main__":
         f"Attempt to run model {model_id} for year {year}, start date {start_date} and end date {end_date}"
     )
 
-    # Ensure that model is not run after model run period (include 2 extra days at the end to replace forecast with reanalysed)
-    if model_end_date + timedelta(days=2) < end_date:
-        logging.error(f"{end_date} is after model period end (+ 2 days) {model_end_date + timedelta(days=2)}. Quit.")
-        sys.exit()
-
     # Ensure that model is not run before model run period
     if today < start_date:
         logging.error("Model period not started. Quit.")
@@ -272,6 +275,11 @@ if __name__ == "__main__":
         filename = os.path.basename(file)
         current_file_date_str = filename.split("_")[1].split(".")[0]
         current_file_date = datetime.strptime(current_file_date_str, "%Y-%m-%d").date()
+
+        if current_file_date > model_end_date:
+            logging.debug(f"Should not calculate for {current_file_date} (after model period).")
+            break
+
         with open("/dev/null", "w") as devnull:
             warning_status_lcc = f"{tmp_dir}result_WARNING_STATUS_{current_file_date_str}_lcc.tif"
             result_lcc = f"{tmp_dir}result_{current_file_date_str}_lcc.tif"
@@ -280,15 +288,6 @@ if __name__ == "__main__":
 
             logging.debug(f"Calculate result for {current_file_date_str}")
 
-            # Ensure warning status 0 for all points on first day after model end date
-            if current_file_date > model_end_date:
-                tmp_file = file + "tmp"
-                os.rename(file, tmp_file)
-                run_command(
-                    command=f'cdo -s -aexpr,"WARNING_STATUS = 0" {tmp_file} {file}',
-                )
-                os.remove(tmp_file)
-
             # For warning status:
             run_command(
                 command=f"gdal_translate -ot Int16 -of GTiff  NETCDF:'{file}':WARNING_STATUS {warning_status_lcc}",
@@ -310,10 +309,6 @@ if __name__ == "__main__":
                 stdout=devnull,
             )
 
-            # Should not generate grey files for more than one day after model end date
-            if current_file_date > model_end_date:
-                break
-
     # Generate mapfile
     # Building data sets for language specific legends
     languages = []

README.md

@@ -65,18 +65,19 @@ MAPSERVER_IMAGE_PATH=/foo/mapserver/image/
 MAPSERVER_EXTENT="-23.5 29.5 62.5 70.5"
 # OPTIONAL Path to CSV file with polygons for masking result. Default: no masking performed
 MASK_FILE=europe_coastline.csv
-# OPTIONAL If model should run for a previous year. Default: current year
-YEAR=2024
 ```
 
 ...this is the contents of the `env-sample` file. 
 
+Run the script with the year as parameter:
 
 ```bash
-$ ./run_ADASMELIAE.sh
+$ ./run_ADASMELIAE.sh 2024
 ```
 
-This creates a Python virtualenv, installs all the Python dependencies, runs the model and stores output in a log file.
+Running the script without the parameter defaults to current year, and is primarily meant for the daily cron job.
+
+The script creates a Python virtualenv, installs all the Python dependencies, runs the model and stores output in a log file.
 
 All intermediary files are stored in the `TMP_DIR` folder, and they are all deleted when the model is done calculating. The GeoTIFF files are stored in the `RESULT_TIF_DIR` folder, and the generated mapfile is stored in the `RESULT_MAPFILE_DIR` folder
 

env-sample

@@ -7,7 +7,7 @@ MODEL_ID="ADASMELIAE"
 # The start date MM-DD of the model
 START_DATE_MM_DD=03-01
 # The end date MM-DD of the model
-END_DATE_MM_DD=04-01
+END_DATE_MM_DD=07-01
 # Where your script resides
 HOME_DIR=/foo/home/ADASMELIAE/
 # Path to folder for intermediary results
@@ -38,5 +38,3 @@ MAPSERVER_IMAGE_PATH=/foo/mapserver/image/
 MAPSERVER_EXTENT="-23.5 29.5 62.5 70.5"
 # OPTIONAL Path to CSV file with polygons for masking result. Default: no masking performed
 MASK_FILE=europe_coastline.csv
-# OPTIONAL If model should run for a previous year. Default: current year
-YEAR=2024

run_ADASMELIAE.sh

@@ -17,6 +17,23 @@
 
 # Configures environment and logging before running the model
 
+validate_year() {
+    if [[ $1 =~ ^[0-9]{4}$ ]]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+# Check if the year parameter is passed and validate it
+if [ -n "$1" ]; then
+    if validate_year "$1"; then
+        year=$1
+    else
+        echo "Invalid year: $1. Please provide a valid 4-digit year."
+        exit 1
+    fi
+fi
 
 # First: Test that we have CDO and GDAL installed
 if ! command -v cdo &> /dev/null
@@ -42,12 +59,6 @@ then
     exit
 fi
 
-if [ -z "${YEAR}" ]
-then
-    YEAR=$(date +%Y)
-    echo "YEAR is not set. Using the current year: ${YEAR}"
-fi
-
 # Paths to scripts and requirements
 LOG_FILE=${HOME_DIR}log/ADASMELIAE.log
 REQUIREMENTS=${HOME_DIR}requirements.txt
@@ -65,8 +76,11 @@ if [ -n "$VIRTUAL_ENV" ]; then
     python3 -m pip install -q --upgrade pip
     pip install -q -r $REQUIREMENTS
 
-    # Run the model
-    python3 ${HOME_DIR}ADASMELIAE.py "$YEAR" >> "$LOG_FILE" 2>&1
+    if [ -z "${year}" ]; then
+        python3 ${HOME_DIR}ADASMELIAE.py >> "$LOG_FILE" 2>&1
+    else
+        python3 ${HOME_DIR}ADASMELIAE.py "$year" >> "$LOG_FILE" 2>&1
+    fi
 
     # Deactivate the virtual environment
     deactivate