Merge branch 'main' into release

NAERSTADMO.py

@@ -20,6 +20,7 @@
 # Author: Brita Linnestad <brita.linnestad@nibio.no>
 
 import os
+import sys
 import subprocess,glob
 from dotenv import load_dotenv
 from datetime import datetime, timedelta
@@ -27,38 +28,64 @@ from jinja2 import Environment, FileSystemLoader
 
 import logging
 import pytz
+import shutil
 import configparser
 import netCDF4 as nc
 import numpy as np
 
-logging.basicConfig(level=logging.INFO)
-
 load_dotenv()
+
+DEBUG = (
+    False
+    if os.getenv("DEBUG") is None or os.getenv("DEBUG").lower() == "false"
+    else True
+)
+logging.basicConfig(
+    level=logging.DEBUG if DEBUG else logging.INFO,
+    format="%(asctime)s - %(levelname).4s - (%(filename)s:%(lineno)d) - %(message)s",
+)
+
 # Get language stuff
 config = configparser.ConfigParser()
 config.read("NAERSTADMO.cfg")
 
+local_timezone = pytz.timezone(os.getenv("LOCAL_TIMEZONE"))
+today = datetime.now(local_timezone)
+if len(sys.argv) > 1:
+    year = int(sys.argv[1])
+else:
+    year = today.year
+
+# Don't run if before start_date
+recurring_start_date = os.getenv("RECURRING_START_DATE")
+model_start_date = datetime.strptime(f"{year}-{recurring_start_date}", "%Y-%m-%d")
+if datetime.now() <= model_start_date:
+    logging.error(f"Today is before the configured start date of {model_start_date}. Exiting.")
+    exit(0)
 
-infile_path = os.getenv("WEATHER_DATA_DIR")
-outfile_path = os.getenv("DATA_DIR")
-outtmp_path = "out/"
-tmpfile_path = "tmp/"
+home_dir = os.getenv('HOME_DIR')
+infile_path = f"{os.getenv('WEATHER_DATA_DIR')}{year}/"
+
+outtmp_path = f"out/{year}/"
+os.makedirs(outtmp_path, exist_ok=True)
+tmpfile_path = f"tmp/{year}/"
+os.makedirs(tmpfile_path, exist_ok=True)
+outfile_path = f"{os.getenv('DATA_DIR')}{year}/"
+os.makedirs(outfile_path, exist_ok=True)
+mapfile_outdir = f"{os.getenv('MAPFILE_DIR')}{year}/"
+os.makedirs(mapfile_outdir, exist_ok=True)
 
 bg_filename = f"{tmpfile_path}background_data.nc"
 tmp_filename = f"{tmpfile_path}background_data_tmp.nc"
 prepareWHS = f"{tmpfile_path}prepare_WHS.nc"
 
-utc_offset = "+02:00"
-local_timezone = pytz.timezone("Europe/Oslo")
-
 filename = f"{tmpfile_path}weather_data.nc"
 
-
 def create_dataset():
     # Find the latest file from previous run to create a start date
     last_final_date = None
     list_of_files = glob.glob(
-        f"{outtmp_path}final_2[0-9][0-9][0-9]-[01][0-9]-[0123][0-9].nc", recursive=True
+        f"{outtmp_path}final_{year}-[01][0-9]-[0123][0-9].nc", recursive=True
     )
 
     if list_of_files:
@@ -70,16 +97,16 @@ def create_dataset():
         last_final_date = file_date
 
     if last_final_date is None or last_final_date < file_date:
-        start_date = datetime.strptime(os.getenv("START_DATE"), "%Y-%m-%d")
+        start_date = model_start_date
     if last_final_date is not None:
         start_date = datetime.strptime(last_final_date, "%Y-%m-%d") - timedelta(days=4)
-    print(
+    logging.info(
         f"Last date of final calculations is {last_final_date}. Start date = {start_date}"
     )
 
     # Find the set of data to merge and use as input file based on start date
     list_weatherdata_files = glob.glob(
-        f"{infile_path}/met_1_0km_nordic-2[0-9][0-9][0-9]-[01][0-9]-[0123][0-9].nc"
+        f"{infile_path}/met_1_0km_nordic-{year}-[01][0-9]-[0123][0-9].nc"
     )
 
     for file in list_weatherdata_files:
@@ -87,14 +114,14 @@ def create_dataset():
         file_date = file_name[
             file_name.index("nordic-") + 7 : file_name.index("nordic-") + 17
         ]
-
         end_date = None
         end_date = start_date + timedelta(days=5)
 
         if file_date >= start_date.strftime(
             "%Y-%m-%d"
         ) and file_date <= end_date.strftime("%Y-%m-%d"):
-            if os.path.exists(f"{tmpfile_path}weather_data.nc") != True:
+            logging.info(f"Work on date {file_date}")
+            if not os.path.exists(f"{tmpfile_path}weather_data.nc"):
                 subprocess.run(f"cp {file} {tmpfile_path}weather_data.nc", shell=True)
             else:
                 subprocess.run(
@@ -109,7 +136,7 @@ def create_dataset():
 
     # Ensure that model is not run if weather data is not available
     if not os.path.exists(f"{tmpfile_path}weather_data.nc"):
-        print(f"{tmpfile_path}weather_data.nc does not exist. Exit.")
+        logging.error(f"{tmpfile_path}weather_data.nc does not exist. Exit.")
         return
 
     subprocess.run(f"rm {outtmp_path}final_*", shell=True)
@@ -146,7 +173,7 @@ def create_warning_status(start_date):
         # Env variable MASK_FILE must be set
         if os.getenv("MASK_FILE") is not None:
             mask_file = os.getenv("MASK_FILE")
-            print(f"Applying mask file {mask_file} to result.nc")
+            logging.info(f"Applying mask file {mask_file} to result.nc")
             subprocess.run(
                 f"cdo -maskregion,{mask_file} {tmpfile_path}result_unmasked.nc {tmpfile_path}result_{file_date}.nc",
                 shell=True,
@@ -226,9 +253,15 @@ def create_warning_status(start_date):
             "language_codes": language_codes,
         }
     )
-    mapfile_outdir = os.getenv("MAPFILE_DIR")
-    with open(f"{mapfile_outdir}/NAERSTADMO.map", "w") as f:
+    with open(f"{mapfile_outdir}NAERSTADMO.map", "w") as f:
         f.write(output)
+    
+    query_template = os.path.join(home_dir, "mapfile/query_template.xml")
+    query_template_IR = os.path.join(home_dir, "mapfile/query_template_IR.xml")
+    query_template_RISK = os.path.join(home_dir, "mapfile/query_template_RISK.xml")
+    shutil.copy(query_template, mapfile_outdir)
+    shutil.copy(query_template_IR, mapfile_outdir)
+    shutil.copy(query_template_RISK, mapfile_outdir)
 
     subprocess.run(f"rm {tmpfile_path}*", shell=True)
 
@@ -329,6 +362,8 @@ def create_matrix():
 
 
 def create_WHS_WH(time):
+    logging.info(f"Create WHS_WH for {time}")
+
     for j in range(5):
         subprocess.run(
             f"cdo -O -chname,WVD,WVDLastHour -selname,WVD -seltimestep,{str(time+j+1)} {tmpfile_path}prepare_WHS.nc {tmpfile_path}WVD_LastHourtmp.nc",
@@ -426,6 +461,7 @@ def create_WHS_WH(time):
 
 
 def create_VRS(time):
+    logging.info("Create VRS")
     subprocess.run(
         f'cdo -aexpr,"RTA=(((Q0-Q0LastHour)>7)?1:0)+((WVD-WVDLastHour)>=15?1:0);IRTA=(1-(BT/80));SFRS=((1-(Q0-270)/540)/1.5);" {tmpfile_path}this_hour.nc {tmpfile_path}this_hr.nc',
         shell=True,
@@ -438,11 +474,12 @@ def create_VRS(time):
 
 
 def create_TSSH_VAS(time):
+    logging.info("Create TSSH_VAS")
     subprocess.run(
         f'cdo -O -aexpr,"TSSH=((HH1==1)?HH2*(TSSHLastHour+TM):HH2*0.75*TSSHLastHour);SPH=((TSSH>87)?1:0);VAS=((VASLastHour*0.95*(1-((WVD-220)/6000))+SPH)/(1+0.3*RR))" {tmpfile_path}this_hour.nc {tmpfile_path}this_hour_tmp.nc',
         shell=True,
     )
-    os.rename(src="tmp/this_hour_tmp.nc", dst="tmp/this_hour.nc")
+    os.rename(src=f"{tmpfile_path}this_hour_tmp.nc", dst=f"{tmpfile_path}this_hour.nc")
 
 
 def create_HH1_HH2(time):
@@ -454,6 +491,7 @@ def create_HH1_HH2(time):
 
 
 def create_saturation():
+    logging.info("Create saturation")
     # This is fixed for all hours and should be available in the background data
     expr = "aexpr,SP=(0.61078*exp(17.269*TM/(TM+237.3)))"
     cdo_command = [
@@ -466,6 +504,7 @@ def create_saturation():
 
 
 def create_pressure():
+    logging.info("Create pressure")
     # This is fixed for all hours and should be available in the background data
     expr = "aexpr,PP=UM*SP/100"
     cdo_command = [
@@ -479,6 +518,7 @@ def create_pressure():
 
 
 def create_wvd():
+    logging.info("Create wvd")
     # This is fixed for all hours and should be available in the background data
     expr = "aexpr,WVD=(SP-PP)*1000"
     cdo_command = [
@@ -493,7 +533,7 @@ def create_wvd():
 
 def create_BT():
     # BT is not available in the dataset and need to be calculted and added to background_data.nc
-
+    logging.info("Create BT")
     expr = "aexpr,BT=((RR > 0 || (((100-UM)/100)*6.112*exp(17.67*TM/(TM+243.5))) < 2)) ? 60 : 0"
     cdo_command = [
         "cdo",
@@ -507,6 +547,7 @@ def create_BT():
 
 def prepare_WHS():
     # system("cdo selname,RR,BT,WVD background_data.nc prepareWHS.nc")
+    logging.info("Prepare WHS")
     my_variables = ["TM", "RR", "BT", "WVD"]
     variable_list = ",".join(my_variables)
     cdo_command = [
@@ -520,18 +561,11 @@ def prepare_WHS():
 
 def run_cdo(cdo_command):
     try:
-        print(cdo_command)
         subprocess.run(cdo_command, check=True)
         logging.info(f"CDO command {cdo_command[1]} executed successfully.")
     except subprocess.CalledProcessError as e:
         logging.error(f"CDO command {cdo_command[1]} failed:", e)
         quit()
 
-# Don't run if before start_date
-start_date = datetime.strptime(os.getenv("START_DATE"), "%Y-%m-%d")
-if datetime.now() <= start_date:
-    print(f"Today is before the configured start date of {start_date}. Exiting.")
-    exit(0)
-
 # Run model
 create_dataset()

README.md

@@ -24,12 +24,14 @@ The model assumes weather data files named `met_1_0km_nordic-[YYYY-MM-DD].nc` wi
 It is required that you have set the following environment variables:
 
 ```bash
-# Path to this code (HOME_DIR + NAERSTADMO)
-HOME_DIR=/foobar/gridmodels/
+# Path to this code
+HOME_DIR=/foobar/gridmodels/NAERSTADMO/
 # Path to the weather data files. Expecting hourly values in files named met_1_0km_nordic-[YYYY-MM-DD].nc
 WEATHER_DATA_DIR=/foobar/met_1_0km_nordic/2024/
-# Start date for the model 
-START_DATE=2024-05-15
+# Start date for the model (MM-DD)
+RECURRING_START_DATE=05-15
+# Local time zone
+LOCAL_TIMEZONE=Europe/Oslo
 # Use this file to crop the output of the grid
 MASK_FILE=Norge_landomrader.csv
 # Where the GeoTIFF files will be output
@@ -46,21 +48,24 @@ MAPSERVER_LOG_FILE=/foobar2/mapserver/log/NAERSTADMO.log
 MAPSERVER_IMAGE_PATH=/foobar2/mapserver/tmp/
 # Extent of map  (written to mapfile)
 MAPSERVER_EXTENT="-1.5831861262936526 52.4465003983706595 39.2608060398730458 71.7683216082912736"
+# Whether or not to debug log. Default value is False
+DEBUG=False
 ```
 
 ...this is the contents of the `env-sample` file
 
 ```bash
-$ ./run_PSILARTEMP.sh
+$ ./run_NAERSTADMO.sh
 ```
 This creates a Python virtualenv, installs all the Python dependencies, runs the model and stores output in a log file. 
 
-Alternatively, primarily for development purposes, you can run the Python script PSILARTEMP directly:
+The script can be run for a specific year like this:
 
 ```bash
-$ ./PSILARTEMP.py
+$ ./run_NAERSTADMO.sh 2024
 ```
 
+
 #### Viewing the result of the model
 
 **TODO** Add more details

env-sample

 # Use this example to create your own .env file
-# Path to this code (HOME_DIR + NAERSTADMO)
-HOME_DIR=/foobar/gridmodels/
+# Path to this code
+HOME_DIR=/foobar/gridmodels/NAERSTADMO/
 # Path to the weather data files. Expecting hourly values in files named met_1_0km_nordic-[YYYY-MM-DD].nc
 WEATHER_DATA_DIR=/foobar/met_1_0km_nordic/2024/
-# Start date for the model 
-START_DATE=2024-05-15
+# Start date for the model (MM-DD)
+RECURRING_START_DATE=05-15
+# Local time zone
+LOCAL_TIMEZONE=Europe/Oslo
 # Use this file to crop the output of the grid
 MASK_FILE=Norge_landomrader.csv
 # Where the GeoTIFF files will be output
@@ -21,3 +23,5 @@ MAPSERVER_LOG_FILE=/foobar2/mapserver/log/NAERSTADMO.log
 MAPSERVER_IMAGE_PATH=/foobar2/mapserver/tmp/
 # Extent of map  (written to mapfile)
 MAPSERVER_EXTENT="-1.5831861262936526 52.4465003983706595 39.2608060398730458 71.7683216082912736"
+# Whether or not to debug log. Default value is False
+DEBUG=False
\ No newline at end of file

run_NAERSTADMO.sh

@@ -18,9 +18,25 @@
 # Configures environment and logging before running the model
 # @author: Tor-Einar Skog <tor-einar.skog@nibio.no>
 
+validate_year() {
+    if [[ $1 =~ ^[0-9]{4}$ ]]; then
+        return 0
+    else
+        return 1
+    fi
+}
 
+# Check if the year parameter is passed and validate it
+if [ -n "$1" ]; then
+    if validate_year "$1"; then
+        year=$1
+    else
+        echo "Invalid year: $1. Please provide a valid 4-digit year."
+        exit 1
+    fi
+fi
 
-# First: Test that we have CDO and GDAL installed
+# Test that we have CDO and GDAL installed
 if ! command -v cdo &> /dev/null
 then
     echo "ERROR: CDO could not be found. Exiting."
@@ -44,11 +60,10 @@ then
 fi
 
 # Paths to scripts and requirements
-APP_PATH=${HOME_DIR}NAERSTADMO/
-LOG_FILE=${APP_PATH}log/NAERSTADMO.log
-REQUIREMENTS=${APP_PATH}requirements.txt
+LOG_FILE=${HOME_DIR}log/NAERSTADMO.log
+REQUIREMENTS=${HOME_DIR}requirements.txt
 
-cd $APP_PATH
+cd $HOME_DIR
 
 # Create and activate the virtual environment
 python3 -m venv .venv
@@ -56,10 +71,14 @@ python3 -m venv .venv
 python3 -m pip install -q --upgrade pip
 pip install -q -r $REQUIREMENTS
 
-# Run the model
-echo "==== `date`: Running model" &>> "$LOG_FILE"
-python3 $APP_PATH/NAERSTADMO.py &>> "$LOG_FILE"
-echo "==== `date`: DONE running model" &>> "$LOG_FILE"
+if [ -z "${year}" ]; then
+    echo "==== `date`: Run model for current year" >> "$LOG_FILE" 2>&1
+    python3 ${HOME_DIR}NAERSTADMO.py >> "$LOG_FILE" 2>&1
+else
+    echo "==== `date`: Run model for $year" >> "$LOG_FILE" 2>&1
+    python3 ${HOME_DIR}NAERSTADMO.py "$year" >> "$LOG_FILE" 2>&1
+fi
+echo "==== `date`: DONE running model" >> "$LOG_FILE" 2>&1
 
 # Deactivate the virtual environment
 deactivate