diff --git a/NAERSTADMO.py b/NAERSTADMO.py
index a236de59a3e54646636ee5a79240a0467e99b202..d233a19c6d6123af97ca3ef635db5ff34b122cd9 100644
--- a/NAERSTADMO.py
+++ b/NAERSTADMO.py
@@ -20,6 +20,7 @@
# Author: Brita Linnestad <brita.linnestad@nibio.no>
import os
+import sys
import subprocess,glob
from dotenv import load_dotenv
from datetime import datetime, timedelta
@@ -27,38 +28,64 @@ from jinja2 import Environment, FileSystemLoader
import logging
import pytz
+import shutil
import configparser
import netCDF4 as nc
import numpy as np
-logging.basicConfig(level=logging.INFO)
-
load_dotenv()
+
+DEBUG = (
+ False
+ if os.getenv("DEBUG") is None or os.getenv("DEBUG").lower() == "false"
+ else True
+)
+logging.basicConfig(
+ level=logging.DEBUG if DEBUG else logging.INFO,
+ format="%(asctime)s - %(levelname).4s - (%(filename)s:%(lineno)d) - %(message)s",
+)
+
# Get language stuff
config = configparser.ConfigParser()
config.read("NAERSTADMO.cfg")
+local_timezone = pytz.timezone(os.getenv("LOCAL_TIMEZONE"))
+today = datetime.now(local_timezone)
+if len(sys.argv) > 1:
+ year = int(sys.argv[1])
+else:
+ year = today.year
+
+# Don't run if before start_date
+recurring_start_date = os.getenv("RECURRING_START_DATE")
+model_start_date = datetime.strptime(f"{year}-{recurring_start_date}", "%Y-%m-%d")
+if datetime.now() <= model_start_date:
+ logging.error(f"Today is before the configured start date of {model_start_date}. Exiting.")
+ exit(0)
-infile_path = os.getenv("WEATHER_DATA_DIR")
-outfile_path = os.getenv("DATA_DIR")
-outtmp_path = "out/"
-tmpfile_path = "tmp/"
+home_dir = os.getenv('HOME_DIR')
+infile_path = f"{os.getenv('WEATHER_DATA_DIR')}{year}/"
+
+outtmp_path = f"out/{year}/"
+os.makedirs(outtmp_path, exist_ok=True)
+tmpfile_path = f"tmp/{year}/"
+os.makedirs(tmpfile_path, exist_ok=True)
+outfile_path = f"{os.getenv('DATA_DIR')}{year}/"
+os.makedirs(outfile_path, exist_ok=True)
+mapfile_outdir = f"{os.getenv('MAPFILE_DIR')}{year}/"
+os.makedirs(mapfile_outdir, exist_ok=True)
bg_filename = f"{tmpfile_path}background_data.nc"
tmp_filename = f"{tmpfile_path}background_data_tmp.nc"
prepareWHS = f"{tmpfile_path}prepare_WHS.nc"
-utc_offset = "+02:00"
-local_timezone = pytz.timezone("Europe/Oslo")
-
filename = f"{tmpfile_path}weather_data.nc"
-
def create_dataset():
# Find the latest file from previous run to create a start date
last_final_date = None
list_of_files = glob.glob(
- f"{outtmp_path}final_2[0-9][0-9][0-9]-[01][0-9]-[0123][0-9].nc", recursive=True
+ f"{outtmp_path}final_{year}-[01][0-9]-[0123][0-9].nc", recursive=True
)
if list_of_files:
@@ -70,16 +97,16 @@ def create_dataset():
last_final_date = file_date
if last_final_date is None or last_final_date < file_date:
- start_date = datetime.strptime(os.getenv("START_DATE"), "%Y-%m-%d")
+ start_date = model_start_date
if last_final_date is not None:
start_date = datetime.strptime(last_final_date, "%Y-%m-%d") - timedelta(days=4)
- print(
+ logging.info(
f"Last date of final calculations is {last_final_date}. Start date = {start_date}"
)
# Find the set of data to merge and use as input file based on start date
list_weatherdata_files = glob.glob(
- f"{infile_path}/met_1_0km_nordic-2[0-9][0-9][0-9]-[01][0-9]-[0123][0-9].nc"
+ f"{infile_path}/met_1_0km_nordic-{year}-[01][0-9]-[0123][0-9].nc"
)
for file in list_weatherdata_files:
@@ -87,14 +114,14 @@ def create_dataset():
file_date = file_name[
file_name.index("nordic-") + 7 : file_name.index("nordic-") + 17
]
-
end_date = None
end_date = start_date + timedelta(days=5)
if file_date >= start_date.strftime(
"%Y-%m-%d"
) and file_date <= end_date.strftime("%Y-%m-%d"):
- if os.path.exists(f"{tmpfile_path}weather_data.nc") != True:
+ logging.info(f"Work on date {file_date}")
+ if not os.path.exists(f"{tmpfile_path}weather_data.nc"):
subprocess.run(f"cp {file} {tmpfile_path}weather_data.nc", shell=True)
else:
subprocess.run(
@@ -109,7 +136,7 @@ def create_dataset():
# Ensure that model is not run if weather data is not available
if not os.path.exists(f"{tmpfile_path}weather_data.nc"):
- print(f"{tmpfile_path}weather_data.nc does not exist. Exit.")
+ logging.error(f"{tmpfile_path}weather_data.nc does not exist. Exit.")
return
subprocess.run(f"rm {outtmp_path}final_*", shell=True)
@@ -146,7 +173,7 @@ def create_warning_status(start_date):
# Env variable MASK_FILE must be set
if os.getenv("MASK_FILE") is not None:
mask_file = os.getenv("MASK_FILE")
- print(f"Applying mask file {mask_file} to result.nc")
+ logging.info(f"Applying mask file {mask_file} to result.nc")
subprocess.run(
f"cdo -maskregion,{mask_file} {tmpfile_path}result_unmasked.nc {tmpfile_path}result_{file_date}.nc",
shell=True,
@@ -226,9 +253,15 @@ def create_warning_status(start_date):
"language_codes": language_codes,
}
)
- mapfile_outdir = os.getenv("MAPFILE_DIR")
- with open(f"{mapfile_outdir}/NAERSTADMO.map", "w") as f:
+ with open(f"{mapfile_outdir}NAERSTADMO.map", "w") as f:
f.write(output)
+
+ query_template = os.path.join(home_dir, "mapfile/query_template.xml")
+ query_template_IR = os.path.join(home_dir, "mapfile/query_template_IR.xml")
+ query_template_RISK = os.path.join(home_dir, "mapfile/query_template_RISK.xml")
+ shutil.copy(query_template, mapfile_outdir)
+ shutil.copy(query_template_IR, mapfile_outdir)
+ shutil.copy(query_template_RISK, mapfile_outdir)
subprocess.run(f"rm {tmpfile_path}*", shell=True)
@@ -329,6 +362,8 @@ def create_matrix():
def create_WHS_WH(time):
+ logging.info(f"Create WHS_WH for {time}")
+
for j in range(5):
subprocess.run(
f"cdo -O -chname,WVD,WVDLastHour -selname,WVD -seltimestep,{str(time+j+1)} {tmpfile_path}prepare_WHS.nc {tmpfile_path}WVD_LastHourtmp.nc",
@@ -426,6 +461,7 @@ def create_WHS_WH(time):
def create_VRS(time):
+ logging.info("Create VRS")
subprocess.run(
f'cdo -aexpr,"RTA=(((Q0-Q0LastHour)>7)?1:0)+((WVD-WVDLastHour)>=15?1:0);IRTA=(1-(BT/80));SFRS=((1-(Q0-270)/540)/1.5);" {tmpfile_path}this_hour.nc {tmpfile_path}this_hr.nc',
shell=True,
@@ -438,11 +474,12 @@ def create_VRS(time):
def create_TSSH_VAS(time):
+ logging.info("Create TSSH_VAS")
subprocess.run(
f'cdo -O -aexpr,"TSSH=((HH1==1)?HH2*(TSSHLastHour+TM):HH2*0.75*TSSHLastHour);SPH=((TSSH>87)?1:0);VAS=((VASLastHour*0.95*(1-((WVD-220)/6000))+SPH)/(1+0.3*RR))" {tmpfile_path}this_hour.nc {tmpfile_path}this_hour_tmp.nc',
shell=True,
)
- os.rename(src="tmp/this_hour_tmp.nc", dst="tmp/this_hour.nc")
+ os.rename(src=f"{tmpfile_path}this_hour_tmp.nc", dst=f"{tmpfile_path}this_hour.nc")
def create_HH1_HH2(time):
@@ -454,6 +491,7 @@ def create_HH1_HH2(time):
def create_saturation():
+ logging.info("Create saturation")
# This is fixed for all hours and should be available in the background data
expr = "aexpr,SP=(0.61078*exp(17.269*TM/(TM+237.3)))"
cdo_command = [
@@ -466,6 +504,7 @@ def create_saturation():
def create_pressure():
+ logging.info("Create pressure")
# This is fixed for all hours and should be available in the background data
expr = "aexpr,PP=UM*SP/100"
cdo_command = [
@@ -479,6 +518,7 @@ def create_pressure():
def create_wvd():
+ logging.info("Create wvd")
# This is fixed for all hours and should be available in the background data
expr = "aexpr,WVD=(SP-PP)*1000"
cdo_command = [
@@ -493,7 +533,7 @@ def create_wvd():
def create_BT():
# BT is not available in the dataset and need to be calculted and added to background_data.nc
-
+ logging.info("Create BT")
expr = "aexpr,BT=((RR > 0 || (((100-UM)/100)*6.112*exp(17.67*TM/(TM+243.5))) < 2)) ? 60 : 0"
cdo_command = [
"cdo",
@@ -507,6 +547,7 @@ def create_BT():
def prepare_WHS():
# system("cdo selname,RR,BT,WVD background_data.nc prepareWHS.nc")
+ logging.info("Prepare WHS")
my_variables = ["TM", "RR", "BT", "WVD"]
variable_list = ",".join(my_variables)
cdo_command = [
@@ -520,18 +561,11 @@ def prepare_WHS():
def run_cdo(cdo_command):
try:
- print(cdo_command)
subprocess.run(cdo_command, check=True)
logging.info(f"CDO command {cdo_command[1]} executed successfully.")
except subprocess.CalledProcessError as e:
logging.error(f"CDO command {cdo_command[1]} failed:", e)
quit()
-# Don't run if before start_date
-start_date = datetime.strptime(os.getenv("START_DATE"), "%Y-%m-%d")
-if datetime.now() <= start_date:
- print(f"Today is before the configured start date of {start_date}. Exiting.")
- exit(0)
-
# Run model
create_dataset()
diff --git a/README.md b/README.md
index c7d2f2c620e3d9c3b420a2cece40e38838fbb19e..2eb0032134faab283b7ca757e1797fb013991aad 100644
--- a/README.md
+++ b/README.md
@@ -24,12 +24,14 @@ The model assumes weather data files named `met_1_0km_nordic-[YYYY-MM-DD].nc` wi
It is required that you have set the following environment variables:
```bash
-# Path to this code (HOME_DIR + NAERSTADMO)
-HOME_DIR=/foobar/gridmodels/
+# Path to this code
+HOME_DIR=/foobar/gridmodels/NAERSTADMO/
# Path to the weather data files. Expecting hourly values in files named met_1_0km_nordic-[YYYY-MM-DD].nc
WEATHER_DATA_DIR=/foobar/met_1_0km_nordic/2024/
-# Start date for the model
-START_DATE=2024-05-15
+# Start date for the model (MM-DD)
+RECURRING_START_DATE=05-15
+# Local time zone
+LOCAL_TIMEZONE=Europe/Oslo
# Use this file to crop the output of the grid
MASK_FILE=Norge_landomrader.csv
# Where the GeoTIFF files will be output
@@ -46,21 +48,24 @@ MAPSERVER_LOG_FILE=/foobar2/mapserver/log/NAERSTADMO.log
MAPSERVER_IMAGE_PATH=/foobar2/mapserver/tmp/
# Extent of map (written to mapfile)
MAPSERVER_EXTENT="-1.5831861262936526 52.4465003983706595 39.2608060398730458 71.7683216082912736"
+# Whether or not to debug log. Default value is False
+DEBUG=False
```
...this is the contents of the `env-sample` file
```bash
-$ ./run_PSILARTEMP.sh
+$ ./run_NAERSTADMO.sh
```
This creates a Python virtualenv, installs all the Python dependencies, runs the model and stores output in a log file.
-Alternatively, primarily for development purposes, you can run the Python script PSILARTEMP directly:
+The script can be run for a specific year like this:
```bash
-$ ./PSILARTEMP.py
+$ ./run_NAERSTADMO.sh 2024
```
+
#### Viewing the result of the model
**TODO** Add more details
diff --git a/env-sample b/env-sample
index d96ced325ca284ccc0f28227433f9bf67239cb8b..89a44884a4ee6485e202c8305fed86413f887a43 100644
--- a/env-sample
+++ b/env-sample
@@ -1,10 +1,12 @@
# Use this example to create your own .env file
-# Path to this code (HOME_DIR + NAERSTADMO)
-HOME_DIR=/foobar/gridmodels/
+# Path to this code
+HOME_DIR=/foobar/gridmodels/NAERSTADMO/
# Path to the weather data files. Expecting hourly values in files named met_1_0km_nordic-[YYYY-MM-DD].nc
WEATHER_DATA_DIR=/foobar/met_1_0km_nordic/2024/
-# Start date for the model
-START_DATE=2024-05-15
+# Start date for the model (MM-DD)
+RECURRING_START_DATE=05-15
+# Local time zone
+LOCAL_TIMEZONE=Europe/Oslo
# Use this file to crop the output of the grid
MASK_FILE=Norge_landomrader.csv
# Where the GeoTIFF files will be output
@@ -21,3 +23,5 @@ MAPSERVER_LOG_FILE=/foobar2/mapserver/log/NAERSTADMO.log
MAPSERVER_IMAGE_PATH=/foobar2/mapserver/tmp/
# Extent of map (written to mapfile)
MAPSERVER_EXTENT="-1.5831861262936526 52.4465003983706595 39.2608060398730458 71.7683216082912736"
+# Whether or not to debug log. Default value is False
+DEBUG=False
\ No newline at end of file
diff --git a/run_NAERSTADMO.sh b/run_NAERSTADMO.sh
index f25253d424ed5b1c01e512781d9cf57c898bfe42..9341470430f8c6b66af90e7830e0064203e938a1 100755
--- a/run_NAERSTADMO.sh
+++ b/run_NAERSTADMO.sh
@@ -18,9 +18,25 @@
# Configures environment and logging before running the model
# @author: Tor-Einar Skog <tor-einar.skog@nibio.no>
+validate_year() {
+ if [[ $1 =~ ^[0-9]{4}$ ]]; then
+ return 0
+ else
+ return 1
+ fi
+}
+# Check if the year parameter is passed and validate it
+if [ -n "$1" ]; then
+ if validate_year "$1"; then
+ year=$1
+ else
+ echo "Invalid year: $1. Please provide a valid 4-digit year."
+ exit 1
+ fi
+fi
-# First: Test that we have CDO and GDAL installed
+# Test that we have CDO and GDAL installed
if ! command -v cdo &> /dev/null
then
echo "ERROR: CDO could not be found. Exiting."
@@ -44,11 +60,10 @@ then
fi
# Paths to scripts and requirements
-APP_PATH=${HOME_DIR}NAERSTADMO/
-LOG_FILE=${APP_PATH}log/NAERSTADMO.log
-REQUIREMENTS=${APP_PATH}requirements.txt
+LOG_FILE=${HOME_DIR}log/NAERSTADMO.log
+REQUIREMENTS=${HOME_DIR}requirements.txt
-cd $APP_PATH
+cd $HOME_DIR
# Create and activate the virtual environment
python3 -m venv .venv
@@ -56,10 +71,14 @@ python3 -m venv .venv
python3 -m pip install -q --upgrade pip
pip install -q -r $REQUIREMENTS
-# Run the model
-echo "==== `date`: Running model" &>> "$LOG_FILE"
-python3 $APP_PATH/NAERSTADMO.py &>> "$LOG_FILE"
-echo "==== `date`: DONE running model" &>> "$LOG_FILE"
+if [ -z "${year}" ]; then
+ echo "==== `date`: Run model for current year" >> "$LOG_FILE" 2>&1
+ python3 ${HOME_DIR}NAERSTADMO.py >> "$LOG_FILE" 2>&1
+else
+ echo "==== `date`: Run model for $year" >> "$LOG_FILE" 2>&1
+ python3 ${HOME_DIR}NAERSTADMO.py "$year" >> "$LOG_FILE" 2>&1
+fi
+echo "==== `date`: DONE running model" >> "$LOG_FILE" 2>&1
# Deactivate the virtual environment
deactivate