diff --git a/.gitignore b/.gitignore
index f190cfdad2843a02c5b907feefadfadc649a64bb..3025c3d0ca99a59f3ddc953932e8468f22c01f18 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,6 @@ in
out
log
tmp
+.pytest_cache
+run_cleanup.sh
+.idea
\ No newline at end of file
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 503118a63d53fd11bb65cd93fed939a273d31e8f..6063682dc33dc1dc5356ddaf426dc2397caec9e3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,6 +8,8 @@ variables:
deploy_to_remote_server:
stage: deploy
+ tags:
+ - python
before_script:
- apk add --no-cache rsync openssh
- mkdir -p ~/.ssh
@@ -21,23 +23,26 @@ deploy_to_remote_server:
# Create .env file
- echo "MODEL_ID=$MODEL_ID" > $ENV_FILE
- - echo "START_DATE=$START_DATE" >> $ENV_FILE
- - echo "END_DATE=$END_DATE" >> $ENV_FILE
+ - echo "START_DATE_MM_DD=$START_DATE_MM_DD" >> $ENV_FILE
+ - echo "END_DATE_MM_DD=$END_DATE_MM_DD" >> $ENV_FILE
- echo "HOME_DIR=$HOME_DIR" >> $ENV_FILE
+ - echo "TMP_DIR=$TMP_DIR" >> $ENV_FILE
- echo "WEATHER_DATA_DIR=$WEATHER_DATA_DIR" >> $ENV_FILE
- - echo "WEATHER_DATA_FILENAME_PATTERN=$WEATHER_DATA_FILENAME_PATTERN" >> $ENV_FILE
- echo "WEATHER_DATA_FILENAME_DATEFORMAT=$WEATHER_DATA_FILENAME_DATEFORMAT" >> $ENV_FILE
- - echo "LOCAL_TIMEZONE=$LOCAL_TIMEZONE" >> $ENV_FILE
- - echo "DATA_DIR=$DATA_DIR" >> $ENV_FILE
- - echo "MAPFILE_DIR=$MAPFILE_DIR" >> $ENV_FILE
- - echo "MAPSERVER_DATA_DIR=$MAPSERVER_DATA_DIR" >> $ENV_FILE
- - echo "MAPSERVER_MAPFILE_DIR=$MAPSERVER_MAPFILE_DIR" >> $ENV_FILE
+ - echo "RESULT_TIF_DIR=$RESULT_TIF_DIR" >> $ENV_FILE
+ - echo "RESULT_MAPFILE_DIR=$RESULT_MAPFILE_DIR" >> $ENV_FILE
+ - echo "MAPSERVER_RESULT_TIF_DIR=$MAPSERVER_RESULT_TIF_DIR" >> $ENV_FILE
+ - echo "MAPSERVER_RESULT_MAPFILE_DIR=$MAPSERVER_RESULT_MAPFILE_DIR" >> $ENV_FILE
- echo "MAPSERVER_LOG_FILE=$MAPSERVER_LOG_FILE" >> $ENV_FILE
- echo "MAPSERVER_IMAGE_PATH=$MAPSERVER_IMAGE_PATH" >> $ENV_FILE
- echo "MAPSERVER_EXTENT=$MAPSERVER_EXTENT" >> $ENV_FILE
+ - echo "MASK_FILE=$MASK_FILE" >> $ENV_FILE
# Copy files to server
- - rsync -avz --progress $ENV_FILE $MODEL_ID.cfg $MODEL_ID.py run_$MODEL_ID.sh test_$MODEL_ID.py requirements.txt requirements.txt $SERVER_USER@$SERVER_IP:$HOME_DIR
- - rsync -avz --progress --rsync-path="mkdir -p $HOME_DIR/mapfile && rsync" mapfile/ $SERVER_USER@$SERVER_IP:$HOME_DIR/mapfile/
+ - rsync -avz --progress $ENV_FILE ADASMELIAE.cfg ADASMELIAE.py run_ADASMELIAE.sh test_ADASMELIAE.py requirements.txt europe_coastline.csv $SERVER_USER@$SERVER_IP:$HOME_DIR
+ - rsync -avz --progress --rsync-path="mkdir -p $TMP_DIR && mkdir -p $HOME_DIR/log && mkdir -p $HOME_DIR/mapfile && rsync" mapfile/ $SERVER_USER@$SERVER_IP:$HOME_DIR/mapfile/
+
+ # Give nibio user (member of deployer group) rwx access
+ - ssh $SERVER_USER@$SERVER_IP "chmod -R 775 $HOME_DIR"
only:
- release
diff --git a/ADASMELIAE.cfg b/ADASMELIAE.cfg
index 7c35058243dde6e664e7b918bd5ad6e56ddcdd16..5b2a9c744a0a5170c9735092ecdf6e4f783c228b 100644
--- a/ADASMELIAE.cfg
+++ b/ADASMELIAE.cfg
@@ -3,11 +3,11 @@
languages=en,nb
[i18n.en]
-low_risk = Low infection risk
-high_risk = High infection risk
-temperature = Temperature
+low_risk = Low risk of migration of adult pollen beetle
+high_risk = High risk of migration of adult pollen beetle
+temperature = Maximum air temperature
[i18n.nb]
-low_risk = Lav infeksjonsrisiko
-high_risk = Høy infeksjonsrisiko
-temperature = Temperatur
\ No newline at end of file
+low_risk = Lav migreringsrisiko
+high_risk = Høy migreringsrisiko
+temperature = Maksimum lufttemperatur
\ No newline at end of file
diff --git a/ADASMELIAE.py b/ADASMELIAE.py
index b7796da72ccf393a2a55a991fe2f4466a4e1cd6e..b2c07238a6818956b87446cb57c749298fe3da93 100755
--- a/ADASMELIAE.py
+++ b/ADASMELIAE.py
@@ -22,7 +22,6 @@
# * GDAL >= v 3.4.3 built with Python support
# * For Python: See requirements.txt
-
import os
import time
import subprocess
@@ -31,9 +30,9 @@ from dotenv import load_dotenv
from datetime import datetime, timedelta
from jinja2 import Environment, FileSystemLoader
import logging
-import pytz
-import netCDF4 as nc
+import re
import configparser
+import sys
load_dotenv()
@@ -44,47 +43,47 @@ DEBUG = (
)
model_id = os.getenv("MODEL_ID")
-filename_pattern = os.getenv("WEATHER_DATA_FILENAME_PATTERN")
-filename_dateformat = os.getenv("WEATHER_DATA_FILENAME_DATEFORMAT")
+home_dir = os.getenv("HOME_DIR")
+weather_data_dir = os.getenv("WEATHER_DATA_DIR")
+result_tif_dir = os.getenv("RESULT_TIF_DIR")
+result_mapfile_dir = os.getenv("RESULT_MAPFILE_DIR")
+tmp_dir = os.getenv("TMP_DIR")
+template_dir = f"{home_dir}mapfile/"
-# TODO Should start before the first European crops reach growth stage 51 and end when the northernmost regions have passed stage 59
-current_year = datetime.now().year
-local_timezone = pytz.timezone(os.getenv("LOCAL_TIMEZONE"))
+weather_data_filename_pattern = os.getenv("WEATHER_DATA_FILENAME_DATEFORMAT")
+start_MM_DD = os.getenv("START_DATE_MM_DD")
+end_MM_DD = os.getenv("END_DATE_MM_DD")
-MODEL_START_DATE = datetime.strptime(
- f"{current_year}-{os.getenv('START_DATE')}", "%Y-%m-%d"
-)
-MODEL_END_DATE = datetime.strptime(
- f"{current_year}-{os.getenv('END_DATE')}", "%Y-%m-%d"
-)
-weather_data_dir = os.getenv("WEATHER_DATA_DIR")
-tmp_dir = os.getenv("TMP_DIR")
-data_dir = os.getenv("DATA_DIR")
+config_file_path = f"{home_dir}ADASMELIAE.cfg"
+mask_file_path = (
+ f"{home_dir}{os.getenv('MASK_FILE')}" if os.getenv("MASK_FILE") else None
+)
+template_file_name = "template.j2"
# Get language stuff
config = configparser.ConfigParser()
-config.read("ADASMELIAE.cfg")
+config.read(config_file_path)
logging.basicConfig(
level=logging.DEBUG if DEBUG else logging.INFO,
format="%(asctime)s - %(levelname).4s - (%(filename)s:%(lineno)d) - %(message)s",
)
-TM = "air_temperature_2m"
-TM_MAX = "TM_MAX"
+tm_max = "air_temperature_2m_max"
+
THRESHOLD = 15
-# Common method for running commands using subprocess.run. Handles logging.
-def run_command(command, shell=True, stdout=subprocess.PIPE):
+# Run given command using subprocess.run. Handle logging.
+def run_command(command, stdout=subprocess.PIPE):
logging.debug(f"{command}")
try:
result = subprocess.run(
command,
- shell=shell,
stdout=stdout,
- stderr=subprocess.PIPE,
+ stderr=stdout,
+ shell=True,
text=True,
check=True,
)
@@ -97,142 +96,164 @@ def run_command(command, shell=True, stdout=subprocess.PIPE):
for line in result.stderr.splitlines():
logging.error(line.strip())
except subprocess.CalledProcessError as e:
- logging.error(f"Command failed: '{command}'")
logging.error(f"{e}")
quit()
-# Iterate the set of previously calculated result files. Find latest result date,
-# and return result date + 1 day. If no files exist, return MODEL_START_DATE.
-def find_start_date(default_date):
- result_file_names = os.listdir(data_dir)
- result_file_names = [
- file for file in result_file_names if file.startswith("result_2")
- ]
+# Run given command with capture_output=True, return result
+def run_command_get_output(command):
+ logging.debug(f"{command}")
try:
- dates = [
- datetime.strptime(file.split("_")[1].split(".")[0], "%Y-%m-%d")
- for file in result_file_names
- ]
- logging.debug(f"Found results for the following dates: {dates}")
- except ValueError as e:
- logging.error(f"Error parsing dates: {e}")
- return MODEL_START_DATE
- latest_result_date = max(dates, default=None)
- return (
- latest_result_date + timedelta(days=1) if latest_result_date else default_date
- )
+ return subprocess.run(
+ command,
+ capture_output=True,
+ shell=True,
+ text=True,
+ check=True,
+ )
+ except subprocess.CalledProcessError as e:
+ logging.error(f"{e}")
+ quit()
+
+
+# Remove all temporary/intermediary files
+def remove_temporary_files():
+ logging.info("Remove temporary/intermediary files")
+ if glob.glob(f"{tmp_dir}*.nc"):
+ run_command(command=f"rm {tmp_dir}*.nc")
+ if glob.glob(f"{tmp_dir}*.tif"):
+ run_command(command=f"rm {tmp_dir}*.tif")
+
+
+# Remove previously calculated results
+def remove_old_results():
+ logging.info("Remove previously calculated results")
+ if glob.glob(f"{result_tif_dir}*.tif"):
+ run_command(command=f"rm {result_tif_dir}*.tif")
+ if glob.glob(f"rm {result_mapfile_dir}*.map"):
+ run_command(command=f"rm {result_mapfile_dir}*.map")
+
+# Get unique dates in given file
+def get_unique_dates(input_file):
+ result = run_command_get_output(f"cdo showdate {input_file}")
+ dates_line = re.findall(r"\d{4}-\d{2}-\d{2}", result.stdout)
+ dates = " ".join(dates_line).split()
+ unique_dates = list(set(dates))
+ unique_dates.sort()
+ logging.info(f"Found {len(unique_dates)} unique dates in file")
+ return unique_dates
-def find_end_date(default_date):
- today = datetime.now()
- return today if today < default_date else default_date
+
+# Split file into one file for each unique date
+def split_by_date(input_file, prefix):
+ logging.info(f"Split {input_file} into individual files for each date")
+ unique_dates = get_unique_dates(input_file)
+ for date in unique_dates:
+ output_filename = f"{prefix}{date}.nc"
+ run_command(command=f"cdo seldate,{date} {input_file} {output_filename}")
+ logging.debug(f"Data for {date} extracted to {output_filename}")
if __name__ == "__main__":
- logging.info(f"Start model {model_id}")
- start_time = time.time()
+ start_time = time.time() # For logging execution time
+ today = datetime.now().date()
+
+ year = today.year
+ model_start_date = datetime.strptime(f"{year}-{start_MM_DD}", "%Y-%m-%d").date()
+ model_end_date = datetime.strptime(f"{year}-{end_MM_DD}", "%Y-%m-%d").date()
+
+ start_date = model_start_date
+ end_date = today + timedelta(days=2)
+
+ if start_date > end_date:
+ logging.error("Model period not started. Quit.")
+ sys.exit()
- start_date = find_start_date(MODEL_START_DATE)
- end_date = find_end_date(MODEL_END_DATE)
logging.info(
- f"Start running model for start date {start_date.date()} and end date {end_date.date()}"
+ f"Start running model {model_id} for start date {start_date} and end date {end_date}"
)
- weather_data_file_pattern = f"{weather_data_dir}{filename_pattern}"
- weather_data_files = glob.glob(weather_data_file_pattern)
- logging.debug(f"Find weather data files: {weather_data_file_pattern}")
- logging.debug(weather_data_files)
-
- timestep_dates = []
- for weather_data_file_path in sorted(weather_data_files):
- weather_data_file_name = os.path.basename(weather_data_file_path)
-
- # Skip if we don't have a valid date
- try:
- file_date = datetime.strptime(weather_data_file_name, filename_dateformat)
- except ValueError as e:
- logging.info(f"{weather_data_file_name} - Skip file due to invalid date")
- logging.debug(e)
- continue
-
- # Set up file names based on date YYYY-MM-DD
- date_YYYY_MM_DD = file_date.strftime("%Y-%m-%d")
- max_temp_file_path = f"{tmp_dir}max_temp_{date_YYYY_MM_DD}.nc"
- result_unmasked_path = f"{tmp_dir}result_unmasked_{date_YYYY_MM_DD}.nc"
- result_path = f"{tmp_dir}result_{date_YYYY_MM_DD}.nc"
-
- # Only process files from within valid interval
- if file_date < start_date or file_date > end_date:
- logging.debug(
- f"{weather_data_file_name} - Skip file with date outside calculation period"
- )
- continue
-
- # Check that the file has at least 23 timesteps
- with nc.Dataset(weather_data_file_path, "r") as weatherdata_file:
- timestep_count = len(weatherdata_file.variables["time"])
- if timestep_count < 23:
- logging.info(
- f"{weather_data_file_name} - Skip file with {timestep_count} timesteps"
- )
- continue
-
- # Skip if result files for date already exist
- if os.path.isfile(
- f"{data_dir}result_WARNING_STATUS_{date_YYYY_MM_DD}.tif"
- ) and os.path.isfile(f"{data_dir}result_{date_YYYY_MM_DD}.tif"):
- logging.info(f"Result files for {date_YYYY_MM_DD} already exist, skip")
- continue
-
- # Produce daily files with MAX_TEMP, which is the highest recorded temperature within the given day
- run_command(
- command=f"cdo -s -O -L -settime,00:00:00 -setdate,{date_YYYY_MM_DD} -chname,{TM},{TM_MAX} -timmax -selvar,{TM} {weather_data_file_path} {max_temp_file_path}"
- )
+ weather_data_file = weather_data_dir + today.strftime(weather_data_filename_pattern)
+ max_temp_in_period_file = f"{tmp_dir}{year}_max_temp_for_period.nc"
+ unmasked_result_file = f"{tmp_dir}{year}_unmasked_result.nc"
+ final_result_file = f"{tmp_dir}{year}_result.nc"
+
+ logging.info(
+ f"Extract {tm_max} for the required period into {os.path.basename(max_temp_in_period_file)}"
+ )
+ run_command(
+ command=f"cdo -L seldate,{start_date},{end_date} -selname,{tm_max} {weather_data_file} {max_temp_in_period_file}"
+ )
- # Classifying warning status for the model
- # temperature < 15 --> 2 (Yellow)
- # temperature >= 15 --> 4 (Red)
+ # Classifying warning status for the model
+ # temperature < 15 --> 2 (Green)
+ # temperature >= 15 --> 4 (Red)
+ logging.info(
+ f"Calculate warning status by checking if {tm_max} is below or above {THRESHOLD}"
+ )
+ run_command(
+ command=f'cdo -s -aexpr,"WARNING_STATUS = {tm_max} < {THRESHOLD} ? 2 : 4" {max_temp_in_period_file} {unmasked_result_file}',
+ )
+
+ # Mask results using a CSV file with polygons. Env variable MASK_FILE must be set, or else we use the file as it is.
+ if mask_file_path is not None:
+ logging.info(f"Mask file with {os.path.basename(mask_file_path)}")
run_command(
- command=f'cdo -s -aexpr,"WARNING_STATUS = {TM_MAX} < {THRESHOLD} ? 2 : 4" {max_temp_file_path} {result_unmasked_path}',
+ command=f"cdo -P 6 -maskregion,{mask_file_path} {unmasked_result_file} {final_result_file}",
)
+ else:
+ os.rename(unmasked_result_file, final_result_file)
- # Mask results using a CSV file with polygons. Env variable MASK_FILE must be set, or else we use the file as it is.
- if os.getenv("MASK_FILE") is not None:
- mask_file = os.getenv("MASK_FILE")
- logging.info(f"Applying mask file {mask_file} to result file")
- run_command(
- command=f"cdo -maskregion,{mask_file} {result_unmasked_path} {result_path}",
- )
- else:
- os.rename(result_unmasked_path, result_path)
+ prefix = f"{tmp_dir}day_"
+ split_by_date(final_result_file, prefix)
+ timestep_files = sorted(glob.glob(f"{prefix}*nc"))
- # Convert to GeoTIFF
- # We only need WHS and WARNING_STATUS
- # Merge the WARNING_STATUS and WHS variables into one GeoTIFF file with two bands.
- # The WARNING_STATUS should always be band #1
+ # Split result file into separate files for each timestep
+ # prefix = f"{tmp_dir}day_"
+ # run_command(command=f"cdo -s splitday {final_result_file} {prefix}")
- # We must delete the GeoTIFF file before merging WHY?
- # run_command(command=f"rm {data_dir}result_*{file_date_str}*.tif")
+ # timestep_file_pattern = f"{prefix}*.nc"
+ # add_dates_to_filenames(timestep_file_pattern)
- timestep_dates.append(date_YYYY_MM_DD)
+ # timestep_files = sorted(glob.glob(timestep_file_pattern))
+ timestep_filenames = [os.path.basename(file) for file in timestep_files]
+ timestep_dates = [
+ filename.split("_")[1].split(".")[0] for filename in timestep_filenames
+ ]
+ logging.info(f"Split into {len(timestep_filenames)} timestep files")
+
+ remove_old_results()
+
+ for file in timestep_files:
+ filename = os.path.basename(file)
+ date_str = filename.split("_")[1].split(".")[0]
with open("/dev/null", "w") as devnull:
+ warning_status_lcc = f"{tmp_dir}result_WARNING_STATUS_{date_str}_lcc.tif"
+ result_lcc = f"{tmp_dir}result_{date_str}_lcc.tif"
+ warning_status = f"{result_tif_dir}result_WARNING_STATUS_{date_str}.tif"
+ result = f"{result_tif_dir}result_{date_str}.tif"
+
+ logging.debug(f"Calculate result for {date_str}")
+
+ # For warning status:
run_command(
- command=f'gdal_translate -ot Int16 -of GTiff NETCDF:"{tmp_dir}result_{date_YYYY_MM_DD}.nc":WARNING_STATUS {tmp_dir}result_WARNING_STATUS_{date_YYYY_MM_DD}_lcc.tif',
+ command=f"gdal_translate -ot Int16 -of GTiff NETCDF:'{file}':WARNING_STATUS {warning_status_lcc}",
stdout=devnull,
)
+ # For max temperature:
run_command(
- command=f'gdal_translate -ot Float32 -of GTiff NETCDF:"{tmp_dir}result_{date_YYYY_MM_DD}.nc":{TM_MAX} {tmp_dir}result_{date_YYYY_MM_DD}_lcc.tif',
+ command=f"gdal_translate -ot Float32 -of GTiff NETCDF:'{file}':{tm_max} {result_lcc}",
stdout=devnull,
)
# Need to reproject the files, to ensure we have the projection given in the generted mapfile. We always use EPSG:4326 for this
run_command(
- command=f"gdalwarp -t_srs EPSG:4326 {tmp_dir}result_WARNING_STATUS_{date_YYYY_MM_DD}_lcc.tif {data_dir}result_WARNING_STATUS_{date_YYYY_MM_DD}.tif",
+ command=f"gdalwarp -t_srs EPSG:4326 {warning_status_lcc} {warning_status}",
stdout=devnull,
)
run_command(
- command=f"gdalwarp -t_srs EPSG:4326 {tmp_dir}result_{date_YYYY_MM_DD}_lcc.tif {data_dir}result_{date_YYYY_MM_DD}.tif",
+ command=f"gdalwarp -t_srs EPSG:4326 {result_lcc} {result}",
stdout=devnull,
)
@@ -247,15 +268,14 @@ if __name__ == "__main__":
language[keyword] = config["i18n.%s" % language_code][keyword]
languages.append(language)
- # The paths should be set in a .env file
- env = Environment(loader=FileSystemLoader("."))
- template = env.get_template("mapfile/template.j2")
+ env = Environment(loader=FileSystemLoader(template_dir))
+ template = env.get_template(template_file_name)
output = template.render(
{
"model_id": model_id,
"timestep_dates": timestep_dates,
- "mapserver_data_dir": os.getenv("MAPSERVER_DATA_DIR"),
- "mapserver_mapfile_dir": os.getenv("MAPSERVER_MAPFILE_DIR"),
+ "mapserver_data_dir": os.getenv("MAPSERVER_RESULT_TIF_DIR"),
+ "mapserver_mapfile_dir": os.getenv("MAPSERVER_RESULT_MAPFILE_DIR"),
"mapserver_log_file": os.getenv("MAPSERVER_LOG_FILE"),
"mapserver_image_path": os.getenv("MAPSERVER_IMAGE_PATH"),
"mapserver_extent": os.getenv("MAPSERVER_EXTENT"),
@@ -263,15 +283,11 @@ if __name__ == "__main__":
"language_codes": language_codes,
}
)
- mapfile_outdir = os.getenv("MAPFILE_DIR")
- with open(f"{mapfile_outdir}/{model_id}.map", "w") as f:
+ with open(f"{result_mapfile_dir}/{model_id}.map", "w") as f:
f.write(output)
-
- # Remove all temporary/intermediary files
- # run_command(command=f"rm {tmp_dir}result*.nc")
- # run_command(command=f"rm {tmp_dir}result*.tif")
+ # remove_temporary_files()
end_time = time.time()
logging.info(
- f"End model {model_id} - time spent: {'Execution time: {:.2f} seconds'.format(end_time - start_time)}"
+ f"End model {model_id} - Execution time: {'{:.2f} seconds'.format(end_time - start_time)}"
)
diff --git a/README.md b/README.md
index 5642dbd2ed74621e1ab55612ab4960c74fe19080..a9465fcda85d4544f6cfbd935b14bdbda88ab5e9 100644
--- a/README.md
+++ b/README.md
@@ -1,47 +1,12 @@
# Pollen beetle migration model - spatial version
-This model is based on the work of Ferguson et al., (2015) to predict migration risk of the Pollen Beetle. The default value of 15 degrees celsius is used to predict migration risk, as that is the temperature advised in the UK at which pollen beetles will fly Threshold however can also be overwritten by the user. The model can only be used between growth stage 51 and 59.
-(From the file MELIAEController.cs)
-
-Input data for opprinnelig modell:
-
-DSSInput_Parameters
-
-- weatherData weatherData (required)
-- int growthStage (required)
-- optionalData optionalData
-
-weatherData
-
-- string timeStart
-- string timeEnd
-- int Interval
-- List<int> weatherParameters
-- List<locationWeatherData> LocationWeatherData z
-
-locationWeatherData
-
-- double longitude
-- double latitude
-- double altitude
-- List<double> amalgamation
-- List<List<double?>> data
-- int width
-- List<int> qc
-- int length
-
-optionalData
-
-- double temp_threshold
-- DateTime? startDate
-- DateTime? endDate
-
-Se MELIAEController.cs for oppstart av applikasjonen
-Se MELIAE_DSS.cs for kjøring av modellen
+This model is based on the work of Ferguson et al., (2015) to predict migration risk of the Pollen Beetle.
+The default value of 15 degrees Celsius is used to predict migration risk, as that is the temperature advised
+in the UK at which pollen beetles will fly. The model can only be used between growth stage 51 and 59.
## Technical description
-The model has been implemented by [Tor-Einar Skog](https://nibio.no/en/employees/tor-einar-skog), [NIBIO](https://nibio.no/en). It is designed to fit into the gridded pest prediction models of [VIPS](https://nibio.no/en/services/vips).
+The model has been implemented by [NIBIO](https://nibio.no/en). It is designed to fit into the gridded pest prediction models of [VIPS](https://nibio.no/en/services/vips).
### Software requirements
@@ -57,49 +22,45 @@ The Python requirements are specified in `requirements.txt` file, and are includ
### Input data requirements
-The model (as per 2023-10-25) assumes that weather data files named `met_1_0km_nordic-[YYYY-MM-DD].nc` are available in the `in/` folder. The files must contain hourly timesteps with the following weather parameters:
-
-- RR (Rainfall in mm)
-- UM (Relative humidity in %)
+The model (as per 2024-04-19) assumes that a weather data file named `YYYY_with_forecast.nc` is available in the `WEATHER_DATA_DIR` folder. The file must contain daily timesteps with the weather parameter `air_temperature_2m_max`.
### Running the model
-It is required that you have set the following environment variables:
+In order to run the model, the following environment variables need to be set:
```bash
+# Default value is False
+DEBUG=False
# This is used to auto generate some variables and file names
MODEL_ID="ADASMELIAE"
-# Where your application resides
-HOME_DIR=/home/foo/2023_vips_in_space/
+# The start date MM-DD of the model
+START_DATE_MM_DD=03-01
+# The end date MM-DD of the model
+END_DATE_MM_DD=04-01
+# Where your script resides
+HOME_DIR=/foo/home/ADASMELIAE/
+# Path to folder for intermediary results
+TMP_DIR=/foo/home/ADASMELIAE/tmp/
# Path to the weather data
-WEATHER_DATA_DIR=in/
-# Used for iterating the weather data files
-WEATHER_DATA_FILENAME_PATTERN="met_1_0km_nordic-*.nc"
-# Used to extract date info from the filename
-WEATHER_DATA_FILENAME_DATEFORMAT="met_1_0km_nordic-%Y-%m-%d.nc"
-# Names of weather parameters in NetCDF files
-# Hourly precipitation
-RR="RR"
-# Relative humidity (2m)
-UM="UM"
-# Timezone for weather data/daily aggregations
-LOCAL_TIMEZONE="Europe/Oslo"
-# Path to optional CSV file with polygons for masking result.
-MASK_FILE=Norge_landomrader.csv
+WEATHER_DATA_DIR=/foo/weather/
+# Used to construct filename from current year
+WEATHER_DATA_FILENAME_DATEFORMAT="%Y_with_forecast.nc"
# Path to the output (GeoTIFF) files as seen from the running model code
-DATA_DIR=out/
+RESULT_TIF_DIR=/foo/out/tif/
# Path to the generated mapfile as seen from the running model code
-MAPFILE_DIR=mapfile/
+RESULT_MAPFILE_DIR=/foo/out/mapfile/
# The path to the output (GeoTIFF files) as seen from Mapserver
-MAPSERVER_DATA_DIR=/foo/mapserver/data/ADASMELIAE/
+MAPSERVER_RESULT_TIF_DIR=/foo/mapserver/data/ADASMELISAE/
# Where your generated MAPFILE and query templates should be placed
-MAPSERVER_MAPFILE_DIR=/foo/mapserver/wms/ADASMELIAE/
+MAPSERVER_RESULT_MAPFILE_DIR=/foo/mapserver/wms/ADASMELISAE/
# Where mapserver logs for this WMS are written
-MAPSERVER_LOG_FILE=/foo/mapserver/log/ADASMELIAE.log
+MAPSERVER_LOG_FILE=/foo/mapserver/log/ADASMELISAE.log
# Path to the temporary directory for writing temporary files and images. Must be writable by the user the web server is running as
-MAPSERVER_IMAGE_PATH=/foo/mapserver/tmp/
+MAPSERVER_IMAGE_PATH=/foo/mapserver/image/
# The value of the EXTENT parameter in Mapserver's mapfile. Units are DD (Decimal degrees)
-MAPSERVER_EXTENT="-1.5831861262936526 52.4465003983706595 39.2608060398730458 71.7683216082912736"
+MAPSERVER_EXTENT="-23.5 29.5 62.5 70.5"
+# Path to optional CSV file with polygons for masking result.
+MASK_FILE=europe_coastline.csv
```
...this is the contents of the `env-sample` file
@@ -110,35 +71,24 @@ $ ./run_ADASMELIAE.sh
This creates a Python virtualenv, installs all the Python dependencies, runs the model and stores output in a log file.
-Alternatively, primarily for development purposes, you can run the Python script ADASMELIAE directly:
-
-```bash
-$ ./ADASMELIAE.py
-```
-
-All intermediary files are stored in the `tmp/` folder, and they are all deleted when the model is done calculating. The GeoTIFF files are stored in the `out/` folder, and the generated mapfile is stored in the `mapfile/` folder
+All intermediary files are stored in the `TMP_DIR` folder, and they are all deleted when the model is done calculating. The GeoTIFF files are stored in the `RESULT_TIF_DIR` folder, and the generated mapfile is stored in the `RESULT_MAPFILE_DIR` folder
### Viewing the result of the model
The model outputs GeoTIFF files, two per day in the season/period of calculation:
-- `result_WARNING_STATUS_[YYYY-MM-DD].tif`, wich indicates infection risk of Septoria.
- - 0 = No infection risk (grey)
- - 2 = Low infection risk (yellow)
- - 3 = Medium infection risk (orange)
- - 4 = High risk (red)
-- `result_[YYYY-MM-DD].tif`, which contains the wet hour sum (WHS) - which is the sum of wet hours for "yesterday", "today" and "tomorrow", relative to the current file date.
-
-A [Jinja2](https://pypi.org/project/Jinja2/) template mapfile (for [Mapserver](https://mapserver.org/)) with separate layers (WARNING_STATUS and WHS) for each date is found in the `mapfile/` folder.
+- `result_WARNING_STATUS_[YYYY-MM-DD].tif`, which indicates migration risk.
+ - 2 = Low migration risk (green)
+ - 4 = High migration risk (red)
+- `result_[YYYY-MM-DD].tif`, which contains the daily maximum air temperature
-Examples of the two layers are shown below
+A [Jinja2](https://pypi.org/project/Jinja2/) template mapfile (for [Mapserver](https://mapserver.org/)) with separate layers (warning status and temperature) for each date is found in the `mapfile/` folder.
-
-_WARNING_STATUS example. Showing Norway_
+
-
+_Warning status example. Showing Europe and the surrounding areas._
-_WHS (Wet Hour Sum) example. Showing Norway_
+
-## Notes to self
+_Temperature example. Showing Europe and the surrounding areas._
diff --git a/WARNING_STATUS_layer_example.png b/WARNING_STATUS_layer_example.png
deleted file mode 100644
index ca87fc7d6920b28e2f78542dd3e7dafcf7a052b6..0000000000000000000000000000000000000000
Binary files a/WARNING_STATUS_layer_example.png and /dev/null differ
diff --git a/WHS_layer_example.png b/WHS_layer_example.png
deleted file mode 100644
index 36c6ef49c4c9653b4a33822cdd6996ff0738df9e..0000000000000000000000000000000000000000
Binary files a/WHS_layer_example.png and /dev/null differ
diff --git a/env-sample b/env-sample
index 071dd7953996410664bc72ad62a705ba06b80853..a86993f6a23ce435004129246c9fa35f49655e3c 100644
--- a/env-sample
+++ b/env-sample
@@ -1,38 +1,34 @@
# Use this as an example to create your own .env file
+# Default value is False
+DEBUG=False
# This is used to auto generate some variables and file names
MODEL_ID="ADASMELIAE"
-# Where your application resides
-HOME_DIR=/home/foo/2023_vips_in_space/ADASMELIAE/
+# The start date MM-DD of the model
+START_DATE_MM_DD=03-01
+# The end date MM-DD of the model
+END_DATE_MM_DD=04-01
+# Where your script resides
+HOME_DIR=/foo/home/ADASMELIAE/
+# Path to folder for intermediary results
+TMP_DIR=/foo/home/ADASMELIAE/tmp/
# Path to the weather data
-WEATHER_DATA_DIR=in/
-# Used for iterating the weather data files
-WEATHER_DATA_FILENAME_PATTERN="met_1_0km_nordic-*.nc"
-# Used to extract date info from the filename
-WEATHER_DATA_FILENAME_DATEFORMAT="met_1_0km_nordic-%Y-%m-%d.nc"
-# Names of weather parameters in NetCDF files
-# Hourly precipitation
-RR="RR"
-# Relative humidity (2m)
-UM="UM"
-# Timezone for weather data/daily aggregations
-LOCAL_TIMEZONE="Europe/Oslo"
-# Path to optional CSV file with polygons for masking result.
-# MASK_FILE=Norge_landomrader.csv
+WEATHER_DATA_DIR=/foo/weather/
+# Used to construct filename from current year
+WEATHER_DATA_FILENAME_DATEFORMAT="%Y_with_forecast.nc"
# Path to the output (GeoTIFF) files as seen from the running model code
-DATA_DIR=out/
+RESULT_TIF_DIR=/foo/out/tif/
# Path to the generated mapfile as seen from the running model code
-MAPFILE_DIR=mapfile/
+RESULT_MAPFILE_DIR=/foo/out/mapfile/
# The path to the output (GeoTIFF files) as seen from Mapserver
-MAPSERVER_DATA_DIR=/foo/mapserver/data/ADASMELIAE/
+MAPSERVER_RESULT_TIF_DIR=/foo/mapserver/data/ADASMELISAE/
# Where your generated MAPFILE and query templates should be placed
-MAPSERVER_MAPFILE_DIR=/foo/mapserver/wms/ADASMELIAE/
+MAPSERVER_RESULT_MAPFILE_DIR=/foo/mapserver/wms/ADASMELISAE/
# Where mapserver logs for this WMS are written
-MAPSERVER_LOG_FILE=/foo/mapserver/log/ADASMELIAE.log
+MAPSERVER_LOG_FILE=/foo/mapserver/log/ADASMELISAE.log
# Path to the temporary directory for writing temporary files and images. Must be writable by the user the web server is running as
-MAPSERVER_IMAGE_PATH=/foo/mapserver/tmp/
+MAPSERVER_IMAGE_PATH=/foo/mapserver/image/
# The value of the EXTENT parameter in Mapserver's mapfile. Units are DD (Decimal degrees)
-MAPSERVER_EXTENT="-1.5831861262936526 52.4465003983706595 39.2608060398730458 71.7683216082912736"
-
-# Default value is false
-#DEBUG=True
\ No newline at end of file
+MAPSERVER_EXTENT="-23.5 29.5 62.5 70.5"
+# Path to optional CSV file with polygons for masking result.
+MASK_FILE=europe_coastline.csv
\ No newline at end of file
diff --git a/layer_example_temperature.png b/layer_example_temperature.png
new file mode 100644
index 0000000000000000000000000000000000000000..087c2c0f2cc2779e67b0c4caa46db1fdaad3541e
Binary files /dev/null and b/layer_example_temperature.png differ
diff --git a/layer_example_warning_status.png b/layer_example_warning_status.png
new file mode 100644
index 0000000000000000000000000000000000000000..43595869bc82202b67a61017cd3d9f63b0175ba1
Binary files /dev/null and b/layer_example_warning_status.png differ
diff --git a/mapfile/template.j2 b/mapfile/template.j2
index b4d81e62cb126541d55d103e34a5a0efcf4e7af4..7d723eb9be0a111fe63ae2f631a272bf53332303 100644
--- a/mapfile/template.j2
+++ b/mapfile/template.j2
@@ -1,3 +1,5 @@
+{# This Jinja2 template is used for generating a map file for the model #}
+
MAP
# This mapfile is generated using Jinja2 templates
@@ -40,35 +42,32 @@ WEB
"wms_inspire_capabilities" "embed"
"wms_languages" "{{ language_codes|join(",")}}" # The first is the default
{% endif %}
- "wms_abstract" "<div id='preamble'>
- <p>Pollen beetle (Meligethes spp.) adults are approximately 2.5 mm,
- metallic greenish-black. Females bite oilseed rape buds and lay their eggs
- inside. Adults and larvae attack buds and flowers, resulting in withered
- buds and reduced pod set. In oilseed rape, adult and larval feeding can
- lead to bud abortion and reduced pod set. However, damage rarely results
- in reduced yields for winter crops. Spring crops are more vulnerable, as
- the susceptible green/yellow bud stage often coincides with beetle
- migration. </p>
+ "wms_abstract" "
+ <div id='preamble'>
+ <p>Pollen beetle (Meligethes spp.) adults are approximately 2.5 mm, metallic greenish-black. Oilseed
+ rape is only vulnerable to pollen beetle damage if large numbers of adult beetles migrate into the
+ crop during green bud stage (BBCH Growth stages 51-59). Adult females bite oilseed rape buds and
+ lay their eggs inside, resulting in withered buds and reduced pod set. Migration can be predicted
+ based on daily maximum air temperature; migrations begin above 12 degrees Celsius, and large numbers
+ migrate above 15 degrees Celsius. Where a high risk of migration coincides with the vulnerable
+ growth stage, crops should be monitored and an appropriate threshold used to inform management
+ decisions.
+ </p>
</div>
<div id='body'>
- <p>
- Oilseed rape is only vulnerable if large numbers of pollen
- beetle migrate into the crop during green bud stage. This DSS predicts
- migration into crops based on air temperature, and so can be used to
- evaluate risk to crop. Daily maximum air temperature is used to predict Migration
- Risk. The default value of 15 degrees celsius is used, as that is the
- temperature advised in the UK at which pollen beetles will fly.
- </p>
- <p>This DSS was adapted from work carried out in the UK, and is
- considered applicable, but not yet validated in, Belgium, Luxembourg,
- Netherlands, France, Germany, Rep. Ireland, and Denmark. Only to be used during Oilseed rape growth stages 51-59. This
- model is a simplification of a more detailed model described in Ferguson et al. (2015) Pest Management Science 72, 609-317.
- <a href="https://doi.org/10.1002/ps.4069">https://doi.org/10.1002/ps.4069</a></p>
-
+ <p>This DSS was adapted from work carried out in the UK, and is considered applicable, but not yet
+ validated in, Belgium, Luxembourg, Netherlands, France, Germany, Rep. Ireland, and Denmark. Only
+ to be used during Oilseed rape growth stages 51-59. This model is a simplification of a more
+ detailed model described in Ferguson et al. (2015) Pest Management Science 72, 609-317.
+ <a href='https://doi.org/10.1002/ps.4069'>https://doi.org/10.1002/ps.4069</a></p>
+ <p>Login and see the DSS Use dashboard for a more in-depth assessment.</p>
<h3>Explanation of parameters</h3>
- <ul>
- <li>TM_MAX = <span itemprop=\"TM_MAX\">Maximum Air Temperature at 2m</span></li>
- </ul>
+ <p>
+ <ul>
+ <li>Warning status = Green signifies mean daily temperatures below 15°C and low migration risk. Red signifies mean daily temperatures above 15°C and high migration risk.</li>
+ <li><span itemprop='temperature'>Maximum air temperature</span> = Areas with daily maximum air temperatures below 10°C are shaded in blue, indicating cooler weather. As temperatures increase, the color shifts gradually to red, with a full red shade marking areas above 20°C.</li>
+ </ul>
+ </p>
</div>
"
"wms_enable_request" "*"
@@ -123,12 +122,12 @@ LAYER
{
\"classification\": 2,
\"legendLabel\": \"{{ language.low_risk }}\",
- \"legendIconCSS\": \"width: 25px; background-color: #FFCC00;\"
+ \"legendIconCSS\": \"width: 25px; background-color: #3ac47d;\"
},
{
\"classification\": 4,
\"legendLabel\": \"{{ language.high_risk }}\",
- \"legendIconCSS\": \"width: 25px; background-color: #FF0000;\"
+ \"legendIconCSS\": \"width: 25px; background-color: #d92550;\"
}
]
}
@@ -141,16 +140,16 @@ LAYER
CLASS
NAME "Low infection risk"
- EXPRESSION ([pixel] < 3)
+ EXPRESSION ([pixel] >= 0 AND [pixel] < 3)
STYLE
- COLOR 112 112 112
+ COLOR 58 196 125 # Green
END
END
CLASS
NAME "High infection risk"
EXPRESSION ([pixel] >= 3)
STYLE
- COLOR 255 0 0
+ COLOR 217 37 80 # Red
END
END
END # Layer
@@ -175,7 +174,7 @@ LAYER
\"legendItems\": [
{
\"legendLabel\": \"{{ language.temperature }}\",
- \"legendIconCSS\": \"width: 25px; background: linear-gradient(to right, #FFFF00, #0000FF);\"
+ \"legendIconCSS\": \"width: 25px; background: linear-gradient(to right, #0000FF, #FF0000);\"
}
]
}
@@ -183,15 +182,28 @@ LAYER
{% endfor %}
END
CLASSITEM "[pixel]"
+ CLASS
+ NAME "Below 10"
+ EXPRESSION ([pixel] < 10)
+ STYLE
+ COLOR 0 0 255 # RGB for blue
+ END
+ END
CLASS
NAME "Temperature range"
- EXPRESSION ([pixel] >= -40 AND [pixel] <= 40)
+ EXPRESSION ([pixel] >= 10 AND [pixel] <= 20)
STYLE
- DATARANGE -40 40
- COLORRANGE 255 255 0 0 0 255
+ DATARANGE 10 20
+ COLORRANGE 0 0 255 255 0 0
END
END
-
+ CLASS
+ NAME "Above 20"
+ EXPRESSION ([pixel] > 20)
+ STYLE
+ COLOR 255 0 0 # RGB for red
+ END
+ END
END # Layer
{% endfor %}
diff --git a/run_ADASMELIAE.sh b/run_ADASMELIAE.sh
index 7f0db4b5adca05d61b006b8b98cec2810b583e23..caa9d0c309a7934b11c6fa6f33822b48841947cf 100755
--- a/run_ADASMELIAE.sh
+++ b/run_ADASMELIAE.sh
@@ -16,7 +16,6 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
# Configures environment and logging before running the model
-# @author: Tor-Einar Skog <tor-einar.skog@nibio.no>
# First: Test that we have CDO and GDAL installed
@@ -32,8 +31,9 @@ then
exit
fi
-# Defines HOME_DIR
-source .env
+# Get script location to be able to read .env regardless of where script is run from
+script_location=$(dirname "$(readlink -f "$0")")
+source $script_location/.env
# Check for HOME_DIR
if [ -z "${HOME_DIR}" ]
@@ -46,7 +46,7 @@ fi
LOG_FILE=${HOME_DIR}log/ADASMELIAE.log
REQUIREMENTS=${HOME_DIR}requirements.txt
-cd $HOME_DIR
+echo "Start model. Follow log at $LOG_FILE"
# Create and activate the virtual environment
echo "Activate virtual environment .venv" >> "$LOG_FILE"
@@ -67,5 +67,6 @@ if [ -n "$VIRTUAL_ENV" ]; then
else
echo "Virtual environment was not successfully activated." >> "$LOG_FILE"
fi
+echo "" >> "$LOG_FILE"
-echo "Run complete. Check log at $LOG_FILE"
\ No newline at end of file
+echo "Run complete"
diff --git a/test_ADASMELIAE.py b/test_ADASMELIAE.py
index 3b4c06681b0f73473533f93e5b0b28c6e6e6eec5..fe002cbf87cbf800653f03c5d9316660f32fd3ae 100644
--- a/test_ADASMELIAE.py
+++ b/test_ADASMELIAE.py
@@ -1,43 +1,30 @@
import pytest
-from datetime import datetime
-from ADASMELIAE import find_start_date
+import logging
+from ADASMELIAE import run_command
-@pytest.fixture
-def example_result_files():
- return [
- "result_2023-04-15.nc",
- "result_2023-04-16.nc",
- "result_2023-04-17.nc",
- "result_WARNING_STATUS_2023-04-15.nc",
- "result_WARNING_STATUS_2023-04-16.nc",
- "result_WARNING_STATUS_2023-04-17.nc",
- "result_WARNING_STATUS_2023-04-18.nc",
- ]
+# Define a fixture for setting up logging
+@pytest.fixture(autouse=True)
+def setup_logging(caplog):
+ caplog.set_level(logging.DEBUG)
-def test_find_start_date_with_previous_results(example_result_files, monkeypatch):
- MODEL_START_DATE = datetime(2023, 3, 1)
- monkeypatch.setenv("DATA_DIR", "out")
+# Test case for successful command execution
+def test_run_command_success(caplog):
+ result = run_command("echo 'Hello, World!'")
+ assert result == ["Hello, World!"]
+ assert "Hello, World!" in caplog.text
- # Mock os.listdir to return the example result files
- with monkeypatch.context() as m:
- m.setattr("os.listdir", lambda _: example_result_files)
- start_date = find_start_date(MODEL_START_DATE)
- # Assert the expected start date
- expected_start_date = datetime(2023, 4, 18)
- assert start_date == expected_start_date
+# Test case for command failure
+def test_run_command_failure(caplog):
+ with pytest.raises(SystemExit):
+ run_command("nonexistent_command")
+ assert "nonexistent_command" in caplog.text
-def test_find_start_date_without_previous_results(monkeypatch):
- MODEL_START_DATE = datetime(2023, 3, 1)
- monkeypatch.setenv("DATA_DIR", "out")
- # Mock os.listdir to return the example result files
- with monkeypatch.context() as m:
- m.setattr("os.listdir", lambda _: [])
- start_date = find_start_date(MODEL_START_DATE)
-
- # Assert the expected start date
- assert start_date == MODEL_START_DATE
+# Test case for handling stderr
+def test_run_command_stderr(caplog):
+ run_command("echo 'This is an error' >&2")
+ assert "This is an error" in caplog.text