diff --git a/.gitignore b/.gitignore index 7db8917d079a2249f5153b1681a6b80e0679bae5..0221f8c7a67bff2321d886db32a004d3f8aa9dde 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ *.tif.aux.xml .env .venv -mapfile/SEPTREFHUM.map +mapfile/*.map diff --git a/README.md b/README.md index 536b39cdeb0f18f3abaca8f72bff02ceac09adf3..ce1932c663ee1bdee4a20b7f420a0e8376c891c1 100644 --- a/README.md +++ b/README.md @@ -53,10 +53,23 @@ The model (as per 2023-10-25) assumes that weather data files named `met_1_0km_n It is required that you have set the following environment variables: ```bash +# This is used to auto generate some variables and file names +MODEL_ID="SEPTREFHUM" # Where your application resides -HOME_DIR=/home/foo/ +HOME_DIR=/home/foo/2023_vips_in_space/ # Path to the weather data WEATHER_DATA_DIR=in/ +# Used for iterating the weather data files +FILENAME_PATTERN="met_1_0km_nordic-*.nc" +# Used to extract date info from the filename +FILENAME_DATEFORMAT="met_1_0km_nordic-%Y-%m-%d.nc" +# Names of weather parameters in NetCDF files +# Hourly precipitation +RR="RR" +# Relative humidity (2m) +UM="UM" +# Timezone for weather data/daily aggregations +LOCAL_TIMEZONE="Europe/Oslo" # Path to optional CSV file with polygons for masking result. MASK_FILE=Norge_landomrader.csv # Path to the output (GeoTIFF) files as seen from the running model code @@ -68,7 +81,7 @@ MAPSERVER_DATA_DIR=/foo/mapserver/data/SEPTREFHUM/ # Where your generated MAPFILE and query templates should be placed MAPSERVER_MAPFILE_DIR=/foo/mapserver/wms/SEPTREFHUM/ # Where mapserver logs for this WMS are written -MAPSERVER_LOG_FILE=/foo/log/SEPTREFHUM.log +MAPSERVER_LOG_FILE=/foo/mapserver/log/SEPTREFHUM.log # Path to the temporary directory for writing temporary files and images. Must be writable by the user the web server is running as MAPSERVER_IMAGE_PATH=/foo/mapserver/tmp/ # The value of the EXTENT parameter in Mapserver's mapfile. Units are DD (Decimal degrees) diff --git a/SEPTREFHUM.py b/SEPTREFHUM.py index 3021e9e12e40cedf2860f15c44d65151a8554153..fdac7172885da3bd3f79c282dfead13fc4ce7e02 100755 --- a/SEPTREFHUM.py +++ b/SEPTREFHUM.py @@ -39,11 +39,20 @@ config = configparser.ConfigParser() config.read("SEPTREFHUM.cfg") # Path to weather data +model_id = os.getenv("MODEL_ID") infile_path = os.getenv("WEATHER_DATA_DIR") +# Used for iterating the weather data files +filename_pattern = os.getenv("FILENAME_PATTERN") +# Date format of weather data filenames +filename_dateformat = os.getenv("FILENAME_DATEFORMAT") # Path to store generated GeoTIFF files outfile_path = os.getenv("DATA_DIR") # Where to store intermediary calculations tmpfile_path = "tmp/" +# Names of weather parameters in NetCDF files +RR = os.getenv("RR") +UM = os.getenv("UM") + # TODO: Make this configurable (.env) local_timezone = pytz.timezone("Europe/Oslo") @@ -64,17 +73,27 @@ if last_wh_date is not None: -# TODO: let filename/pattern be configurable -weatherdata_files = glob.glob(f"{infile_path}met_1_0km_nordic-*.nc") -for file_path in weatherdata_files: +# TODO: let filename/pattern be configurable <== DONE?? +weatherdata_files = glob.glob(f"{infile_path}{filename_pattern}") +#print(f"{infile_path}{filename_pattern}") +#print("What are the weatherdata files?") +#print(weatherdata_files) +for file_path in sorted(weatherdata_files): + print("Examining %s" % file_path) # TODO: When filename/pattern is configurable: make the string search adaptable file_name = os.path.basename(file_path) - file_date = file_name[file_name.index("nordic")+7:file_name.index("nordic")+17] - # Skip if we don't have a complete date (10 characters), which could indicate that we are looking at a yearly file of daily data - if len(file_date) != 10: + # Skip if we don't have a complete date, which could indicate that we are looking at a yearly file of daily data + try: + wh_sum_date = local_timezone.localize(datetime.strptime(file_name, filename_dateformat)) + except ValueError as e: + print(e) continue + #file_date = file_name[file_name.index("nordic")+7:file_name.index("nordic")+17] + # Skip if we don't have a complete date (10 characters), which could indicate that we are looking at a yearly file of daily data + #if len(file_date) != 10: + # continue - wh_sum_date = local_timezone.localize(datetime.strptime(f"{file_date}", "%Y-%m-%d")) + #wh_sum_date = local_timezone.localize(datetime.strptime(f"{file_date}", "%Y-%m-%d")) # Only process files from the three last days (if this is not a work from scratch) if start_date is not None and wh_sum_date < start_date: continue @@ -91,7 +110,7 @@ for file_path in weatherdata_files: wh_sum_date_str = wh_sum_date.strftime("%Y-%m-%d") wh_sum_hour_str = wh_sum_date.strftime("%H") subprocess.run( - f'cdo -s -O -setdate,{wh_sum_date_str} -settime,{wh_sum_hour_str}:00:00 -chname,WH,WH_DAYSUM -timsum -selname,WH -aexpr,"WH = RR > 0.2 || UM > 88.0 ? 1 : 0;" {file_path} {tmpfile_path}wh_{file_date}.nc', + f'cdo -s -O -setdate,{wh_sum_date_str} -settime,{wh_sum_hour_str}:00:00 -chname,WH,WH_DAYSUM -timsum -selname,WH -aexpr,"WH = {RR} > 0.2 || {UM} > 88.0 ? 1 : 0;" {file_path} {tmpfile_path}wh_{wh_sum_date_str}.nc', shell=True ) @@ -223,6 +242,7 @@ for language_code in language_codes: env = Environment(loader=FileSystemLoader('.')) template = env.get_template("mapfile/template.j2") output = template.render({ + "model_id":model_id, "timestep_dates": timestep_dates, "mapserver_data_dir": os.getenv("MAPSERVER_DATA_DIR"), "mapserver_mapfile_dir": os.getenv("MAPSERVER_MAPFILE_DIR"), @@ -233,7 +253,7 @@ output = template.render({ "language_codes": language_codes }) mapfile_outdir = os.getenv("MAPFILE_DIR") -with open(f"{mapfile_outdir}/SEPTREFHUM.map", 'w') as f: +with open(f"{mapfile_outdir}/{model_id}.map", 'w') as f: f.write(output) diff --git a/env-sample b/env-sample index 5b69fb0e4f5493da87833e0276198e52255af538..b96bf037c0b75e1362da61bda2b3aac3ae12d4ea 100644 --- a/env-sample +++ b/env-sample @@ -1,9 +1,22 @@ # Use this as an example to create your own .env file +# This is used to auto generate some variables and file names +MODEL_ID="SEPTREFHUM" # Where your application resides HOME_DIR=/home/foo/2023_vips_in_space/ # Path to the weather data WEATHER_DATA_DIR=in/ +# Used for iterating the weather data files +FILENAME_PATTERN="met_1_0km_nordic-*.nc" +# Used to extract date info from the filename +FILENAME_DATEFORMAT="met_1_0km_nordic-%Y-%m-%d.nc" +# Names of weather parameters in NetCDF files +# Hourly precipitation +RR="RR" +# Relative humidity (2m) +UM="UM" +# Timezone for weather data/daily aggregations +LOCAL_TIMEZONE="Europe/Oslo" # Path to optional CSV file with polygons for masking result. # MASK_FILE=Norge_landomrader.csv # Path to the output (GeoTIFF) files as seen from the running model code diff --git a/europe_coastline.csv b/europe_coastline.csv new file mode 100644 index 0000000000000000000000000000000000000000..4edfdce61ad6dff11c1d40a222c7b4daff1ebcda Binary files /dev/null and b/europe_coastline.csv differ diff --git a/mapfile/template.j2 b/mapfile/template.j2 index 14f1a7916190edf9922ee9bfee58cfe297595e87..aa6e72c6935d55457b3e5147df10e9347a40ca85 100644 --- a/mapfile/template.j2 +++ b/mapfile/template.j2 @@ -83,7 +83,6 @@ WEB " "wms_enable_request" "*" "wms_title" "Septoria Reference Humidity Model" - #"wms_onlineresource" "https://testvips.nibio.no/cgi-bin/SEPTREFHUM" "wms_getfeatureinfo_formatlist" "text/plain,text/html,text/xml" "wms_accessconstraints" "none" "wms_addresstype" "" @@ -115,7 +114,7 @@ END {% for timestep_date in timestep_dates %} LAYER - NAME "SEPTREFHUM.WARNING_STATUS.{{ timestep_date }}" + NAME "{{model_id}}.WARNING_STATUS.{{ timestep_date }}" DATA "{{mapserver_data_dir}}result_WARNING_STATUS_{{ timestep_date }}.tif" TEMPLATE "{{mapserver_mapfile_dir}}query_template.xml" TOLERANCE 1 TOLERANCEUNITS PIXELS TYPE RASTER @@ -191,7 +190,7 @@ LAYER END # Layer LAYER - NAME "SEPTREFHUM.WHS.{{ timestep_date }}" + NAME "{{model_id}}.WHS.{{ timestep_date }}" DATA "{{mapserver_data_dir}}result_{{ timestep_date }}.tif" TEMPLATE "{{mapserver_mapfile_dir}}query_template_WHS.xml" TOLERANCE 1 TOLERANCEUNITS PIXELS TYPE RASTER