Initial commit

.gitignore

+*venv/

app/app.py

+from flask import Flask
+from flask import request
+from flask import render_template
+import json
+from weatherdata_utils import *
+
+from models import WeatherData
+
+app = Flask(__name__)
+
+
+@app.route("/")
+def index():
+    return render_template("usage.html")
+
+@app.route("/weather_data")
+def get_weather_data():
+    longitude = request.args.get("longitude", None) # WGS84
+    latitude = request.args.get("latitude", None) # WGS84
+    parameters = request.args.get("parameters", None) # Comma separated list
+    try:
+        parameters = None if parameters == None else [int(i) for i in parameters.split(",")]
+    except ValueError as e:
+        return "BAD REQUEST: Error in specified weather parameters: %s" % e, 403
+    interval = int(request.args.get("interval", 3600))
+    time_start = request.args.get("timeStart", "2021-01-01")
+    time_end = request.args.get("timeEnd", "2021-12-31")
+    # Input check
+    if longitude is None or latitude is None:
+        return "BAD REQUEST: longitude (%s) and/or latitude (%s) is not set" % (longitude, latitude), 403
+    wd_file = open("../data_files/2021_apelsvoll_redigert.json")
+    weather_data = WeatherData(**json.load(wd_file))
+    wd_file.close()
+
+    # FILTER THE DATA
+    # Period first
+    weather_data = filter_period(weather_data,time_start,time_end)
+    # Parameters next
+    weather_data = filter_params(weather_data, parameters)
+    # Aggregation?
+
+    return weather_data.as_dict()
+    #return "<p>Hello, World!</p>"

app/models.py

+#!/usr/bin/python3
+
+#    Copyright (C) 2021  Tor-Einar Skog,  NIBIO
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from datetime import datetime
+from _datetime import date
+
+from shapely import wkb
+from shapely.geometry import Point
+
+class WeatherData:
+    def __init__(self, *args, **kwargs):
+        # Get the times
+        self.timeStart = WeatherData.to_epoch_seconds(kwargs.get("timeStart", None))
+        self.timeEnd = WeatherData.to_epoch_seconds(kwargs.get("timeEnd", None))
+        self.interval = kwargs.get("interval", 3600)
+        self.weatherParameters = kwargs.get("weatherParameters", None)
+        self.locationWeatherData = []
+        lwd_tmp = kwargs.get("locationWeatherData", [])
+        if len(lwd_tmp) > 0:
+            for lwd in lwd_tmp:
+                self.locationWeatherData.append(LocationWeatherData(**lwd) if not isinstance(lwd, LocationWeatherData) else lwd)
+         
+    @classmethod
+    def to_epoch_seconds(self,some_kind_of_date):
+        # Epochs and None are returned as is
+        # We only try to convert strings
+        try:
+            # if the date is an invalid string, the ValueError is propagated
+            return datetime.fromisoformat(some_kind_of_date.replace("Z","+00:00")).timestamp()
+        except (TypeError, AttributeError):
+            if some_kind_of_date is None or isinstance(some_kind_of_date, int):
+                return some_kind_of_date
+            else:
+                raise TypeError("Date (timeStart or timeEnd) is neither None, int or String. Please check your input!")
+        
+    def set_value(self, parameter, timestamp, value):
+        col = self.weatherParameters.index(parameter)
+        row = int((timestamp - self.timeStart) / 3600)
+        self.locationWeatherData[0].data[row][col] = value
+
+    def as_dict(self):
+        retval = vars(self)
+        retval["timeStart"] = None if self.timeStart is None else "%sZ" % datetime.utcfromtimestamp(self.timeStart).isoformat()
+        retval["timeEnd"] = None if self.timeEnd is None else "%sZ" % datetime.utcfromtimestamp(self.timeEnd).isoformat()
+        lwds_dict = []
+        for lwd in self.locationWeatherData:
+            lwds_dict.append(lwd.as_dict())
+        retval["locationWeatherData"] = lwds_dict
+        return retval
+
+    # In which row can I get data for this point in time?
+    def get_index_from_epoch_seconds(self, epoch_seconds):
+        if epoch_seconds % 1 != 0:
+            # Check that epoch_seconds is int
+            raise ValueError("Timestamp %s is not an integer" % epoch_seconds)
+        # Check that return value is an int
+        index = (epoch_seconds - self.timeStart) / self.interval
+        if index %1 != 0:
+            raise ValueError("Timestamp %s is not divisible by %s" % (epoch_seconds,self.timeStart))
+        return int(index)
+
+
+class Parameter:
+    #/** Method for how to create e.g. daily values from hourly values */
+    AGGREGATION_TYPE_AVERAGE = "AVG";
+    #    /** Method for how to create e.g. daily values from hourly values */
+    AGGREGATION_TYPE_MINIMUM = "MIN";
+    #    /** Method for how to create e.g. daily values from hourly values */
+    AGGREGATION_TYPE_MAXIMUM = "MAX";
+    #    /** Method for how to create e.g. daily values from hourly values */
+    AGGREGATION_TYPE_SUM = "SUM";
+
+class LocationWeatherData:
+    def __init__(self, *args, **kwargs):
+        self.altitude = kwargs.get("altitude", None)
+        self.longitude = kwargs.get("longitude", None)
+        self.latitude = kwargs.get("latitude", None)
+        self.qc = kwargs.get("qc", None)
+        self.data = kwargs.get("data",[])
+
+
+    def as_dict(self):
+        retval = vars(self)
+        # Add location weather data
+        return retval 
+
+class Site:
+    def __init__(self, *args, **kwargs):
+        self.site_id = kwargs.get("site_id", None)
+        self.location = kwargs.get("location", None)
+        if self.location is not None and not isinstance(self.location, Point):
+            self.location = wkb.loads(self.location,hex=True)
+            
+            
+            
\ No newline at end of file

app/templates/usage.html

+<!DOCTYPE html>
+<html>
+    <head>
+        <title>Mock weather data source</title>
+    </head>
+    <body>
+		<p><a href="https://nibio.no/en/projects/malawi-digital-plant-health-service-madiphs" target="new"><img src="/static/images/MaDiPHS_logo.png" style="width: 250px;"/></a></p>
+		<h1>Mock weather data source</h1>
+		<p>This a weather data source with no live data. It returns data from 2021 from one virtual location. It can be used for testing the API.</p>
+		<p>The weather data source returns weather data following the <a href="https://github.com/H2020-IPM-Decisions/WeatherService/blob/develop/docs/weather_service.md" target="new">IPM Decisions weather data format</a>.</p>
+		<p>The parameters provided are [parameter_id in square brackets]:
+		</p>
+		<ul>
+			<li>Mean air temperature at 2m (&deg;C) [1002]</li>
+			<li>Minimum air temperature at 2m (&deg;C) [1003]</li>
+			<li>Maximum air temperature at 2m (&deg;C) [1004]</li>
+			<li>Mean air temperature at -10cm (&deg;C) [1112]</li>
+			<li>Precipitation (mm) [2001]</li>
+			<li>Mean relative humidity at 2m (%) [3002]</li>
+			<li>Leaf wetness at 2m (minutes/h) [3101]</li>
+			<li>Instantaneous wind speed at 2m (m/s) [4002]</li>
+			<li>Mean wind speed at 2m (m/s) [4003]</li>
+			<li>Solar radiation (W/sqm) [5001]</li>
+		</ul>
+		<p>The log interval is either hourly (default) or daily</p>
+
+        <h1>Usage</h1>
+        <p>
+            /weather_data?latitude=[]&amp;longitude=[]&amp;parameters=[,,]&amp;timeStart=[]&amp;timeEnd=[]&amp;interval=[]
+        </p>
+		<ul>
+			<li>latitude and longitude are mandatory parameters. Coordinate system: WGS84 (Decimal degrees). <em>It does however not affect the data returned from this service</em></li>
+			<li>parameters are optional <a href="https://raw.githubusercontent.com/H2020-IPM-Decisions/formats/master/weather_data/weather_parameters_draft_v2.yaml" target="new">Parameters list</a></li>
+			<li>timeStart and timeEnd are optional. Format: YYYY-mm-dd, e.g. 2021-10-28. If omitted, data from Jan-Dec of 2021 will be returned</li>
+			<li>interval is optional. The default value is 3600 (hourly values). To get daily values, set interval=86400.</li> 
+		</ul>
+		<h2>Example</h2>
+		<p><a href="/weather_data?latitude=50.109&amp;longitude=10.961&amp;parameters=1001,2001&timeStart=2021-10-28&timeEnd=2021-11-15">/weather_data?latitude=50.109&amp;longitude=10.961&amp;parameters=1001,2001</a></p>
+    </body>
+</html>
\ No newline at end of file

app/weatherdata_utils.py

+#!/usr/bin/python3
+
+from tracemalloc import start
+from models import WeatherData
+
+
+def filter_period(weather_data, time_start, time_end):
+    # Get the start and end index
+    #print(time_start)
+    start_index = weather_data.get_index_from_epoch_seconds(WeatherData.to_epoch_seconds(time_start))
+    #print(start_index)
+    end_index = weather_data.get_index_from_epoch_seconds(WeatherData.to_epoch_seconds(time_end))
+    for lwd in weather_data.locationWeatherData:
+        lwd.data = lwd.data[start_index:end_index]
+    # Adjust timeStart and timeEnd
+    weather_data.timeStart = WeatherData.to_epoch_seconds(time_start)
+    weather_data.timeEnd = WeatherData.to_epoch_seconds(time_end)
+    return weather_data
+
+def filter_params(weather_data, params):
+    #print(params)
+    include_columns_indexes = []
+    for param in params:
+        include_columns_indexes.append(weather_data.weatherParameters.index(param))
+    for lwd in weather_data.locationWeatherData:
+        # Transpose the matrix
+        # Referring to this: https://stackoverflow.com/questions/8421337/rotating-a-two-dimensional-array-in-python
+        data_transposed = list(zip(*lwd.data, strict=True))
+        #print(data_transposed)
+        filtered_data_transposed = []
+        for include_column_index in include_columns_indexes:
+            filtered_data_transposed.append(data_transposed[include_column_index])
+        lwd.data = list(zip(*filtered_data_transposed))
+        lwd.length = len(lwd.data)
+    # Adjust parameters index
+    weather_data.weatherParameters = [weather_data.weatherParameters[include_column_index] for include_column_index in include_columns_indexes ]
+    return weather_data

data_files/datamanipulation.py

+#!/usr/bin/python3
+# Adding 10 degrees C to all temperatures - to "simulate" Malawi
+# Removing the NULL columns
+
+import json
+
+# Load from file
+wd_file = open("./2021_apelsvoll.json")
+weather_data = json.load(wd_file)
+wd_file.close()
+
+# Get parameters
+parameters = weather_data["weatherParameters"]
+# Which columns in the data array are temperature parameters?
+columns = []
+for param in [1002,1003,1004]:
+    columns.append(parameters.index(param))
+# print(columns)
+data = weather_data["locationWeatherData"][0]["data"]
+
+for row_idx, row in enumerate(data):
+    #print(row_idx)
+    for column in columns:
+        if data[row_idx][column] is None:
+            print("%s: Null found in row %s, column %s" % (data[row_idx], row_idx, column))
+            continue
+        data[row_idx][column] = data[row_idx][column] + 10.0
+    # Remove the two NULL columns
+    data[row_idx] = data[row_idx][2:]
+
+weather_data["locationWeatherData"][0]["width"] = 10
+
+# Remove the two NULL parameters
+weather_data["weatherParameters"] = parameters[2:]
+#print(parameters)
+
+print(json.dumps(weather_data))
\ No newline at end of file

init_flask_dev

+#!/bin/bash
+# Usage: source init_flask_dev
+export FLASK_DEBUG=True
+source mwdsvenv/bin/activate

requirements.txt

+Flask
+shapely>=1.8.0