diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000000000000000000000000000000000000..d73198fd985df3ed4a7cf516c93af8d4a6df5734
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,11 @@
+{
+ "python.testing.unittestArgs": [
+ "-v",
+ "-s",
+ "./src",
+ "-p",
+ "test_*.py"
+ ],
+ "python.testing.pytestEnabled": false,
+ "python.testing.unittestEnabled": true
+}
\ No newline at end of file
diff --git a/docs/illustrations/vipslogo_512.png b/docs/illustrations/vipslogo_512.png
new file mode 100644
index 0000000000000000000000000000000000000000..f4e416ef1adf3fa60969a5cfb1fd6387ee867ae3
Binary files /dev/null and b/docs/illustrations/vipslogo_512.png differ
diff --git a/src/vipscore_common/data_utils.py b/src/vipscore_common/data_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..10098ecc837c05b0da783010835a099717e787ec
--- /dev/null
+++ b/src/vipscore_common/data_utils.py
@@ -0,0 +1,89 @@
+#!/usr/bin/python3
+"""
+Copyright (c) 2023 NIBIO <http://www.nibio.no/>.
+
+This file is part of VIPSCore-Python-Common.
+VIPSCore-Python-Common is free software: you can redistribute it and/or modify
+it under the terms of the NIBIO Open Source License as published by
+NIBIO, either version 1 of the License, or (at your option) any
+later version.
+
+VIPSCore-Python-Common is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+NIBIO Open Source License for more details.
+
+You should have received a copy of the NIBIO Open Source License
+along with VIPSCore-Python-Common. If not, see <http://www.nibio.no/licenses/>.
+"""
+
+"""
+Util methods for data manipulation
+Author: Tor-Einar Skog <tor-einar.skog@nibio.no>
+"""
+
+import numpy as np
+import pandas as pd
+from pandas import DataFrame
+import json
+from entities import *
+
+def get_dataframe_from_weather_observations(weather_observations: list, timezone:datetime.tzinfo) -> DataFrame:
+ """
+ Create a Pandas data frame with timeseries from VIPS weather observations
+ """
+ # Analyze the input data
+ # Which weather parameters are included
+ # Get first and last time_measured
+ # What's the most specific log interval (hour? day?)
+ params = set()
+ start_date = None
+ end_date = None
+ min_log_interval = None
+ for obs in weather_observations:
+ start_date = obs.timeMeasured if start_date is None else obs.timeMeasured if obs.timeMeasured < start_date else start_date
+ end_date = obs.timeMeasured if end_date is None else obs.timeMeasured if obs.timeMeasured > end_date else end_date
+ params.add(obs.elementMeasurementTypeId)
+ if min_log_interval is None:
+ min_log_interval = obs.logIntervalId
+ elif min_log_interval == WeatherObservation.LOG_INTERVAL_ID_1D and obs.logIntervalId == WeatherObservation.LOG_INTERVAL_ID_1H:
+ min_log_interval = WeatherObservation.LOG_INTERVAL_ID_1H
+
+ # Start date must be converted to the given timezone
+ tz_start_date = start_date.astimezone(timezone)
+ # What's the frequency? Hourly? Daily? TODO: Support other log intervals
+ freq = "H" if min_log_interval == 1 else "D"
+ # This creates a datetime_range in the given timezone
+ datetime_range = pd.date_range(tz_start_date,periods = len(weather_observations),freq=freq,tz=timezone)
+ # Create the dataframe with the correct columns
+ df = pd.DataFrame(index=datetime_range, columns=list(params))
+
+ # Loop through the observations again, adding values to the frame
+ for obs in weather_observations:
+ df.at[obs.timeMeasured, obs.elementMeasurementTypeId] = obs.value
+
+ return df
+
+
+def get_weather_observations_from_json(weather_data_raw: str) -> list:
+ """
+ Convert a raw json string with VIPS of weather data to a list of WeatherObservation objects
+ """
+ obs_json_list = []
+ #print(weather_data_raw)
+ weather_data_json = json.loads(weather_data_raw)
+
+ return get_weather_observations_from_json_list(weather_data_json)
+
+def get_weather_observations_from_json_list(weather_data: list) -> list:
+ """
+ Convert a json list of VIPS of weather data to a list of WeatherObservation objects
+ """
+ retval = []
+ for node in weather_data:
+ retval.append(WeatherObservation(**node))
+ return retval
+
+def get_temp_adjusted_for_base(temp: float, base_temp = 0.0) -> float:
+ adjusted = temp - base_temp
+ return max(0.0, adjusted)
\ No newline at end of file
diff --git a/src/vipscore_common/entities.py b/src/vipscore_common/entities.py
index d9be988a1c0a190ed6d5eb4b9b49ad139f10c0be..43b17a8005066978803aabc30bfb046cb018f02f 100755
--- a/src/vipscore_common/entities.py
+++ b/src/vipscore_common/entities.py
@@ -24,8 +24,9 @@ along with VIPSCore-Python-Common. If not, see <http://www.nibio.no/licenses/>.
from datetime import datetime
from shapely.geometry import Point, Polygon
-from pydantic import BaseModel, validator, constr
-from typing import Any, Union
+from pydantic import BaseModel, validator, constr
+from typing import Any, Union, ClassVar
+import pytz
class Result(BaseModel):
"""Represents a set of DSS model result values for a given point in space (Point, Polygon, MultiPolygon) and time (Period or immediate) """
@@ -35,6 +36,14 @@ class Result(BaseModel):
warning_status: int
all_values: dict
+
+ WARNING_STATUS_NO_WARNING: ClassVar[int] = 0
+ WARNING_STATUS_NO_WARNING_MISSING_DATA: ClassVar[int] = 1
+ WARNING_STATUS_NO_RISK: ClassVar[int] = 2
+ WARNING_STATUS_MINOR_RISK: ClassVar[int] = 3
+ WARNING_STATUS_HIGH_RISK: ClassVar[int] = 4
+
+
def get_keys(self):
return set(self.all_values.keys) if self.all_values is not None else set()
@@ -54,6 +63,50 @@ class ModelConfiguration(BaseModel):
model_id: constr(min_length=10, max_length=10)
config_parameters: dict
+ # Can we do this and still serialize the object??
+ def get_config_parameter_as_date(self, param_name: str, required=True) -> datetime:
+ """
+ Input check and typing of date (ISO format)
+ """
+ param_value = self.config_parameters.get(param_name, None)
+ if required and param_value is None:
+ raise ModelConfigurationException("%s is required" % param_name)
+ try:
+ return datetime.fromisoformat(param_value)
+ except ValueError:
+ raise ModelConfigurationException("%s=%s, which is not a valid date." % (param_name, param_value))
+
+ # Can we do this and still serialize the object??
+ def get_config_parameter_as_string(self, param_name: str, required=True) -> datetime:
+ """
+ Input check
+ """
+ param_value = self.config_parameters.get(param_name, None)
+ if required and param_value is None:
+ raise ModelConfigurationException("%s is required" % param_name)
+ return param_value
+
+ # Can we do this and still serialize the object??
+ def get_config_parameter_as_timezone(self, param_name: str, required=True) -> datetime:
+ """
+ Input check
+ """
+ param_value = self.config_parameters.get(param_name, None)
+ if required and param_value is None:
+ raise ModelConfigurationException("%s is required" % param_name)
+
+ try:
+ return pytz.timezone(param_value)
+ except pytz.exceptions.UnknownTimeZoneError:
+ raise ModelConfigurationException("%s=%s, which is not a valid timezone." % (param_name, param_value))
+
+
+class ModelConfigurationException(Exception):
+ """
+ Should be raised when there's something wrong with the VIPS model configuration
+ passed to VIPSModel.set_configuration
+ """
+
class WeatherObservation(BaseModel):
"""Data class for a single weather observation"""
elementMeasurementTypeId: str
@@ -62,18 +115,19 @@ class WeatherObservation(BaseModel):
value: float
# Log interval categories
- LOG_INTERVAL_ID_1M = 8;
- LOG_INTERVAL_ID_5M = 9;
- LOG_INTERVAL_ID_10M = 6;
- LOG_INTERVAL_ID_15M = 5;
- LOG_INTERVAL_ID_30M = 4;
- LOG_INTERVAL_ID_1H = 1;
- LOG_INTERVAL_ID_3H = 3;
- LOG_INTERVAL_ID_6H = 7;
- LOG_INTERVAL_ID_1D = 2;
+ LOG_INTERVAL_ID_1M: ClassVar[int] = 8
+ LOG_INTERVAL_ID_5M: ClassVar[int] = 9
+ LOG_INTERVAL_ID_10M: ClassVar[int] = 6
+ LOG_INTERVAL_ID_15M: ClassVar[int] = 5
+ LOG_INTERVAL_ID_30M: ClassVar[int] = 4
+ LOG_INTERVAL_ID_1H: ClassVar[int] = 1
+ LOG_INTERVAL_ID_3H: ClassVar[int] = 3
+ LOG_INTERVAL_ID_6H: ClassVar[int] = 7
+ LOG_INTERVAL_ID_1D: ClassVar[int] = 2
@validator("timeMeasured")
def ensure_timezone(cls, v):
if v.tzinfo is None or v.tzinfo.utcoffset(v) is None:
raise ValueError("%s must be timezone aware" % v)
return v
+
diff --git a/src/vipscore_common/reference_model.py b/src/vipscore_common/reference_model.py
index 54d3c3229df6c6b43f1747693b495153dfe6a652..da96f082adeae5eadb306ed3b63315223ee5dcd5 100755
--- a/src/vipscore_common/reference_model.py
+++ b/src/vipscore_common/reference_model.py
@@ -20,25 +20,69 @@ along with VIPSCore-Python-Common. If not, see <http://www.nibio.no/licenses/>.
from vips_model import VIPSModel
from entities import Result, ModelConfiguration, WeatherObservation
+import data_utils
+import numpy as np
class ReferenceModel(VIPSModel):
+ """
+ A reference implementation of the VIPSModel abstract class. Showcasing core functionality
+ and best practices
+ """
MODEL_ID = "REFERENCEM"
COPYRIGHT = "(c) 2023 NIBIO"
+
+ THRESHOLD_LOW = 100.0
+ THRESHOLD_MEDIUM = 300.0
+ THRESHOLD_HIGH = 500.0
def set_configuration(self, model_configuration: ModelConfiguration):
if not isinstance(model_configuration, ModelConfiguration):
raise ValueError("%s is not a ModelConfiguration object" % model_configuration)
if model_configuration.model_id != ReferenceModel.MODEL_ID:
raise ValueError("%s is not the correct model ID!" % model_configuration.model_id)
- self.sowing_date = model_configuration.config_parameters["sowingDate"]
- self.observations = []
- for node in model_configuration.config_parameters["observations"]:
- self.observations.append(WeatherObservation(**node))
+
+ # Input data check
+ self.sowing_date = model_configuration.get_config_parameter_as_date("sowingDate")
+ self.timezone = model_configuration.get_config_parameter_as_timezone("timeZone")
+ # Weather data is turned into Pandas dataframe
+ self.df = data_utils.get_dataframe_from_weather_observations(
+ data_utils.get_weather_observations_from_json_list(
+ model_configuration.config_parameters["observations"]
+ ),
+ self.timezone
+ )
+
+
+
+ def determine_warning_status(self, TMDD: float) -> int:
+ if TMDD < ReferenceModel.THRESHOLD_MEDIUM:
+ return Result.WARNING_STATUS_NO_WARNING
+ if TMDD < ReferenceModel.THRESHOLD_HIGH:
+ return Result.WARNING_STATUS_MINOR_RISK
+ else:
+ return Result.WARNING_STATUS_HIGH_RISK
+
+
def get_result(self) -> list[Result]:
"""Get the results as a list of Result objects (TODO ref)"""
- pass
+
+ # Calculate day degrees from sowingDate and as far as weather data goes
+ # Adjusting for base temperature
+ self.df["TMContrib"] = self.df["TM"].apply(data_utils.get_temp_adjusted_for_base, args=(5,))
+ # Aggregating the day degrees
+ self.df["TMDD"] = self.df["TMContrib"].cumsum()
+ # Adding the thresholds to the data frame
+ self.df["THRESHOLD_LOW"] = ReferenceModel.THRESHOLD_LOW
+ self.df["THRESHOLD_MEDIUM"] = ReferenceModel.THRESHOLD_MEDIUM
+ self.df["THRESHOLD_HIGH"] = ReferenceModel.THRESHOLD_HIGH
+ self.df["WARNING_STATUS"] = self.df["TMDD"].apply(self.determine_warning_status)
+ print(self.df)
+ # For each day: check accumulated day-degrees and decide warning status
+
+
+
@property
def model_id(self) -> str:
@@ -91,8 +135,8 @@ class ReferenceModel(VIPSModel):
Gray status (warning status == 0): Warning not applicable
Blue status (warning status == 1): Missing data
Green status (warning status == 2): No risk. Sleep well
- Yellow status (warning status == 3): Medium risk. Be on the alert, inspect your field
- Red status (warning status == 4): High risk. When the going gets tough, the tough get going
+ Yellow status (warning status == 3): The day-degree hreshold for medium risk has been passed. Be on the alert, inspect your field
+ Red status (warning status == 4): The day-degree threshold for high risk has been passed. When the going gets tough, the tough get going
"""
def get_model_usage(self, language = VIPSModel.default_language) -> str:
diff --git a/src/vipscore_common/tests/test_data_utils.py b/src/vipscore_common/tests/test_data_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..09dab010aa0f266db0bf92d6cd21a780a5319577
--- /dev/null
+++ b/src/vipscore_common/tests/test_data_utils.py
@@ -0,0 +1,31 @@
+import unittest
+
+
+import data_utils
+from reference_model import *
+from entities import *
+import pytz
+
+class TestDataUtils(unittest.TestCase):
+ def test_get_weather_observations_from_json(self):
+ """
+ The method can convert a json dict to a list of WeatherObservation objects
+ """
+ with open ("src/vipscore_common/tests/weather_data_2015_NO_aas_TMD.json") as f:
+ result = data_utils.get_weather_observations_from_json(f.read())
+ # Length should be > 0
+ self.assertGreater(len(result), 0)
+ # All items should be a WeatherObservation class
+ for item in result:
+ self.assertIsInstance(item, WeatherObservation)
+
+ def test_getdataframe_from_weather_observations(self):
+ """
+ Lorem Ipsum osv
+ """
+ with open ("src/vipscore_common/tests/weather_data_2015_NO_aas_TMD.json") as f:
+ weather_observations = data_utils.get_weather_observations_from_json(f.read())
+ result = data_utils.get_dataframe_from_weather_observations(weather_observations, pytz.timezone("Europe/Oslo"))
+ self.assertIsNotNone(result)
+ start_date = datetime.fromisoformat("2015-03-01T00:00:00+01:00")
+ self.assertEqual(1.41025,result.at[start_date,"TM"])
diff --git a/src/vipscore_common/tests/test_reference_model.py b/src/vipscore_common/tests/test_reference_model.py
index 9732894028a296ef615c80baf9eff8bbc8115bc5..161a04b6738cc19ffaae8df4844f550a43c8ef10 100644
--- a/src/vipscore_common/tests/test_reference_model.py
+++ b/src/vipscore_common/tests/test_reference_model.py
@@ -6,15 +6,16 @@ from reference_model import *
from entities import *
def get_model_configuration():
- with open ("src/vipscore_common/tests/weather_data_2015_NO_aas_TMD.json") as f:
- weather_observations = json.load(f)
- return ModelConfiguration(
- model_id=ReferenceModel.MODEL_ID,
- config_parameters={
- "sowingDate": "2022-03-01",
- "observations": weather_observations
- }
- )
+ with open ("src/vipscore_common/tests/weather_data_2015_NO_aas_TMD.json") as f:
+ weather_observations = json.load(f)
+ return ModelConfiguration(
+ model_id=ReferenceModel.MODEL_ID,
+ config_parameters={
+ "sowingDate": "2022-03-01",
+ "timeZone" : "Europe/Oslo",
+ "observations": weather_observations
+ }
+ )
class TestReferenceModel(unittest.TestCase):
def test_set_configuration(self):
@@ -25,6 +26,14 @@ class TestReferenceModel(unittest.TestCase):
instance = ReferenceModel()
instance.set_configuration(get_model_configuration())
+ def test_get_result(self):
+ """
+ We get a series of results from the calculation
+ """
+ instance = ReferenceModel()
+ instance.set_configuration(get_model_configuration())
+ result_list = instance.get_result()
+
def test_get_model_id(self):
"""
The model returns the correct ID