Reference model produces dataframe with all results

.vscode/settings.json

+{
+    "python.testing.unittestArgs": [
+        "-v",
+        "-s",
+        "./src",
+        "-p",
+        "test_*.py"
+    ],
+    "python.testing.pytestEnabled": false,
+    "python.testing.unittestEnabled": true
+}
\ No newline at end of file

src/vipscore_common/data_utils.py

+#!/usr/bin/python3
+"""
+Copyright (c) 2023 NIBIO <http://www.nibio.no/>. 
+
+This file is part of VIPSCore-Python-Common.
+VIPSCore-Python-Common is free software: you can redistribute it and/or modify
+it under the terms of the NIBIO Open Source License as published by 
+NIBIO, either version 1 of the License, or (at your option) any
+later version.
+ 
+VIPSCore-Python-Common is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+NIBIO Open Source License for more details.
+
+You should have received a copy of the NIBIO Open Source License
+along with VIPSCore-Python-Common.  If not, see <http://www.nibio.no/licenses/>.
+"""
+
+"""
+Util methods for data manipulation
+Author: Tor-Einar Skog <tor-einar.skog@nibio.no>
+"""
+
+import numpy as np
+import pandas as pd
+from pandas import DataFrame
+import json
+from entities import *
+
+def get_dataframe_from_weather_observations(weather_observations: list, timezone:datetime.tzinfo) -> DataFrame:
+    """
+    Create a Pandas data frame with timeseries from VIPS weather observations
+    """
+    # Analyze the input data
+    # Which weather parameters are included
+    # Get first and last time_measured
+    # What's the most specific log interval (hour? day?)
+    params = set()
+    start_date = None
+    end_date = None
+    min_log_interval = None
+    for obs in weather_observations:
+        start_date = obs.timeMeasured if start_date is None else obs.timeMeasured if obs.timeMeasured < start_date else start_date
+        end_date = obs.timeMeasured if end_date is None else obs.timeMeasured if obs.timeMeasured > end_date else end_date
+        params.add(obs.elementMeasurementTypeId)
+        if min_log_interval is None:
+            min_log_interval = obs.logIntervalId 
+        elif min_log_interval == WeatherObservation.LOG_INTERVAL_ID_1D and obs.logIntervalId == WeatherObservation.LOG_INTERVAL_ID_1H:
+            min_log_interval = WeatherObservation.LOG_INTERVAL_ID_1H
+    
+    # Start date must be converted to the given timezone
+    tz_start_date = start_date.astimezone(timezone) 
+    # What's the frequency? Hourly? Daily? TODO: Support other log intervals
+    freq = "H" if min_log_interval == 1 else "D"
+    # This creates a datetime_range in the given timezone
+    datetime_range = pd.date_range(tz_start_date,periods = len(weather_observations),freq=freq,tz=timezone)
+    # Create the dataframe with the correct columns
+    df = pd.DataFrame(index=datetime_range, columns=list(params))
+    
+    # Loop through the observations again, adding values to the frame
+    for obs in weather_observations:
+        df.at[obs.timeMeasured, obs.elementMeasurementTypeId] = obs.value
+    
+    return df
+
+
+def get_weather_observations_from_json(weather_data_raw: str) -> list:
+    """
+    Convert a raw json string with VIPS of weather data to a list of WeatherObservation objects
+    """
+    obs_json_list = []
+    #print(weather_data_raw)
+    weather_data_json = json.loads(weather_data_raw)
+    
+    return get_weather_observations_from_json_list(weather_data_json)
+
+def get_weather_observations_from_json_list(weather_data: list) -> list:
+    """
+    Convert a json list of VIPS of weather data to a list of WeatherObservation objects
+    """
+    retval = []
+    for node in weather_data:
+        retval.append(WeatherObservation(**node))
+    return retval
+
+def get_temp_adjusted_for_base(temp: float, base_temp = 0.0) -> float:
+    adjusted = temp - base_temp
+    return max(0.0, adjusted)
\ No newline at end of file

src/vipscore_common/entities.py

@@ -24,8 +24,9 @@ along with VIPSCore-Python-Common.  If not, see <http://www.nibio.no/licenses/>.
 
 from datetime import datetime
 from shapely.geometry import Point, Polygon
-from pydantic import BaseModel, validator, constr
-from typing import Any, Union
+from pydantic import BaseModel,  validator, constr
+from typing import Any, Union, ClassVar
+import pytz
 
 class Result(BaseModel):
     """Represents a set of DSS model result values for a given point in space (Point, Polygon, MultiPolygon) and time (Period or immediate) """
@@ -35,6 +36,14 @@ class Result(BaseModel):
     warning_status: int
     all_values: dict
 
+
+    WARNING_STATUS_NO_WARNING:              ClassVar[int] = 0
+    WARNING_STATUS_NO_WARNING_MISSING_DATA: ClassVar[int] = 1
+    WARNING_STATUS_NO_RISK:                 ClassVar[int] = 2
+    WARNING_STATUS_MINOR_RISK:              ClassVar[int] = 3
+    WARNING_STATUS_HIGH_RISK:               ClassVar[int] = 4
+    
+
     def get_keys(self):
         return set(self.all_values.keys) if self.all_values is not None else set()
 
@@ -54,6 +63,50 @@ class ModelConfiguration(BaseModel):
     model_id: constr(min_length=10, max_length=10)
     config_parameters: dict
 
+    # Can we do this and still serialize the object??
+    def get_config_parameter_as_date(self, param_name: str, required=True) -> datetime:
+        """
+        Input check and typing of date (ISO format)
+        """
+        param_value = self.config_parameters.get(param_name, None)
+        if required and param_value is None:
+            raise ModelConfigurationException("%s is required" % param_name)
+        try:
+            return datetime.fromisoformat(param_value)
+        except ValueError:
+            raise ModelConfigurationException("%s=%s, which is not a valid date." % (param_name, param_value))
+
+    # Can we do this and still serialize the object??
+    def get_config_parameter_as_string(self, param_name: str, required=True) -> datetime:
+        """
+        Input check
+        """
+        param_value = self.config_parameters.get(param_name, None)
+        if required and param_value is None:
+            raise ModelConfigurationException("%s is required" % param_name)
+        return param_value
+    
+    # Can we do this and still serialize the object??
+    def get_config_parameter_as_timezone(self, param_name: str, required=True) -> datetime:
+        """
+        Input check
+        """
+        param_value = self.config_parameters.get(param_name, None)
+        if required and param_value is None:
+            raise ModelConfigurationException("%s is required" % param_name)
+        
+        try:
+            return pytz.timezone(param_value)
+        except pytz.exceptions.UnknownTimeZoneError:
+            raise ModelConfigurationException("%s=%s, which is not a valid timezone." % (param_name, param_value))
+
+
+class ModelConfigurationException(Exception):
+    """
+    Should be raised when there's something wrong with the VIPS model configuration
+    passed to VIPSModel.set_configuration
+    """
+
 class WeatherObservation(BaseModel):
     """Data class for a single weather observation"""
     elementMeasurementTypeId: str
@@ -62,18 +115,19 @@ class WeatherObservation(BaseModel):
     value: float
 
     # Log interval categories
-    LOG_INTERVAL_ID_1M = 8;
-    LOG_INTERVAL_ID_5M = 9;
-    LOG_INTERVAL_ID_10M = 6;
-    LOG_INTERVAL_ID_15M = 5;
-    LOG_INTERVAL_ID_30M = 4;
-    LOG_INTERVAL_ID_1H = 1;
-    LOG_INTERVAL_ID_3H = 3;
-    LOG_INTERVAL_ID_6H = 7;
-    LOG_INTERVAL_ID_1D = 2;
+    LOG_INTERVAL_ID_1M: ClassVar[int] = 8
+    LOG_INTERVAL_ID_5M: ClassVar[int] = 9
+    LOG_INTERVAL_ID_10M: ClassVar[int] = 6
+    LOG_INTERVAL_ID_15M: ClassVar[int] = 5
+    LOG_INTERVAL_ID_30M: ClassVar[int] = 4
+    LOG_INTERVAL_ID_1H: ClassVar[int] = 1
+    LOG_INTERVAL_ID_3H: ClassVar[int] = 3
+    LOG_INTERVAL_ID_6H: ClassVar[int] = 7
+    LOG_INTERVAL_ID_1D: ClassVar[int] = 2
 
     @validator("timeMeasured")
     def ensure_timezone(cls, v):
         if v.tzinfo is None or v.tzinfo.utcoffset(v) is None:
             raise ValueError("%s must be timezone aware" % v)
         return v
+

src/vipscore_common/reference_model.py

@@ -20,25 +20,69 @@ along with VIPSCore-Python-Common.  If not, see <http://www.nibio.no/licenses/>.
 
 from vips_model import VIPSModel
 from entities import Result, ModelConfiguration, WeatherObservation
+import data_utils
+import numpy as np
 
 class ReferenceModel(VIPSModel):
+    """
+        A reference implementation of the VIPSModel abstract class. Showcasing core functionality
+        and best practices
+    """
 
     MODEL_ID        = "REFERENCEM"
     COPYRIGHT       = "(c) 2023 NIBIO"
+
+    THRESHOLD_LOW = 100.0
+    THRESHOLD_MEDIUM = 300.0
+    THRESHOLD_HIGH = 500.0
  
     def set_configuration(self, model_configuration: ModelConfiguration):
         if not isinstance(model_configuration, ModelConfiguration):
             raise ValueError("%s is not a ModelConfiguration object" % model_configuration)
         if model_configuration.model_id != ReferenceModel.MODEL_ID:
             raise ValueError("%s is not the correct model ID!" % model_configuration.model_id)
-        self.sowing_date = model_configuration.config_parameters["sowingDate"]
-        self.observations = []
-        for node in model_configuration.config_parameters["observations"]:
-            self.observations.append(WeatherObservation(**node))
+        
+        # Input data check
+        self.sowing_date = model_configuration.get_config_parameter_as_date("sowingDate")
+        self.timezone = model_configuration.get_config_parameter_as_timezone("timeZone")
+        # Weather data is turned into Pandas dataframe
+        self.df = data_utils.get_dataframe_from_weather_observations(
+                data_utils.get_weather_observations_from_json_list(
+                    model_configuration.config_parameters["observations"]
+                    ),
+                self.timezone
+            )
+
+    
+
+    def determine_warning_status(self, TMDD: float) -> int:
+        if TMDD < ReferenceModel.THRESHOLD_MEDIUM:
+            return  Result.WARNING_STATUS_NO_WARNING
+        if TMDD < ReferenceModel.THRESHOLD_HIGH:
+            return Result.WARNING_STATUS_MINOR_RISK
+        else:
+            return Result.WARNING_STATUS_HIGH_RISK
+
+        
     
     def get_result(self) -> list[Result]:
         """Get the results as a list of Result objects (TODO ref)"""
-        pass
+        
+        # Calculate day degrees from sowingDate and as far as weather data goes
+        # Adjusting for base temperature
+        self.df["TMContrib"] = self.df["TM"].apply(data_utils.get_temp_adjusted_for_base, args=(5,))
+        # Aggregating the day degrees
+        self.df["TMDD"] = self.df["TMContrib"].cumsum()
+        # Adding the thresholds to the data frame
+        self.df["THRESHOLD_LOW"] = ReferenceModel.THRESHOLD_LOW
+        self.df["THRESHOLD_MEDIUM"] = ReferenceModel.THRESHOLD_MEDIUM
+        self.df["THRESHOLD_HIGH"] = ReferenceModel.THRESHOLD_HIGH
+        self.df["WARNING_STATUS"] = self.df["TMDD"].apply(self.determine_warning_status)
+        print(self.df)
+        # For each day: check accumulated day-degrees and decide warning status
+
+
+
 
     @property
     def model_id(self) -> str:
@@ -91,8 +135,8 @@ class ReferenceModel(VIPSModel):
         Gray   status (warning status == 0): Warning not applicable
         Blue   status (warning status == 1): Missing data
         Green  status (warning status == 2): No risk. Sleep well
-        Yellow status (warning status == 3): Medium risk. Be on the alert, inspect your field
-        Red status    (warning status == 4): High risk. When the going gets tough, the tough get going
+        Yellow status (warning status == 3): The day-degree hreshold for medium risk has been passed. Be on the alert, inspect your field
+        Red status    (warning status == 4): The day-degree threshold for high risk has been passed. When the going gets tough, the tough get going
         """
 
     def get_model_usage(self, language = VIPSModel.default_language) -> str:

src/vipscore_common/tests/test_data_utils.py

+import unittest
+
+
+import data_utils
+from reference_model import *
+from entities import *
+import pytz
+
+class TestDataUtils(unittest.TestCase):
+    def test_get_weather_observations_from_json(self):
+        """
+        The method can convert a json dict to a list of WeatherObservation objects
+        """
+        with open ("src/vipscore_common/tests/weather_data_2015_NO_aas_TMD.json") as f:
+            result = data_utils.get_weather_observations_from_json(f.read())
+            # Length should be > 0
+            self.assertGreater(len(result), 0)
+            # All items should be a WeatherObservation class
+            for item in result:
+                self.assertIsInstance(item, WeatherObservation)
+    
+    def test_getdataframe_from_weather_observations(self):
+        """
+        Lorem Ipsum osv
+        """
+        with open ("src/vipscore_common/tests/weather_data_2015_NO_aas_TMD.json") as f:
+            weather_observations = data_utils.get_weather_observations_from_json(f.read())
+            result = data_utils.get_dataframe_from_weather_observations(weather_observations, pytz.timezone("Europe/Oslo"))
+            self.assertIsNotNone(result)
+            start_date = datetime.fromisoformat("2015-03-01T00:00:00+01:00")
+            self.assertEqual(1.41025,result.at[start_date,"TM"])

src/vipscore_common/tests/test_reference_model.py

@@ -6,15 +6,16 @@ from reference_model import *
 from entities import *
 
 def get_model_configuration():
-        with open ("src/vipscore_common/tests/weather_data_2015_NO_aas_TMD.json") as f:
-            weather_observations = json.load(f)
-            return ModelConfiguration(
-                model_id=ReferenceModel.MODEL_ID,
-                config_parameters={
-                    "sowingDate": "2022-03-01",
-                    "observations": weather_observations
-                }
-            )
+    with open ("src/vipscore_common/tests/weather_data_2015_NO_aas_TMD.json") as f:
+        weather_observations = json.load(f)
+        return ModelConfiguration(
+            model_id=ReferenceModel.MODEL_ID,
+            config_parameters={
+                "sowingDate": "2022-03-01",
+                "timeZone" : "Europe/Oslo",
+                "observations": weather_observations
+            }
+        )
 
 class TestReferenceModel(unittest.TestCase):
     def test_set_configuration(self):
@@ -25,6 +26,14 @@ class TestReferenceModel(unittest.TestCase):
         instance = ReferenceModel()
         instance.set_configuration(get_model_configuration())
 
+    def test_get_result(self):
+        """
+        We get a series of results from the calculation
+        """
+        instance = ReferenceModel()
+        instance.set_configuration(get_model_configuration())
+        result_list = instance.get_result()
+
     def test_get_model_id(self):
          """
          The model returns the correct ID