Basic phenology model (General)

Added required changes to related to phenology 

src/main/java/no/nibio/vips/logic/scheduling/model/preprocessor/PhenologyModelPreprocessor.java

+ 
+package no.nibio.vips.logic.scheduling.model.preprocessor;
+
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.MappingJsonFactory;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import no.nibio.vips.entity.ModelConfiguration;
+import no.nibio.vips.entity.WeatherObservation;
+import no.nibio.vips.logic.entity.ForecastConfiguration;
+import no.nibio.vips.logic.entity.PointOfInterestWeatherStation;
+import no.nibio.vips.logic.scheduling.model.ModelRunPreprocessor;
+import no.nibio.vips.logic.scheduling.model.PreprocessorException;
+import no.nibio.vips.logic.util.Plantation;
+import no.nibio.vips.logic.util.SessionControllerGetter;
+import no.nibio.vips.logic.util.SystemTime;
+import no.nibio.vips.model.ConfigValidationException;
+import no.nibio.vips.util.WeatherElements;
+import no.nibio.vips.util.WeatherObservationListException;
+import no.nibio.vips.util.WeatherUtil;
+import no.nibio.vips.util.weather.WeatherDataSourceException;
+import no.nibio.vips.util.weather.WeatherDataSourceUtil;
+
+/**
+ *
+ * @author wildfly
+ */
+
+
+public class PhenologyModelPreprocessor extends ModelRunPreprocessor{
+     public final           Logger      LOGGER              =   Logger.getLogger(this.getClass().getName());
+    private final           boolean     DEBUG               =   false;
+    private final   static  String      MODEL_ID            =   "PHENOLOGYM";
+    private final   static  String      MAIZE_VARITY        =   "maizeVarity";
+    private final   static  String      SOWING_DATE         =   "sowingDate";
+    
+    private final   static  String      SOWING_DATA         =   "sowingData";
+    private final   static  String      PLANTATION_DATA     =   "plantationData";
+    private final   static  String      FILE_PLANTATION     =   "/dataset/plantationData.json";
+    
+    @Override
+    public ModelConfiguration getModelConfiguration(ForecastConfiguration configuration) throws PreprocessorException {
+        String      PHENOLOGY_MAZE_VARIETY                  = SessionControllerGetter.getForecastBean().getDeCamelizedFieldName(MODEL_ID, MAIZE_VARITY);
+        String      PHENOLOGY_SOWING_DATE                   = SessionControllerGetter.getForecastBean().getDeCamelizedFieldName(MODEL_ID, SOWING_DATE);
+        
+        String      paramMazeVariety                        = null;          
+        Date        paramSowingDate                         = null;
+        
+        
+        String                         paramConfigValue     =   null;
+        
+        ModelConfiguration              retVal              =   new ModelConfiguration();
+        PointOfInterestWeatherStation   weatherStation      =   (PointOfInterestWeatherStation)  configuration.getWeatherStationPointOfInterestId();
+        WeatherDataSourceUtil           wdsUtil             =   new WeatherDataSourceUtil();
+        WeatherUtil                     wUtil               =   new WeatherUtil();
+        TimeZone                        timeZone            =   TimeZone.getTimeZone(weatherStation.getTimeZone());
+        SimpleDateFormat                format              =   new SimpleDateFormat("yyyy-MM-dd");
+        format.setTimeZone(timeZone);
+        // Getting date three days after "today"
+        Calendar cal = Calendar.getInstance(timeZone);
+        cal.setTime(SystemTime.getSystemTime());
+        cal.add(Calendar.DATE, 3);
+        Date endDate = wUtil.normalizeToExactDate(configuration.getDateEnd(), timeZone);
+        cal.set(Calendar.MONTH, Calendar.JANUARY);
+        cal.set(Calendar.DATE, 1);
+        
+        Plantation  sowdata  = new Plantation();
+        
+        try
+        {
+        paramConfigValue    = configuration.getForecastModelConfigurationValue(PHENOLOGY_SOWING_DATE);
+        paramSowingDate     = format.parse(paramConfigValue);
+        }
+        catch(ParseException ex)
+        {
+            throw new PreprocessorException("Could not parse date for Sowing Date for "+MODEL_ID+". Given string was: "+ paramConfigValue);
+        }
+        
+        Date                            startDate       =   wUtil.normalizeToExactDate(
+                                                                        paramSowingDate
+                                                                    ,   timeZone
+                                                            );
+        
+        List<WeatherObservation>        observations    =   null;
+        
+        try {
+            observations    = wdsUtil.getWeatherObservations(
+                                                              weatherStation
+                                                            , WeatherObservation.LOG_INTERVAL_ID_1H
+                                                            , new String[]{
+                                                                            WeatherElements.TEMPERATURE_MEAN,
+                                                                            WeatherElements.PRECIPITATION,
+                                                                            WeatherElements.LEAF_WETNESS_DURATION,
+                                                                            WeatherElements.RELATIVE_HUMIDITY_MEAN
+                                                                        }
+                                                            , startDate
+                                                            , endDate);
+        }
+        catch(WeatherDataSourceException ex)
+        {
+            throw new PreprocessorException(ex.getMessage());
+        }
+        
+
+        
+        
+        paramMazeVariety    = configuration.getForecastModelConfigurationValue(PHENOLOGY_MAZE_VARIETY);
+    
+ 
+        Collections.sort(observations);
+
+        try {
+            List<WeatherObservation>  calcObs  = wdsUtil.getWeatherObservations(
+                                                                                    weatherStation
+                                                                                ,   WeatherObservation.LOG_INTERVAL_ID_1H
+                                                                                ,   new String[] {
+                                                                                                WeatherElements.GLOBAL_RADIATION,
+                                                                                                WeatherElements.WIND_SPEED_2M
+                                                                                                }
+                                                                                , startDate
+                                                                                , endDate
+                                                                                );
+            observations.addAll(calcObs);
+        }catch (WeatherDataSourceException ex)
+        {
+            // TODO nothing reported
+        }
+        
+       if(DEBUG)
+        {
+            LOGGER.log(Level.CONFIG, "Finished getting weather data at  "+new Date().toString());
+        }
+        try {
+            observations = validateAndSanitizeObservations(observations, startDate);
+        } catch (ConfigValidationException | WeatherObservationListException ex) {
+            //ex.printStackTrace();
+            throw new PreprocessorException(ex.getMessage());
+        } 
+        if(DEBUG)
+        {        
+            LOGGER.log(Level.CONFIG, "Observations=" + observations.toString());
+        }
+    
+        sowdata.setVarietyId(paramMazeVariety);
+        sowdata.setStartDate(paramSowingDate);
+        
+        retVal.setModelId(this.getModelId());
+        retVal.setConfigParameter("timeZone", timeZone.getID());
+        retVal.setConfigParameter("observations", observations);
+        retVal.setConfigParameter(SOWING_DATA, sowdata);
+        retVal.setConfigParameter(PLANTATION_DATA, getConfigurationPlantation(FILE_PLANTATION));
+
+        
+        return retVal;
+    }
+
+    
+    
+    
+    @Override
+    public String getModelId() {
+        return MODEL_ID;
+    }
+    
+    
+       private List<WeatherObservation> validateAndSanitizeObservations(List<WeatherObservation> observations, Date firstTimeStamp) throws ConfigValidationException, WeatherObservationListException 
+       {
+       
+         if(DEBUG)
+        {
+            LOGGER.log(Level.CONFIG, "validateAndSanitizeObservations");
+        }
+        
+        WeatherUtil wUtil = new WeatherUtil();
+        
+        // First we remove all duplicates
+        observations = wUtil.removeDuplicateWeatherObservations(observations, null);
+        
+        // Fix weather data
+        List<WeatherObservation> fixedObservations = wUtil.fixHourlyValuesForParameters(
+                observations, 
+                new HashSet(Arrays.asList("TM","RR")), 
+                firstTimeStamp, 
+                null
+        );
+        
+        
+        // Now we need to validate and possibly try to fix the weather data
+        List<WeatherObservation> TM = new ArrayList<>();
+        List<WeatherObservation> RR = new ArrayList<>();
+        
+        for(WeatherObservation o:fixedObservations)
+        {
+            switch(o.getElementMeasurementTypeId())
+            {
+                case WeatherElements.TEMPERATURE_MEAN:
+                    TM.add(o); 
+                    break;
+                case WeatherElements.PRECIPITATION:
+                    RR.add(o); 
+                    break;
+                    
+            }
+        }
+        
+        List<WeatherObservation> BT = new ArrayList<>();
+        List<WeatherObservation> UM = new ArrayList<>();
+        List<WeatherObservation> Q0 = new ArrayList<>();
+        List<WeatherObservation> FM2 = new ArrayList<>();
+        for(WeatherObservation o:observations)
+        {
+            switch(o.getElementMeasurementTypeId())
+            {
+                case WeatherElements.LEAF_WETNESS_DURATION:
+                    BT.add(o); 
+                    break;
+                case WeatherElements.RELATIVE_HUMIDITY_MEAN:
+                    UM.add(o); 
+                    break;
+                case WeatherElements.GLOBAL_RADIATION:
+                    Q0.add(o);
+                    break;
+                case WeatherElements.WIND_SPEED_2M:
+                    FM2.add(o);
+                    break;
+                default:
+                    // Let it pass in silence 
+                    break;
+            }
+        }
+        
+        // Problems with weather observations
+        
+        // Holes in series
+        if(DEBUG)
+        {
+            LOGGER.log(Level.CONFIG, "checkForAndFixHourlyTimeSeriesHoles");
+        }
+
+        
+        
+        // Unequal length of lists
+        if  (        
+                RR.size() != TM.size()
+                || BT.size() != TM.size()
+                || RR.size() != TM.size()
+            )
+        {
+            UM = wUtil.fixHourlyValuesForParameters(
+                    UM, 
+                    new HashSet(Arrays.asList("UM")), 
+                    firstTimeStamp, 
+                    null
+            );
+            
+            
+            
+            // Fallback if missing leaf wetness: If we have relative humidity. leaf wetness may be calculated
+            if((BT.size() != TM.size()) && (UM.size() == TM.size()))
+            {
+                BT = wUtil.calculateLeafWetnessHourSeriesBestEffort(BT,TM, RR, Q0, FM2, UM);
+                
+                if(BT.size() != TM.size())
+                {
+                   throw new ConfigValidationException("Missing leaf wetness data. Also, attempting to calculate leaf wetness from other weather parameters failed.");
+                }
+            }
+            else
+            {
+                LOGGER.log(Level.WARNING, "TM starts " + TM.get(0).getTimeMeasured() + ", ends " + TM.get(TM.size()-1).getTimeMeasured());
+                throw new ConfigValidationException("Incorrect number of weather data. TM.size() = " + TM.size() + ", BT.size()=" + BT.size() +  ", RR.size()=" + RR.size() + ", UM.size()=" + UM.size() );
+            }
+        }
+        List<WeatherObservation> retVal = new ArrayList<>();
+        retVal.addAll(TM);
+        retVal.addAll(RR);
+        retVal.addAll(BT);
+        return retVal;
+    }
+    
+    private List<Plantation> getConfigurationPlantation(String fileName)
+    {
+        List<Plantation> plantations    = new ArrayList<Plantation>();
+        
+        
+        try {
+            BufferedInputStream inputStream = new BufferedInputStream(this.getClass().getResourceAsStream(fileName));
+            JsonFactory f = new MappingJsonFactory();
+            JsonParser jp = f.createParser(inputStream);
+            JsonNode all = jp.readValueAsTree();
+            JsonNode nodePhaseInfo = all.path("phaseInfo");
+            if(nodePhaseInfo.isArray())
+            {
+                for(JsonNode node : nodePhaseInfo)
+                {
+                     Plantation   plantation  =   new Plantation
+                                                (           all.get("typeName").asText()
+                                                        ,   node.get("phaseName").asText()
+                                                       ,    node.get("heatReq").asDouble()
+                                                );
+                     plantations.add(plantation);
+                }
+            }
+            
+            
+            /*
+            if(all.isArray())
+            {
+               for(JsonNode node : all)
+               {
+                   Plantation   plantation  =   new Plantation
+                                                (
+                                                            node.get("maizeVarity").asText()
+                                                       ,    node.get("phaseName").asText()
+                                                       ,    node.get("baseTemp").asText()
+                                                       ,    node.get("heatReq").asDouble()
+                                                );
+                  plantations.add(plantation);
+                  
+                } 
+            }
+            
+            */
+            
+            
+        } catch (IOException ex) {
+            Logger.getLogger(this.getClass().getName()).log(Level.SEVERE, null, ex);
+        }
+        
+        return plantations;
+    }
+       
+}

src/main/java/no/nibio/vips/logic/util/Plantation.java

+
+package no.nibio.vips.logic.util;
+
+import java.util.Date;
+
+/**
+ *
+ * @author wildfly
+ */
+
+
+public class Plantation {
+     private String  varietyId;
+    private String  phaseName;
+    private String  baseTemp;
+    private Double  heatReq;
+    private Date    startDate;
+
+    public Plantation() {
+    }
+
+    
+    
+    public Plantation(String varietyId, String phaseName, String baseTemp, Double heatReq) {
+        this.varietyId = varietyId;
+        this.phaseName = phaseName;
+        this.baseTemp = baseTemp;
+        this.heatReq = heatReq;
+    }
+
+    public Plantation(String varietyId, String phaseName, Double heatReq) {
+        this.varietyId = varietyId;
+        this.phaseName = phaseName;
+        this.heatReq = heatReq;
+        } 
+    
+      public Plantation(String phaseName, Double heatReq) {
+        this.phaseName = phaseName;
+        this.heatReq = heatReq;
+        }  
+    
+    public String getVarietyId() {
+        return varietyId;
+    }
+
+    public void setVarietyId(String varietyId) {
+        this.varietyId = varietyId;
+    }
+
+    public String getPhaseName() {
+        return phaseName;
+    }
+
+    public void setPhaseName(String phaseName) {
+        this.phaseName = phaseName;
+    }
+
+    public String getBaseTemp() {
+        return baseTemp;
+    }
+
+    public void setBaseTemp(String baseTemp) {
+        this.baseTemp = baseTemp;
+    }
+
+    public Double getHeatReq() {
+        return heatReq;
+    }
+
+    public void setHeatReq(Double heatReq) {
+        this.heatReq = heatReq;
+    }
+
+    public Date getStartDate() {
+        return startDate;
+    }
+
+    public void setStartDate(Date startDate) {
+        this.startDate = startDate;
+    }
+
+    @Override
+    public String toString() {
+        return "Plantation{" + "varietyId=" + varietyId + ", phaseName=" + phaseName + ", baseTemp=" + baseTemp + ", heatSum=" + heatReq + ", sowingDate=" + startDate + '}';
+    }
+
+    
+
+    
+    
+
+}

src/main/resources/dataset/plantationData.json

+{
+    "speciesLatinName"        : "Zea mays",
+    "typeName"                : "TESTVARIETY 2",
+    "baseTemp"                : "10",
+    "phaseInfo":
+[
+    {
+        "phaseName": "VE",
+        "heatReq"  : "66.67"
+    },
+    {
+        "phaseName": "V2",
+         "heatReq"  : "111.11"
+    },
+     {
+        "phaseName": "V3",
+        "heatReq"  : "194.44"
+    },
+     {
+        "phaseName": "V4-V6",
+        "heatReq"  : "263.89"
+    }, 
+    {
+        "phaseName": "V7-V9",
+        "heatReq"  : "338.89"
+    }, 
+    {
+        "phaseName": "V10",
+        "heatReq"  : "411.11"
+    }, 
+    {
+        "phaseName": "VT",
+        "heatReq"  : "630.55"
+    }, 
+    {
+        "phaseName": "R2",
+        "heatReq"  : "922.22"
+    }, 
+    {
+        "phaseName": "R5",
+        "heatReq"  : "1361.11"
+    }, 
+    {
+        "phaseName": "R6",
+        "heatReq"  : "1500"
+    }
+ 
+]
+}
\ No newline at end of file

src/main/resources/no/nibio/vips/logic/i18n/vipslogictexts.properties

@@ -494,3 +494,6 @@ sprayingDate01=Spraying Date 01
 sprayingDate02=Spraying Date 02
 sprayingDate03=Spraying Date 03
 sprayingDate04=Spraying Date 04
+PHENOLOGYM=Phenology Model
+maizeVarity=Maize Variety
+sowingDate=Sowing Date

src/main/webapp/formdefinitions/models/PHENOLOGYM.json

+{
+    "_licenseNote": [
+        "Copyright (c) 2014 NIBIO <http://www.nibio.no/>. ",
+        "",
+        "This file is part of VIPSLogic. ",
+        "VIPSLogic is free software: you can redistribute it and/or modify ",
+        "it under the terms of the NIBIO Open Source License as published by ",
+        "NIBIO, either version 1 of the License, or (at your option) any ",
+        "later version. ",
+        "",
+        "VIPSLogic is distributed in the hope that it will be useful, ",
+        "but WITHOUT ANY WARRANTY; without even the implied warranty of ",
+        "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the ",
+        "NIBIO Open Source License for more details. ",
+        "",
+        "You should have received a copy of the NIBIO Open Source License ",
+        "along with VIPSLogic.  If not, see <http://www.nibio.no/licenses/>. "
+    ],
+    "_comment" : "Structure of the specific fields for FAWPHENOLO",
+        "fields": [
+            {
+                "name" : "maizeVarity",
+                "dataType" : "STRING",
+                "fieldType" : "SELECT_SINGLE",
+                "required" : false,
+                "required" : true,
+                            "selectOptions": [
+                                    {"value":"SOTUBAKA", "label":{"en":"SOTUBAKA Variety","nb":"SOTUBAKA Variety"}, "selected":"true"},
+                                    {"value":"TESTVARIETY 2", "label":{"en":"TESTVARIETY 2","nb":"TESTVARIETY 2"}},
+                                    {"value":"NIELENI", "label":{"en" : "NIELENI Variety", "nb":"NIELENI Variety"}}
+                            ]
+            },
+            {
+                "name" : "sowingDate",
+                "dataType" : "DATE",
+                "dateFormat" : "yyyy-MM-dd",
+                "required" : false
+            }
+       ]
+}