From 1cfaf27e45ed610643260a27849129757e78ba4f Mon Sep 17 00:00:00 2001
From: Bhabesh <bhabesh.mukhopadhyay@nibio.no>
Date: Mon, 19 Oct 2020 13:45:55 +0200
Subject: [PATCH] Basic phenology model (General)
Added required changes to related to phenology
---
.../PhenologyModelPreprocessor.java | 348 ++++++++++++++++++
.../no/nibio/vips/logic/util/Plantation.java | 92 +++++
.../resources/dataset/plantationData.json | 49 +++
.../vips/logic/i18n/vipslogictexts.properties | 3 +
.../formdefinitions/models/PHENOLOGYM.json | 40 ++
5 files changed, 532 insertions(+)
create mode 100644 src/main/java/no/nibio/vips/logic/scheduling/model/preprocessor/PhenologyModelPreprocessor.java
create mode 100644 src/main/java/no/nibio/vips/logic/util/Plantation.java
create mode 100644 src/main/resources/dataset/plantationData.json
create mode 100644 src/main/webapp/formdefinitions/models/PHENOLOGYM.json
diff --git a/src/main/java/no/nibio/vips/logic/scheduling/model/preprocessor/PhenologyModelPreprocessor.java b/src/main/java/no/nibio/vips/logic/scheduling/model/preprocessor/PhenologyModelPreprocessor.java
new file mode 100644
index 00000000..78f47281
--- /dev/null
+++ b/src/main/java/no/nibio/vips/logic/scheduling/model/preprocessor/PhenologyModelPreprocessor.java
@@ -0,0 +1,348 @@
+
+package no.nibio.vips.logic.scheduling.model.preprocessor;
+
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.MappingJsonFactory;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import no.nibio.vips.entity.ModelConfiguration;
+import no.nibio.vips.entity.WeatherObservation;
+import no.nibio.vips.logic.entity.ForecastConfiguration;
+import no.nibio.vips.logic.entity.PointOfInterestWeatherStation;
+import no.nibio.vips.logic.scheduling.model.ModelRunPreprocessor;
+import no.nibio.vips.logic.scheduling.model.PreprocessorException;
+import no.nibio.vips.logic.util.Plantation;
+import no.nibio.vips.logic.util.SessionControllerGetter;
+import no.nibio.vips.logic.util.SystemTime;
+import no.nibio.vips.model.ConfigValidationException;
+import no.nibio.vips.util.WeatherElements;
+import no.nibio.vips.util.WeatherObservationListException;
+import no.nibio.vips.util.WeatherUtil;
+import no.nibio.vips.util.weather.WeatherDataSourceException;
+import no.nibio.vips.util.weather.WeatherDataSourceUtil;
+
+/**
+ *
+ * @author wildfly
+ */
+
+
+public class PhenologyModelPreprocessor extends ModelRunPreprocessor{
+ public final Logger LOGGER = Logger.getLogger(this.getClass().getName());
+ private final boolean DEBUG = false;
+ private final static String MODEL_ID = "PHENOLOGYM";
+ private final static String MAIZE_VARITY = "maizeVarity";
+ private final static String SOWING_DATE = "sowingDate";
+
+ private final static String SOWING_DATA = "sowingData";
+ private final static String PLANTATION_DATA = "plantationData";
+ private final static String FILE_PLANTATION = "/dataset/plantationData.json";
+
+ @Override
+ public ModelConfiguration getModelConfiguration(ForecastConfiguration configuration) throws PreprocessorException {
+ String PHENOLOGY_MAZE_VARIETY = SessionControllerGetter.getForecastBean().getDeCamelizedFieldName(MODEL_ID, MAIZE_VARITY);
+ String PHENOLOGY_SOWING_DATE = SessionControllerGetter.getForecastBean().getDeCamelizedFieldName(MODEL_ID, SOWING_DATE);
+
+ String paramMazeVariety = null;
+ Date paramSowingDate = null;
+
+
+ String paramConfigValue = null;
+
+ ModelConfiguration retVal = new ModelConfiguration();
+ PointOfInterestWeatherStation weatherStation = (PointOfInterestWeatherStation) configuration.getWeatherStationPointOfInterestId();
+ WeatherDataSourceUtil wdsUtil = new WeatherDataSourceUtil();
+ WeatherUtil wUtil = new WeatherUtil();
+ TimeZone timeZone = TimeZone.getTimeZone(weatherStation.getTimeZone());
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
+ format.setTimeZone(timeZone);
+ // Getting date three days after "today"
+ Calendar cal = Calendar.getInstance(timeZone);
+ cal.setTime(SystemTime.getSystemTime());
+ cal.add(Calendar.DATE, 3);
+ Date endDate = wUtil.normalizeToExactDate(configuration.getDateEnd(), timeZone);
+ cal.set(Calendar.MONTH, Calendar.JANUARY);
+ cal.set(Calendar.DATE, 1);
+
+ Plantation sowdata = new Plantation();
+
+ try
+ {
+ paramConfigValue = configuration.getForecastModelConfigurationValue(PHENOLOGY_SOWING_DATE);
+ paramSowingDate = format.parse(paramConfigValue);
+ }
+ catch(ParseException ex)
+ {
+ throw new PreprocessorException("Could not parse date for Sowing Date for "+MODEL_ID+". Given string was: "+ paramConfigValue);
+ }
+
+ Date startDate = wUtil.normalizeToExactDate(
+ paramSowingDate
+ , timeZone
+ );
+
+ List<WeatherObservation> observations = null;
+
+ try {
+ observations = wdsUtil.getWeatherObservations(
+ weatherStation
+ , WeatherObservation.LOG_INTERVAL_ID_1H
+ , new String[]{
+ WeatherElements.TEMPERATURE_MEAN,
+ WeatherElements.PRECIPITATION,
+ WeatherElements.LEAF_WETNESS_DURATION,
+ WeatherElements.RELATIVE_HUMIDITY_MEAN
+ }
+ , startDate
+ , endDate);
+ }
+ catch(WeatherDataSourceException ex)
+ {
+ throw new PreprocessorException(ex.getMessage());
+ }
+
+
+
+
+ paramMazeVariety = configuration.getForecastModelConfigurationValue(PHENOLOGY_MAZE_VARIETY);
+
+
+ Collections.sort(observations);
+
+ try {
+ List<WeatherObservation> calcObs = wdsUtil.getWeatherObservations(
+ weatherStation
+ , WeatherObservation.LOG_INTERVAL_ID_1H
+ , new String[] {
+ WeatherElements.GLOBAL_RADIATION,
+ WeatherElements.WIND_SPEED_2M
+ }
+ , startDate
+ , endDate
+ );
+ observations.addAll(calcObs);
+ }catch (WeatherDataSourceException ex)
+ {
+ // TODO nothing reported
+ }
+
+ if(DEBUG)
+ {
+ LOGGER.log(Level.CONFIG, "Finished getting weather data at "+new Date().toString());
+ }
+ try {
+ observations = validateAndSanitizeObservations(observations, startDate);
+ } catch (ConfigValidationException | WeatherObservationListException ex) {
+ //ex.printStackTrace();
+ throw new PreprocessorException(ex.getMessage());
+ }
+ if(DEBUG)
+ {
+ LOGGER.log(Level.CONFIG, "Observations=" + observations.toString());
+ }
+
+ sowdata.setVarietyId(paramMazeVariety);
+ sowdata.setStartDate(paramSowingDate);
+
+ retVal.setModelId(this.getModelId());
+ retVal.setConfigParameter("timeZone", timeZone.getID());
+ retVal.setConfigParameter("observations", observations);
+ retVal.setConfigParameter(SOWING_DATA, sowdata);
+ retVal.setConfigParameter(PLANTATION_DATA, getConfigurationPlantation(FILE_PLANTATION));
+
+
+ return retVal;
+ }
+
+
+
+
+ @Override
+ public String getModelId() {
+ return MODEL_ID;
+ }
+
+
+ private List<WeatherObservation> validateAndSanitizeObservations(List<WeatherObservation> observations, Date firstTimeStamp) throws ConfigValidationException, WeatherObservationListException
+ {
+
+ if(DEBUG)
+ {
+ LOGGER.log(Level.CONFIG, "validateAndSanitizeObservations");
+ }
+
+ WeatherUtil wUtil = new WeatherUtil();
+
+ // First we remove all duplicates
+ observations = wUtil.removeDuplicateWeatherObservations(observations, null);
+
+ // Fix weather data
+ List<WeatherObservation> fixedObservations = wUtil.fixHourlyValuesForParameters(
+ observations,
+ new HashSet(Arrays.asList("TM","RR")),
+ firstTimeStamp,
+ null
+ );
+
+
+ // Now we need to validate and possibly try to fix the weather data
+ List<WeatherObservation> TM = new ArrayList<>();
+ List<WeatherObservation> RR = new ArrayList<>();
+
+ for(WeatherObservation o:fixedObservations)
+ {
+ switch(o.getElementMeasurementTypeId())
+ {
+ case WeatherElements.TEMPERATURE_MEAN:
+ TM.add(o);
+ break;
+ case WeatherElements.PRECIPITATION:
+ RR.add(o);
+ break;
+
+ }
+ }
+
+ List<WeatherObservation> BT = new ArrayList<>();
+ List<WeatherObservation> UM = new ArrayList<>();
+ List<WeatherObservation> Q0 = new ArrayList<>();
+ List<WeatherObservation> FM2 = new ArrayList<>();
+ for(WeatherObservation o:observations)
+ {
+ switch(o.getElementMeasurementTypeId())
+ {
+ case WeatherElements.LEAF_WETNESS_DURATION:
+ BT.add(o);
+ break;
+ case WeatherElements.RELATIVE_HUMIDITY_MEAN:
+ UM.add(o);
+ break;
+ case WeatherElements.GLOBAL_RADIATION:
+ Q0.add(o);
+ break;
+ case WeatherElements.WIND_SPEED_2M:
+ FM2.add(o);
+ break;
+ default:
+ // Let it pass in silence
+ break;
+ }
+ }
+
+ // Problems with weather observations
+
+ // Holes in series
+ if(DEBUG)
+ {
+ LOGGER.log(Level.CONFIG, "checkForAndFixHourlyTimeSeriesHoles");
+ }
+
+
+
+ // Unequal length of lists
+ if (
+ RR.size() != TM.size()
+ || BT.size() != TM.size()
+ || RR.size() != TM.size()
+ )
+ {
+ UM = wUtil.fixHourlyValuesForParameters(
+ UM,
+ new HashSet(Arrays.asList("UM")),
+ firstTimeStamp,
+ null
+ );
+
+
+
+ // Fallback if missing leaf wetness: If we have relative humidity. leaf wetness may be calculated
+ if((BT.size() != TM.size()) && (UM.size() == TM.size()))
+ {
+ BT = wUtil.calculateLeafWetnessHourSeriesBestEffort(BT,TM, RR, Q0, FM2, UM);
+
+ if(BT.size() != TM.size())
+ {
+ throw new ConfigValidationException("Missing leaf wetness data. Also, attempting to calculate leaf wetness from other weather parameters failed.");
+ }
+ }
+ else
+ {
+ LOGGER.log(Level.WARNING, "TM starts " + TM.get(0).getTimeMeasured() + ", ends " + TM.get(TM.size()-1).getTimeMeasured());
+ throw new ConfigValidationException("Incorrect number of weather data. TM.size() = " + TM.size() + ", BT.size()=" + BT.size() + ", RR.size()=" + RR.size() + ", UM.size()=" + UM.size() );
+ }
+ }
+ List<WeatherObservation> retVal = new ArrayList<>();
+ retVal.addAll(TM);
+ retVal.addAll(RR);
+ retVal.addAll(BT);
+ return retVal;
+ }
+
+ private List<Plantation> getConfigurationPlantation(String fileName)
+ {
+ List<Plantation> plantations = new ArrayList<Plantation>();
+
+
+ try {
+ BufferedInputStream inputStream = new BufferedInputStream(this.getClass().getResourceAsStream(fileName));
+ JsonFactory f = new MappingJsonFactory();
+ JsonParser jp = f.createParser(inputStream);
+ JsonNode all = jp.readValueAsTree();
+ JsonNode nodePhaseInfo = all.path("phaseInfo");
+ if(nodePhaseInfo.isArray())
+ {
+ for(JsonNode node : nodePhaseInfo)
+ {
+ Plantation plantation = new Plantation
+ ( all.get("typeName").asText()
+ , node.get("phaseName").asText()
+ , node.get("heatReq").asDouble()
+ );
+ plantations.add(plantation);
+ }
+ }
+
+
+ /*
+ if(all.isArray())
+ {
+ for(JsonNode node : all)
+ {
+ Plantation plantation = new Plantation
+ (
+ node.get("maizeVarity").asText()
+ , node.get("phaseName").asText()
+ , node.get("baseTemp").asText()
+ , node.get("heatReq").asDouble()
+ );
+ plantations.add(plantation);
+
+ }
+ }
+
+ */
+
+
+ } catch (IOException ex) {
+ Logger.getLogger(this.getClass().getName()).log(Level.SEVERE, null, ex);
+ }
+
+ return plantations;
+ }
+
+}
diff --git a/src/main/java/no/nibio/vips/logic/util/Plantation.java b/src/main/java/no/nibio/vips/logic/util/Plantation.java
new file mode 100644
index 00000000..b50afe70
--- /dev/null
+++ b/src/main/java/no/nibio/vips/logic/util/Plantation.java
@@ -0,0 +1,92 @@
+
+package no.nibio.vips.logic.util;
+
+import java.util.Date;
+
+/**
+ *
+ * @author wildfly
+ */
+
+
+public class Plantation {
+ private String varietyId;
+ private String phaseName;
+ private String baseTemp;
+ private Double heatReq;
+ private Date startDate;
+
+ public Plantation() {
+ }
+
+
+
+ public Plantation(String varietyId, String phaseName, String baseTemp, Double heatReq) {
+ this.varietyId = varietyId;
+ this.phaseName = phaseName;
+ this.baseTemp = baseTemp;
+ this.heatReq = heatReq;
+ }
+
+ public Plantation(String varietyId, String phaseName, Double heatReq) {
+ this.varietyId = varietyId;
+ this.phaseName = phaseName;
+ this.heatReq = heatReq;
+ }
+
+ public Plantation(String phaseName, Double heatReq) {
+ this.phaseName = phaseName;
+ this.heatReq = heatReq;
+ }
+
+ public String getVarietyId() {
+ return varietyId;
+ }
+
+ public void setVarietyId(String varietyId) {
+ this.varietyId = varietyId;
+ }
+
+ public String getPhaseName() {
+ return phaseName;
+ }
+
+ public void setPhaseName(String phaseName) {
+ this.phaseName = phaseName;
+ }
+
+ public String getBaseTemp() {
+ return baseTemp;
+ }
+
+ public void setBaseTemp(String baseTemp) {
+ this.baseTemp = baseTemp;
+ }
+
+ public Double getHeatReq() {
+ return heatReq;
+ }
+
+ public void setHeatReq(Double heatReq) {
+ this.heatReq = heatReq;
+ }
+
+ public Date getStartDate() {
+ return startDate;
+ }
+
+ public void setStartDate(Date startDate) {
+ this.startDate = startDate;
+ }
+
+ @Override
+ public String toString() {
+ return "Plantation{" + "varietyId=" + varietyId + ", phaseName=" + phaseName + ", baseTemp=" + baseTemp + ", heatSum=" + heatReq + ", sowingDate=" + startDate + '}';
+ }
+
+
+
+
+
+
+}
diff --git a/src/main/resources/dataset/plantationData.json b/src/main/resources/dataset/plantationData.json
new file mode 100644
index 00000000..5e041fff
--- /dev/null
+++ b/src/main/resources/dataset/plantationData.json
@@ -0,0 +1,49 @@
+{
+ "speciesLatinName" : "Zea mays",
+ "typeName" : "TESTVARIETY 2",
+ "baseTemp" : "10",
+ "phaseInfo":
+[
+ {
+ "phaseName": "VE",
+ "heatReq" : "66.67"
+ },
+ {
+ "phaseName": "V2",
+ "heatReq" : "111.11"
+ },
+ {
+ "phaseName": "V3",
+ "heatReq" : "194.44"
+ },
+ {
+ "phaseName": "V4-V6",
+ "heatReq" : "263.89"
+ },
+ {
+ "phaseName": "V7-V9",
+ "heatReq" : "338.89"
+ },
+ {
+ "phaseName": "V10",
+ "heatReq" : "411.11"
+ },
+ {
+ "phaseName": "VT",
+ "heatReq" : "630.55"
+ },
+ {
+ "phaseName": "R2",
+ "heatReq" : "922.22"
+ },
+ {
+ "phaseName": "R5",
+ "heatReq" : "1361.11"
+ },
+ {
+ "phaseName": "R6",
+ "heatReq" : "1500"
+ }
+
+]
+}
\ No newline at end of file
diff --git a/src/main/resources/no/nibio/vips/logic/i18n/vipslogictexts.properties b/src/main/resources/no/nibio/vips/logic/i18n/vipslogictexts.properties
index 602449bc..d9e661de 100755
--- a/src/main/resources/no/nibio/vips/logic/i18n/vipslogictexts.properties
+++ b/src/main/resources/no/nibio/vips/logic/i18n/vipslogictexts.properties
@@ -494,3 +494,6 @@ sprayingDate01=Spraying Date 01
sprayingDate02=Spraying Date 02
sprayingDate03=Spraying Date 03
sprayingDate04=Spraying Date 04
+PHENOLOGYM=Phenology Model
+maizeVarity=Maize Variety
+sowingDate=Sowing Date
diff --git a/src/main/webapp/formdefinitions/models/PHENOLOGYM.json b/src/main/webapp/formdefinitions/models/PHENOLOGYM.json
new file mode 100644
index 00000000..23ffe221
--- /dev/null
+++ b/src/main/webapp/formdefinitions/models/PHENOLOGYM.json
@@ -0,0 +1,40 @@
+{
+ "_licenseNote": [
+ "Copyright (c) 2014 NIBIO <http://www.nibio.no/>. ",
+ "",
+ "This file is part of VIPSLogic. ",
+ "VIPSLogic is free software: you can redistribute it and/or modify ",
+ "it under the terms of the NIBIO Open Source License as published by ",
+ "NIBIO, either version 1 of the License, or (at your option) any ",
+ "later version. ",
+ "",
+ "VIPSLogic is distributed in the hope that it will be useful, ",
+ "but WITHOUT ANY WARRANTY; without even the implied warranty of ",
+ "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ",
+ "NIBIO Open Source License for more details. ",
+ "",
+ "You should have received a copy of the NIBIO Open Source License ",
+ "along with VIPSLogic. If not, see <http://www.nibio.no/licenses/>. "
+ ],
+ "_comment" : "Structure of the specific fields for FAWPHENOLO",
+ "fields": [
+ {
+ "name" : "maizeVarity",
+ "dataType" : "STRING",
+ "fieldType" : "SELECT_SINGLE",
+ "required" : false,
+ "required" : true,
+ "selectOptions": [
+ {"value":"SOTUBAKA", "label":{"en":"SOTUBAKA Variety","nb":"SOTUBAKA Variety"}, "selected":"true"},
+ {"value":"TESTVARIETY 2", "label":{"en":"TESTVARIETY 2","nb":"TESTVARIETY 2"}},
+ {"value":"NIELENI", "label":{"en" : "NIELENI Variety", "nb":"NIELENI Variety"}}
+ ]
+ },
+ {
+ "name" : "sowingDate",
+ "dataType" : "DATE",
+ "dateFormat" : "yyyy-MM-dd",
+ "required" : false
+ }
+ ]
+}
--
GitLab