From afcbab84ad3b3516c20d484aed4e560917adc2e2 Mon Sep 17 00:00:00 2001
From: Tor-Einar Skog <tor-einar.skog@nibio.no>
Date: Mon, 21 Jan 2019 13:10:00 +0100
Subject: [PATCH] Fixed handling of weather data from forecasts, particularly
for the GRID humidity model
---
...ptoriaSimpleRiskGridModelPreprocessor.java | 8 ++--
.../util/weather/WeatherDataSourceUtil.java | 48 ++++++++++++++++---
.../weather/YrWeatherForecastProvider.java | 11 +++--
.../dnmipointweb/DMIPointWebDataParser.java | 46 ++++++++++++------
.../nordic_septoria_whs.js | 2 +-
.../YrWeatherForecastProviderTest.java | 18 +++----
6 files changed, 93 insertions(+), 40 deletions(-)
diff --git a/src/main/java/no/nibio/vips/logic/scheduling/model/grid/preprocessor/ZymoseptoriaSimpleRiskGridModelPreprocessor.java b/src/main/java/no/nibio/vips/logic/scheduling/model/grid/preprocessor/ZymoseptoriaSimpleRiskGridModelPreprocessor.java
index e2e28760..2abdd427 100644
--- a/src/main/java/no/nibio/vips/logic/scheduling/model/grid/preprocessor/ZymoseptoriaSimpleRiskGridModelPreprocessor.java
+++ b/src/main/java/no/nibio/vips/logic/scheduling/model/grid/preprocessor/ZymoseptoriaSimpleRiskGridModelPreprocessor.java
@@ -23,7 +23,6 @@ import com.vividsolutions.jts.geom.Coordinate;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.temporal.ChronoUnit;
-import java.time.temporal.TemporalUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
@@ -128,11 +127,12 @@ public class ZymoseptoriaSimpleRiskGridModelPreprocessor extends ModelRunPreproc
},
Date.from(aWeekAgo.toInstant()),
Date.from(aWeekAhead.toInstant()),
- true
+ true,
+ new HashSet<>(Arrays.asList(WeatherObservation.LOG_INTERVAL_ID_1H,WeatherObservation.LOG_INTERVAL_ID_3H, WeatherObservation.LOG_INTERVAL_ID_6H))
);
- stationObs = wUtil.checkForAndFixHourlyTimeSeriesHoles(stationObs);
+ //stationObs = wUtil.checkForAndFixHourlyTimeSeriesHoles(stationObs,6);
return stationObs;
- } catch (WeatherDataSourceException | WeatherObservationListException ex ) {
+ } catch (WeatherDataSourceException ex ) {
throw new PreprocessorException(ex.getMessage());
}
}
diff --git a/src/main/java/no/nibio/vips/util/weather/WeatherDataSourceUtil.java b/src/main/java/no/nibio/vips/util/weather/WeatherDataSourceUtil.java
index 406dc962..ad85e6ae 100755
--- a/src/main/java/no/nibio/vips/util/weather/WeatherDataSourceUtil.java
+++ b/src/main/java/no/nibio/vips/util/weather/WeatherDataSourceUtil.java
@@ -33,8 +33,10 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.TimeZone;
import no.nibio.vips.entity.WeatherObservation;
import no.nibio.vips.logic.entity.PointOfInterestWeatherStation;
@@ -68,13 +70,16 @@ public class WeatherDataSourceUtil {
* @return
* @throws PreprocessorException
*/
- public List<WeatherObservation> getWeatherObservations(PointOfInterestWeatherStation station, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, Boolean ignoreErrors) throws WeatherDataSourceException {
- List<WeatherObservation> observations = this.getWeatherObservations(station.getDataFetchUri(), logIntervalId, elementMeasurementTypes, startTime, endTime, TimeZone.getTimeZone(station.getTimeZone()), ignoreErrors);
+ public List<WeatherObservation> getWeatherObservations(PointOfInterestWeatherStation station, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, Boolean ignoreErrors, Set<Integer> toleratedLogIntervalIds) throws WeatherDataSourceException {
+ // Get measured (and possibly forecasted, depending on the data source) observations
+ List<WeatherObservation> observations = this.getWeatherObservations(station.getDataFetchUri(), logIntervalId, elementMeasurementTypes, startTime, endTime, TimeZone.getTimeZone(station.getTimeZone()), ignoreErrors, toleratedLogIntervalIds);
Collections.sort(observations);
+ // Append forecasts, if available
Date latestTimeOfMeasuredObservations = observations.isEmpty() ? null : observations.get(observations.size() - 1).getTimeMeasured();
//System.out.println("latestTimeOfMeasuredObservations = " + latestTimeOfMeasuredObservations);
// Todo: We don't collect forecast data when the endTime is before the actual, current date (not systemdate)
- if (station.getWeatherForecastProviderId() != null && ! SystemTime.isSystemTimeOffsetFromNow()) {
+ if (station.getWeatherForecastProviderId() != null && ! SystemTime.isSystemTimeOffsetFromNow())
+ {
try {
WeatherForecastProvider forecastProvider = WeatherStationProviderFactory.getWeatherForecastProvider(station.getWeatherForecastProviderId().getWeatherForecastProviderId());
List<WeatherObservation> forecasts = forecastProvider.getWeatherForecasts(station);
@@ -83,7 +88,7 @@ public class WeatherDataSourceUtil {
obsMap.put(elementMeasurementType, new ArrayList<>());
}
forecasts.stream().filter((obs) -> (
- obs.getLogIntervalId().equals(logIntervalId) && obsMap.get(obs.getElementMeasurementTypeId()) != null)
+ toleratedLogIntervalIds.contains(obs.getLogIntervalId()) && obsMap.get(obs.getElementMeasurementTypeId()) != null)
).forEachOrdered((obs) -> {
obsMap.get(obs.getElementMeasurementTypeId()).add(obs);
});
@@ -120,6 +125,21 @@ public class WeatherDataSourceUtil {
}
return observations;
}
+
+ /**
+ * The intolerant version. Only accept observations with same logIntervalId as requested
+ * @param station
+ * @param logIntervalId
+ * @param elementMeasurementTypes
+ * @param startTime
+ * @param endTime
+ * @param ignoreErrors
+ * @return
+ * @throws WeatherDataSourceException
+ */
+ public List<WeatherObservation> getWeatherObservations(PointOfInterestWeatherStation station, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, Boolean ignoreErrors) throws WeatherDataSourceException {
+ return this.getWeatherObservations(station, logIntervalId, elementMeasurementTypes, startTime, endTime, ignoreErrors, new HashSet<>(Arrays.asList(logIntervalId)));
+ }
/**
* Fetches measured data from the stations weather data source, and optionally
@@ -149,6 +169,22 @@ public class WeatherDataSourceUtil {
return new ObjectMapper().readValue(JSONtext, new TypeReference<List<WeatherObservation>>() {
});
}
+
+ /**
+ * The intolerant version. Only accept observations with same logIntervalId as requested
+ * @param fetchURI
+ * @param logIntervalId
+ * @param elementMeasurementTypes
+ * @param startTime
+ * @param endTime
+ * @param timeZone
+ * @param ignoreErrors
+ * @return
+ * @throws WeatherDataSourceException
+ */
+ public List<WeatherObservation> getWeatherObservations(String fetchURI, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, TimeZone timeZone, Boolean ignoreErrors) throws WeatherDataSourceException {
+ return this.getWeatherObservations(fetchURI, logIntervalId, elementMeasurementTypes, startTime, endTime, timeZone, ignoreErrors, new HashSet<>(Arrays.asList(logIntervalId)));
+ }
/**
* Collects weather observations from a data source
@@ -160,7 +196,7 @@ public class WeatherDataSourceUtil {
* @param timeZone
* @return
*/
- public List<WeatherObservation> getWeatherObservations(String fetchURI, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, TimeZone timeZone, Boolean ignoreErrors) throws WeatherDataSourceException {
+ public List<WeatherObservation> getWeatherObservations(String fetchURI, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, TimeZone timeZone, Boolean ignoreErrors, Set<Integer> toleratedLogIntervalIds) throws WeatherDataSourceException {
SimpleDateFormat dateOutput = new SimpleDateFormat("yyyy-MM-dd");
dateOutput.setTimeZone(timeZone);
SimpleDateFormat hourOutput = new SimpleDateFormat("H");
@@ -199,7 +235,7 @@ public class WeatherDataSourceUtil {
List<WeatherObservation> filteredObservations = new ArrayList<>();
//System.out.println(this.getClass().getName() + "/preliminaryResult.size()=" + preliminaryResult.size());
preliminaryResult.stream().filter((candidateObs) -> (
- candidateObs.getLogIntervalId().equals(logIntervalId)
+ toleratedLogIntervalIds.contains(candidateObs.getLogIntervalId())
&& Arrays.asList(elementMeasurementTypes).contains(candidateObs.getElementMeasurementTypeId())
)
).forEachOrdered((candidateObs) -> {
diff --git a/src/main/java/no/nibio/vips/util/weather/YrWeatherForecastProvider.java b/src/main/java/no/nibio/vips/util/weather/YrWeatherForecastProvider.java
index 5b17b7f1..c3db63a0 100755
--- a/src/main/java/no/nibio/vips/util/weather/YrWeatherForecastProvider.java
+++ b/src/main/java/no/nibio/vips/util/weather/YrWeatherForecastProvider.java
@@ -77,7 +77,7 @@ public class YrWeatherForecastProvider implements WeatherForecastProvider{
YrWeatherForecastProvider.YR_API_URL,
latitude,
longitude,
- altitude)
+ altitude.intValue())
);
//System.out.println("yrURL=" + yrURL.toString());
@@ -94,6 +94,7 @@ public class YrWeatherForecastProvider implements WeatherForecastProvider{
Long sixHoursDelta = 6 * hourDelta;
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX");
Map<Date, WeatherObservation> RRMap = new HashMap<>();
+ Date earliestHourlyPrecipitationObservation = null;
for(int i=0;i<nodes.getLength();i++)
{
Node node = nodes.item(i);
@@ -138,14 +139,14 @@ public class YrWeatherForecastProvider implements WeatherForecastProvider{
RR.setElementMeasurementTypeId(WeatherElements.PRECIPITATION);
RR.setValue(Double.parseDouble(DOMUtils.getNodeAttr("precipitation","value",node2.getChildNodes())));
//System.out.println("Timediff=" + (toTime.getTime() - fromTime.getTime()));
- Date earliestHourlyObservation = null;
+
if(toTime.getTime() - fromTime.getTime() == hourDelta)
{
//System.out.println("Found 1 hour record at " + fromTime);
RR.setLogIntervalId(WeatherObservation.LOG_INTERVAL_ID_1H);
- if(earliestHourlyObservation == null)
+ if(earliestHourlyPrecipitationObservation == null)
{
- earliestHourlyObservation = RR.getTimeMeasured();
+ earliestHourlyPrecipitationObservation = RR.getTimeMeasured();
}
}
else if(toTime.getTime() - fromTime.getTime() == threeHoursDelta)
@@ -165,7 +166,7 @@ public class YrWeatherForecastProvider implements WeatherForecastProvider{
// In order to avoid overestimation of rain in the beginning,
// We skip ahead until we find observations with 1 hour resolution
if(!RR.getLogIntervalId().equals(WeatherObservation.LOG_INTERVAL_ID_1H)
- && (earliestHourlyObservation == null || earliestHourlyObservation.after(RR.getTimeMeasured()))
+ && (earliestHourlyPrecipitationObservation == null || earliestHourlyPrecipitationObservation.after(RR.getTimeMeasured()))
)
{
continue;
diff --git a/src/main/java/no/nibio/vips/util/weather/dnmipointweb/DMIPointWebDataParser.java b/src/main/java/no/nibio/vips/util/weather/dnmipointweb/DMIPointWebDataParser.java
index 36e7a08a..cff4c3f0 100644
--- a/src/main/java/no/nibio/vips/util/weather/dnmipointweb/DMIPointWebDataParser.java
+++ b/src/main/java/no/nibio/vips/util/weather/dnmipointweb/DMIPointWebDataParser.java
@@ -22,19 +22,23 @@ package no.nibio.vips.util.weather.dnmipointweb;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.TimeZone;
import java.util.logging.Level;
import java.util.logging.Logger;
+import java.util.stream.Collectors;
import javax.xml.datatype.DatatypeFactory;
import javax.xml.datatype.XMLGregorianCalendar;
import no.nibio.vips.entity.WeatherObservation;
import no.nibio.vips.util.WeatherElements;
+import no.nibio.vips.util.WeatherUtil;
/**
* Gets data from the Danish Meteorological Institute's Point Web service
@@ -58,7 +62,7 @@ public class DMIPointWebDataParser {
public List<WeatherObservation> getData(Double longitude, Double latitude, Date dateFrom, Date dateTo)
{
- List<WeatherObservation> allObservations = new ArrayList<>();
+ List<WeatherObservation> rawObservations = new ArrayList<>();
TimeZone danishTZ = TimeZone.getTimeZone("Europe/Copenhagen");
try {
IWeatherService proxy = new WeatherService().getSslOffloadedBasicHttpBindingIWeatherService();
@@ -106,7 +110,7 @@ public class DMIPointWebDataParser {
WeatherObservation obs = this.getWeatherObservation(VIPSParam, weatherDataModel);
if(obs != null)
{
- allObservations.add(obs);
+ rawObservations.add(obs);
}
}
);
@@ -118,22 +122,34 @@ public class DMIPointWebDataParser {
}
//System.out.println("Number of extracted weather data = " + retVal.size());
// After "now", the DMI service provides forecast values seamlessly. After approx 48 hours these
- // values turn into 6 hour intervals, which we can't use. We need to weed them out
- Collections.sort(allObservations);
+ // values turn into 6 hour intervals
+ // 2019-01-18: We keep all observations, leave it up to the client to decide what to do with the data.
+ // But we mark them with the correct measuring interval
+
ZonedDateTime now = ZonedDateTime.now();
- ZonedDateTime lastValidObsTime = null;
ZoneId danishZ = ZoneId.of("Europe/Copenhagen");
- List<WeatherObservation> filteredObservations = new ArrayList<>();
- for(WeatherObservation obs:allObservations)
- {
- ZonedDateTime obsTime = ZonedDateTime.ofInstant(obs.getTimeMeasured().toInstant(),danishZ);
- if(lastValidObsTime == null || obsTime.isBefore(now) || obsTime.minusHours(6).isBefore(lastValidObsTime))
+ //List<WeatherObservation> filteredObservations = new ArrayList<>();
+ WeatherUtil wUtil = new WeatherUtil();
+ List<WeatherObservation> retVal = new ArrayList<>();
+ Set<String> parametersInResult = rawObservations.stream().map(obs->obs.getElementMeasurementTypeId()).collect(Collectors.toSet());
+ parametersInResult.forEach((param) -> {
+ ZonedDateTime lastObsTime = null;
+ List<WeatherObservation> paramObs = wUtil.filterWeatherObservationsByParameter(rawObservations, param);
+ Collections.sort(paramObs);
+ for(WeatherObservation obs:paramObs)
{
- filteredObservations.add(obs);
- lastValidObsTime = obsTime;
- }
- }
- return allObservations;
+ ZonedDateTime obsTime = ZonedDateTime.ofInstant(obs.getTimeMeasured().toInstant(),danishZ);
+
+ if(lastObsTime != null && obsTime.isAfter(now) && !obsTime.minusHours(6).isBefore(lastObsTime))
+ {
+ obs.setLogIntervalId(WeatherObservation.LOG_INTERVAL_ID_6H);
+ }
+ retVal.add(obs);
+ lastObsTime = obsTime;
+ }
+ });
+ Collections.sort(retVal);
+ return retVal;
}
private WeatherObservation getWeatherObservation(String VIPSParam, WeatherDataModel wDataModel) {
diff --git a/src/main/webapp/public/nordic_septoria_whs/nordic_septoria_whs.js b/src/main/webapp/public/nordic_septoria_whs/nordic_septoria_whs.js
index a11212bb..509bee5e 100644
--- a/src/main/webapp/public/nordic_septoria_whs/nordic_septoria_whs.js
+++ b/src/main/webapp/public/nordic_septoria_whs/nordic_septoria_whs.js
@@ -192,7 +192,7 @@ function getTodayAtMidnight()
return new Date(midnightString);
// OR RETURN A FIXED DATE FOR TESTING
- //return new Date("2017-09-07T22:00:00.000+0000");
+ //return new Date("2018-08-16T22:00:00.000+0000");
}
diff --git a/src/test/java/no/nibio/vips/util/weather/YrWeatherForecastProviderTest.java b/src/test/java/no/nibio/vips/util/weather/YrWeatherForecastProviderTest.java
index 8cc2534d..b7294e0c 100755
--- a/src/test/java/no/nibio/vips/util/weather/YrWeatherForecastProviderTest.java
+++ b/src/test/java/no/nibio/vips/util/weather/YrWeatherForecastProviderTest.java
@@ -87,26 +87,26 @@ public class YrWeatherForecastProviderTest {
//weatherStation.setAltitude(110.0);
//weatherStation.setLongitude(10.83474668);
//weatherStation.setLatitude(59.63851390);
- weatherStation.setAltitude(19.0);
- weatherStation.setLongitude(6.57052);
- weatherStation.setLatitude(60.2542);
+ weatherStation.setAltitude(102.0);
+ weatherStation.setLongitude(11.39);
+ weatherStation.setLatitude(59.388);
result = instance.getWeatherForecasts(weatherStation);
assertNotNull(result);
-
- /*for(WeatherObservation obs:result)
+ Collections.sort(result);
+ for(WeatherObservation obs:result)
{
- if(obs.getElementMeasurementTypeId().equals("RR"))
+ if(obs.getElementMeasurementTypeId().equals("TM"))
System.out.println(obs.toString());
- }*/
+ }
WeatherUtil wUtil = new WeatherUtil();
- wUtil.checkForAndFixHourlyTimeSeriesHoles(result);
+ //wUtil.checkForAndFixHourlyTimeSeriesHoles(result);
// Testing Jablanica (Republika Srpska, Bosnia)
weatherStation.setAltitude(219.0);
weatherStation.setLongitude(16.988056);
weatherStation.setLatitude(45.115);
- result = instance.getWeatherForecasts(weatherStation);
+ //result = instance.getWeatherForecasts(weatherStation);
assertNotNull(result);
/**Collections.sort(result);
--
GitLab