Skip to content
Snippets Groups Projects
Commit afcbab84 authored by Tor-Einar Skog's avatar Tor-Einar Skog
Browse files

Fixed handling of weather data from forecasts, 

particularly for the GRID humidity model
parent 8e8e27ab
No related branches found
No related tags found
2 merge requests!17Develop,!10Fixed handling of weather data from forecasts
This commit is part of merge request !17. Comments created here will be created in the context of that merge request.
Hide whitespace changes
Inline Side-by-side
Showing
with 93 additions and 40 deletions
src/main/java/no/nibio/vips/logic/scheduling/model/grid/preprocessor/ZymoseptoriaSimpleRiskGridModelPreprocessor.java
+ 4
− 4
Edit View file @ afcbab84
......@@ -23,7 +23,6 @@ import com.vividsolutions.jts.geom.Coordinate;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
......@@ -128,11 +127,12 @@ public class ZymoseptoriaSimpleRiskGridModelPreprocessor extends ModelRunPreproc
},
Date.from(aWeekAgo.toInstant()),
Date.from(aWeekAhead.toInstant()),
true
true,
new HashSet<>(Arrays.asList(WeatherObservation.LOG_INTERVAL_ID_1H,WeatherObservation.LOG_INTERVAL_ID_3H, WeatherObservation.LOG_INTERVAL_ID_6H))
);
stationObs = wUtil.checkForAndFixHourlyTimeSeriesHoles(stationObs);
//stationObs = wUtil.checkForAndFixHourlyTimeSeriesHoles(stationObs,6);
return stationObs;
} catch (WeatherDataSourceException | WeatherObservationListException ex ) {
} catch (WeatherDataSourceException ex ) {
throw new PreprocessorException(ex.getMessage());
}
}
......
src/main/java/no/nibio/vips/util/weather/WeatherDataSourceUtil.java
+ 42
− 6
Edit View file @ afcbab84
......@@ -33,8 +33,10 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
import no.nibio.vips.entity.WeatherObservation;
import no.nibio.vips.logic.entity.PointOfInterestWeatherStation;
......@@ -68,13 +70,16 @@ public class WeatherDataSourceUtil {
* @return
* @throws PreprocessorException
*/
public List<WeatherObservation> getWeatherObservations(PointOfInterestWeatherStation station, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, Boolean ignoreErrors) throws WeatherDataSourceException {
List<WeatherObservation> observations = this.getWeatherObservations(station.getDataFetchUri(), logIntervalId, elementMeasurementTypes, startTime, endTime, TimeZone.getTimeZone(station.getTimeZone()), ignoreErrors);
public List<WeatherObservation> getWeatherObservations(PointOfInterestWeatherStation station, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, Boolean ignoreErrors, Set<Integer> toleratedLogIntervalIds) throws WeatherDataSourceException {
// Get measured (and possibly forecasted, depending on the data source) observations
List<WeatherObservation> observations = this.getWeatherObservations(station.getDataFetchUri(), logIntervalId, elementMeasurementTypes, startTime, endTime, TimeZone.getTimeZone(station.getTimeZone()), ignoreErrors, toleratedLogIntervalIds);
Collections.sort(observations);
// Append forecasts, if available
Date latestTimeOfMeasuredObservations = observations.isEmpty() ? null : observations.get(observations.size() - 1).getTimeMeasured();
//System.out.println("latestTimeOfMeasuredObservations = " + latestTimeOfMeasuredObservations);
// Todo: We don't collect forecast data when the endTime is before the actual, current date (not systemdate)
if (station.getWeatherForecastProviderId() != null && ! SystemTime.isSystemTimeOffsetFromNow()) {
if (station.getWeatherForecastProviderId() != null && ! SystemTime.isSystemTimeOffsetFromNow())
{
try {
WeatherForecastProvider forecastProvider = WeatherStationProviderFactory.getWeatherForecastProvider(station.getWeatherForecastProviderId().getWeatherForecastProviderId());
List<WeatherObservation> forecasts = forecastProvider.getWeatherForecasts(station);
......@@ -83,7 +88,7 @@ public class WeatherDataSourceUtil {
obsMap.put(elementMeasurementType, new ArrayList<>());
}
forecasts.stream().filter((obs) -> (
obs.getLogIntervalId().equals(logIntervalId) && obsMap.get(obs.getElementMeasurementTypeId()) != null)
toleratedLogIntervalIds.contains(obs.getLogIntervalId()) && obsMap.get(obs.getElementMeasurementTypeId()) != null)
).forEachOrdered((obs) -> {
obsMap.get(obs.getElementMeasurementTypeId()).add(obs);
});
......@@ -120,6 +125,21 @@ public class WeatherDataSourceUtil {
}
return observations;
}
/**
* The intolerant version. Only accept observations with same logIntervalId as requested
* @param station
* @param logIntervalId
* @param elementMeasurementTypes
* @param startTime
* @param endTime
* @param ignoreErrors
* @return
* @throws WeatherDataSourceException
*/
public List<WeatherObservation> getWeatherObservations(PointOfInterestWeatherStation station, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, Boolean ignoreErrors) throws WeatherDataSourceException {
return this.getWeatherObservations(station, logIntervalId, elementMeasurementTypes, startTime, endTime, ignoreErrors, new HashSet<>(Arrays.asList(logIntervalId)));
}
/**
* Fetches measured data from the stations weather data source, and optionally
......@@ -149,6 +169,22 @@ public class WeatherDataSourceUtil {
return new ObjectMapper().readValue(JSONtext, new TypeReference<List<WeatherObservation>>() {
});
}
/**
* The intolerant version. Only accept observations with same logIntervalId as requested
* @param fetchURI
* @param logIntervalId
* @param elementMeasurementTypes
* @param startTime
* @param endTime
* @param timeZone
* @param ignoreErrors
* @return
* @throws WeatherDataSourceException
*/
public List<WeatherObservation> getWeatherObservations(String fetchURI, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, TimeZone timeZone, Boolean ignoreErrors) throws WeatherDataSourceException {
return this.getWeatherObservations(fetchURI, logIntervalId, elementMeasurementTypes, startTime, endTime, timeZone, ignoreErrors, new HashSet<>(Arrays.asList(logIntervalId)));
}
/**
* Collects weather observations from a data source
......@@ -160,7 +196,7 @@ public class WeatherDataSourceUtil {
* @param timeZone
* @return
*/
public List<WeatherObservation> getWeatherObservations(String fetchURI, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, TimeZone timeZone, Boolean ignoreErrors) throws WeatherDataSourceException {
public List<WeatherObservation> getWeatherObservations(String fetchURI, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, TimeZone timeZone, Boolean ignoreErrors, Set<Integer> toleratedLogIntervalIds) throws WeatherDataSourceException {
SimpleDateFormat dateOutput = new SimpleDateFormat("yyyy-MM-dd");
dateOutput.setTimeZone(timeZone);
SimpleDateFormat hourOutput = new SimpleDateFormat("H");
......@@ -199,7 +235,7 @@ public class WeatherDataSourceUtil {
List<WeatherObservation> filteredObservations = new ArrayList<>();
//System.out.println(this.getClass().getName() + "/preliminaryResult.size()=" + preliminaryResult.size());
preliminaryResult.stream().filter((candidateObs) -> (
candidateObs.getLogIntervalId().equals(logIntervalId)
toleratedLogIntervalIds.contains(candidateObs.getLogIntervalId())
&& Arrays.asList(elementMeasurementTypes).contains(candidateObs.getElementMeasurementTypeId())
)
).forEachOrdered((candidateObs) -> {
......
src/main/java/no/nibio/vips/util/weather/YrWeatherForecastProvider.java
+ 6
− 5
Edit View file @ afcbab84
......@@ -77,7 +77,7 @@ public class YrWeatherForecastProvider implements WeatherForecastProvider{
YrWeatherForecastProvider.YR_API_URL,
latitude,
longitude,
altitude)
altitude.intValue())
);
//System.out.println("yrURL=" + yrURL.toString());
......@@ -94,6 +94,7 @@ public class YrWeatherForecastProvider implements WeatherForecastProvider{
Long sixHoursDelta = 6 * hourDelta;
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX");
Map<Date, WeatherObservation> RRMap = new HashMap<>();
Date earliestHourlyPrecipitationObservation = null;
for(int i=0;i<nodes.getLength();i++)
{
Node node = nodes.item(i);
......@@ -138,14 +139,14 @@ public class YrWeatherForecastProvider implements WeatherForecastProvider{
RR.setElementMeasurementTypeId(WeatherElements.PRECIPITATION);
RR.setValue(Double.parseDouble(DOMUtils.getNodeAttr("precipitation","value",node2.getChildNodes())));
//System.out.println("Timediff=" + (toTime.getTime() - fromTime.getTime()));
Date earliestHourlyObservation = null;
if(toTime.getTime() - fromTime.getTime() == hourDelta)
{
//System.out.println("Found 1 hour record at " + fromTime);
RR.setLogIntervalId(WeatherObservation.LOG_INTERVAL_ID_1H);
if(earliestHourlyObservation == null)
if(earliestHourlyPrecipitationObservation == null)
{
earliestHourlyObservation = RR.getTimeMeasured();
earliestHourlyPrecipitationObservation = RR.getTimeMeasured();
}
}
else if(toTime.getTime() - fromTime.getTime() == threeHoursDelta)
......@@ -165,7 +166,7 @@ public class YrWeatherForecastProvider implements WeatherForecastProvider{
// In order to avoid overestimation of rain in the beginning,
// We skip ahead until we find observations with 1 hour resolution
if(!RR.getLogIntervalId().equals(WeatherObservation.LOG_INTERVAL_ID_1H)
&& (earliestHourlyObservation == null || earliestHourlyObservation.after(RR.getTimeMeasured()))
&& (earliestHourlyPrecipitationObservation == null || earliestHourlyPrecipitationObservation.after(RR.getTimeMeasured()))
)
{
continue;
......
src/main/java/no/nibio/vips/util/weather/dnmipointweb/DMIPointWebDataParser.java
+ 31
− 15
Edit View file @ afcbab84
......@@ -22,19 +22,23 @@ package no.nibio.vips.util.weather.dnmipointweb;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import javax.xml.datatype.DatatypeFactory;
import javax.xml.datatype.XMLGregorianCalendar;
import no.nibio.vips.entity.WeatherObservation;
import no.nibio.vips.util.WeatherElements;
import no.nibio.vips.util.WeatherUtil;
/**
* Gets data from the Danish Meteorological Institute's Point Web service
......@@ -58,7 +62,7 @@ public class DMIPointWebDataParser {
public List<WeatherObservation> getData(Double longitude, Double latitude, Date dateFrom, Date dateTo)
{
List<WeatherObservation> allObservations = new ArrayList<>();
List<WeatherObservation> rawObservations = new ArrayList<>();
TimeZone danishTZ = TimeZone.getTimeZone("Europe/Copenhagen");
try {
IWeatherService proxy = new WeatherService().getSslOffloadedBasicHttpBindingIWeatherService();
......@@ -106,7 +110,7 @@ public class DMIPointWebDataParser {
WeatherObservation obs = this.getWeatherObservation(VIPSParam, weatherDataModel);
if(obs != null)
{
allObservations.add(obs);
rawObservations.add(obs);
}
}
);
......@@ -118,22 +122,34 @@ public class DMIPointWebDataParser {
}
//System.out.println("Number of extracted weather data = " + retVal.size());
// After "now", the DMI service provides forecast values seamlessly. After approx 48 hours these
// values turn into 6 hour intervals, which we can't use. We need to weed them out
Collections.sort(allObservations);
// values turn into 6 hour intervals
// 2019-01-18: We keep all observations, leave it up to the client to decide what to do with the data.
// But we mark them with the correct measuring interval
ZonedDateTime now = ZonedDateTime.now();
ZonedDateTime lastValidObsTime = null;
ZoneId danishZ = ZoneId.of("Europe/Copenhagen");
List<WeatherObservation> filteredObservations = new ArrayList<>();
for(WeatherObservation obs:allObservations)
{
ZonedDateTime obsTime = ZonedDateTime.ofInstant(obs.getTimeMeasured().toInstant(),danishZ);
if(lastValidObsTime == null || obsTime.isBefore(now) || obsTime.minusHours(6).isBefore(lastValidObsTime))
//List<WeatherObservation> filteredObservations = new ArrayList<>();
WeatherUtil wUtil = new WeatherUtil();
List<WeatherObservation> retVal = new ArrayList<>();
Set<String> parametersInResult = rawObservations.stream().map(obs->obs.getElementMeasurementTypeId()).collect(Collectors.toSet());
parametersInResult.forEach((param) -> {
ZonedDateTime lastObsTime = null;
List<WeatherObservation> paramObs = wUtil.filterWeatherObservationsByParameter(rawObservations, param);
Collections.sort(paramObs);
for(WeatherObservation obs:paramObs)
{
filteredObservations.add(obs);
lastValidObsTime = obsTime;
}
}
return allObservations;
ZonedDateTime obsTime = ZonedDateTime.ofInstant(obs.getTimeMeasured().toInstant(),danishZ);
if(lastObsTime != null && obsTime.isAfter(now) && !obsTime.minusHours(6).isBefore(lastObsTime))
{
obs.setLogIntervalId(WeatherObservation.LOG_INTERVAL_ID_6H);
}
retVal.add(obs);
lastObsTime = obsTime;
}
});
Collections.sort(retVal);
return retVal;
}
private WeatherObservation getWeatherObservation(String VIPSParam, WeatherDataModel wDataModel) {
......
src/main/webapp/public/nordic_septoria_whs/nordic_septoria_whs.js
+ 1
− 1
Edit View file @ afcbab84
......@@ -192,7 +192,7 @@ function getTodayAtMidnight()
return new Date(midnightString);
// OR RETURN A FIXED DATE FOR TESTING
//return new Date("2017-09-07T22:00:00.000+0000");
//return new Date("2018-08-16T22:00:00.000+0000");
}
......
src/test/java/no/nibio/vips/util/weather/YrWeatherForecastProviderTest.java
+ 9
− 9
Edit View file @ afcbab84
......@@ -87,26 +87,26 @@ public class YrWeatherForecastProviderTest {
//weatherStation.setAltitude(110.0);
//weatherStation.setLongitude(10.83474668);
//weatherStation.setLatitude(59.63851390);
weatherStation.setAltitude(19.0);
weatherStation.setLongitude(6.57052);
weatherStation.setLatitude(60.2542);
weatherStation.setAltitude(102.0);
weatherStation.setLongitude(11.39);
weatherStation.setLatitude(59.388);
result = instance.getWeatherForecasts(weatherStation);
assertNotNull(result);
/*for(WeatherObservation obs:result)
Collections.sort(result);
for(WeatherObservation obs:result)
{
if(obs.getElementMeasurementTypeId().equals("RR"))
if(obs.getElementMeasurementTypeId().equals("TM"))
System.out.println(obs.toString());
}*/
}
WeatherUtil wUtil = new WeatherUtil();
wUtil.checkForAndFixHourlyTimeSeriesHoles(result);
//wUtil.checkForAndFixHourlyTimeSeriesHoles(result);
// Testing Jablanica (Republika Srpska, Bosnia)
weatherStation.setAltitude(219.0);
weatherStation.setLongitude(16.988056);
weatherStation.setLatitude(45.115);
result = instance.getWeatherForecasts(weatherStation);
//result = instance.getWeatherForecasts(weatherStation);
assertNotNull(result);
/**Collections.sort(result);
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment