Skip to content
Snippets Groups Projects
Commit afcbab84 authored by Tor-Einar Skog's avatar Tor-Einar Skog
Browse files

Fixed handling of weather data from forecasts, 

particularly for the GRID humidity model
parent 8e8e27ab
Branches
Tags
2 merge requests!17Develop,!10Fixed handling of weather data from forecasts
Hide whitespace changes
Inline Side-by-side
Showing
with 93 additions and 40 deletions
src/main/java/no/nibio/vips/logic/scheduling/model/grid/preprocessor/ZymoseptoriaSimpleRiskGridModelPreprocessor.java
+ 4
− 4
View file @ afcbab84
......@@ -23,7 +23,6 @@ import com.vividsolutions.jts.geom.Coordinate;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
......@@ -128,11 +127,12 @@ public class ZymoseptoriaSimpleRiskGridModelPreprocessor extends ModelRunPreproc
},
Date.from(aWeekAgo.toInstant()),
Date.from(aWeekAhead.toInstant()),
true
true,
new HashSet<>(Arrays.asList(WeatherObservation.LOG_INTERVAL_ID_1H,WeatherObservation.LOG_INTERVAL_ID_3H, WeatherObservation.LOG_INTERVAL_ID_6H))
);
stationObs = wUtil.checkForAndFixHourlyTimeSeriesHoles(stationObs);
//stationObs = wUtil.checkForAndFixHourlyTimeSeriesHoles(stationObs,6);
return stationObs;
} catch (WeatherDataSourceException | WeatherObservationListException ex ) {
} catch (WeatherDataSourceException ex ) {
throw new PreprocessorException(ex.getMessage());
}
}
......
src/main/java/no/nibio/vips/util/weather/WeatherDataSourceUtil.java
+ 42
− 6
View file @ afcbab84
......@@ -33,8 +33,10 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
import no.nibio.vips.entity.WeatherObservation;
import no.nibio.vips.logic.entity.PointOfInterestWeatherStation;
......@@ -68,13 +70,16 @@ public class WeatherDataSourceUtil {
* @return
* @throws PreprocessorException
*/
public List<WeatherObservation> getWeatherObservations(PointOfInterestWeatherStation station, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, Boolean ignoreErrors) throws WeatherDataSourceException {
List<WeatherObservation> observations = this.getWeatherObservations(station.getDataFetchUri(), logIntervalId, elementMeasurementTypes, startTime, endTime, TimeZone.getTimeZone(station.getTimeZone()), ignoreErrors);
public List<WeatherObservation> getWeatherObservations(PointOfInterestWeatherStation station, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, Boolean ignoreErrors, Set<Integer> toleratedLogIntervalIds) throws WeatherDataSourceException {
// Get measured (and possibly forecasted, depending on the data source) observations
List<WeatherObservation> observations = this.getWeatherObservations(station.getDataFetchUri(), logIntervalId, elementMeasurementTypes, startTime, endTime, TimeZone.getTimeZone(station.getTimeZone()), ignoreErrors, toleratedLogIntervalIds);
Collections.sort(observations);
// Append forecasts, if available
Date latestTimeOfMeasuredObservations = observations.isEmpty() ? null : observations.get(observations.size() - 1).getTimeMeasured();
//System.out.println("latestTimeOfMeasuredObservations = " + latestTimeOfMeasuredObservations);
// Todo: We don't collect forecast data when the endTime is before the actual, current date (not systemdate)
if (station.getWeatherForecastProviderId() != null && ! SystemTime.isSystemTimeOffsetFromNow()) {
if (station.getWeatherForecastProviderId() != null && ! SystemTime.isSystemTimeOffsetFromNow())
{
try {
WeatherForecastProvider forecastProvider = WeatherStationProviderFactory.getWeatherForecastProvider(station.getWeatherForecastProviderId().getWeatherForecastProviderId());
List<WeatherObservation> forecasts = forecastProvider.getWeatherForecasts(station);
......@@ -83,7 +88,7 @@ public class WeatherDataSourceUtil {
obsMap.put(elementMeasurementType, new ArrayList<>());
}
forecasts.stream().filter((obs) -> (
obs.getLogIntervalId().equals(logIntervalId) && obsMap.get(obs.getElementMeasurementTypeId()) != null)
toleratedLogIntervalIds.contains(obs.getLogIntervalId()) && obsMap.get(obs.getElementMeasurementTypeId()) != null)
).forEachOrdered((obs) -> {
obsMap.get(obs.getElementMeasurementTypeId()).add(obs);
});
......@@ -120,6 +125,21 @@ public class WeatherDataSourceUtil {
}
return observations;
}
/**
* The intolerant version. Only accept observations with same logIntervalId as requested
* @param station
* @param logIntervalId
* @param elementMeasurementTypes
* @param startTime
* @param endTime
* @param ignoreErrors
* @return
* @throws WeatherDataSourceException
*/
public List<WeatherObservation> getWeatherObservations(PointOfInterestWeatherStation station, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, Boolean ignoreErrors) throws WeatherDataSourceException {
return this.getWeatherObservations(station, logIntervalId, elementMeasurementTypes, startTime, endTime, ignoreErrors, new HashSet<>(Arrays.asList(logIntervalId)));
}
/**
* Fetches measured data from the stations weather data source, and optionally
......@@ -149,6 +169,22 @@ public class WeatherDataSourceUtil {
return new ObjectMapper().readValue(JSONtext, new TypeReference<List<WeatherObservation>>() {
});
}
/**
* The intolerant version. Only accept observations with same logIntervalId as requested
* @param fetchURI
* @param logIntervalId
* @param elementMeasurementTypes
* @param startTime
* @param endTime
* @param timeZone
* @param ignoreErrors
* @return
* @throws WeatherDataSourceException
*/
public List<WeatherObservation> getWeatherObservations(String fetchURI, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, TimeZone timeZone, Boolean ignoreErrors) throws WeatherDataSourceException {
return this.getWeatherObservations(fetchURI, logIntervalId, elementMeasurementTypes, startTime, endTime, timeZone, ignoreErrors, new HashSet<>(Arrays.asList(logIntervalId)));
}
/**
* Collects weather observations from a data source
......@@ -160,7 +196,7 @@ public class WeatherDataSourceUtil {
* @param timeZone
* @return
*/
public List<WeatherObservation> getWeatherObservations(String fetchURI, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, TimeZone timeZone, Boolean ignoreErrors) throws WeatherDataSourceException {
public List<WeatherObservation> getWeatherObservations(String fetchURI, Integer logIntervalId, String[] elementMeasurementTypes, Date startTime, Date endTime, TimeZone timeZone, Boolean ignoreErrors, Set<Integer> toleratedLogIntervalIds) throws WeatherDataSourceException {
SimpleDateFormat dateOutput = new SimpleDateFormat("yyyy-MM-dd");
dateOutput.setTimeZone(timeZone);
SimpleDateFormat hourOutput = new SimpleDateFormat("H");
......@@ -199,7 +235,7 @@ public class WeatherDataSourceUtil {
List<WeatherObservation> filteredObservations = new ArrayList<>();
//System.out.println(this.getClass().getName() + "/preliminaryResult.size()=" + preliminaryResult.size());
preliminaryResult.stream().filter((candidateObs) -> (
candidateObs.getLogIntervalId().equals(logIntervalId)
toleratedLogIntervalIds.contains(candidateObs.getLogIntervalId())
&& Arrays.asList(elementMeasurementTypes).contains(candidateObs.getElementMeasurementTypeId())
)
).forEachOrdered((candidateObs) -> {
......
src/main/java/no/nibio/vips/util/weather/YrWeatherForecastProvider.java
+ 6
− 5
View file @ afcbab84
......@@ -77,7 +77,7 @@ public class YrWeatherForecastProvider implements WeatherForecastProvider{
YrWeatherForecastProvider.YR_API_URL,
latitude,
longitude,
altitude)
altitude.intValue())
);
//System.out.println("yrURL=" + yrURL.toString());
......@@ -94,6 +94,7 @@ public class YrWeatherForecastProvider implements WeatherForecastProvider{
Long sixHoursDelta = 6 * hourDelta;
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX");
Map<Date, WeatherObservation> RRMap = new HashMap<>();
Date earliestHourlyPrecipitationObservation = null;
for(int i=0;i<nodes.getLength();i++)
{
Node node = nodes.item(i);
......@@ -138,14 +139,14 @@ public class YrWeatherForecastProvider implements WeatherForecastProvider{
RR.setElementMeasurementTypeId(WeatherElements.PRECIPITATION);
RR.setValue(Double.parseDouble(DOMUtils.getNodeAttr("precipitation","value",node2.getChildNodes())));
//System.out.println("Timediff=" + (toTime.getTime() - fromTime.getTime()));
Date earliestHourlyObservation = null;
if(toTime.getTime() - fromTime.getTime() == hourDelta)
{
//System.out.println("Found 1 hour record at " + fromTime);
RR.setLogIntervalId(WeatherObservation.LOG_INTERVAL_ID_1H);
if(earliestHourlyObservation == null)
if(earliestHourlyPrecipitationObservation == null)
{
earliestHourlyObservation = RR.getTimeMeasured();
earliestHourlyPrecipitationObservation = RR.getTimeMeasured();
}
}
else if(toTime.getTime() - fromTime.getTime() == threeHoursDelta)
......@@ -165,7 +166,7 @@ public class YrWeatherForecastProvider implements WeatherForecastProvider{
// In order to avoid overestimation of rain in the beginning,
// We skip ahead until we find observations with 1 hour resolution
if(!RR.getLogIntervalId().equals(WeatherObservation.LOG_INTERVAL_ID_1H)
&& (earliestHourlyObservation == null || earliestHourlyObservation.after(RR.getTimeMeasured()))
&& (earliestHourlyPrecipitationObservation == null || earliestHourlyPrecipitationObservation.after(RR.getTimeMeasured()))
)
{
continue;
......
src/main/java/no/nibio/vips/util/weather/dnmipointweb/DMIPointWebDataParser.java
+ 31
− 15
View file @ afcbab84
......@@ -22,19 +22,23 @@ package no.nibio.vips.util.weather.dnmipointweb;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import javax.xml.datatype.DatatypeFactory;
import javax.xml.datatype.XMLGregorianCalendar;
import no.nibio.vips.entity.WeatherObservation;
import no.nibio.vips.util.WeatherElements;
import no.nibio.vips.util.WeatherUtil;
/**
* Gets data from the Danish Meteorological Institute's Point Web service
......@@ -58,7 +62,7 @@ public class DMIPointWebDataParser {
public List<WeatherObservation> getData(Double longitude, Double latitude, Date dateFrom, Date dateTo)
{
List<WeatherObservation> allObservations = new ArrayList<>();
List<WeatherObservation> rawObservations = new ArrayList<>();
TimeZone danishTZ = TimeZone.getTimeZone("Europe/Copenhagen");
try {
IWeatherService proxy = new WeatherService().getSslOffloadedBasicHttpBindingIWeatherService();
......@@ -106,7 +110,7 @@ public class DMIPointWebDataParser {
WeatherObservation obs = this.getWeatherObservation(VIPSParam, weatherDataModel);
if(obs != null)
{
allObservations.add(obs);
rawObservations.add(obs);
}
}
);
......@@ -118,22 +122,34 @@ public class DMIPointWebDataParser {
}
//System.out.println("Number of extracted weather data = " + retVal.size());
// After "now", the DMI service provides forecast values seamlessly. After approx 48 hours these
// values turn into 6 hour intervals, which we can't use. We need to weed them out
Collections.sort(allObservations);
// values turn into 6 hour intervals
// 2019-01-18: We keep all observations, leave it up to the client to decide what to do with the data.
// But we mark them with the correct measuring interval
ZonedDateTime now = ZonedDateTime.now();
ZonedDateTime lastValidObsTime = null;
ZoneId danishZ = ZoneId.of("Europe/Copenhagen");
List<WeatherObservation> filteredObservations = new ArrayList<>();
for(WeatherObservation obs:allObservations)
{
ZonedDateTime obsTime = ZonedDateTime.ofInstant(obs.getTimeMeasured().toInstant(),danishZ);
if(lastValidObsTime == null || obsTime.isBefore(now) || obsTime.minusHours(6).isBefore(lastValidObsTime))
//List<WeatherObservation> filteredObservations = new ArrayList<>();
WeatherUtil wUtil = new WeatherUtil();
List<WeatherObservation> retVal = new ArrayList<>();
Set<String> parametersInResult = rawObservations.stream().map(obs->obs.getElementMeasurementTypeId()).collect(Collectors.toSet());
parametersInResult.forEach((param) -> {
ZonedDateTime lastObsTime = null;
List<WeatherObservation> paramObs = wUtil.filterWeatherObservationsByParameter(rawObservations, param);
Collections.sort(paramObs);
for(WeatherObservation obs:paramObs)
{
filteredObservations.add(obs);
lastValidObsTime = obsTime;
}
}
return allObservations;
ZonedDateTime obsTime = ZonedDateTime.ofInstant(obs.getTimeMeasured().toInstant(),danishZ);
if(lastObsTime != null && obsTime.isAfter(now) && !obsTime.minusHours(6).isBefore(lastObsTime))
{
obs.setLogIntervalId(WeatherObservation.LOG_INTERVAL_ID_6H);
}
retVal.add(obs);
lastObsTime = obsTime;
}
});
Collections.sort(retVal);
return retVal;
}
private WeatherObservation getWeatherObservation(String VIPSParam, WeatherDataModel wDataModel) {
......
src/main/webapp/public/nordic_septoria_whs/nordic_septoria_whs.js
+ 1
− 1
View file @ afcbab84
......@@ -192,7 +192,7 @@ function getTodayAtMidnight()
return new Date(midnightString);
// OR RETURN A FIXED DATE FOR TESTING
//return new Date("2017-09-07T22:00:00.000+0000");
//return new Date("2018-08-16T22:00:00.000+0000");
}
......
src/test/java/no/nibio/vips/util/weather/YrWeatherForecastProviderTest.java
+ 9
− 9
View file @ afcbab84
......@@ -87,26 +87,26 @@ public class YrWeatherForecastProviderTest {
//weatherStation.setAltitude(110.0);
//weatherStation.setLongitude(10.83474668);
//weatherStation.setLatitude(59.63851390);
weatherStation.setAltitude(19.0);
weatherStation.setLongitude(6.57052);
weatherStation.setLatitude(60.2542);
weatherStation.setAltitude(102.0);
weatherStation.setLongitude(11.39);
weatherStation.setLatitude(59.388);
result = instance.getWeatherForecasts(weatherStation);
assertNotNull(result);
/*for(WeatherObservation obs:result)
Collections.sort(result);
for(WeatherObservation obs:result)
{
if(obs.getElementMeasurementTypeId().equals("RR"))
if(obs.getElementMeasurementTypeId().equals("TM"))
System.out.println(obs.toString());
}*/
}
WeatherUtil wUtil = new WeatherUtil();
wUtil.checkForAndFixHourlyTimeSeriesHoles(result);
//wUtil.checkForAndFixHourlyTimeSeriesHoles(result);
// Testing Jablanica (Republika Srpska, Bosnia)
weatherStation.setAltitude(219.0);
weatherStation.setLongitude(16.988056);
weatherStation.setLatitude(45.115);
result = instance.getWeatherForecasts(weatherStation);
//result = instance.getWeatherForecasts(weatherStation);
assertNotNull(result);
/**Collections.sort(result);
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment