From 82af2d017522526520c9f1bf488dc8dc1da691ef Mon Sep 17 00:00:00 2001 From: Tor-Einar Skog <tor-einar.skog@nibio.no> Date: Wed, 21 Mar 2018 11:01:19 +0100 Subject: [PATCH] Getting ready for running forecasts based on weather data from Davis stations --- .../controller/session/ForecastBean.java | 1 - ...castConfigurationsForOrganizationTask.java | 6 +-- .../RunAllForecastConfigurationsTask.java | 10 ++--- .../logic/service/WeatherProxyService.java | 8 +++- .../no/nibio/vips/logic/util/SystemTime.java | 11 ++++++ .../util/weather/FruitWebDavisDataParser.java | 27 +++++++------- .../util/weather/WeatherDataSourceUtil.java | 10 ++++- .../weather/YrWeatherForecastProvider.java | 37 ++++++++++++++++++- .../weather/FruitWebDavisDataParserTest.java | 14 ++++--- 9 files changed, 89 insertions(+), 35 deletions(-) diff --git a/src/main/java/no/nibio/vips/logic/controller/session/ForecastBean.java b/src/main/java/no/nibio/vips/logic/controller/session/ForecastBean.java index 848a987f..71d487cf 100755 --- a/src/main/java/no/nibio/vips/logic/controller/session/ForecastBean.java +++ b/src/main/java/no/nibio/vips/logic/controller/session/ForecastBean.java @@ -594,7 +594,6 @@ public class ForecastBean { ex.printStackTrace(); }*/ Response resp = this.getManagerResource().runModel(config.getModelId(), request); - if(resp.getStatus() == Response.Status.OK.getStatusCode()) { List<Result> results = (List<Result>) resp.readEntity(new GenericType<List<Result>>(){}); diff --git a/src/main/java/no/nibio/vips/logic/scheduling/tasks/RunAllForecastConfigurationsForOrganizationTask.java b/src/main/java/no/nibio/vips/logic/scheduling/tasks/RunAllForecastConfigurationsForOrganizationTask.java index 3d1daeeb..eebc9e72 100644 --- a/src/main/java/no/nibio/vips/logic/scheduling/tasks/RunAllForecastConfigurationsForOrganizationTask.java +++ b/src/main/java/no/nibio/vips/logic/scheduling/tasks/RunAllForecastConfigurationsForOrganizationTask.java @@ -29,7 +29,6 @@ import no.nibio.vips.logic.entity.ModelInformation; import no.nibio.vips.logic.entity.PointOfInterest; import no.nibio.vips.logic.entity.PointOfInterestWeatherStation; import no.nibio.vips.logic.scheduling.SchedulingUtil; -import no.nibio.vips.logic.scheduling.VipsLogicTaskFactory; import no.nibio.vips.logic.scheduling.model.PreprocessorException; import no.nibio.vips.logic.util.RunModelException; import no.nibio.vips.logic.util.SessionControllerGetter; @@ -37,7 +36,7 @@ import no.nibio.vips.logic.util.SystemTime; import no.nibio.web.forms.FormField; /** - * @copyright 2017 <a href="http://www.nibio.no/">NIBIO</a> + * @copyright 2018 <a href="http://www.nibio.no/">NIBIO</a> * @author Tor-Einar Skog <tor-einar.skog@nibio.no> */ public class RunAllForecastConfigurationsForOrganizationTask extends RunAllForecastConfigurationsTask{ @@ -70,7 +69,8 @@ public class RunAllForecastConfigurationsForOrganizationTask extends RunAllForec if( weatherStationPointOfInterestId == null || weatherStationPointOfInterestId <= 0 - || forecastConfiguration.getWeatherStationPointOfInterestId().getPointOfInterestId().equals(weatherStationPointOfInterestId)) + || (forecastConfiguration.getWeatherStationPointOfInterestId() != null && forecastConfiguration.getWeatherStationPointOfInterestId().getPointOfInterestId().equals(weatherStationPointOfInterestId)) + ) { try { diff --git a/src/main/java/no/nibio/vips/logic/scheduling/tasks/RunAllForecastConfigurationsTask.java b/src/main/java/no/nibio/vips/logic/scheduling/tasks/RunAllForecastConfigurationsTask.java index d9d7c703..70d882c8 100755 --- a/src/main/java/no/nibio/vips/logic/scheduling/tasks/RunAllForecastConfigurationsTask.java +++ b/src/main/java/no/nibio/vips/logic/scheduling/tasks/RunAllForecastConfigurationsTask.java @@ -32,7 +32,6 @@ import no.nibio.vips.logic.entity.PointOfInterest; import no.nibio.vips.logic.entity.PointOfInterestWeatherStation; import no.nibio.vips.logic.scheduling.SchedulingUtil; import no.nibio.vips.logic.scheduling.VipsLogicTask; -import no.nibio.vips.logic.scheduling.VipsLogicTaskFactory; import no.nibio.vips.logic.scheduling.model.PreprocessorException; import no.nibio.vips.logic.util.RunModelException; import no.nibio.vips.logic.util.SessionControllerGetter; @@ -40,7 +39,7 @@ import no.nibio.vips.logic.util.SystemTime; import no.nibio.web.forms.FormField; /** - * @copyright 2013-2016 <a href="http://www.nibio.no/">NIBIO</a> + * @copyright 2013-2018 <a href="http://www.nibio.no/">NIBIO</a> * @author Tor-Einar Skog <tor-einar.skog@nibio.no> */ public class RunAllForecastConfigurationsTask extends VipsLogicTask{ @@ -78,7 +77,7 @@ public class RunAllForecastConfigurationsTask extends VipsLogicTask{ boolean noForecastConfigurationsFound = true; Map<String, ModelInformation> modelInformationMap = SessionControllerGetter.getForecastBean().getIndexedBatchableModelInformation(); - + for(Organization organization : organizations) { List<ForecastConfiguration> currentForecastConfigurations = SessionControllerGetter.getForecastBean().getForecastConfigurationsValidAtTime(organization, SystemTime.getSystemTime()); @@ -91,7 +90,8 @@ public class RunAllForecastConfigurationsTask extends VipsLogicTask{ if( weatherStationPointOfInterestId == null || weatherStationPointOfInterestId <= 0 - || forecastConfiguration.getWeatherStationPointOfInterestId().getPointOfInterestId().equals(weatherStationPointOfInterestId)) + || (forecastConfiguration.getWeatherStationPointOfInterestId() != null && forecastConfiguration.getWeatherStationPointOfInterestId().getPointOfInterestId().equals(weatherStationPointOfInterestId)) + ) { try { @@ -133,13 +133,11 @@ public class RunAllForecastConfigurationsTask extends VipsLogicTask{ } } - if(noForecastConfigurationsFound) { tec.setCompleteness(1.0); tec.setStatusMessage("No current forecast configurations were found"); } - if(tec.getTaskExecutor().getCompleteness() != 1.0) { //System.out.println("Error detected"); diff --git a/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java b/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java index 7be58c49..842641bc 100755 --- a/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java +++ b/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java @@ -129,7 +129,9 @@ public class WeatherProxyService { @FormParam("timeZone") String timeZonePOST, @QueryParam("timeZone") String timeZoneGET, @FormParam("startDate") String startDatePOST, - @QueryParam("startDate") String startDateGET + @QueryParam("startDate") String startDateGET, + @FormParam("endDate") String endDatePOST, + @QueryParam("endDate") String endDateGET ) { List<WeatherObservation> observations; @@ -141,7 +143,9 @@ public class WeatherProxyService { format.setTimeZone(timeZone); String startDateParam = startDatePOST != null ? startDatePOST : startDateGET; Date startDate1 = format.parse(startDateParam); - observations = new FruitWebDavisDataParser().getWeatherObservations(URLDecoder.decode(stationId, "UTF-8"), timeZone, startDate1); + String endDateParam = endDatePOST != null ? endDatePOST : endDateGET != null ? endDateGET : null; + Date endDate1 = endDateParam != null ? format.parse(endDateParam) : null; + observations = new FruitWebDavisDataParser().getWeatherObservations(URLDecoder.decode(stationId, "UTF-8"), timeZone, startDate1, endDate1); } catch (ParseException | ParseWeatherDataException | NullPointerException | UnsupportedEncodingException ex) { return Response.serverError().entity(ex).build(); } diff --git a/src/main/java/no/nibio/vips/logic/util/SystemTime.java b/src/main/java/no/nibio/vips/logic/util/SystemTime.java index b9afee55..06fb8b70 100755 --- a/src/main/java/no/nibio/vips/logic/util/SystemTime.java +++ b/src/main/java/no/nibio/vips/logic/util/SystemTime.java @@ -51,6 +51,17 @@ public final class SystemTime { return getExactDate() != null ? getExactDate(): cal.getTime(); } + /** + * Simple check to see if the system time is offset from now. The threshold + * is 5 seconds + * @return + */ + public static boolean isSystemTimeOffsetFromNow(){ + Date systemTime = SystemTime.getSystemTime(); + Date now = new Date(); + return Math.abs(now.getTime() - systemTime.getTime()) > 5000; + } + private static Integer getOffsetMonths() { return System.getProperty("no.nibio.vips.logic.SYSTEM_TIME_OFFSET_MONTHS") != null ? diff --git a/src/main/java/no/nibio/vips/util/weather/FruitWebDavisDataParser.java b/src/main/java/no/nibio/vips/util/weather/FruitWebDavisDataParser.java index c7dec864..73ee3562 100755 --- a/src/main/java/no/nibio/vips/util/weather/FruitWebDavisDataParser.java +++ b/src/main/java/no/nibio/vips/util/weather/FruitWebDavisDataParser.java @@ -33,6 +33,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TimeZone; +import java.util.stream.Collectors; import no.nibio.vips.entity.WeatherObservation; /** @@ -41,9 +42,10 @@ import no.nibio.vips.entity.WeatherObservation; * @author Tor-Einar Skog <tor-einar.skog@nibio.no> */ public class FruitWebDavisDataParser { - // Sample: http://wds.fruitweb.info/sc/getFile.php?id=283&date=2016-01-01&pw=Kghr528Ub5 + // Sample: https://www.fruitweb.info/sc/getFile.php?id=283&date=2016-01-01&pw=Kghr528Ub5 // id and password combined must be the "remote ID" in VIPSLogic form - public final static String FRUITWEB_URL_TEMPLATE = "http://wds.fruitweb.info/sc/getFile.php?{0}&date={1}"; + //public final static String FRUITWEB_URL_TEMPLATE = "http://wds.fruitweb.info/sc/getFile.php?{0}&date={1}"; + public final static String FRUITWEB_URL_TEMPLATE = "https://www.fruitweb.info/sc/getFile.php?{0}&date={1}"; // Metos parameters, including name and aggregation type private final static String[][] ELEMENT_MEASUREMENT_TYPES = { {"RAIN","RR","SUM"}, @@ -61,7 +63,7 @@ public class FruitWebDavisDataParser { * @param startDate * @return */ - public List<WeatherObservation> getWeatherObservations(String stationID, TimeZone timeZone, Date startDate) throws ParseWeatherDataException + public List<WeatherObservation> getWeatherObservations(String stationID, TimeZone timeZone, Date startDate, Date endDate) throws ParseWeatherDataException { List<WeatherObservation> retVal = new ArrayList<>(); SimpleDateFormat urlDFormat = new SimpleDateFormat("yyyy-MM-dd"); @@ -81,7 +83,6 @@ public class FruitWebDavisDataParser { String inputLine; Date testTimestamp; - // We need to collect all the lines first, because we need to analyze // If the resolution is 30 minutes or 1 hour while ((inputLine = in.readLine()) != null) @@ -192,12 +193,12 @@ public class FruitWebDavisDataParser { aggregateValue = (value00 + value30); } - WeatherObservation obs = new WeatherObservation(); - obs.setTimeMeasured(timestamp); - obs.setLogIntervalId(WeatherObservation.LOG_INTERVAL_ID_1H); - obs.setElementMeasurementTypeId(FruitWebDavisDataParser.ELEMENT_MEASUREMENT_TYPES[elementMeasurementTypeIndex][1]); - obs.setValue(aggregateValue); - retVal.add(obs); + WeatherObservation obs = new WeatherObservation(); + obs.setTimeMeasured(timestamp); + obs.setLogIntervalId(WeatherObservation.LOG_INTERVAL_ID_1H); + obs.setElementMeasurementTypeId(FruitWebDavisDataParser.ELEMENT_MEASUREMENT_TYPES[elementMeasurementTypeIndex][1]); + obs.setValue(aggregateValue); + retVal.add(obs); } } } @@ -238,9 +239,9 @@ public class FruitWebDavisDataParser { } } } - - - return retVal; + return retVal.stream().filter(obs -> + (obs.getTimeMeasured().compareTo(startDate) >= 0 && (endDate == null || obs.getTimeMeasured().compareTo(endDate) <= 0)) + ).collect(Collectors.toList()); } } diff --git a/src/main/java/no/nibio/vips/util/weather/WeatherDataSourceUtil.java b/src/main/java/no/nibio/vips/util/weather/WeatherDataSourceUtil.java index 6369833c..3e94f1be 100755 --- a/src/main/java/no/nibio/vips/util/weather/WeatherDataSourceUtil.java +++ b/src/main/java/no/nibio/vips/util/weather/WeatherDataSourceUtil.java @@ -28,6 +28,7 @@ import java.net.URL; import java.net.URLConnection; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -37,6 +38,7 @@ import java.util.TimeZone; import no.nibio.vips.entity.WeatherObservation; import no.nibio.vips.logic.entity.PointOfInterestWeatherStation; import no.nibio.vips.logic.scheduling.model.PreprocessorException; +import no.nibio.vips.logic.util.SystemTime; import org.apache.commons.io.IOUtils; /** @@ -67,7 +69,9 @@ public class WeatherDataSourceUtil { List<WeatherObservation> observations = this.getWeatherObservations(station.getDataFetchUri(), logIntervalId, elementMeasurementTypes, startTime, endTime, TimeZone.getTimeZone(station.getTimeZone()), ignoreErrors); Collections.sort(observations); Date latestTimeOfMeasuredObservations = observations.isEmpty() ? null : observations.get(observations.size() - 1).getTimeMeasured(); - if (station.getWeatherForecastProviderId() != null) { + //System.out.println("latestTimeOfMeasuredObservations = " + latestTimeOfMeasuredObservations); + // Todo: We don't collect forecast data when the endTime is before the actual, current date (not systemdate) + if (station.getWeatherForecastProviderId() != null && ! SystemTime.isSystemTimeOffsetFromNow()) { try { WeatherForecastProvider forecastProvider = WeatherStationProviderFactory.getWeatherForecastProvider(station.getWeatherForecastProviderId().getWeatherForecastProviderId()); List<WeatherObservation> forecasts = forecastProvider.getWeatherForecasts(station); @@ -186,7 +190,9 @@ public class WeatherDataSourceUtil { List<WeatherObservation> preliminaryResult = this.getWeatherObservations(URLOutput); List<WeatherObservation> filteredObservations = new ArrayList<>(); preliminaryResult.stream().filter((candidateObs) -> ( - candidateObs.getLogIntervalId().equals(logIntervalId)) + candidateObs.getLogIntervalId().equals(logIntervalId) + && Arrays.asList(elementMeasurementTypes).contains(candidateObs.getElementMeasurementTypeId()) + ) ).forEachOrdered((candidateObs) -> { filteredObservations.add(candidateObs); }); diff --git a/src/main/java/no/nibio/vips/util/weather/YrWeatherForecastProvider.java b/src/main/java/no/nibio/vips/util/weather/YrWeatherForecastProvider.java index aa0640e7..3c15a6ec 100755 --- a/src/main/java/no/nibio/vips/util/weather/YrWeatherForecastProvider.java +++ b/src/main/java/no/nibio/vips/util/weather/YrWeatherForecastProvider.java @@ -19,7 +19,10 @@ package no.nibio.vips.util.weather; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.net.URL; import java.text.MessageFormat; import java.text.ParseException; @@ -49,7 +52,7 @@ import org.xml.sax.SAXException; */ public class YrWeatherForecastProvider implements WeatherForecastProvider{ - private final static String YR_API_URL = "http://api.yr.no/weatherapi/locationforecastlts/1.3/?lat={0};lon={1};msl={2}"; + private final static String YR_API_URL = "https://api.met.no/weatherapi/locationforecastlts/1.3/?lat={0};lon={1};msl={2}"; @Override public List<WeatherObservation> getWeatherForecasts(PointOfInterest location) throws ParseWeatherDataException @@ -68,8 +71,9 @@ public class YrWeatherForecastProvider implements WeatherForecastProvider{ public List<WeatherObservation> getWeatherForecasts(Double longitude, Double latitude, Double altitude) throws ParseWeatherDataException { List<WeatherObservation> yrValues = new ArrayList<>(); + URL yrURL; try { - URL yrURL = new URL(MessageFormat.format( + yrURL = new URL(MessageFormat.format( YrWeatherForecastProvider.YR_API_URL, latitude, longitude, @@ -77,6 +81,8 @@ public class YrWeatherForecastProvider implements WeatherForecastProvider{ ); //System.out.println("yrURL=" + yrURL.toString()); + //System.out.println(getStringFromInputStream(yrURL.openStream())); + // TODO: Parse with DOM parser DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); @@ -275,5 +281,32 @@ public class YrWeatherForecastProvider implements WeatherForecastProvider{ } return retVal; } + + private String getStringFromInputStream(InputStream is) { + BufferedReader br = null; + StringBuilder sb = new StringBuilder(); + + String line; + try { + + br = new BufferedReader(new InputStreamReader(is)); + while ((line = br.readLine()) != null) { + sb.append(line); + } + + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (br != null) { + try { + br.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + return sb.toString(); + } } diff --git a/src/test/java/no/nibio/vips/util/weather/FruitWebDavisDataParserTest.java b/src/test/java/no/nibio/vips/util/weather/FruitWebDavisDataParserTest.java index 0b2887c3..1a0d884d 100755 --- a/src/test/java/no/nibio/vips/util/weather/FruitWebDavisDataParserTest.java +++ b/src/test/java/no/nibio/vips/util/weather/FruitWebDavisDataParserTest.java @@ -61,18 +61,20 @@ public class FruitWebDavisDataParserTest { @Test public void testGetWeatherObservations() throws Exception { System.out.println("getWeatherObservations"); - String stationID = "id=283&pw=Kghr528Ub5"; + //String stationID = "id=283&pw=Kghr528Ub5"; + String stationID = "id=537&pw=Kgsf72k388"; TimeZone timeZone = TimeZone.getTimeZone("UTC"); Calendar cal = Calendar.getInstance(timeZone); - cal.set(2016, Calendar.JANUARY, 1, 0, 0, 0); + cal.set(2017, Calendar.JANUARY, 1, 0, 0, 0); Date startDate = cal.getTime(); FruitWebDavisDataParser instance = new FruitWebDavisDataParser(); + cal.set(Calendar.MONTH, Calendar.AUGUST); + Date endDate = cal.getTime(); - List<WeatherObservation> result = instance.getWeatherObservations(stationID, timeZone, startDate); - for(WeatherObservation obs:result) - { + List<WeatherObservation> result = instance.getWeatherObservations(stationID, timeZone, startDate, endDate); + result.forEach((obs) -> { System.out.println(obs); - } + }); assertNotNull( result); } -- GitLab