diff --git a/pom.xml b/pom.xml
index 11af3368334cfc59612dfd4c6e85773a8a9cdd22..5b77c25756503e1dad5955c8656f1ac3dcbd1044 100644
--- a/pom.xml
+++ b/pom.xml
@@ -66,8 +66,16 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- <version>4.5.2</version>
+ <version>4.5</version>
<type>jar</type>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>4.4.1</version>
+ <type>jar</type>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
diff --git a/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java b/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java
index 0cdd6d80459874c5af87c9f70cea68ff9de41072..29b52dc568d3d917f4fb9ef56ec4c610c9220cea 100644
--- a/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java
+++ b/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java
@@ -39,6 +39,7 @@ import no.nibio.vips.util.weather.ALabDataParser;
import no.nibio.vips.util.weather.FruitWebDavisDataParser;
import no.nibio.vips.util.weather.MetosDataParser;
import no.nibio.vips.util.weather.ParseWeatherDataException;
+import no.nibio.vips.util.weather.USPestDataParser;
import org.jboss.resteasy.annotations.GZIP;
/**
@@ -141,4 +142,33 @@ public class WeatherProxyService {
return Response.ok().entity(observations).build();
}
+ @GET
+ @POST
+ @Path("uspest/{stationId}")
+ @GZIP
+ @Produces("application/json;charset=UTF-8")
+ public Response getUSPestWeatherData(
+ @PathParam("stationId") String stationId,
+ @FormParam("timeZone") String timeZonePOST,
+ @QueryParam("timeZone") String timeZoneGET,
+ @FormParam("startDate") String startDatePOST,
+ @QueryParam("startDate") String startDateGET
+ )
+ {
+ List<WeatherObservation> observations;
+ try
+ {
+ String timeZoneParam = timeZonePOST != null ? timeZonePOST : timeZoneGET != null ? timeZoneGET : "UTC";
+ TimeZone timeZone = TimeZone.getTimeZone(timeZoneParam);
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
+ format.setTimeZone(timeZone);
+ String startDateParam = startDatePOST != null ? startDatePOST : startDateGET;
+ Date startDate1 = format.parse(startDateParam);
+ observations = new USPestDataParser().getWeatherObservations(stationId, timeZone, startDate1);
+ } catch (ParseException | ParseWeatherDataException | NullPointerException ex) {
+ return Response.serverError().entity(ex).build();
+ }
+ return Response.ok().entity(observations).build();
+ }
+
}
diff --git a/src/main/java/no/nibio/vips/logic/util/Globals.java b/src/main/java/no/nibio/vips/logic/util/Globals.java
index 0ab2fc2f2478bfabf1cc2d3ede4b607d03a186a7..8c1ec555399abf02e482d6a2b5b71d39939ec200 100644
--- a/src/main/java/no/nibio/vips/logic/util/Globals.java
+++ b/src/main/java/no/nibio/vips/logic/util/Globals.java
@@ -74,7 +74,8 @@ public class Globals {
"Europe/Stockholm",
"Europe/Helsinki",
"Europe/Riga",
- "Europe/Sofia"
+ "Europe/Sofia",
+ "America/Los_Angeles"
};
diff --git a/src/main/java/no/nibio/vips/util/weather/USPestDataParser.java b/src/main/java/no/nibio/vips/util/weather/USPestDataParser.java
new file mode 100644
index 0000000000000000000000000000000000000000..f47d8c3ac971b96394de77d07bebee0df90b4f55
--- /dev/null
+++ b/src/main/java/no/nibio/vips/util/weather/USPestDataParser.java
@@ -0,0 +1,227 @@
+/*
+ * Copyright (c) 2016 NIBIO <http://www.nibio.no/>.
+ *
+ * This file is part of VIPSLogic.
+ * VIPSLogic is free software: you can redistribute it and/or modify
+ * it under the terms of the NIBIO Open Source License as published by
+ * NIBIO, either version 1 of the License, or (at your option) any
+ * later version.
+ *
+ * VIPSLogic is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * NIBIO Open Source License for more details.
+ *
+ * You should have received a copy of the NIBIO Open Source License
+ * along with VIPSLogic. If not, see <http://www.nibio.no/licenses/>.
+ *
+ */
+
+package no.nibio.vips.util.weather;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import no.nibio.vips.entity.WeatherObservation;
+import no.nibio.vips.logic.util.SystemTime;
+import no.nibio.vips.util.WeatherUtil;
+import no.nibio.vips.util.WebUtil;
+
+/**
+ * @copyright 2016 <a href="http://www.nibio.no/">NIBIO</a>
+ * @author Tor-Einar Skog <tor-einar.skog@nibio.no>
+ */
+public class USPestDataParser {
+ public final static String USPEST_URL_TEMPLATE = "http://uspest.org/risk/models";
+ // Parameters, including name and aggregation type
+ private final static String[][] elementMeasurementTypes = {
+ {"rain","RR","SUM"},
+ {"temp","TM","AVG"},
+ {"dewpt","DEWPOINT","AVG"},
+ {"rel_hum","UM","AVG"},
+ {"windspeed","FM2","AVG"},
+ {"leafwet","BT","AVG"}
+ };
+
+ /**
+ * Makes sure only data after the requested date is returned
+ * @param stationID
+ * @param timeZone
+ * @param startDate
+ * @return
+ * @throws ParseWeatherDataException
+ */
+ public List<WeatherObservation> getWeatherObservations(String stationID, TimeZone timeZone, Date startDate) throws ParseWeatherDataException
+ {
+ List<WeatherObservation> retVal = new ArrayList<>();
+ Map<Integer, Integer> elementOrdering = new HashMap<>();
+ WeatherUtil wUtil = new WeatherUtil();
+ SimpleDateFormat usPestDFormat = new SimpleDateFormat("MMM dd yyyy HH");
+ usPestDFormat.setTimeZone(timeZone);
+ SimpleDateFormat postFormDFormat = new SimpleDateFormat("yyyyMMdd");
+ //Date today = SystemTime.getSystemTime();
+ Date today = new Date();
+ try
+ {
+ today = SystemTime.getSystemTime();
+ }
+ catch(NumberFormatException ex)
+ {
+
+ }
+
+
+ Map<String, String> queryParams = new HashMap<>();
+ queryParams.put("station", stationID);
+ Calendar cal = Calendar.getInstance(timeZone);
+ cal.setTime(startDate);
+ int startDOY = cal.get(Calendar.DAY_OF_YEAR);
+ queryParams.put("stm",String.valueOf(cal.get(Calendar.MONTH)+1)); // Start month
+ queryParams.put("std",String.valueOf(cal.get(Calendar.DATE))); // Start dayOfMonth
+ queryParams.put("sty",String.valueOf(cal.get(Calendar.YEAR))); // Start year
+ cal.setTime(today);
+ int todayDOY = cal.get(Calendar.DAY_OF_YEAR);
+ queryParams.put("span",String.valueOf(todayDOY + 10 - startDOY)); // Number of days from start // Seems as it needs to be not to far in the future
+ queryParams.put("fsm",String.valueOf(cal.get(Calendar.MONTH)+1)); // Forecast start month
+ queryParams.put("fsd",String.valueOf(cal.get(Calendar.DATE))); // Forecast start dayOfMonth
+ // fcast, Type of forecast
+ // 1 = after 7day use 10 year averages (default)
+ // 2 = after 7day use 30 year averages
+ // 3 = after 7day use data from 1 year ago (if available)
+ // 4 = after 7day use data from 2 years ago (if available)
+ // 5 = after 7day use NMME extended seasonal forecast
+ // 6 = after 7day use the CFSv2 extended seasonal forecast
+ queryParams.put("fcast","1");
+ // Bad weather data detection and replacement config
+ queryParams.put("en_v2_tdp_missing", "on"); // Replace missing Temperature and Dew Point
+ queryParams.put("en_v2_tdp_flatline", "on"); // Replace flat line Temperature and Dew Point
+ queryParams.put("en_v2_tdp_range", "on"); // Replace range exceed Temperature and Dew Point
+ //queryParams.put("en_v2_tdp_extrap", "on"); // NOT RECOMMENDED
+ queryParams.put("en_v2_tdp_interp", "on"); // Replace data out of bounds of an interpolation
+ //queryParams.put("en_v2_tdp_elevreg", "on"); // NOT RECOMMENDED
+ queryParams.put("en_v2_windspeed_missing", "on"); // Replace missing wind speed
+ queryParams.put("en_v2_rain_missing", "on"); // Replance missing precipitation
+ queryParams.put("download_data","Download Data");
+ queryParams.put("stdt",postFormDFormat.format(startDate)); // Probably start date, but why the double info (see stm/std)
+ queryParams.put("date",postFormDFormat.format(today)); // Should be "today"
+
+ String[] weatherParams = {"temp", "dewpt","rel_hum","windspeed","rain","leafwet"};
+ Map<String, String[]> arrP = new HashMap<>();
+ arrP.put("wp", weatherParams);
+ // Keep track of which timestamps we already have,
+ // to make sure overlapping forecast values don't overwrite measured values
+ Set<Date> measuredDataTimestamps = new HashSet<>();
+ //queryParams.keySet().stream().forEach(key -> System.out.println(key + "=" + queryParams.get(key)) );
+ try
+ {
+ String output = WebUtil.getOutputFromPostRequest(USPestDataParser.USPEST_URL_TEMPLATE, queryParams, arrP);
+ try (BufferedReader reader = new BufferedReader(new StringReader(output))) {
+ reader.lines().forEach(
+ inputLine -> {
+ String[] lineData = inputLine.split(",");
+ //System.out.println(inputLine);
+
+ // Skip empty lines
+ if(lineData.length <= 1)
+ {
+ return;
+ }
+ // Check for valid start of line
+ try {
+
+ String dateAndTimeStr = lineData[0] + " " + lineData[1];
+ Date timeMeasured = usPestDFormat.parse(dateAndTimeStr);
+ // If this time stamp already has been processed, this must be
+ // an overlapping forecast value. We skip it.
+ if(measuredDataTimestamps.contains(timeMeasured))
+ {
+ return;
+ }
+ else
+ {
+ measuredDataTimestamps.add(timeMeasured);
+ }
+ // Do stuff
+ for(Integer i=2;i<lineData.length;i++)
+ {
+ // Skip empty columns
+ Double value = 0.0;
+ try
+ {
+ value = Double.valueOf(lineData[i].replaceAll(",","."));
+ }
+ catch(NumberFormatException ex) {/*System.out.println("Error formatting \"" + lineData[i] + "\" lineData[" + i + "]");*/}
+
+ Integer elementMeasurementTypeIndex = elementOrdering.get(i);
+ // This means there is an element type we don't collect
+ if(elementMeasurementTypeIndex == null)
+ {
+ continue;
+ }
+ WeatherObservation obs = new WeatherObservation();
+ obs.setTimeMeasured(timeMeasured);
+ obs.setLogIntervalId(WeatherObservation.LOG_INTERVAL_ID_1H);
+ obs.setElementMeasurementTypeId(USPestDataParser.elementMeasurementTypes[elementMeasurementTypeIndex][1]);
+ // Converting from US system to VIPS
+ // Temps measured in F -> C
+ if(obs.getElementMeasurementTypeId().equals("TM"))
+ {
+ obs.setValue(wUtil.getCelciusFromFahrenheit(value));
+ }
+ // Rain measured in inches -> mm
+ else if(obs.getElementMeasurementTypeId().equals("RR"))
+ {
+ obs.setValue(wUtil.getMillimetersFromInches(value));
+ }
+ // Leaf wetness [0...10] -> [0...60]
+ else if(obs.getElementMeasurementTypeId().equals("BT"))
+ {
+ obs.setValue(wUtil.getLeafWetnessValueFromUSPest(value));
+ }
+ else
+ {
+ obs.setValue(value);
+ }
+ retVal.add(obs);
+ }
+ } catch (ParseException ex) {
+
+ // Is this the heading line?
+ // Then we parse it to set the ordering of elements
+ if(lineData[0].equals("Date"))
+ {
+ // Date and hour should always be the two first ones
+ for(int i=2;i<lineData.length;i++)
+ {
+ for(int j=0;j<elementMeasurementTypes.length;j++)
+ {
+ if(elementMeasurementTypes[j][0].equals(lineData[i]))
+ {
+ elementOrdering.put(i,j);
+ }
+ }
+ }
+ }
+ }
+ }
+ );
+ }
+ }
+ catch(IOException ex)
+ {
+ ex.printStackTrace();
+ }
+ return retVal;
+ }
+}
diff --git a/src/test/java/no/nibio/vips/util/weather/USPestDataParserTest.java b/src/test/java/no/nibio/vips/util/weather/USPestDataParserTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..a47ff992b2dd8585e14ad560fd888e76cd655a0c
--- /dev/null
+++ b/src/test/java/no/nibio/vips/util/weather/USPestDataParserTest.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2016 NIBIO <http://www.nibio.no/>.
+ *
+ * This file is part of VIPSLogic.
+ * VIPSLogic is free software: you can redistribute it and/or modify
+ * it under the terms of the NIBIO Open Source License as published by
+ * NIBIO, either version 1 of the License, or (at your option) any
+ * later version.
+ *
+ * VIPSLogic is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * NIBIO Open Source License for more details.
+ *
+ * You should have received a copy of the NIBIO Open Source License
+ * along with VIPSLogic. If not, see <http://www.nibio.no/licenses/>.
+ *
+ */
+package no.nibio.vips.util.weather;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
+import no.nibio.vips.entity.WeatherObservation;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author treinar
+ */
+public class USPestDataParserTest {
+
+ public USPestDataParserTest() {
+ }
+
+ @BeforeClass
+ public static void setUpClass() {
+ }
+
+ @AfterClass
+ public static void tearDownClass() {
+ }
+
+ @Before
+ public void setUp() {
+ }
+
+ @After
+ public void tearDown() {
+ }
+
+ /**
+ * Test of getWeatherObservations method, of class USPestDataParser.
+ */
+ @Test
+ public void testGetWeatherObservations() throws Exception {
+ System.out.println("getWeatherObservations");
+ String stationID = "FNWO3";
+ TimeZone timeZone = TimeZone.getTimeZone("America/Los_Angeles");
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
+ f.setTimeZone(timeZone);
+ Date startDate = f.parse("2016-10-01");
+ USPestDataParser instance = new USPestDataParser();
+
+ List<WeatherObservation> result = instance.getWeatherObservations(stationID, timeZone, startDate);
+ //System.out.println("Result.size()=" + result.size());
+ //result.stream().filter(obs->obs.getElementMeasurementTypeId().equals("RR")).forEach(obs->System.out.println(obs.toString()));
+ assertNotNull(result);
+
+ }
+
+}