From 561183d2755d121c11d9413f42a9d1a6c481ff68 Mon Sep 17 00:00:00 2001
From: Tor-Einar Skog <tor-einar.skog@bioforsk.no>
Date: Thu, 3 Nov 2016 17:11:20 -0700
Subject: [PATCH] Added tools to help download and convert data from USPest.org
---
pom.xml | 16 ++-
.../java/no/nibio/vips/util/WeatherUtil.java | 32 +++++
src/main/java/no/nibio/vips/util/WebUtil.java | 63 +++++++++
.../java/no/nibio/vips/util/WebUtilTest.java | 132 ++++++++++++++++++
4 files changed, 241 insertions(+), 2 deletions(-)
create mode 100644 src/test/java/no/nibio/vips/util/WebUtilTest.java
diff --git a/pom.xml b/pom.xml
index ae28fa4..7927f07 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,11 +46,23 @@
<artifactId>jackson-databind</artifactId>
<version>2.4.1</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5.2</version>
+ <type>jar</type>
+ </dependency>
+ <dependency>
+ <groupId>javax</groupId>
+ <artifactId>javaee-api</artifactId>
+ <version>7.0</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>javax</groupId>
<artifactId>javaee-web-api</artifactId>
- <version>6.0</version>
- <type>jar</type>
+ <version>7.0</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
diff --git a/src/main/java/no/nibio/vips/util/WeatherUtil.java b/src/main/java/no/nibio/vips/util/WeatherUtil.java
index 3253b19..b868ddd 100644
--- a/src/main/java/no/nibio/vips/util/WeatherUtil.java
+++ b/src/main/java/no/nibio/vips/util/WeatherUtil.java
@@ -693,6 +693,18 @@ public class WeatherUtil {
return LE;
}
+
+ /**
+ * USpest.org's leaf wetness values range from 0 to 10. Based on explanation from
+ * Len Coop, associate director of IPPC and documentation found here: http://uspest.org/wea/weanew.html#LWDOC
+ * ... we simplify this to denote all USPest.org values > 2 to be considered wet hours
+ * @param value
+ * @return
+ */
+ public Double getLeafWetnessValueFromUSPest(Double value)
+ {
+ return value >= 2.0 ? 60.0 : 0.0;
+ }
/**
* Fixes by
@@ -1541,4 +1553,24 @@ public class WeatherUtil {
{
return (celciusTemp * 1.8) + 32;
}
+
+ /**
+ * Convert from inches to mm
+ * @param inches
+ * @return
+ */
+ public Double getMillimetersFromInches(Double inches)
+ {
+ return inches * 25.4;
+ }
+
+ /**
+ * Convert from mm to inches
+ * @param millimeters
+ * @return
+ */
+ public Double getInchesFromMillimeters(Double millimeters)
+ {
+ return millimeters / 25.4;
+ }
}
diff --git a/src/main/java/no/nibio/vips/util/WebUtil.java b/src/main/java/no/nibio/vips/util/WebUtil.java
index fba3fa8..06c4805 100644
--- a/src/main/java/no/nibio/vips/util/WebUtil.java
+++ b/src/main/java/no/nibio/vips/util/WebUtil.java
@@ -19,9 +19,25 @@
package no.nibio.vips.util;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import static java.util.stream.Collectors.toList;
import javax.mail.internet.AddressException;
import javax.mail.internet.InternetAddress;
import org.apache.commons.validator.UrlValidator;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicNameValuePair;
/**
* Static convenience methods for common web-related tasks.
@@ -108,4 +124,51 @@ public final class WebUtil {
//return text.replaceAll("(.*://[^<>[:space:]]+[[:alnum:]/])", "<a href=\"$1\">$1</a>");
return text.replaceAll("(\\A|\\s)((http|https|ftp|mailto):\\S+)(\\s|\\z)", "$1<a href=\"$2\" target=\"" + target + "\">$2</a>$4");
}
+
+ /**
+ *
+ * @param URL
+ * @param queryParams
+ * @param arrayQueryParams
+ * @return
+ * @throws IOException
+ */
+ public static String getOutputFromPostRequest(String URL, Map<String,String> queryParams, Map<String, String[]> arrayQueryParams) throws IOException
+ {
+ CloseableHttpClient client = HttpClients.createDefault();
+ String output = "";
+ try
+ {
+ HttpPost post = new HttpPost(URL);
+ post.setHeader("User-Agent", "VIPS");
+ // Getting the
+ List<BasicNameValuePair> params = queryParams.keySet().stream().map(
+ key -> new BasicNameValuePair(key, queryParams.get(key))
+ )
+ .collect(toList());
+
+ arrayQueryParams.keySet().stream().forEach((key) -> {
+ Arrays.asList(arrayQueryParams.get(key)).stream().forEach(value -> params.add(new BasicNameValuePair(key, value)));
+
+ });
+
+ post.setEntity(new UrlEncodedFormEntity(params));
+
+
+ HttpResponse response = client.execute(post);
+ BufferedReader rd = new BufferedReader(
+ new InputStreamReader(response.getEntity().getContent())
+ );
+ String line = "";
+ while ((line = rd.readLine()) != null) {
+ output += line + "\n";
+ }
+ }
+ catch (UnsupportedEncodingException ex) {
+ Logger.getLogger(WebUtil.class.getName()).log(Level.SEVERE, null, ex);
+ } finally{
+ client.close();
+ }
+ return output;
+ }
}
diff --git a/src/test/java/no/nibio/vips/util/WebUtilTest.java b/src/test/java/no/nibio/vips/util/WebUtilTest.java
new file mode 100644
index 0000000..d696966
--- /dev/null
+++ b/src/test/java/no/nibio/vips/util/WebUtilTest.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2016 NIBIO <http://www.nibio.no/>.
+ *
+ * This file is part of VIPSCommon.
+ * VIPSCommon is free software: you can redistribute it and/or modify
+ * it under the terms of the NIBIO Open Source License as published by
+ * NIBIO, either version 1 of the License, or (at your option) any
+ * later version.
+ *
+ * VIPSCommon is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * NIBIO Open Source License for more details.
+ *
+ * You should have received a copy of the NIBIO Open Source License
+ * along with VIPSCommon. If not, see <http://www.nibio.no/licenses/>.
+ *
+ */
+package no.nibio.vips.util;
+
+import java.util.HashMap;
+import java.util.Map;
+import junit.framework.TestCase;
+
+/**
+ *
+ * @author treinar
+ */
+public class WebUtilTest extends TestCase {
+
+ public WebUtilTest(String testName) {
+ super(testName);
+ }
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ /**
+ * Test of isValidEmailAddress method, of class WebUtil.
+ */
+ public void IsValidEmailAddress() {
+ System.out.println("isValidEmailAddress");
+ String aEmailAddress = "";
+ boolean expResult = false;
+ boolean result = WebUtil.isValidEmailAddress(aEmailAddress);
+ assertEquals(expResult, result);
+ // TODO review the generated test code and remove the default call to fail.
+ fail("The test case is a prototype.");
+ }
+
+ /**
+ * Test of isValidWebAddress method, of class WebUtil.
+ */
+ public void IsValidWebAddress() {
+ System.out.println("isValidWebAddress");
+ String aWebAddress = "";
+ boolean expResult = false;
+ boolean result = WebUtil.isValidWebAddress(aWebAddress);
+ assertEquals(expResult, result);
+ // TODO review the generated test code and remove the default call to fail.
+ fail("The test case is a prototype.");
+ }
+
+ /**
+ * Test of replaceTextURLWithLink method, of class WebUtil.
+ */
+ public void ReplaceTextURLWithLink() {
+ System.out.println("replaceTextURLWithLink");
+ String text = "";
+ String target = "";
+ String expResult = "";
+ String result = WebUtil.replaceTextURLWithLink(text, target);
+ assertEquals(expResult, result);
+ // TODO review the generated test code and remove the default call to fail.
+ fail("The test case is a prototype.");
+ }
+
+ /**
+ * Test of getOutputFromPostRequest method, of class WebUtil.
+ */
+ public void testGetOutputFromPostRequest() throws Exception {
+ System.out.println("getOutputFromPostRequest");
+ String URL = "http://uspest.org/risk/models";
+ Map<String, String> queryParams = new HashMap<>();
+ queryParams.put("station", "FNWO3");
+ queryParams.put("stm","10"); // Start month
+ queryParams.put("std","1"); // Start dayOfMonth
+ queryParams.put("sty","2016"); // Start year
+ queryParams.put("span","60"); // Number of days from start
+ queryParams.put("fsm","11"); // Forecast start month
+ queryParams.put("fsd","1"); // Forecast start dayOfMonth
+ // fcast, Type of forecast
+ // 1 = after 7day use 10 year averages (default)
+ // 2 = after 7day use 30 year averages
+ // 3 = after 7day use data from 1 year ago (if available)
+ // 4 = after 7day use data from 2 years ago (if available)
+ // 5 = after 7day use NMME extended seasonal forecast
+ // 6 = after 7day use the CFSv2 extended seasonal forecast
+ queryParams.put("fcast","1");
+ queryParams.put("en_v2_tdp_missing", "on"); // Replace missing Temperature and Dew Point
+ queryParams.put("en_v2_tdp_flatline", "on"); // Replace flat line Temperature and Dew Point
+ queryParams.put("en_v2_tdp_range", "on"); // Replace range exceed Temperature and Dew Point
+ queryParams.put("en_v2_tdp_extrap", "on"); // Weather data fallback config ASK LEN
+ queryParams.put("en_v2_tdp_interp", "on"); // Weather data fallback config ASK LEN
+ queryParams.put("en_v2_tdp_elevreg", "on"); // Weather data fallback config ASK LEN
+ queryParams.put("en_v2_windspeed_missing", "on"); // Replace missing wind speed
+ queryParams.put("en_v2_rain_missing", "on"); // Replance missing precipitation
+ queryParams.put("download_data","Download Data");
+ queryParams.put("last_date_of_refresh","20161101"); // ??
+ queryParams.put("stdt","20161001"); // Probably start date, but why the double info (see stm/std)
+ queryParams.put("date","20161101"); // Should be "today"
+ queryParams.put("tbl","0"); // Does this mean don't display table, ie irrelevant? ASK LEN
+ queryParams.put("Y", "464"); // ASK LEN
+
+ String expResult = "";
+ String[] weatherParams = {"temp", "dewpt","rel_hum","windspeed","rain","leafwet"};
+ Map<String, String[]> arrP = new HashMap<>();
+ arrP.put("wp", weatherParams);
+ String result = WebUtil.getOutputFromPostRequest(URL, queryParams, arrP);
+ System.out.println(result);
+ assertNotNull(expResult, result);
+
+ }
+
+}
--
GitLab