diff --git a/nbactions.xml b/nbactions.xml
index b86e6dae997d8163ce37edc2b227745c9a53a3f8..8b8525cb0c098afb214566e0aec3e35318f3bca7 100755
--- a/nbactions.xml
+++ b/nbactions.xml
@@ -46,10 +46,6 @@
<no.nibio.vips.logic.weather.FIELDCLIMATE_API_CLIENT_SECRET>aa8f4b62b72986bac7c84be78836c2c6</no.nibio.vips.logic.weather.FIELDCLIMATE_API_CLIENT_SECRET>
<no.nibio.vips.logic.weather.METNOTHREDDS_TMP_FILE_PATH>/home/treinar/prosjekter/vips/projects/2017_SpotIT/Task 3.2/</no.nibio.vips.logic.weather.METNOTHREDDS_TMP_FILE_PATH>
-
-
-
-
</properties>
</action>
</actions>
diff --git a/pom.xml b/pom.xml
index 117d5281a0f9407ea034da9937e52d244798ea6f..4cfa5b0bacdfa21ad7eb9a221fe386a7646458d8 100755
--- a/pom.xml
+++ b/pom.xml
@@ -19,6 +19,11 @@
<id>jitpack.io</id>
<url>https://jitpack.io</url>
</repository>
+ <repository>
+ <id>bedatadriven</id>
+ <name>bedatadriven public repo</name>
+ <url>https://nexus.bedatadriven.com/content/groups/public/</url>
+ </repository>
<repository>
<id>unidata-releases</id>
<name>Unidata Releases</name>
@@ -225,7 +230,21 @@
<version>9.4-1211</version>
<scope>provided</scope>
</dependency-->
-
+<dependency>
+ <groupId>org.openjdk.jol</groupId>
+ <artifactId>jol-core</artifactId>
+ <version>0.9</version>
+</dependency>
+<dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-api</artifactId>
+ <version>17.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-epsg-hsql</artifactId>
+ <version>17.2</version>
+ </dependency>
</dependencies>
<build>
@@ -304,6 +323,14 @@
</webResources>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.10</version>
+ <configuration>
+ <argLine>-Xmx6048m</argLine>
+ </configuration>
+ </plugin>
</plugins>
</build>
diff --git a/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java b/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java
index 1728e80fbf6c3bb493f089ccb1f2f5b0ed6a7a65..792acb49157b774b4947e276b6368c948d4bbd1b 100755
--- a/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java
+++ b/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java
@@ -52,6 +52,7 @@ import no.nibio.vips.util.weather.USPestDataParser;
import no.nibio.vips.util.weather.YrWeatherForecastProvider;
import no.nibio.vips.util.weather.dnmipointweb.DMIPointWebDataParser;
import no.nibio.vips.util.weather.metnothredds.MetNoThreddsDataParser;
+import no.nibio.vips.util.weather.metnothredds.TooMuchDataToAskForException;
import org.jboss.resteasy.annotations.GZIP;
/**
@@ -383,9 +384,9 @@ public class WeatherProxyService {
List<PointWeatherObservationList> retVal = dp.getGridData(gf.toGeometry(envelope), 10.0, startDate, endDate, Arrays.asList(elementMeasurementTypes));
return Response.ok().entity(retVal).build();
}
- catch(ParseException pe)
+ catch(ParseException | TooMuchDataToAskForException e)
{
- return Response.status(Response.Status.BAD_REQUEST).entity(pe.getMessage()).build();
+ return Response.status(Response.Status.BAD_REQUEST).entity(e.getMessage()).build();
}
}
}
diff --git a/src/main/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParser.java b/src/main/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParser.java
index 723d84d5b79ec4d4a17a7cab8a299606d9bac260..5688971da1930b031e0c9d4141431d6be5c3b142 100644
--- a/src/main/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParser.java
+++ b/src/main/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParser.java
@@ -41,9 +41,15 @@ import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
+import javax.measure.Measure;
+import javax.measure.quantity.Area;
+import javax.measure.unit.SI;
+import javax.measure.unit.Unit;
import no.nibio.vips.entity.PointWeatherObservationList;
import no.nibio.vips.entity.WeatherObservation;
+import no.nibio.vips.gis.GISUtil;
import no.nibio.vips.gis.SimpleWGS84Coordinate;
+import no.nibio.vips.util.SystemUtil;
import no.nibio.vips.util.WeatherElements;
import no.nibio.vips.util.WeatherUtil;
import org.apache.commons.io.FileUtils;
@@ -73,6 +79,9 @@ public class MetNoThreddsDataParser {
private final SimpleDateFormat pathFormat;
private final SimpleDateFormat fileTimeStampFormat;
+ // Approximate number of memory bytes per observation
+ // See the test class for more info
+ private final int AVERAGE_OBS_OBJ_SIZE = 40;
private final WeatherUtil weatherUtil;
@@ -93,8 +102,69 @@ public class MetNoThreddsDataParser {
this.fileTimeStampFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
}
- public List<PointWeatherObservationList> getGridData(Geometry geometry, Double resolution, Date dateFrom, Date dateTo, List<String> VIPSWeatherParameters)
+ /**
+ * If you want a lot of data, split it into smaller, manageable parts
+ * @param geometry
+ * @param resolution
+ * @return
+ */
+ public Geometry[] getManageableGeometries(Geometry geometry, Double resolution, Date dateFrom, Date dateTo, List<String> VIPSWeatherParameters)
+ {
+ Long presumableFreeMemory = SystemUtil.getPresumableFreeMemory();
+ Integer divideBy = 1;
+ while(this.getMemoryNeededForRequest(this.getGeometryFraction(geometry, divideBy, 0, 0), resolution, dateFrom, dateTo, VIPSWeatherParameters) > presumableFreeMemory * 0.3)
+ {
+ divideBy++;
+ }
+
+ Geometry[] retVal = new Geometry[divideBy * divideBy];
+ for(int x=0;x<divideBy;x++)
+ {
+ for(int y=0;y<divideBy;y++)
+ {
+ retVal[x*divideBy + y] = this.getGeometryFraction(geometry, divideBy, x,y);
+ }
+ }
+ //System.out.println("divideBy=" + divideBy);
+ return retVal;
+ }
+
+ public Geometry getGeometryFraction(Geometry geometry, Integer divideBy, Integer x, Integer y)
+ {
+ Polygon p = (Polygon) geometry.getEnvelope();
+ Double lonLength = (this.getEast(p) - this.getWest(p)) / divideBy;
+ Double latLength = (this.getNorth(p) - this.getSouth(p)) / divideBy;
+
+ //Double newNorth = ((this.getNorth(p) - this.getSouth(p))/divideBy) + this.getSouth(p);
+ //Double newEast = (this.getEast(p) - this.getWest(p)/divideBy) + this.getWest(p);
+ return new GISUtil().createRectangleFromBounds(
+ this.getSouth(p) + (latLength * (y +1)),
+ this.getSouth(p) + (latLength * y),
+ this.getWest(p) + (lonLength * (x +1)),
+ this.getWest(p) + (lonLength * x)
+ );
+ }
+
+ /**
+ *
+ * @param geometry
+ * @param resolution
+ * @param dateFrom
+ * @param dateTo
+ * @param VIPSWeatherParameters
+ * @return
+ * @throws TooMuchDataToAskForException
+ */
+ public List<PointWeatherObservationList> getGridData(Geometry geometry, Double resolution, Date dateFrom, Date dateTo, List<String> VIPSWeatherParameters) throws TooMuchDataToAskForException
{
+ Long memoryNeededForRequest = this.getMemoryNeededForRequest(geometry, resolution, dateFrom, dateTo, VIPSWeatherParameters);
+ Long presumableFreeMemory = SystemUtil.getPresumableFreeMemory();
+ //System.out.println("memoryNeeded=" + memoryNeededForRequest + ", free=" + presumableFreeMemory);
+ if(memoryNeededForRequest > presumableFreeMemory * 0.6)
+ {
+ throw new TooMuchDataToAskForException("The request is estimated to require " + memoryNeededForRequest + " bytes of memory, this is too much. Please ask for less!");
+ }
+
Set<String> metParamNames = VIPSWeatherParameters.stream().map(
param -> paramInfo.getProperty(param)
).collect(Collectors.toSet());
@@ -104,8 +174,8 @@ public class MetNoThreddsDataParser {
Collections.sort(archiveDates);
//new GeometryFactory().
Polygon gridBounds = (Polygon) geometry.getEnvelope();
- System.out.println("geometryType=" + gridBounds.getGeometryType());
- System.out.println("gridBounds=" + gridBounds);
+ //System.out.println("geometryType=" + gridBounds.getGeometryType());
+ //System.out.println("gridBounds=" + gridBounds);
Map<Coordinate, Map<Long, WeatherObservation>> tempRetVal = new HashMap<>();
archiveDates.stream().forEach(archiveDate -> {
@@ -121,7 +191,7 @@ public class MetNoThreddsDataParser {
"&north=" + this.getNorth(gridBounds) + "&west=" + this.getWest(gridBounds) + "&east=" + this.getEast(gridBounds) + "&south=" + this.getSouth(gridBounds) +
"&var=" + String.join(",", metParamNames) +
"&addLatLon=true");
- System.out.println("URL: " + threddsURL.toString());
+ //System.out.println("URL: " + threddsURL.toString());
Long timestamp = new Date().getTime();
tmp = new File(this.TMP_FILE_PATH + timestamp + ".nc");
FileUtils.copyURLToFile(threddsURL, tmp);
@@ -156,14 +226,14 @@ public class MetNoThreddsDataParser {
for(String metParamName:metParamNames)
{
String VIPSParamName = this.findVIPSParamNameFromMetParamName(metParamName);
-
+ //System.out.println("metParamName=" + metParamName);
Variable var = ncFile.findVariable(metParamName);
// All variables have the dims [time, height, y, x]. All
// variables represent only one height, and can therefore be reduced
// from 4 to three dimensions
// Assuming all weather parameters are of type "float"
- ArrayFloat.D3 varArray = (ArrayFloat.D3) var.read().reduce();
+ ArrayFloat.D3 varArray = (ArrayFloat.D3) var.read().reduce(1); // 1 is the height parameter
// We always skip the two first timestamps, since the model needs
// a couple of hours to spin up
for(int i=2;i<timeMeasured.length;i++)
@@ -203,9 +273,9 @@ public class MetNoThreddsDataParser {
}
}
- catch(IOException ioe)
+ catch(IOException | ClassCastException ex)
{
-
+ ex.printStackTrace();
}
finally {
if(ncFile != null)
@@ -254,7 +324,7 @@ public class MetNoThreddsDataParser {
this.fileTimeStampFormat.format(archiveDate) +
NETCDF_EXT + STANDARD_PARAMS + "&longitude=" + longitude + "&latitude=" + latitude
+ "&var=" + String.join(",", metParamNames));
- System.out.println("URL: " + threddsURL.toString());
+ //System.out.println("URL: " + threddsURL.toString());
Long timestamp = new Date().getTime();
tmp = new File(this.TMP_FILE_PATH + timestamp + ".nc");
FileUtils.copyURLToFile(threddsURL, tmp);
@@ -455,4 +525,37 @@ public class MetNoThreddsDataParser {
{
return envelope.getCoordinates()[2].x;
}
+
+ /**
+ * Calculating roughly the number of grid points within an envelope
+ * @param envelope
+ * @param resolution km-resolution of the grid points
+ * @return
+ */
+ public Long getNumberOfGridPointsInEnvelope(Polygon envelope, Double resolution)
+ {
+ GISUtil gisUtil = new GISUtil();
+
+ Measure<Double,Area> m = gisUtil.calcArea(envelope);
+ //System.out.println("measure m = " + m);
+ Unit<Area> sq_km = (Unit<Area>) SI.KILOMETER.pow(2);
+ //System.out.println("Area=" + m.to(sq_km).getValue());
+ Double side = Math.sqrt(m.to(sq_km).getValue());
+
+
+ return Math.round((1 + side / resolution) * (1 + side / resolution));
+ }
+
+ public Long getNumberOfWeatherObservationsRequested(Geometry geometry, Double resolution, Date dateFrom, Date dateTo, List<String> VIPSWeatherParameters)
+ {
+ Polygon gridBounds = (Polygon) geometry.getEnvelope();
+ Long gridPoints = this.getNumberOfGridPointsInEnvelope(gridBounds, resolution);
+ Long hoursInPeriod = Math.round((dateTo.getTime() - dateFrom.getTime()) / (3600.0 * 1000));
+ return gridPoints * hoursInPeriod * VIPSWeatherParameters.size();
+ }
+
+ public Long getMemoryNeededForRequest(Geometry geometry, Double resolution, Date dateFrom, Date dateTo, List<String> VIPSWeatherParameters)
+ {
+ return this.getNumberOfWeatherObservationsRequested(geometry, resolution, dateFrom, dateTo, VIPSWeatherParameters) * this.AVERAGE_OBS_OBJ_SIZE;
+ }
}
diff --git a/src/main/java/no/nibio/vips/util/weather/metnothredds/TooMuchDataToAskForException.java b/src/main/java/no/nibio/vips/util/weather/metnothredds/TooMuchDataToAskForException.java
new file mode 100644
index 0000000000000000000000000000000000000000..3031f9f7a0d79511a11a9d2f287c9cbe3ba01959
--- /dev/null
+++ b/src/main/java/no/nibio/vips/util/weather/metnothredds/TooMuchDataToAskForException.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2017 NIBIO <http://www.nibio.no/>.
+ *
+ * This file is part of VIPSLogic.
+ * VIPSLogic is free software: you can redistribute it and/or modify
+ * it under the terms of the NIBIO Open Source License as published by
+ * NIBIO, either version 1 of the License, or (at your option) any
+ * later version.
+ *
+ * VIPSLogic is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * NIBIO Open Source License for more details.
+ *
+ * You should have received a copy of the NIBIO Open Source License
+ * along with VIPSLogic. If not, see <http://www.nibio.no/licenses/>.
+ *
+ */
+
+package no.nibio.vips.util.weather.metnothredds;
+
+/**
+ * This is simply a message telling that you're asking for too much data,
+ * putting the server at risk of running out of resources, most notably memory.
+ * @copyright 2017 <a href="http://www.nibio.no/">NIBIO</a>
+ * @author Tor-Einar Skog <tor-einar.skog@nibio.no>
+ */
+public class TooMuchDataToAskForException extends Exception {
+
+ public TooMuchDataToAskForException()
+ {
+ super();
+ }
+
+ public TooMuchDataToAskForException(String msg)
+ {
+ super(msg);
+ }
+
+}
diff --git a/src/main/webapp/public/images/station_icon_status_winter.png b/src/main/webapp/public/images/station_icon_status_winter.png
new file mode 100644
index 0000000000000000000000000000000000000000..f2de71d2b1f989688c6f3865dcaba5e8eaedaab2
Binary files /dev/null and b/src/main/webapp/public/images/station_icon_status_winter.png differ
diff --git a/src/main/webapp/public/images/station_icon_status_xmas.png b/src/main/webapp/public/images/station_icon_status_xmas.png
new file mode 100644
index 0000000000000000000000000000000000000000..059c00b378efaca99c5d723f541319c6b1cfc06c
Binary files /dev/null and b/src/main/webapp/public/images/station_icon_status_xmas.png differ
diff --git a/src/test/java/no/nibio/vips/logic/test/mock/DataMatrix.java b/src/test/java/no/nibio/vips/logic/test/mock/DataMatrix.java
new file mode 100644
index 0000000000000000000000000000000000000000..ec6bd70a9fcddce34d3269f1f6c461e696e90ac0
--- /dev/null
+++ b/src/test/java/no/nibio/vips/logic/test/mock/DataMatrix.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2017 NIBIO <http://www.nibio.no/>.
+ *
+ * This file is part of ZymoseptoriaSimpleRiskGridModel.
+ * ZymoseptoriaSimpleRiskGridModel is free software: you can redistribute it and/or modify
+ * it under the terms of the NIBIO Open Source License as published by
+ * NIBIO, either version 1 of the License, or (at your option) any
+ * later version.
+ *
+ * ZymoseptoriaSimpleRiskGridModel is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * NIBIO Open Source License for more details.
+ *
+ * You should have received a copy of the NIBIO Open Source License
+ * along with ZymoseptoriaSimpleRiskGridModel. If not, see <http://www.nibio.no/licenses/>.
+ *
+ */
+
+package no.nibio.vips.logic.test.mock;
+
+import no.nibio.vips.util.DateMap;
+
+/**
+ * @copyright 2017 <a href="http://www.nibio.no/">NIBIO</a>
+ * @author Tor-Einar Skog <tor-einar.skog@nibio.no>
+ */
+public class DataMatrix extends DateMap {
+ public final static String UM = "UM";
+ public final static String BT = "BT";
+ public final static String RR = "RR";
+ public final static String TM = "TM";
+ public final static String WET_HOUR_SUM = "WHS";
+
+}
diff --git a/src/test/java/no/nibio/vips/logic/test/mock/MockModel.java b/src/test/java/no/nibio/vips/logic/test/mock/MockModel.java
new file mode 100644
index 0000000000000000000000000000000000000000..140a7d04549d6ee213905ab9872d32b8c285ae50
--- /dev/null
+++ b/src/test/java/no/nibio/vips/logic/test/mock/MockModel.java
@@ -0,0 +1,226 @@
+/*
+ * Copyright (c) 2017 NIBIO <http://www.nibio.no/>.
+ *
+ * This file is part of VIPSLogic.
+ * VIPSLogic is free software: you can redistribute it and/or modify
+ * it under the terms of the NIBIO Open Source License as published by
+ * NIBIO, either version 1 of the License, or (at your option) any
+ * later version.
+ *
+ * VIPSLogic is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * NIBIO Open Source License for more details.
+ *
+ * You should have received a copy of the NIBIO Open Source License
+ * along with VIPSLogic. If not, see <http://www.nibio.no/licenses/>.
+ *
+ */
+
+package no.nibio.vips.logic.test.mock;
+
+import com.vividsolutions.jts.geom.Coordinate;
+import com.vividsolutions.jts.geom.GeometryFactory;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+import no.nibio.vips.entity.ModelConfiguration;
+import no.nibio.vips.entity.PointWeatherObservationList;
+import no.nibio.vips.entity.Result;
+import no.nibio.vips.entity.ResultImpl;
+import no.nibio.vips.model.ConfigValidationException;
+import no.nibio.vips.model.Model;
+import no.nibio.vips.model.ModelExcecutionException;
+import no.nibio.vips.model.ModelId;
+import no.nibio.vips.util.WeatherElements;
+import no.nibio.vips.util.WeatherUtil;
+
+/**
+ *
+ * This is made for testing purposes!
+ * @copyright 2017 <a href="http://www.nibio.no/">NIBIO</a>
+ * @author Tor-Einar Skog <tor-einar.skog@nibio.no>
+ */
+public class MockModel implements Model{
+
+ // The minimum number of hours accepted for a daily aggregate
+ private final Integer MINIMUM_HOURLY_VALUES = 16;
+
+ private List<PointWeatherObservationList> pointWeatherObservationList;
+ private TimeZone timeZone;
+
+ public final static ModelId MODEL_ID = new ModelId("MOCKMODEL");
+
+ @Override
+ public List<Result> getResult() throws ModelExcecutionException {
+ if(this.pointWeatherObservationList != null)
+ {
+ return this.getGridResult();
+ }
+
+ throw new UnsupportedOperationException("For now, only grid data are supported");
+ }
+
+ /**
+ * A simple wet hour sum calculation for the grid data
+ * @return
+ * @throws ModelExcecutionException
+ */
+ public List<Result> getGridResult() throws ModelExcecutionException{
+ Map<Date, Map<Coordinate, Integer>> resultMap = new HashMap<>();
+
+ // Calculate the model for each point
+ WeatherUtil wUtil = new WeatherUtil();
+ GeometryFactory geometryFactory = new GeometryFactory();
+ this.pointWeatherObservationList.stream().forEach(oList->{
+ //System.out.println("oList coords=" + oList.getCoordinate());
+ // We are going to calculate number of wet (risky) hours for a day
+ // including yesterday and tomorrow (72 hours in total)
+ // Integer[0] = number of input data observation hours (0->24), Integer[1] number of wet hours
+ Map<Date, Integer[]> wetHours = new HashMap<>();
+ DataMatrix pointMatrix = new DataMatrix();
+ oList.getObservations().stream().forEach(obs -> {
+ pointMatrix.setParamDoubleValueForDate(obs.getTimeMeasured(), obs.getElementMeasurementTypeId(), obs.getValue());
+ }
+ );
+ pointMatrix.getSortedDateKeys().stream().forEach(timestamp -> {
+ Date dayStamp = wUtil.normalizeToExactDate(timestamp, timeZone);
+ Integer[] wetHoursForDay = wetHours.get(dayStamp);
+ if(wetHoursForDay == null)
+ {
+ wetHoursForDay = new Integer[2];
+ wetHoursForDay[0] = 0;
+ wetHoursForDay[1] = 0;
+ }
+ Double UM = pointMatrix.getParamDoubleValueForDate(timestamp, WeatherElements.RELATIVE_HUMIDITY_MEAN);
+ Double TM = pointMatrix.getParamDoubleValueForDate(timestamp, WeatherElements.TEMPERATURE_MEAN);
+ if(UM != null && TM != null)
+ {
+ wetHoursForDay[0]++; // We add up the number of hours we actually have weather data for
+ }
+ Double BT = pointMatrix.getParamDoubleValueForDate(timestamp, WeatherElements.LEAF_WETNESS);
+ Double RR = pointMatrix.getParamDoubleValueForDate(timestamp, WeatherElements.PRECIPITATION);
+ if(TM >= 10.0)
+ {
+ wetHoursForDay[1] += (BT != null && RR != null) ?
+ (UM >= 88.0 && BT >= 30.0 && RR > 0.2) ? 1 : 0
+ : UM >= 88.0 ? 1 : 0;
+ }
+ wetHours.put(dayStamp, wetHoursForDay);
+ });
+
+ //wetHours.keySet().stream().sorted().forEach(dayStamp -> System.out.println(dayStamp + ": " + wetHours.get(dayStamp)[0] + "/" + wetHours.get(dayStamp)[1]));
+ List<Date> dayStamps = new ArrayList(wetHours.keySet());
+ Collections.sort(dayStamps);
+ for(int i=1;i<dayStamps.size()-1; i++)
+ {
+ // TODO: Check that each day has enough hourly values as aggregate base
+ if(
+ wetHours.get(dayStamps.get(i-1))[0] >= this.MINIMUM_HOURLY_VALUES
+ && wetHours.get(dayStamps.get(i))[0] >= this.MINIMUM_HOURLY_VALUES
+ && wetHours.get(dayStamps.get(i+1))[0] >= this.MINIMUM_HOURLY_VALUES
+ )
+ {
+ // Wohoo, we have a result!
+ Integer wetHourSumForDayAndPoint =
+ wetHours.get(dayStamps.get(i-1))[1]
+ + wetHours.get(dayStamps.get(i))[1]
+ + wetHours.get(dayStamps.get(i+1))[1];
+ Map<Coordinate, Integer> pointAndDateResult = resultMap.get(dayStamps.get(i));
+ if(pointAndDateResult == null)
+ {
+ pointAndDateResult = new HashMap<>();
+ }
+ pointAndDateResult.put(oList.getCoordinate(), wetHourSumForDayAndPoint);
+ resultMap.put(dayStamps.get(i), pointAndDateResult);
+ }
+ }
+ });
+
+ List<Result> retVal = new ArrayList<>();
+ resultMap.entrySet().stream().forEach(timeEntry -> {
+ timeEntry.getValue().entrySet().stream().forEach(pointEntry -> {
+ Result r = new ResultImpl();
+ r.setValidTimeStart(timeEntry.getKey());
+ Coordinate c = new Coordinate(pointEntry.getKey().x, pointEntry.getKey().y);
+ r.setValidGeometry(geometryFactory.createPoint(c));
+ r.setValue(MockModel.MODEL_ID.toString(), DataMatrix.WET_HOUR_SUM, pointEntry.getValue().toString());
+ retVal.add(r);
+ });
+ });
+
+ // Iterate through dates
+ return retVal;
+ }
+
+ @Override
+ public ModelId getModelId() {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ }
+
+ @Override
+ public String getModelName() {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ }
+
+ @Override
+ public String getModelName(String language) {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ }
+
+ @Override
+ public String getLicense() {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ }
+
+ @Override
+ public String getCopyright() {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ }
+
+ @Override
+ public String getModelDescription() {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ }
+
+ @Override
+ public String getModelDescription(String language) {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ }
+
+ @Override
+ public String getWarningStatusInterpretation() {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ }
+
+ @Override
+ public String getWarningStatusInterpretation(String language) {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ }
+
+ @Override
+ public String getModelUsage() {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ }
+
+ @Override
+ public String getModelUsage(String language) {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ }
+
+ @Override
+ public String getSampleConfig() {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ }
+
+ @Override
+ public void setConfiguration(ModelConfiguration config) throws ConfigValidationException {
+ pointWeatherObservationList = (List<PointWeatherObservationList>) config.getConfigParameter("multiPointWeatherObservations");
+ this.timeZone = TimeZone.getTimeZone((String)config.getConfigParameter("timeZone"));
+ }
+
+}
diff --git a/src/test/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParserTest.java b/src/test/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParserTest.java
index bc433129cb45fd59b5ccfd6ff6a9d9782b7cb3da..f92eb57d2e5d9b37008c28065a640b08ea480d1e 100644
--- a/src/test/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParserTest.java
+++ b/src/test/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParserTest.java
@@ -18,22 +18,37 @@
*/
package no.nibio.vips.util.weather.metnothredds;
+import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.vividsolutions.jts.geom.Coordinate;
+import com.vividsolutions.jts.geom.Geometry;
import com.vividsolutions.jts.geom.GeometryFactory;
import com.vividsolutions.jts.geom.Polygon;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.PrintWriter;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.TimeZone;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import no.nibio.vips.entity.ModelConfiguration;
import no.nibio.vips.entity.PointWeatherObservationList;
+import no.nibio.vips.entity.Result;
import no.nibio.vips.entity.WeatherObservation;
+import no.nibio.vips.logic.test.mock.MockModel;
+import no.nibio.vips.model.ConfigValidationException;
+import no.nibio.vips.model.ModelExcecutionException;
+import no.nibio.vips.util.SystemUtil;
import no.nibio.vips.util.WeatherUtil;
import org.junit.After;
import org.junit.AfterClass;
@@ -41,6 +56,10 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.Assert.*;
+import org.openjdk.jol.info.GraphLayout;
+import org.wololo.geojson.Feature;
+import org.wololo.geojson.FeatureCollection;
+import org.wololo.jts2geojson.GeoJSONWriter;
/**
*
@@ -66,38 +85,297 @@ public class MetNoThreddsDataParserTest {
@After
public void tearDown() {
}
-
- //@Test
- public void testGetGridData()
+
+ @Test
+ public void testGetGridDataInPractice()
{
- System.out.println("testGetGridData");
+ System.out.println("testGetGridDataInPractice");
Coordinate[] coords = new Coordinate[5];
- coords[0] = new Coordinate(10.31,59.91);
- coords[1] = new Coordinate(11.07,59.91);
+ /*coords[0] = new Coordinate(8.31,63.91);
+ coords[1] = new Coordinate(11.07,63.91);
coords[2] = new Coordinate(11.07,59.52);
- coords[3] = new Coordinate(10.31,59.52);
- coords[4] = new Coordinate(10.31,59.91);
+ coords[3] = new Coordinate(8.31,59.52);
+ coords[4] = new Coordinate(8.31,63.91);*/
+ coords[0] = new Coordinate(-14.4103,71.8152);
+ coords[1] = new Coordinate(48.7212,71.8152);
+ coords[2] = new Coordinate(48.7212,51.7262);
+ coords[3] = new Coordinate(-14.4103,51.7262);
+ coords[4] = new Coordinate(-14.4103,71.8152);
GeometryFactory gFac = new GeometryFactory();
Polygon pol = gFac.createPolygon(coords);
-
- List<String> weatherParameters = Arrays.asList("TM","RR", "Q0", "TX","TN","UM");
+
+ List<String> weatherParameters = Arrays.asList("TM","RR","Q0","TX","TN","UM");
MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
-
+
Date start = new Date();
Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("Europe/Oslo"));
cal.setTime(new Date());
- //cal.add(Calendar.DATE, -2);
+ cal.add(Calendar.DATE, -25);
cal.set(Calendar.HOUR_OF_DAY, 5);
Date dateFrom = new WeatherUtil().normalizeToExactHour(cal.getTime(), TimeZone.getDefault());
- cal.add(Calendar.DATE, 5);
+ cal.add(Calendar.DATE, 27);
Date dateTo = cal.getTime();
+
+ List<PointWeatherObservationList> result = null;
+ List<Result> modelResult = new ArrayList<>();
+ try
+ {
+ result = instance.getGridData(pol, 10.0, dateFrom, dateTo, weatherParameters);
+ }
+ catch(TooMuchDataToAskForException ex)
+ {
+ Geometry[] manageableGeometries = instance.getManageableGeometries(pol,10.0, dateFrom, dateTo, weatherParameters);
+ System.out.println("The original polygon has been split into " + manageableGeometries.length + " parts");
+ int counter = 0;
+ for(Geometry manageableGeometry:manageableGeometries)
+ {
+ counter++;
+ try
+ {
+ result = instance.getGridData(manageableGeometry, 10.0, dateFrom, dateTo, weatherParameters);
+ System.out.println("Batch #" + counter + " size=" + result.size());
+ ModelConfiguration config = new ModelConfiguration();
+ config.setConfigParameter("multiPointWeatherObservations", result);
+ config.setConfigParameter("timeZone","Europe/Oslo");
+ MockModel mockModel = new MockModel();
+ mockModel.setConfiguration(config);
+ modelResult = mockModel.getResult();
+ // Simulate that we continue with the next iteration
+ // In a real setting, we'd need to store the modelResults in some
+ // permanent storage like files or DB (most likely)
+ result = null; config = null; mockModel = null;
+ System.gc();
+ }
+ catch(TooMuchDataToAskForException | ConfigValidationException | ModelExcecutionException ex2)
+ {
+ fail(ex2.getMessage());
+ }
+ }
+ }
+ assertNull(result);
+ System.out.println("Antall punkter=" + result.size());
- List<PointWeatherObservationList> result = instance.getGridData(pol, 10.0, dateFrom, dateTo, weatherParameters);
- assertNotNull(result);
Long timeSpent = new Date().getTime() - start.getTime();
System.out.println("Time spent=" + (new Double(timeSpent)/1000) + " seconds");
- result.stream().forEach(mp->System.out.println(mp));
+ /*result.stream().forEach(mp->{
+ System.out.println(mp);
+ //System.out.println(GraphLayout.parseInstance(mp).toPrintable());
+ });
+ */
+ /*PointWeatherObservationList pwol = result.get(0);
+ WeatherObservation wo = pwol.getObservations().get(0);
+ System.out.println(ClassLayout.parseInstance(wo).toPrintable());
+ System.out.println(GraphLayout.parseInstance(wo).toFootprint());*/
+ System.out.println("Total memory used:\n" + GraphLayout.parseInstance(result).toFootprint());
+ }
+
+ //@Test
+ public void testGetGridData()
+ {
+ try {
+ System.out.println("testGetGridData");
+ Coordinate[] coords = new Coordinate[5];
+ /*coords[0] = new Coordinate(8.31,63.91);
+ coords[1] = new Coordinate(11.07,63.91);
+ coords[2] = new Coordinate(11.07,59.52);
+ coords[3] = new Coordinate(8.31,59.52);
+ coords[4] = new Coordinate(8.31,63.91);*/
+ coords[0] = new Coordinate(-14.4103,71.8152);
+ coords[1] = new Coordinate(48.7212,71.8152);
+ coords[2] = new Coordinate(48.7212,51.7262);
+ coords[3] = new Coordinate(-14.4103,51.7262);
+ coords[4] = new Coordinate(-14.4103,71.8152);
+ GeometryFactory gFac = new GeometryFactory();
+ Polygon pol = gFac.createPolygon(coords);
+
+ List<String> weatherParameters = Arrays.asList("TM","RR","Q0","TX","TN","UM");
+ MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
+
+ Date start = new Date();
+ Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("Europe/Oslo"));
+ cal.setTime(new Date());
+ cal.add(Calendar.DATE, -25);
+ cal.set(Calendar.HOUR_OF_DAY, 5);
+ Date dateFrom = new WeatherUtil().normalizeToExactHour(cal.getTime(), TimeZone.getDefault());
+ cal.add(Calendar.DATE, 27);
+ Date dateTo = cal.getTime();
+
+
+ List<PointWeatherObservationList> result = instance.getGridData(pol, 10.0, dateFrom, dateTo, weatherParameters);
+ System.out.println("Antall punkter=" + result.size());
+ assertNotNull(result);
+ Long timeSpent = new Date().getTime() - start.getTime();
+ System.out.println("Time spent=" + (new Double(timeSpent)/1000) + " seconds");
+ /*result.stream().forEach(mp->{
+ System.out.println(mp);
+ //System.out.println(GraphLayout.parseInstance(mp).toPrintable());
+ });
+ */
+ /*PointWeatherObservationList pwol = result.get(0);
+ WeatherObservation wo = pwol.getObservations().get(0);
+ System.out.println(ClassLayout.parseInstance(wo).toPrintable());
+ System.out.println(GraphLayout.parseInstance(wo).toFootprint());*/
+ System.out.println("Total memory used:\n" + GraphLayout.parseInstance(result).toFootprint());
+ } catch (TooMuchDataToAskForException ex) {
+ ex.printStackTrace();
+ fail();
+ }
+ }
+
+ //@Test
+ public void testGetManageableGeometries()
+ {
+ Double resolution = 10.0;
+ System.out.println("testGetManageableGeometries");
+ Coordinate[] coords = new Coordinate[5];
+ /*coords[0] = new Coordinate(8.31,63.91);
+ coords[1] = new Coordinate(11.07,63.91);
+ coords[2] = new Coordinate(11.07,59.52);
+ coords[3] = new Coordinate(8.31,59.52);
+ coords[4] = new Coordinate(8.31,63.91);*/
+ coords[0] = new Coordinate(-14.4103,71.8152);
+ coords[1] = new Coordinate(48.7212,71.8152);
+ coords[2] = new Coordinate(48.7212,51.7262);
+ coords[3] = new Coordinate(-14.4103,51.7262);
+ coords[4] = new Coordinate(-14.4103,71.8152);
+ GeometryFactory gFac = new GeometryFactory();
+ Polygon pol = gFac.createPolygon(coords);
+
+
+ List<String> weatherParameters = Arrays.asList("TM","RR", "Q0", "TX","TN","UM");
+ MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
+
+ Date start = new Date();
+ Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("Europe/Oslo"));
+ cal.setTime(new Date());
+ cal.add(Calendar.DATE, -50);
+ cal.set(Calendar.HOUR_OF_DAY, 5);
+ Date dateFrom = new WeatherUtil().normalizeToExactHour(cal.getTime(), TimeZone.getDefault());
+ cal.add(Calendar.DATE, 52);
+ Date dateTo = cal.getTime();
+
+
+ Geometry[] result = instance.getManageableGeometries(pol, resolution, dateFrom, dateTo, weatherParameters);
+ List<Feature> features = new ArrayList<>();
+ GeoJSONWriter writer = new GeoJSONWriter();
+ Map<String, Object> props = new HashMap<>();
+ for(Geometry g:result)
+ {
+ System.out.println(g.toText());
+ org.wololo.geojson.Geometry gj = writer.write(g);
+ features.add(new Feature(gj, props));
+ }
+ FeatureCollection coll = new FeatureCollection(features.toArray(new Feature[0]));
+ try(PrintWriter out = new PrintWriter("/home/treinar/prosjekter/vips/projects/2017_SpotIT/grid_model/subgeomtest.geojson")){
+ out.println(coll.toString());
+ }
+ catch(FileNotFoundException ex)
+ {
+ ex.printStackTrace();
+ }
+ try
+ {
+ ObjectMapper mp = new ObjectMapper();
+ for(Geometry g:result)
+ {
+ List<PointWeatherObservationList> wData = instance.getGridData(g, 10.0, dateFrom, dateTo, weatherParameters);
+ System.out.println(mp.writeValueAsString(wData));
+ }
+ }
+ catch(TooMuchDataToAskForException| JsonProcessingException ex)
+ {
+ ex.printStackTrace();
+ }
+
+
+ }
+
+ //@Test
+ public void testGetNumberOfWeatherObservationsRequested(){
+ System.out.println("getNumberOfWeatherObservationsRequested");
+ Coordinate[] coords = new Coordinate[5];
+ coords[0] = new Coordinate(8.31,63.91);
+ coords[1] = new Coordinate(11.07,63.91);
+ coords[2] = new Coordinate(11.07,59.52);
+ coords[3] = new Coordinate(8.31,59.52);
+ coords[4] = new Coordinate(8.31,63.91);
+ GeometryFactory gFac = new GeometryFactory();
+ Polygon pol = gFac.createPolygon(coords);
+ pol = (Polygon) pol.getEnvelope();
+
+ List<String> weatherParameters = Arrays.asList("TM","RR", "Q0", "TX","TN","UM");
+ MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
+
+
+ Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("Europe/Oslo"));
+ cal.setTime(new Date());
+ cal.add(Calendar.DATE, -25);
+ cal.set(Calendar.HOUR_OF_DAY, 5);
+ Date dateFrom = new WeatherUtil().normalizeToExactHour(cal.getTime(), TimeZone.getDefault());
+ cal.add(Calendar.DATE, 27);
+ Date dateTo = cal.getTime();
+
+
+ Long result = instance.getNumberOfWeatherObservationsRequested(pol, 10.0, dateFrom, dateTo, weatherParameters);
+ System.out.println("Weather observations requested=" + result);
+ }
+
+ //@Test
+ public void testGetMemoryNeededForRequest(){
+ System.out.println("getMemoryNeededForRequest");
+ Coordinate[] coords = new Coordinate[5];
+ /*coords[0] = new Coordinate(8.31,63.91);
+ coords[1] = new Coordinate(11.07,63.91);
+ coords[2] = new Coordinate(11.07,59.52);
+ coords[3] = new Coordinate(8.31,59.52);
+ coords[4] = new Coordinate(8.31,63.91);*/
+ coords[0] = new Coordinate(-14.4103,71.8152);
+ coords[1] = new Coordinate(48.7212,71.8152);
+ coords[2] = new Coordinate(48.7212,51.7262);
+ coords[3] = new Coordinate(-14.4103,51.7262);
+ coords[4] = new Coordinate(-14.4103,71.8152);
+ GeometryFactory gFac = new GeometryFactory();
+ Polygon pol = gFac.createPolygon(coords);
+ pol = (Polygon) pol.getEnvelope();
+
+ List<String> weatherParameters = Arrays.asList("TM","RR", "Q0", "TX","TN","UM");
+ MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
+
+ Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("Europe/Oslo"));
+ cal.setTime(new Date());
+ cal.add(Calendar.DATE, -25);
+ cal.set(Calendar.HOUR_OF_DAY, 5);
+ Date dateFrom = new WeatherUtil().normalizeToExactHour(cal.getTime(), TimeZone.getDefault());
+ cal.add(Calendar.DATE, 27);
+ Date dateTo = cal.getTime();
+
+
+ Long result = instance.getMemoryNeededForRequest(pol, 10.0, dateFrom, dateTo, weatherParameters);
+ System.out.println("Memory needed for request=" + result + " bytes");
+
+ System.out.println("presumableFreeMemory=" + SystemUtil.getPresumableFreeMemory());
+ }
+
+ //@Test
+ public void testGetNumberOfGridPointsInEnvelope()//(Polygon envelope, Double resolution)
+ {
+ System.out.println("getNumberOfGridPointsInEnvelope");
+ Coordinate[] coords = new Coordinate[5];
+ coords[0] = new Coordinate(8.31,63.91);
+ coords[1] = new Coordinate(11.07,63.91);
+ coords[2] = new Coordinate(11.07,59.52);
+ coords[3] = new Coordinate(8.31,59.52);
+ coords[4] = new Coordinate(8.31,63.91);
+ GeometryFactory gFac = new GeometryFactory();
+ Polygon pol = gFac.createPolygon(coords);
+ pol = (Polygon) pol.getEnvelope();
+
+ MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
+
+ Long result = instance.getNumberOfGridPointsInEnvelope(pol, 10.0);
+
+ System.out.println("Number of grid points in envelope=" + result);
}
/**