First non-failing version of the MetnoThreddsDataParser

nbactions.xml

@@ -46,10 +46,6 @@
                 <no.nibio.vips.logic.weather.FIELDCLIMATE_API_CLIENT_SECRET>aa8f4b62b72986bac7c84be78836c2c6</no.nibio.vips.logic.weather.FIELDCLIMATE_API_CLIENT_SECRET>
                 <no.nibio.vips.logic.weather.METNOTHREDDS_TMP_FILE_PATH>/home/treinar/prosjekter/vips/projects/2017_SpotIT/Task 3.2/</no.nibio.vips.logic.weather.METNOTHREDDS_TMP_FILE_PATH>
                 
-                
-                
-                
-                
             </properties>
         </action>
     </actions>

pom.xml

@@ -19,6 +19,11 @@
         <id>jitpack.io</id>
         <url>https://jitpack.io</url>
     </repository>
+    <repository>
+    <id>bedatadriven</id>
+    <name>bedatadriven public repo</name>
+    <url>https://nexus.bedatadriven.com/content/groups/public/</url>
+  </repository>
     <repository>
         <id>unidata-releases</id>
         <name>Unidata Releases</name>
@@ -225,7 +230,21 @@
 	<version>9.4-1211</version>
         <scope>provided</scope>
 </dependency-->
-
+<dependency>
+    <groupId>org.openjdk.jol</groupId>
+    <artifactId>jol-core</artifactId>
+    <version>0.9</version>
+</dependency>
+<dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-api</artifactId>
+            <version>17.2</version>
+        </dependency>
+        <dependency>
+        <groupId>org.geotools</groupId>
+        <artifactId>gt-epsg-hsql</artifactId>
+        <version>17.2</version>
+    </dependency>
   </dependencies>
 
   <build>
@@ -304,6 +323,14 @@
           </webResources>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>2.10</version>
+        <configuration>
+            <argLine>-Xmx6048m</argLine>
+        </configuration>
+    </plugin>
     </plugins>
   </build>
   

src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java

@@ -52,6 +52,7 @@ import no.nibio.vips.util.weather.USPestDataParser;
 import no.nibio.vips.util.weather.YrWeatherForecastProvider;
 import no.nibio.vips.util.weather.dnmipointweb.DMIPointWebDataParser;
 import no.nibio.vips.util.weather.metnothredds.MetNoThreddsDataParser;
+import no.nibio.vips.util.weather.metnothredds.TooMuchDataToAskForException;
 import org.jboss.resteasy.annotations.GZIP;
 
 /**
@@ -383,9 +384,9 @@ public class WeatherProxyService {
             List<PointWeatherObservationList> retVal = dp.getGridData(gf.toGeometry(envelope), 10.0, startDate, endDate, Arrays.asList(elementMeasurementTypes));
             return Response.ok().entity(retVal).build();
         }
-        catch(ParseException pe)
+        catch(ParseException | TooMuchDataToAskForException e)
         {
-            return Response.status(Response.Status.BAD_REQUEST).entity(pe.getMessage()).build();
+            return Response.status(Response.Status.BAD_REQUEST).entity(e.getMessage()).build();
         }
     }
 }

src/main/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParser.java

@@ -41,9 +41,15 @@ import java.util.Optional;
 import java.util.Properties;
 import java.util.Set;
 import java.util.stream.Collectors;
+import javax.measure.Measure;
+import javax.measure.quantity.Area;
+import javax.measure.unit.SI;
+import javax.measure.unit.Unit;
 import no.nibio.vips.entity.PointWeatherObservationList;
 import no.nibio.vips.entity.WeatherObservation;
+import no.nibio.vips.gis.GISUtil;
 import no.nibio.vips.gis.SimpleWGS84Coordinate;
+import no.nibio.vips.util.SystemUtil;
 import no.nibio.vips.util.WeatherElements;
 import no.nibio.vips.util.WeatherUtil;
 import org.apache.commons.io.FileUtils;
@@ -73,6 +79,9 @@ public class MetNoThreddsDataParser {
     private final SimpleDateFormat pathFormat;
     private final SimpleDateFormat fileTimeStampFormat;
     
+    // Approximate number of memory bytes per observation
+    // See the test class for more info
+    private final int AVERAGE_OBS_OBJ_SIZE = 40; 
     
     private final WeatherUtil weatherUtil;
     
@@ -93,8 +102,69 @@ public class MetNoThreddsDataParser {
         this.fileTimeStampFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
     }
     
-    public List<PointWeatherObservationList> getGridData(Geometry geometry, Double resolution, Date dateFrom, Date dateTo, List<String> VIPSWeatherParameters)
+    /**
+     * If you want a lot of data, split it into smaller, manageable parts
+     * @param geometry
+     * @param resolution
+     * @return 
+     */
+    public Geometry[] getManageableGeometries(Geometry geometry, Double resolution, Date dateFrom, Date dateTo, List<String> VIPSWeatherParameters)
+    {
+        Long presumableFreeMemory = SystemUtil.getPresumableFreeMemory();
+        Integer divideBy = 1;
+        while(this.getMemoryNeededForRequest(this.getGeometryFraction(geometry, divideBy, 0, 0), resolution, dateFrom, dateTo, VIPSWeatherParameters) > presumableFreeMemory * 0.3)
+        {
+            divideBy++;
+        }
+        
+        Geometry[] retVal = new Geometry[divideBy * divideBy];
+        for(int x=0;x<divideBy;x++)
+        {
+            for(int y=0;y<divideBy;y++)
+            {
+                retVal[x*divideBy + y] = this.getGeometryFraction(geometry, divideBy, x,y);
+            }
+        }
+        //System.out.println("divideBy=" + divideBy);
+        return retVal;
+    }
+    
+    public Geometry getGeometryFraction(Geometry geometry, Integer divideBy, Integer x, Integer y)
+    {
+        Polygon p = (Polygon) geometry.getEnvelope();
+        Double lonLength = (this.getEast(p) - this.getWest(p)) / divideBy;
+        Double latLength = (this.getNorth(p) - this.getSouth(p)) / divideBy;
+        
+        //Double newNorth = ((this.getNorth(p) - this.getSouth(p))/divideBy) + this.getSouth(p);
+        //Double newEast = (this.getEast(p) - this.getWest(p)/divideBy) + this.getWest(p);
+        return new GISUtil().createRectangleFromBounds(
+                this.getSouth(p) + (latLength * (y +1)), 
+                this.getSouth(p) + (latLength * y), 
+                this.getWest(p) + (lonLength * (x +1)), 
+                this.getWest(p) + (lonLength * x)
+        );
+    }
+    
+    /**
+     * 
+     * @param geometry
+     * @param resolution
+     * @param dateFrom
+     * @param dateTo
+     * @param VIPSWeatherParameters
+     * @return
+     * @throws TooMuchDataToAskForException 
+     */
+    public List<PointWeatherObservationList> getGridData(Geometry geometry, Double resolution, Date dateFrom, Date dateTo, List<String> VIPSWeatherParameters) throws TooMuchDataToAskForException
     {
+        Long memoryNeededForRequest = this.getMemoryNeededForRequest(geometry, resolution, dateFrom, dateTo, VIPSWeatherParameters);
+        Long presumableFreeMemory = SystemUtil.getPresumableFreeMemory();
+        //System.out.println("memoryNeeded=" + memoryNeededForRequest + ", free=" + presumableFreeMemory);
+        if(memoryNeededForRequest > presumableFreeMemory * 0.6)
+        {
+            throw new TooMuchDataToAskForException("The request is estimated to require " + memoryNeededForRequest + " bytes of memory, this is too much. Please ask for less!");
+        }
+        
         Set<String> metParamNames = VIPSWeatherParameters.stream().map(
                 param -> paramInfo.getProperty(param)
         ).collect(Collectors.toSet());
@@ -104,8 +174,8 @@ public class MetNoThreddsDataParser {
         Collections.sort(archiveDates);
         //new GeometryFactory().
         Polygon gridBounds =  (Polygon) geometry.getEnvelope();
-        System.out.println("geometryType=" + gridBounds.getGeometryType());
-        System.out.println("gridBounds=" + gridBounds);
+        //System.out.println("geometryType=" + gridBounds.getGeometryType());
+        //System.out.println("gridBounds=" + gridBounds);
          
         Map<Coordinate, Map<Long, WeatherObservation>> tempRetVal = new HashMap<>();
         archiveDates.stream().forEach(archiveDate -> {
@@ -121,7 +191,7 @@ public class MetNoThreddsDataParser {
                         "&north=" + this.getNorth(gridBounds) + "&west=" + this.getWest(gridBounds) + "&east=" + this.getEast(gridBounds) + "&south=" + this.getSouth(gridBounds) +
                         "&var=" + String.join(",", metParamNames) +
                         "&addLatLon=true");
-                System.out.println("URL: " + threddsURL.toString());
+                //System.out.println("URL: " + threddsURL.toString());
                 Long timestamp = new Date().getTime();
                 tmp = new File(this.TMP_FILE_PATH + timestamp + ".nc");
                 FileUtils.copyURLToFile(threddsURL, tmp);
@@ -156,14 +226,14 @@ public class MetNoThreddsDataParser {
                 for(String metParamName:metParamNames)
                 {
                     String VIPSParamName = this.findVIPSParamNameFromMetParamName(metParamName);
-
+                    //System.out.println("metParamName=" + metParamName);
                     Variable var = ncFile.findVariable(metParamName);
                     // All variables have the dims [time, height, y, x]. All 
                     // variables represent only one height, and can therefore be reduced 
                     // from 4 to three dimensions
                     // Assuming all weather parameters are of type "float"
 
-                    ArrayFloat.D3 varArray = (ArrayFloat.D3) var.read().reduce();
+                    ArrayFloat.D3 varArray = (ArrayFloat.D3) var.read().reduce(1); // 1 is the height parameter
                     // We always skip the two first timestamps, since the model needs 
                     // a couple of hours to spin up
                     for(int i=2;i<timeMeasured.length;i++)
@@ -203,9 +273,9 @@ public class MetNoThreddsDataParser {
                 }
                 
             }
-            catch(IOException ioe)
+            catch(IOException | ClassCastException ex)
             {
-                
+                ex.printStackTrace();
             }
             finally {
                 if(ncFile != null)
@@ -254,7 +324,7 @@ public class MetNoThreddsDataParser {
                         this.fileTimeStampFormat.format(archiveDate) +
                         NETCDF_EXT + STANDARD_PARAMS + "&longitude=" + longitude + "&latitude=" + latitude
                         + "&var=" + String.join(",", metParamNames));
-                System.out.println("URL: " + threddsURL.toString());
+                //System.out.println("URL: " + threddsURL.toString());
                 Long timestamp = new Date().getTime();
                 tmp = new File(this.TMP_FILE_PATH + timestamp + ".nc");
                 FileUtils.copyURLToFile(threddsURL, tmp);
@@ -455,4 +525,37 @@ public class MetNoThreddsDataParser {
     {
         return envelope.getCoordinates()[2].x;
     }
+    
+    /**
+     * Calculating roughly the number of grid points within an envelope
+     * @param envelope
+     * @param resolution km-resolution of the grid points
+     * @return 
+     */
+    public Long getNumberOfGridPointsInEnvelope(Polygon envelope, Double resolution)
+    {
+        GISUtil gisUtil = new GISUtil();
+        
+        Measure<Double,Area> m = gisUtil.calcArea(envelope);
+        //System.out.println("measure m = " + m);
+        Unit<Area> sq_km = (Unit<Area>) SI.KILOMETER.pow(2);
+        //System.out.println("Area=" + m.to(sq_km).getValue());
+        Double side = Math.sqrt(m.to(sq_km).getValue());
+        
+        
+        return Math.round((1 + side / resolution) * (1 + side / resolution));
+    }
+    
+    public Long getNumberOfWeatherObservationsRequested(Geometry geometry, Double resolution, Date dateFrom, Date dateTo, List<String> VIPSWeatherParameters)
+    {
+        Polygon gridBounds =  (Polygon) geometry.getEnvelope();
+        Long gridPoints = this.getNumberOfGridPointsInEnvelope(gridBounds, resolution);
+        Long hoursInPeriod = Math.round((dateTo.getTime() - dateFrom.getTime()) / (3600.0 * 1000));        
+        return gridPoints * hoursInPeriod * VIPSWeatherParameters.size();
+    }
+    
+    public Long getMemoryNeededForRequest(Geometry geometry, Double resolution, Date dateFrom, Date dateTo, List<String> VIPSWeatherParameters)
+    {
+        return this.getNumberOfWeatherObservationsRequested(geometry, resolution, dateFrom, dateTo, VIPSWeatherParameters) * this.AVERAGE_OBS_OBJ_SIZE;
+    }
 }

src/main/java/no/nibio/vips/util/weather/metnothredds/TooMuchDataToAskForException.java

+/*
+ * Copyright (c) 2017 NIBIO <http://www.nibio.no/>. 
+ * 
+ * This file is part of VIPSLogic.
+ * VIPSLogic is free software: you can redistribute it and/or modify
+ * it under the terms of the NIBIO Open Source License as published by 
+ * NIBIO, either version 1 of the License, or (at your option) any
+ * later version.
+ * 
+ * VIPSLogic is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * NIBIO Open Source License for more details.
+ * 
+ * You should have received a copy of the NIBIO Open Source License
+ * along with VIPSLogic.  If not, see <http://www.nibio.no/licenses/>.
+ * 
+ */
+
+package no.nibio.vips.util.weather.metnothredds;
+
+/**
+ * This is simply a message telling that you're asking for too much data,
+ * putting the server at risk of running out of resources, most notably memory.
+ * @copyright 2017 <a href="http://www.nibio.no/">NIBIO</a>
+ * @author Tor-Einar Skog <tor-einar.skog@nibio.no>
+ */
+public class TooMuchDataToAskForException extends Exception {
+    
+    public TooMuchDataToAskForException()
+    {
+        super();
+    }
+    
+    public TooMuchDataToAskForException(String msg)
+    {
+        super(msg);
+    }
+
+}

src/test/java/no/nibio/vips/logic/test/mock/DataMatrix.java

+/*
+ * Copyright (c) 2017 NIBIO <http://www.nibio.no/>. 
+ * 
+ * This file is part of ZymoseptoriaSimpleRiskGridModel.
+ * ZymoseptoriaSimpleRiskGridModel is free software: you can redistribute it and/or modify
+ * it under the terms of the NIBIO Open Source License as published by 
+ * NIBIO, either version 1 of the License, or (at your option) any
+ * later version.
+ * 
+ * ZymoseptoriaSimpleRiskGridModel is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * NIBIO Open Source License for more details.
+ * 
+ * You should have received a copy of the NIBIO Open Source License
+ * along with ZymoseptoriaSimpleRiskGridModel.  If not, see <http://www.nibio.no/licenses/>.
+ * 
+ */
+
+package no.nibio.vips.logic.test.mock;
+
+import no.nibio.vips.util.DateMap;
+
+/**
+ * @copyright 2017 <a href="http://www.nibio.no/">NIBIO</a>
+ * @author Tor-Einar Skog <tor-einar.skog@nibio.no>
+ */
+public class DataMatrix extends DateMap {
+    public final static String UM = "UM";
+    public final static String BT = "BT";
+    public final static String RR = "RR";
+    public final static String TM = "TM";
+    public final static String WET_HOUR_SUM = "WHS";
+
+}

src/test/java/no/nibio/vips/logic/test/mock/MockModel.java

+/*
+ * Copyright (c) 2017 NIBIO <http://www.nibio.no/>. 
+ * 
+ * This file is part of VIPSLogic.
+ * VIPSLogic is free software: you can redistribute it and/or modify
+ * it under the terms of the NIBIO Open Source License as published by 
+ * NIBIO, either version 1 of the License, or (at your option) any
+ * later version.
+ * 
+ * VIPSLogic is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * NIBIO Open Source License for more details.
+ * 
+ * You should have received a copy of the NIBIO Open Source License
+ * along with VIPSLogic.  If not, see <http://www.nibio.no/licenses/>.
+ * 
+ */
+
+package no.nibio.vips.logic.test.mock;
+
+import com.vividsolutions.jts.geom.Coordinate;
+import com.vividsolutions.jts.geom.GeometryFactory;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+import no.nibio.vips.entity.ModelConfiguration;
+import no.nibio.vips.entity.PointWeatherObservationList;
+import no.nibio.vips.entity.Result;
+import no.nibio.vips.entity.ResultImpl;
+import no.nibio.vips.model.ConfigValidationException;
+import no.nibio.vips.model.Model;
+import no.nibio.vips.model.ModelExcecutionException;
+import no.nibio.vips.model.ModelId;
+import no.nibio.vips.util.WeatherElements;
+import no.nibio.vips.util.WeatherUtil;
+
+/**
+ * 
+ * This is made for testing purposes!
+ * @copyright 2017 <a href="http://www.nibio.no/">NIBIO</a>
+ * @author Tor-Einar Skog <tor-einar.skog@nibio.no>
+ */
+public class MockModel implements Model{
+    
+    // The minimum number of hours accepted for a daily aggregate
+    private final Integer MINIMUM_HOURLY_VALUES = 16;
+    
+    private List<PointWeatherObservationList> pointWeatherObservationList;
+    private TimeZone timeZone;
+    
+    public final static ModelId MODEL_ID = new ModelId("MOCKMODEL");
+
+    @Override
+    public List<Result> getResult() throws ModelExcecutionException {
+        if(this.pointWeatherObservationList != null)
+        {
+            return this.getGridResult();
+        }
+        
+        throw new UnsupportedOperationException("For now, only grid data are supported");
+    }
+    
+    /**
+     * A simple wet hour sum calculation for the grid data
+     * @return
+     * @throws ModelExcecutionException 
+     */
+    public List<Result> getGridResult() throws ModelExcecutionException{
+        Map<Date, Map<Coordinate, Integer>> resultMap = new HashMap<>();
+        
+        // Calculate the model for each point
+        WeatherUtil wUtil = new WeatherUtil();
+        GeometryFactory geometryFactory = new GeometryFactory();
+        this.pointWeatherObservationList.stream().forEach(oList->{
+            //System.out.println("oList coords=" + oList.getCoordinate());
+            // We are going to calculate number of wet (risky) hours for a day 
+            // including yesterday and tomorrow (72 hours in total) 
+            // Integer[0] = number of input data observation hours (0->24), Integer[1] number of wet hours
+            Map<Date, Integer[]> wetHours = new HashMap<>();
+            DataMatrix pointMatrix = new DataMatrix();
+            oList.getObservations().stream().forEach(obs -> {
+                    pointMatrix.setParamDoubleValueForDate(obs.getTimeMeasured(), obs.getElementMeasurementTypeId(), obs.getValue());
+                }
+            );
+            pointMatrix.getSortedDateKeys().stream().forEach(timestamp -> {
+                Date dayStamp = wUtil.normalizeToExactDate(timestamp, timeZone);
+                Integer[] wetHoursForDay = wetHours.get(dayStamp);
+                if(wetHoursForDay == null)
+                {
+                    wetHoursForDay = new Integer[2];
+                    wetHoursForDay[0] = 0;
+                    wetHoursForDay[1] = 0;
+                }
+                Double UM = pointMatrix.getParamDoubleValueForDate(timestamp, WeatherElements.RELATIVE_HUMIDITY_MEAN);
+                Double TM = pointMatrix.getParamDoubleValueForDate(timestamp, WeatherElements.TEMPERATURE_MEAN);
+                if(UM != null && TM != null)
+                {
+                    wetHoursForDay[0]++; // We add up the number of hours we actually have weather data for
+                }
+                Double BT = pointMatrix.getParamDoubleValueForDate(timestamp, WeatherElements.LEAF_WETNESS);
+                Double RR = pointMatrix.getParamDoubleValueForDate(timestamp, WeatherElements.PRECIPITATION);
+                if(TM >= 10.0)
+                {
+                    wetHoursForDay[1] += (BT != null && RR != null) ? 
+                                                (UM >= 88.0 && BT >= 30.0 && RR > 0.2) ? 1 : 0
+                                            : UM >= 88.0 ? 1 : 0;
+                }
+                wetHours.put(dayStamp, wetHoursForDay);
+            });
+            
+            //wetHours.keySet().stream().sorted().forEach(dayStamp -> System.out.println(dayStamp + ": " + wetHours.get(dayStamp)[0] + "/" + wetHours.get(dayStamp)[1]));
+            List<Date> dayStamps = new ArrayList(wetHours.keySet());
+            Collections.sort(dayStamps);
+            for(int i=1;i<dayStamps.size()-1; i++)
+            {
+                // TODO: Check that each day has enough hourly values as aggregate base
+                if(
+                    wetHours.get(dayStamps.get(i-1))[0] >= this.MINIMUM_HOURLY_VALUES
+                    && wetHours.get(dayStamps.get(i))[0] >= this.MINIMUM_HOURLY_VALUES
+                    && wetHours.get(dayStamps.get(i+1))[0] >= this.MINIMUM_HOURLY_VALUES
+                )
+                {
+                    // Wohoo, we have a result!
+                    Integer wetHourSumForDayAndPoint = 
+                            wetHours.get(dayStamps.get(i-1))[1]
+                            + wetHours.get(dayStamps.get(i))[1]
+                            + wetHours.get(dayStamps.get(i+1))[1];
+                    Map<Coordinate, Integer> pointAndDateResult = resultMap.get(dayStamps.get(i));
+                    if(pointAndDateResult == null)
+                    {
+                        pointAndDateResult = new HashMap<>();
+                    }
+                    pointAndDateResult.put(oList.getCoordinate(), wetHourSumForDayAndPoint);
+                    resultMap.put(dayStamps.get(i), pointAndDateResult);
+                }
+            }
+        });
+        
+        List<Result> retVal = new ArrayList<>();
+        resultMap.entrySet().stream().forEach(timeEntry -> {
+            timeEntry.getValue().entrySet().stream().forEach(pointEntry -> {
+                Result r = new ResultImpl();
+                r.setValidTimeStart(timeEntry.getKey());
+                Coordinate c = new Coordinate(pointEntry.getKey().x, pointEntry.getKey().y);
+                r.setValidGeometry(geometryFactory.createPoint(c));
+                r.setValue(MockModel.MODEL_ID.toString(), DataMatrix.WET_HOUR_SUM, pointEntry.getValue().toString());
+                retVal.add(r);
+            });
+        });
+        
+        // Iterate through dates
+        return retVal;
+    }
+
+    @Override
+    public ModelId getModelId() {
+        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+    }
+
+    @Override
+    public String getModelName() {
+        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+    }
+
+    @Override
+    public String getModelName(String language) {
+        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+    }
+
+    @Override
+    public String getLicense() {
+        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+    }
+
+    @Override
+    public String getCopyright() {
+        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+    }
+
+    @Override
+    public String getModelDescription() {
+        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+    }
+
+    @Override
+    public String getModelDescription(String language) {
+        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+    }
+
+    @Override
+    public String getWarningStatusInterpretation() {
+        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+    }
+
+    @Override
+    public String getWarningStatusInterpretation(String language) {
+        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+    }
+
+    @Override
+    public String getModelUsage() {
+        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+    }
+
+    @Override
+    public String getModelUsage(String language) {
+        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+    }
+
+    @Override
+    public String getSampleConfig() {
+        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+    }
+
+    @Override
+    public void setConfiguration(ModelConfiguration config) throws ConfigValidationException {
+        pointWeatherObservationList = (List<PointWeatherObservationList>) config.getConfigParameter("multiPointWeatherObservations");
+        this.timeZone = TimeZone.getTimeZone((String)config.getConfigParameter("timeZone"));
+    }
+
+}

src/test/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParserTest.java

@@ -18,22 +18,37 @@
  */
 package no.nibio.vips.util.weather.metnothredds;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.vividsolutions.jts.geom.Coordinate;
+import com.vividsolutions.jts.geom.Geometry;
 import com.vividsolutions.jts.geom.GeometryFactory;
 import com.vividsolutions.jts.geom.Polygon;
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.PrintWriter;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.TimeZone;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import no.nibio.vips.entity.ModelConfiguration;
 import no.nibio.vips.entity.PointWeatherObservationList;
+import no.nibio.vips.entity.Result;
 import no.nibio.vips.entity.WeatherObservation;
+import no.nibio.vips.logic.test.mock.MockModel;
+import no.nibio.vips.model.ConfigValidationException;
+import no.nibio.vips.model.ModelExcecutionException;
+import no.nibio.vips.util.SystemUtil;
 import no.nibio.vips.util.WeatherUtil;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -41,6 +56,10 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import static org.junit.Assert.*;
+import org.openjdk.jol.info.GraphLayout;
+import org.wololo.geojson.Feature;
+import org.wololo.geojson.FeatureCollection;
+import org.wololo.jts2geojson.GeoJSONWriter;
 
 /**
  *
@@ -66,38 +85,297 @@ public class MetNoThreddsDataParserTest {
     @After
     public void tearDown() {
     }
-
-    //@Test
-    public void testGetGridData()
+    
+    @Test
+    public void testGetGridDataInPractice()
     {
-        System.out.println("testGetGridData");
+        System.out.println("testGetGridDataInPractice");
         Coordinate[] coords = new Coordinate[5];
-        coords[0] = new Coordinate(10.31,59.91);
-        coords[1] = new Coordinate(11.07,59.91);
+        /*coords[0] = new Coordinate(8.31,63.91);
+        coords[1] = new Coordinate(11.07,63.91);
         coords[2] = new Coordinate(11.07,59.52);
-        coords[3] = new Coordinate(10.31,59.52);
-        coords[4] = new Coordinate(10.31,59.91);
+        coords[3] = new Coordinate(8.31,59.52);
+        coords[4] = new Coordinate(8.31,63.91);*/
+        coords[0] = new Coordinate(-14.4103,71.8152);
+        coords[1] = new Coordinate(48.7212,71.8152);
+        coords[2] = new Coordinate(48.7212,51.7262);
+        coords[3] = new Coordinate(-14.4103,51.7262);
+        coords[4] = new Coordinate(-14.4103,71.8152);
         GeometryFactory gFac = new GeometryFactory();
         Polygon pol = gFac.createPolygon(coords);
-        
-        List<String> weatherParameters = Arrays.asList("TM","RR", "Q0", "TX","TN","UM");
+
+        List<String> weatherParameters = Arrays.asList("TM","RR","Q0","TX","TN","UM");
         MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
-        
+
         Date start = new Date();
         Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("Europe/Oslo"));
         cal.setTime(new Date());
-        //cal.add(Calendar.DATE, -2);
+        cal.add(Calendar.DATE, -25);
         cal.set(Calendar.HOUR_OF_DAY, 5);
         Date dateFrom = new WeatherUtil().normalizeToExactHour(cal.getTime(), TimeZone.getDefault());
-        cal.add(Calendar.DATE, 5);
+        cal.add(Calendar.DATE, 27);
         Date dateTo = cal.getTime();
+
+        List<PointWeatherObservationList> result = null;
         
+        List<Result> modelResult = new ArrayList<>();
+        try
+        {
+            result = instance.getGridData(pol, 10.0, dateFrom, dateTo, weatherParameters);
+        }
+        catch(TooMuchDataToAskForException ex)
+        {
+            Geometry[] manageableGeometries = instance.getManageableGeometries(pol,10.0, dateFrom, dateTo, weatherParameters);
+            System.out.println("The original polygon has been split into " + manageableGeometries.length + " parts");
+            int counter = 0;
+            for(Geometry manageableGeometry:manageableGeometries)
+            {
+                counter++;
+                try
+                {
+                        result = instance.getGridData(manageableGeometry, 10.0, dateFrom, dateTo, weatherParameters);
+                        System.out.println("Batch #" + counter + " size=" + result.size());
+                        ModelConfiguration config = new ModelConfiguration();
+                        config.setConfigParameter("multiPointWeatherObservations", result);
+                        config.setConfigParameter("timeZone","Europe/Oslo");
+                        MockModel mockModel = new MockModel();
+                        mockModel.setConfiguration(config);
+                        modelResult = mockModel.getResult();
+                        // Simulate that we continue with the next iteration
+                        // In a real setting, we'd need to store the modelResults in some
+                        // permanent storage like files or DB (most likely)
+                        result = null; config = null; mockModel = null;
+                        System.gc();
+                }
+                catch(TooMuchDataToAskForException | ConfigValidationException | ModelExcecutionException ex2)
+                {    
+                    fail(ex2.getMessage());
+                }
+            }
+        }
+        assertNull(result);
+        System.out.println("Antall punkter=" + result.size());
         
-        List<PointWeatherObservationList> result = instance.getGridData(pol, 10.0, dateFrom, dateTo, weatherParameters);
-        assertNotNull(result);
         Long timeSpent = new Date().getTime() - start.getTime();
         System.out.println("Time spent=" + (new Double(timeSpent)/1000) + " seconds");
-        result.stream().forEach(mp->System.out.println(mp));
+        /*result.stream().forEach(mp->{
+            System.out.println(mp);
+            //System.out.println(GraphLayout.parseInstance(mp).toPrintable());
+        });
+        */
+        /*PointWeatherObservationList pwol = result.get(0);
+        WeatherObservation wo = pwol.getObservations().get(0);
+        System.out.println(ClassLayout.parseInstance(wo).toPrintable());
+        System.out.println(GraphLayout.parseInstance(wo).toFootprint());*/
+        System.out.println("Total memory used:\n" + GraphLayout.parseInstance(result).toFootprint());
+    }
+
+    //@Test
+    public void testGetGridData()
+    {
+        try {
+            System.out.println("testGetGridData");
+            Coordinate[] coords = new Coordinate[5];
+            /*coords[0] = new Coordinate(8.31,63.91);
+            coords[1] = new Coordinate(11.07,63.91);
+            coords[2] = new Coordinate(11.07,59.52);
+            coords[3] = new Coordinate(8.31,59.52);
+            coords[4] = new Coordinate(8.31,63.91);*/
+            coords[0] = new Coordinate(-14.4103,71.8152);
+            coords[1] = new Coordinate(48.7212,71.8152);
+            coords[2] = new Coordinate(48.7212,51.7262);
+            coords[3] = new Coordinate(-14.4103,51.7262);
+            coords[4] = new Coordinate(-14.4103,71.8152);
+            GeometryFactory gFac = new GeometryFactory();
+            Polygon pol = gFac.createPolygon(coords);
+            
+            List<String> weatherParameters = Arrays.asList("TM","RR","Q0","TX","TN","UM");
+            MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
+            
+            Date start = new Date();
+            Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("Europe/Oslo"));
+            cal.setTime(new Date());
+            cal.add(Calendar.DATE, -25);
+            cal.set(Calendar.HOUR_OF_DAY, 5);
+            Date dateFrom = new WeatherUtil().normalizeToExactHour(cal.getTime(), TimeZone.getDefault());
+            cal.add(Calendar.DATE, 27);
+            Date dateTo = cal.getTime();
+            
+            
+            List<PointWeatherObservationList> result = instance.getGridData(pol, 10.0, dateFrom, dateTo, weatherParameters);
+            System.out.println("Antall punkter=" + result.size());
+            assertNotNull(result);
+            Long timeSpent = new Date().getTime() - start.getTime();
+            System.out.println("Time spent=" + (new Double(timeSpent)/1000) + " seconds");
+            /*result.stream().forEach(mp->{
+                System.out.println(mp);
+                //System.out.println(GraphLayout.parseInstance(mp).toPrintable());
+            });
+            */
+            /*PointWeatherObservationList pwol = result.get(0);
+            WeatherObservation wo = pwol.getObservations().get(0);
+            System.out.println(ClassLayout.parseInstance(wo).toPrintable());
+            System.out.println(GraphLayout.parseInstance(wo).toFootprint());*/
+            System.out.println("Total memory used:\n" + GraphLayout.parseInstance(result).toFootprint());
+        } catch (TooMuchDataToAskForException ex) {
+            ex.printStackTrace();
+            fail();
+        }
+    }
+    
+    //@Test
+    public void testGetManageableGeometries()
+    {
+            Double resolution = 10.0;
+            System.out.println("testGetManageableGeometries");
+            Coordinate[] coords = new Coordinate[5];
+            /*coords[0] = new Coordinate(8.31,63.91);
+            coords[1] = new Coordinate(11.07,63.91);
+            coords[2] = new Coordinate(11.07,59.52);
+            coords[3] = new Coordinate(8.31,59.52);
+            coords[4] = new Coordinate(8.31,63.91);*/
+            coords[0] = new Coordinate(-14.4103,71.8152);
+            coords[1] = new Coordinate(48.7212,71.8152);
+            coords[2] = new Coordinate(48.7212,51.7262);
+            coords[3] = new Coordinate(-14.4103,51.7262);
+            coords[4] = new Coordinate(-14.4103,71.8152);
+            GeometryFactory gFac = new GeometryFactory();
+            Polygon pol = gFac.createPolygon(coords);
+            
+            
+            List<String> weatherParameters = Arrays.asList("TM","RR", "Q0", "TX","TN","UM");
+            MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
+            
+            Date start = new Date();
+            Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("Europe/Oslo"));
+            cal.setTime(new Date());
+            cal.add(Calendar.DATE, -50);
+            cal.set(Calendar.HOUR_OF_DAY, 5);
+            Date dateFrom = new WeatherUtil().normalizeToExactHour(cal.getTime(), TimeZone.getDefault());
+            cal.add(Calendar.DATE, 52);
+            Date dateTo = cal.getTime();
+            
+            
+            Geometry[] result = instance.getManageableGeometries(pol, resolution, dateFrom, dateTo, weatherParameters);
+            List<Feature> features = new ArrayList<>();
+            GeoJSONWriter writer = new GeoJSONWriter();
+            Map<String, Object> props = new HashMap<>();
+            for(Geometry g:result)
+            {
+                System.out.println(g.toText());
+                org.wololo.geojson.Geometry gj = writer.write(g);
+                features.add(new Feature(gj, props));
+            }
+            FeatureCollection coll = new FeatureCollection(features.toArray(new Feature[0]));
+            try(PrintWriter out = new PrintWriter("/home/treinar/prosjekter/vips/projects/2017_SpotIT/grid_model/subgeomtest.geojson")){
+                out.println(coll.toString());
+            }
+            catch(FileNotFoundException ex)
+            {
+                ex.printStackTrace();
+            }
+            try
+            {
+                ObjectMapper mp = new ObjectMapper();
+                for(Geometry g:result)
+                {
+                    List<PointWeatherObservationList> wData = instance.getGridData(g, 10.0, dateFrom, dateTo, weatherParameters);
+                    System.out.println(mp.writeValueAsString(wData));
+                }
+            }
+            catch(TooMuchDataToAskForException| JsonProcessingException ex)
+            {
+                ex.printStackTrace();
+            }
+            
+            
+    }
+    
+    //@Test
+    public void testGetNumberOfWeatherObservationsRequested(){
+        System.out.println("getNumberOfWeatherObservationsRequested");
+        Coordinate[] coords = new Coordinate[5];
+           coords[0] = new Coordinate(8.31,63.91);
+            coords[1] = new Coordinate(11.07,63.91);
+            coords[2] = new Coordinate(11.07,59.52);
+            coords[3] = new Coordinate(8.31,59.52);
+            coords[4] = new Coordinate(8.31,63.91);
+            GeometryFactory gFac = new GeometryFactory();
+            Polygon pol = gFac.createPolygon(coords);
+            pol = (Polygon) pol.getEnvelope();
+            
+            List<String> weatherParameters = Arrays.asList("TM","RR", "Q0", "TX","TN","UM");
+            MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
+            
+
+            Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("Europe/Oslo"));
+            cal.setTime(new Date());
+            cal.add(Calendar.DATE, -25);
+            cal.set(Calendar.HOUR_OF_DAY, 5);
+            Date dateFrom = new WeatherUtil().normalizeToExactHour(cal.getTime(), TimeZone.getDefault());
+            cal.add(Calendar.DATE, 27);
+            Date dateTo = cal.getTime();
+            
+            
+            Long result = instance.getNumberOfWeatherObservationsRequested(pol, 10.0, dateFrom, dateTo, weatherParameters);
+            System.out.println("Weather observations requested=" + result);
+    }
+    
+    //@Test
+    public void testGetMemoryNeededForRequest(){
+        System.out.println("getMemoryNeededForRequest");
+        Coordinate[] coords = new Coordinate[5];
+           /*coords[0] = new Coordinate(8.31,63.91);
+            coords[1] = new Coordinate(11.07,63.91);
+            coords[2] = new Coordinate(11.07,59.52);
+            coords[3] = new Coordinate(8.31,59.52);
+            coords[4] = new Coordinate(8.31,63.91);*/
+           coords[0] = new Coordinate(-14.4103,71.8152);
+            coords[1] = new Coordinate(48.7212,71.8152);
+            coords[2] = new Coordinate(48.7212,51.7262);
+            coords[3] = new Coordinate(-14.4103,51.7262);
+            coords[4] = new Coordinate(-14.4103,71.8152);
+            GeometryFactory gFac = new GeometryFactory();
+            Polygon pol = gFac.createPolygon(coords);
+            pol = (Polygon) pol.getEnvelope();
+            
+            List<String> weatherParameters = Arrays.asList("TM","RR", "Q0", "TX","TN","UM");
+            MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
+            
+            Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("Europe/Oslo"));
+            cal.setTime(new Date());
+            cal.add(Calendar.DATE, -25);
+            cal.set(Calendar.HOUR_OF_DAY, 5);
+            Date dateFrom = new WeatherUtil().normalizeToExactHour(cal.getTime(), TimeZone.getDefault());
+            cal.add(Calendar.DATE, 27);
+            Date dateTo = cal.getTime();
+            
+            
+            Long result = instance.getMemoryNeededForRequest(pol, 10.0, dateFrom, dateTo, weatherParameters);
+            System.out.println("Memory needed for request=" + result + " bytes");
+            
+            System.out.println("presumableFreeMemory=" + SystemUtil.getPresumableFreeMemory());
+    }
+    
+    //@Test
+    public void testGetNumberOfGridPointsInEnvelope()//(Polygon envelope, Double resolution)
+    {
+        System.out.println("getNumberOfGridPointsInEnvelope");
+        Coordinate[] coords = new Coordinate[5];
+            coords[0] = new Coordinate(8.31,63.91);
+            coords[1] = new Coordinate(11.07,63.91);
+            coords[2] = new Coordinate(11.07,59.52);
+            coords[3] = new Coordinate(8.31,59.52);
+            coords[4] = new Coordinate(8.31,63.91);
+            GeometryFactory gFac = new GeometryFactory();
+            Polygon pol = gFac.createPolygon(coords);
+            pol = (Polygon) pol.getEnvelope();
+            
+            MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
+            
+            Long result = instance.getNumberOfGridPointsInEnvelope(pol, 10.0);
+            
+            System.out.println("Number of grid points in envelope=" + result);
     }
     
     /**