Adaptations to Danish humidity models

nb-configuration.xml

@@ -26,5 +26,6 @@ Any value defined here will override the pom.xml file value but is only applicab
         <org-netbeans-modules-maven-j2ee.netbeans_2e_hint_2e_deploy_2e_server>WildFly</org-netbeans-modules-maven-j2ee.netbeans_2e_hint_2e_deploy_2e_server>
         <org-netbeans-modules-projectapi.jsf_2e_language>Facelets</org-netbeans-modules-projectapi.jsf_2e_language>
         <org-netbeans-modules-maven-jaxws._5f_C_5f_DMIWeatherService_2e_svc>https://dmiweatherservice-plant.dlbr.dk/DMIWeatherService.svc?wsdl</org-netbeans-modules-maven-jaxws._5f_C_5f_DMIWeatherService_2e_svc>
+        <netbeans.compile.on.save>none</netbeans.compile.on.save>
     </properties>
 </project-shared-configuration>

src/main/java/no/nibio/vips/logic/controller/servlet/PointOfInterestController.java

@@ -311,26 +311,28 @@ public class PointOfInterestController extends HttpServlet {
                             weatherStation = SessionControllerGetter.getPointOfInterestBean().storeWeatherStation(weatherStation);
                             
                             Map<String, FormField> externalResourceIdentifiers = formValidation.getMultipleMapFormFields().get("externalResourceIdentifier");
-                            for(String key:externalResourceIdentifiers.keySet())
+                            if(externalResourceIdentifiers != null)
                             {
-                                Integer externalResourceId = Integer.valueOf(key);
-                                FormField identifierField = externalResourceIdentifiers.get(key);
-                                if(identifierField.getWebValue() == null || identifierField.getWebValue().isEmpty())
+                                for(String key:externalResourceIdentifiers.keySet())
                                 {
-                                    // We delete existing if unset
-                                    SessionControllerGetter.getPointOfInterestBean().deletePointOfInterestExternalResource(weatherStation.getPointOfInterestId(), externalResourceId);
-                                }
-                                else
-                                {
-                                    
-                                    PointOfInterestExternalResource poiExternalResource = new PointOfInterestExternalResource();
-                                    PointOfInterestExternalResourcePK pk = new PointOfInterestExternalResourcePK(weatherStation.getPointOfInterestId(), externalResourceId);
-                                    poiExternalResource.setPointOfInterestExternalResourcePK(pk);
-                                    poiExternalResource.setResourceIdentifier(identifierField.getWebValue());
-                                    SessionControllerGetter.getPointOfInterestBean().storePointOfInterestExternalResource(poiExternalResource);
+                                    Integer externalResourceId = Integer.valueOf(key);
+                                    FormField identifierField = externalResourceIdentifiers.get(key);
+                                    if(identifierField.getWebValue() == null || identifierField.getWebValue().isEmpty())
+                                    {
+                                        // We delete existing if unset
+                                        SessionControllerGetter.getPointOfInterestBean().deletePointOfInterestExternalResource(weatherStation.getPointOfInterestId(), externalResourceId);
+                                    }
+                                    else
+                                    {
+
+                                        PointOfInterestExternalResource poiExternalResource = new PointOfInterestExternalResource();
+                                        PointOfInterestExternalResourcePK pk = new PointOfInterestExternalResourcePK(weatherStation.getPointOfInterestId(), externalResourceId);
+                                        poiExternalResource.setPointOfInterestExternalResourcePK(pk);
+                                        poiExternalResource.setResourceIdentifier(identifierField.getWebValue());
+                                        SessionControllerGetter.getPointOfInterestBean().storePointOfInterestExternalResource(poiExternalResource);
+                                    }
                                 }
                             }
-                            
                             // Redirect to form
                             response.sendRedirect(new StringBuilder(Globals.PROTOCOL + "://")
                                 .append(ServletUtil.getServerName(request))

src/main/java/no/nibio/vips/logic/entity/ForecastConfiguration.java

@@ -44,7 +44,11 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
 import java.util.List;
 import java.util.TimeZone;
 import javax.persistence.Transient;
+import no.nibio.vips.logic.util.IntegerArrayUserType;
 import no.nibio.vips.util.WeatherUtil;
+import org.hibernate.annotations.Type;
+import org.hibernate.annotations.TypeDef;
+import org.hibernate.annotations.TypeDefs;
 
 /**
  * @copyright 2014-2016 <a href="http://www.nibio.no/">NIBIO</a>
@@ -53,6 +57,7 @@ import no.nibio.vips.util.WeatherUtil;
 @Entity
 @Table(name = "forecast_configuration")
 @XmlRootElement
+@TypeDefs( {@TypeDef( name= "IntegerArray", typeClass = IntegerArrayUserType.class)})
 @NamedQueries({
     @NamedQuery(name = "ForecastConfiguration.findAll", query = "SELECT f FROM ForecastConfiguration f WHERE f.isPrivate = FALSE"),
     @NamedQuery(name = "ForecastConfiguration.findByForecastConfigurationId", query = "SELECT f FROM ForecastConfiguration f WHERE f.forecastConfigurationId = :forecastConfigurationId"),
@@ -117,6 +122,10 @@ public class ForecastConfiguration implements Serializable, Comparable {
     @Column(name = "is_private")
     private Boolean isPrivate;
     
+    @Type(type = "IntegerArray")
+    @Column(name = "grid_weather_station_point_of_interest_ids")
+    private Integer[] gridWeatherStationPointOfInterestIds;
+    
     @Transient
     private WeatherUtil weatherUtil;
     
@@ -391,4 +400,18 @@ public class ForecastConfiguration implements Serializable, Comparable {
     public void setIsPrivate(Boolean isPrivate) {
         this.isPrivate = isPrivate;
     }
+
+    /**
+     * @return the gridWeatherStationPointOfInterestIds
+     */
+    public Integer[] getGridWeatherStationPointOfInterestIds() {
+        return gridWeatherStationPointOfInterestIds;
+    }
+
+    /**
+     * @param gridWeatherStationPointOfInterestIds the gridWeatherStationPointOfInterestIds to set
+     */
+    public void setGridWeatherStationPointOfInterestIds(Integer[] gridWeatherStationPointOfInterestIds) {
+        this.gridWeatherStationPointOfInterestIds = gridWeatherStationPointOfInterestIds;
+    }
 }

src/main/java/no/nibio/vips/logic/scheduling/model/grid/preprocessor/ZymoseptoriaSimpleRiskGridModelPreprocessor.java

@@ -46,26 +46,35 @@ public class ZymoseptoriaSimpleRiskGridModelPreprocessor extends ModelRunPreproc
 
     @Override
     public ModelConfiguration getModelConfiguration(ForecastConfiguration configuration) throws PreprocessorException {
+        
         ModelConfiguration modelConfig = new ModelConfiguration();
         // Which weather stations??
         // Ilseng and Apelsvoll to start with ;-)
-        List<PointOfInterestWeatherStation> stations = Stream.of(new Integer[]{50,91,87}).map(
+        List<PointOfInterestWeatherStation> stations = Stream.of(configuration.getGridWeatherStationPointOfInterestIds()).map(
                 stationId -> (PointOfInterestWeatherStation) SessionControllerGetter.getPointOfInterestBean().getPointOfInterest(stationId)
             ).collect(Collectors.toList());
         
         List<PointWeatherObservationList> allObs = new ArrayList<>();
-        
+        WeatherUtil wUtil = new WeatherUtil();
         for(PointOfInterestWeatherStation station:stations)
         {
             Coordinate coordinate = new Coordinate(station.getPointOfInterest().getLongitude(), station.getPointOfInterest().getLatitude());
             List<WeatherObservation> stationObs = getStationObs(station, configuration);
-            PointWeatherObservationList pointObs = new PointWeatherObservationList(coordinate, stationObs);
+            try
+            {
+                stationObs = wUtil.checkForAndFixHourlyTimeSeriesHolesMultiParameter(stationObs, 6);
+            }
+            catch(WeatherObservationListException ex)
+            {
+                throw new PreprocessorException(ex.getMessage());
+            }
+            PointWeatherObservationList pointObs = new PointWeatherObservationList(coordinate, stationObs, station.getTimeZone());
             allObs.add(pointObs);
         }
         
         modelConfig.setConfigParameter("multiPointWeatherObservations", allObs);
         modelConfig.setModelId(this.getModelId());
-        modelConfig.setConfigParameter("timeZone", stations.get(0).getTimeZone());
+        modelConfig.setConfigParameter("timeZone", configuration.getTimeZone());
         return modelConfig;
     }
 

src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java

@@ -23,7 +23,9 @@ import com.vividsolutions.jts.geom.Envelope;
 import com.vividsolutions.jts.geom.GeometryFactory;
 import com.webcohesion.enunciate.metadata.Facet;
 import java.io.UnsupportedEncodingException;
+import java.net.URI;
 import java.net.URLDecoder;
+import java.text.MessageFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
@@ -40,9 +42,11 @@ import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriBuilder;
 import no.nibio.vips.entity.PointWeatherObservationList;
 import no.nibio.vips.entity.WeatherObservation;
 import no.nibio.vips.logic.util.SystemTime;
+import no.nibio.vips.util.XDate;
 import no.nibio.vips.util.weather.ALabDataParser;
 import no.nibio.vips.util.weather.FruitWebDavisDataParser;
 import no.nibio.vips.util.weather.MetosAPIDataParser;
@@ -90,6 +94,41 @@ public class WeatherProxyService {
         return Response.ok().entity(observations).build();
     }
     
+    @GET
+    @POST
+    @Path("fmi/{stationId}")
+    @GZIP
+    @Produces("application/json;charset=UTF-8")
+    public Response getMetosRIMProWeatherData(
+            @PathParam("stationId") String stationId, 
+            @FormParam("timeZone") String timeZonePOST, 
+            @QueryParam("timeZone") String timeZoneGET, 
+            @FormParam("startDate") String startDatePOST,
+            @QueryParam("startDate") String startDateGET,
+            @FormParam("endDate") String endDatePOST,
+            @QueryParam("endDate") String endDateGET
+            )
+    {
+        String FMI_URL_TEMPLATE="http://www.cropinfra.com:8080/weather/resources/fmi/temporal/vips/{0}/{1}/{2}";
+        List<WeatherObservation> observations;
+        try 
+        {
+            String timeZoneParam = timeZonePOST != null ? timeZonePOST : timeZoneGET != null ? timeZoneGET : "UTC";
+            TimeZone timeZone = TimeZone.getTimeZone(timeZoneParam);
+            SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
+            format.setTimeZone(timeZone);
+            String startDateParam = startDatePOST != null ? startDatePOST : startDateGET;
+            XDate startDate1 = new XDate(format.parse(startDateParam));
+            String endDateParam = endDatePOST != null ? endDatePOST : endDateGET;
+            XDate endDate1 = new XDate(format.parse(endDateParam));
+            URI uri = UriBuilder.fromUri(MessageFormat.format(FMI_URL_TEMPLATE, stationId, startDate1.getISOUTCFormat(), endDate1.getISOUTCFormat())).build();
+            //observations = new MetosRIMProDataParser().getWeatherObservations(stationId, timeZone, startDate1);
+            return Response.temporaryRedirect(uri).build();
+        } catch (ParseException | NullPointerException ex) {
+            return Response.serverError().entity(ex).build();
+        }
+    }
+    
     @GET
     @POST
     @Path("metosapi/{stationId}")

src/main/java/no/nibio/vips/logic/util/Globals.java

@@ -70,6 +70,7 @@ public class Globals {
     public static String[] availableTimeZones = {
         "UTC",
         "Etc/GMT-1",
+        "Europe/Copenhagen",
         "Europe/Oslo",
         "Europe/Sarajevo",
         "Europe/Stockholm",

src/main/java/no/nibio/vips/util/weather/MetosAPIDataParser.java

@@ -230,7 +230,7 @@ public class MetosAPIDataParser {
         try
         {
             String accessToken = this.getToken();
-            HttpGet httpget = new HttpGet("https://api.fieldclimate.com/v1/data/optimized/000024A0/hourly/from/" + (startDate.getTime() / 1000)); 
+            HttpGet httpget = new HttpGet("https://api.fieldclimate.com/v1/data/optimized/" + stationId + "/hourly/from/" + (startDate.getTime() / 1000)); 
 
             httpget.addHeader("Accept", "application/json");
             httpget.addHeader("Authorization", "Bearer " + accessToken);

src/main/java/no/nibio/vips/util/weather/dnmipointweb/DMIPointWebDataParser.java

@@ -19,7 +19,10 @@
 
 package no.nibio.vips.util.weather.dnmipointweb;
 
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Date;
 import java.util.GregorianCalendar;
 import java.util.HashMap;
@@ -55,15 +58,16 @@ public class DMIPointWebDataParser {
     
     public List<WeatherObservation> getData(Double longitude, Double latitude, Date dateFrom, Date dateTo)
     {
-        List<WeatherObservation> retVal = new ArrayList<>();
+        List<WeatherObservation> allObservations = new ArrayList<>();
+        TimeZone danishTZ = TimeZone.getTimeZone("Europe/Copenhagen");
         try {
             IWeatherService proxy = new WeatherService().getSslOffloadedBasicHttpBindingIWeatherService();
             UseableArrayOfWeatherDataSource wdsource = new UseableArrayOfWeatherDataSource();
             wdsource.add(WeatherDataSource.OBS);
             wdsource.add(WeatherDataSource.FORECAST);
-            wdsource.add(WeatherDataSource.NORMAL);
-
-            GregorianCalendar gc = new GregorianCalendar(TimeZone.getTimeZone("Europe/Copenhagen"));
+            //wdsource.add(WeatherDataSource.NORMAL);
+            
+            GregorianCalendar gc = new GregorianCalendar(danishTZ);
             gc.setTime(dateFrom);
             XMLGregorianCalendar calFrom = DatatypeFactory.newInstance().newXMLGregorianCalendar(gc);
             XMLGregorianCalendar calTo = null;
@@ -79,7 +83,8 @@ public class DMIPointWebDataParser {
                     listedParam -> wdparam.add(listedParam)
             );
             
-            WeatherResponse result = proxy.getWeatherData(
+            //WeatherResponse result = proxy.getWeatherData(
+            WeatherResponse result = proxy.getWeatherDataExtended(
                 String.valueOf(latitude).replace(".", ","), // Latitude ("X") Decimal separator is comma!
                 String.valueOf(longitude).replace(".", ","), // Longitude ("Y") Decimal separator is comma!
                 false, // true if UTM, false if decimal degrees
@@ -90,18 +95,19 @@ public class DMIPointWebDataParser {
                 calTo, // End of period with data
                 wdparam, // Set of requested parameters
                 0.0, // Base temperature value
-                false // Use base temperature? (t/f)
+                false, // Use base temperature? (t/f)
+                true // Get 6 days of forecasts (only for the proxy.getWeatherDataExtended method)
             );
-             
             
-
             ArrayOfWeatherDataModel value = result.getWeahterDataList().getValue();
             value.getWeatherDataModel().stream().forEach(weatherDataModel -> {
                     DMIPointWebDataParser.PARAM_MAP.keySet().stream().forEach(
                             VIPSParam->{
                                 WeatherObservation obs = this.getWeatherObservation(VIPSParam, weatherDataModel);
                                 if(obs != null)
-                                    retVal.add(obs);
+                                {
+                                    allObservations.add(obs);
+                                }
                             }
                     );
                 }
@@ -111,7 +117,23 @@ public class DMIPointWebDataParser {
             Logger.getLogger(DMIPointWebDataParser.class.getName()).log(Level.SEVERE, null, ex);
         }
         //System.out.println("Number of extracted weather data = " + retVal.size());
-        return retVal;
+        // After "now", the DMI service provides forecast values seamlessly. After approx 48 hours these
+        // values turn into 6 hour intervals, which we can't use. We need to weed them out
+        Collections.sort(allObservations);
+        ZonedDateTime now = ZonedDateTime.now();
+        ZonedDateTime lastValidObsTime = null;
+        ZoneId danishZ = ZoneId.of("Europe/Copenhagen");
+        List<WeatherObservation> filteredObservations = new ArrayList<>();
+        for(WeatherObservation obs:allObservations)
+        {
+            ZonedDateTime obsTime = ZonedDateTime.ofInstant(obs.getTimeMeasured().toInstant(),danishZ);
+            if(lastValidObsTime == null || obsTime.isBefore(now) || obsTime.minusHours(6).isBefore(lastValidObsTime))
+            {
+                filteredObservations.add(obs);
+                lastValidObsTime = obsTime;
+            }            
+        }
+        return allObservations;
     }
 
     private WeatherObservation getWeatherObservation(String VIPSParam, WeatherDataModel wDataModel) {

src/main/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParser.java

@@ -297,7 +297,7 @@ public class MetNoThreddsDataParser {
             Map<Long, WeatherObservation> hashedObsForPoint = tempRetVal.get(c);
             List<WeatherObservation> obsList = new ArrayList(hashedObsForPoint.values());
             Collections.sort(obsList);
-            PointWeatherObservationList obsForPoint = new PointWeatherObservationList(c,obsList);
+            PointWeatherObservationList obsForPoint = new PointWeatherObservationList(c,obsList,"UTC");
             retVal.add(obsForPoint);
         }
         return retVal;

src/main/webapp/formdefinitions/models/FINNCEREAL.json

@@ -73,5 +73,5 @@
 			]
         }
                 
-    ]
+    ] 
 }
\ No newline at end of file

src/main/webapp/public/nordic_septoria_whs/nordic_septoria_whs.js

@@ -178,18 +178,15 @@ function parseJSONDate(theDateString)
 
 function getTodayAtMidnight()
 {
-    var retVal = new Date();
-    retVal.setHours(0);
-    retVal.setMinutes(0);
-    retVal.setSeconds(0);
-    retVal.setMilliseconds(0);
+    var now = new Date();
     
-    //return retVal;
+    var midnightString = now.getFullYear() + "-" + 
+                            ('0' + (now.getMonth()+1)).slice(-2) + "-" +
+                            ('0' + (now.getDate()-1)).slice(-2) + 
+                            "T22:00:00.000+0000";
+    return new Date(midnightString);
     // OR RETURN A FIXED DATE FOR TESTING
-    retVal.setFullYear(2017);
-    retVal.setMonth(8); // September
-    retVal.setDate(9);
-    return retVal;//new Date("2017-09-09T22:00:00.000+0000");
+    return new Date("2017-09-07T22:00:00.000+0000");
 }
 
 

src/test/java/no/nibio/vips/util/weather/MetosAPIDataParserTest.java

@@ -79,7 +79,7 @@ public class MetosAPIDataParserTest {
         MetosAPIDataParser instance = new MetosAPIDataParser();
         List<WeatherObservation> result = instance.getWeatherObservations(stationId, timeZone, startDate);
         assertNotNull( result);
-        //result.stream().forEach(obs -> System.out.println(obs.toString()));
+        result.stream().forEach(obs -> System.out.println(obs.toString()));
         
     }