From c38735b76aba4e2e160626684aea238fbd5ab2a2 Mon Sep 17 00:00:00 2001
From: Tor-Einar Skog <tor-einar.skog@nibio.no>
Date: Mon, 9 Oct 2017 09:41:32 +0200
Subject: [PATCH] First version of the parser service for the Norwegian
Meteorological service's Thredds server
---
nbactions.xml | 1 +
pom.xml | 10 +
.../logic/service/WeatherProxyService.java | 85 ++++
.../metnothredds/MetNoThreddsDataParser.java | 455 ++++++++++++++++++
.../metnothredds/parametersources.properties | 22 +
.../MetNoThreddsDataParserTest.java | 166 +++++++
6 files changed, 739 insertions(+)
create mode 100644 src/main/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParser.java
create mode 100644 src/main/resources/no/nibio/vips/util/weather/metnothredds/parametersources.properties
create mode 100644 src/test/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParserTest.java
diff --git a/nbactions.xml b/nbactions.xml
index cd76a198..b86e6dae 100755
--- a/nbactions.xml
+++ b/nbactions.xml
@@ -44,6 +44,7 @@
<no.nibio.vips.logic.weather.FIELDCLIMATE_API_PASSWORD>q22bspFVPwkaohImV21m</no.nibio.vips.logic.weather.FIELDCLIMATE_API_PASSWORD>
<no.nibio.vips.logic.weather.FIELDCLIMATE_API_CLIENT_ID>MetosDemo</no.nibio.vips.logic.weather.FIELDCLIMATE_API_CLIENT_ID>
<no.nibio.vips.logic.weather.FIELDCLIMATE_API_CLIENT_SECRET>aa8f4b62b72986bac7c84be78836c2c6</no.nibio.vips.logic.weather.FIELDCLIMATE_API_CLIENT_SECRET>
+ <no.nibio.vips.logic.weather.METNOTHREDDS_TMP_FILE_PATH>/home/treinar/prosjekter/vips/projects/2017_SpotIT/Task 3.2/</no.nibio.vips.logic.weather.METNOTHREDDS_TMP_FILE_PATH>
diff --git a/pom.xml b/pom.xml
index 7ee404c2..696f8bee 100755
--- a/pom.xml
+++ b/pom.xml
@@ -19,8 +19,18 @@
<id>jitpack.io</id>
<url>https://jitpack.io</url>
</repository>
+ <repository>
+ <id>unidata-releases</id>
+ <name>Unidata Releases</name>
+ <url>https://artifacts.unidata.ucar.edu/content/repositories/unidata-releases/</url>
+ </repository>
</repositories>
<dependencies>
+ <dependency>
+ <groupId>edu.ucar</groupId>
+ <artifactId>cdm</artifactId>
+ <version>4.6.10</version>
+ </dependency>
<dependency>
<groupId>com.github.bjornharrtell</groupId>
<!--groupId>org.wololo</groupId-->
diff --git a/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java b/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java
index 6803e60c..7c9fdcb1 100755
--- a/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java
+++ b/src/main/java/no/nibio/vips/logic/service/WeatherProxyService.java
@@ -19,11 +19,15 @@
package no.nibio.vips.logic.service;
+import com.vividsolutions.jts.geom.Envelope;
+import com.vividsolutions.jts.geom.GeometryFactory;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.text.ParseException;
import java.text.SimpleDateFormat;
+import java.util.Arrays;
import java.util.Calendar;
+import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.TimeZone;
@@ -35,6 +39,7 @@ import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Response;
+import no.nibio.vips.entity.PointWeatherObservationList;
import no.nibio.vips.entity.WeatherObservation;
import no.nibio.vips.logic.util.SessionControllerGetter;
import no.nibio.vips.logic.util.SystemTime;
@@ -46,6 +51,7 @@ import no.nibio.vips.util.weather.ParseWeatherDataException;
import no.nibio.vips.util.weather.USPestDataParser;
import no.nibio.vips.util.weather.YrWeatherForecastProvider;
import no.nibio.vips.util.weather.dnmipointweb.DMIPointWebDataParser;
+import no.nibio.vips.util.weather.metnothredds.MetNoThreddsDataParser;
import org.jboss.resteasy.annotations.GZIP;
/**
@@ -287,4 +293,83 @@ public class WeatherProxyService {
return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(ex.getMessage()).build();
}
}
+
+ @GET
+ @Path("metno/thredds/point/")
+ @GZIP
+ @Produces("application/json;charset=UTF-8")
+ public Response getMetNoThreddsPointData(
+ @QueryParam("longitude") Double longitude,
+ @QueryParam("latitude") Double latitude,
+ @QueryParam("timeZone") String timeZoneStr,
+ @QueryParam("startDate") String startDateStr,
+ @QueryParam("startTime") String startTimeStr,
+ @QueryParam("endDate") String endDateStr,
+ @QueryParam("endTime") String endTimeStr,
+ @QueryParam("elementMeasurementTypes") String[] elementMeasurementTypes
+
+ )
+ {
+ try
+ {
+ TimeZone timeZone = TimeZone.getTimeZone(timeZoneStr);
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH");
+ format.setTimeZone(timeZone);
+ startTimeStr = startTimeStr != null ? startTimeStr : "00";
+ endTimeStr = endTimeStr != null ? endTimeStr : "23";
+
+ Date startDate = format.parse(startDateStr + " " + startTimeStr);
+ Date endDate = format.parse(endDateStr + " " + endTimeStr);
+ MetNoThreddsDataParser dp = new MetNoThreddsDataParser();
+ List<WeatherObservation> retVal = dp.getPointData(longitude, latitude, startDate, endDate, Arrays.asList(elementMeasurementTypes));
+ Collections.sort(retVal);
+ return Response.ok().entity(retVal).build();
+ }
+ catch(ParseException pe)
+ {
+ return Response.status(Response.Status.BAD_REQUEST).entity(pe.getMessage()).build();
+ }
+ }
+
+ @GET
+ @Path("metno/thredds/grid/")
+ @GZIP
+ @Produces("application/json;charset=UTF-8")
+ public Response getMetNoThreddsGridData(
+ @QueryParam("north") Double north,
+ @QueryParam("south") Double south,
+ @QueryParam("east") Double east,
+ @QueryParam("west") Double west,
+ @QueryParam("timeZone") String timeZoneStr,
+ @QueryParam("startDate") String startDateStr,
+ @QueryParam("startTime") String startTimeStr,
+ @QueryParam("endDate") String endDateStr,
+ @QueryParam("endTime") String endTimeStr,
+ @QueryParam("elementMeasurementTypes") String[] elementMeasurementTypes
+
+ )
+ {
+ try
+ {
+ TimeZone timeZone = TimeZone.getTimeZone(timeZoneStr);
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH");
+ format.setTimeZone(timeZone);
+ startTimeStr = startTimeStr != null ? startTimeStr : "00";
+ endTimeStr = endTimeStr != null ? endTimeStr : "23";
+
+ Date startDate = format.parse(startDateStr + " " + startTimeStr);
+ Date endDate = format.parse(endDateStr + " " + endTimeStr);
+ MetNoThreddsDataParser dp = new MetNoThreddsDataParser();
+
+ // Creating an envelope with the given bounds
+ Envelope envelope = new Envelope(west, east, south, north);
+ GeometryFactory gf = new GeometryFactory();
+ List<PointWeatherObservationList> retVal = dp.getGridData(gf.toGeometry(envelope), startDate, endDate, Arrays.asList(elementMeasurementTypes));
+ return Response.ok().entity(retVal).build();
+ }
+ catch(ParseException pe)
+ {
+ return Response.status(Response.Status.BAD_REQUEST).entity(pe.getMessage()).build();
+ }
+ }
}
diff --git a/src/main/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParser.java b/src/main/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParser.java
new file mode 100644
index 00000000..609f06f3
--- /dev/null
+++ b/src/main/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParser.java
@@ -0,0 +1,455 @@
+/*
+ * Copyright (c) 2017 NIBIO <http://www.nibio.no/>.
+ *
+ * This file is part of VIPSLogic.
+ * VIPSLogic is free software: you can redistribute it and/or modify
+ * it under the terms of the NIBIO Open Source License as published by
+ * NIBIO, either version 1 of the License, or (at your option) any
+ * later version.
+ *
+ * VIPSLogic is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * NIBIO Open Source License for more details.
+ *
+ * You should have received a copy of the NIBIO Open Source License
+ * along with VIPSLogic. If not, see <http://www.nibio.no/licenses/>.
+ *
+ */
+
+package no.nibio.vips.util.weather.metnothredds;
+
+import com.vividsolutions.jts.geom.Coordinate;
+import com.vividsolutions.jts.geom.Geometry;
+import com.vividsolutions.jts.geom.Polygon;
+import java.util.TimeZone;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.Set;
+import java.util.stream.Collectors;
+import no.nibio.vips.entity.PointWeatherObservationList;
+import no.nibio.vips.entity.WeatherObservation;
+import no.nibio.vips.gis.SimpleWGS84Coordinate;
+import no.nibio.vips.util.WeatherElements;
+import no.nibio.vips.util.WeatherUtil;
+import org.apache.commons.io.FileUtils;
+import ucar.ma2.Array;
+import ucar.ma2.ArrayDouble;
+import ucar.ma2.ArrayFloat;
+import ucar.nc2.NetcdfFile;
+import ucar.nc2.Variable;
+
+/**
+ * Digging into the Thredds archive of the Norwegian Meteorological Institute,
+ * collecting
+ * @copyright 2017 <a href="http://www.nibio.no/">NIBIO</a>
+ * @author Tor-Einar Skog <tor-einar.skog@nibio.no>
+ */
+public class MetNoThreddsDataParser {
+
+ private final String BASE_URL = "http://thredds.met.no/thredds/";
+ private final String NCSS_URL = BASE_URL + "ncss/";
+ private final String MEPS25_PATH = "meps25detarchive/";
+ private final String MEPS25_FILEBASENAME = "meps_mbr0_extracted_2_5km_";
+ private final String NETCDF_EXT = ".nc";
+ private final String STANDARD_PARAMS = "?accept=netcdf&temporal=all&&horizStride=1&disableLLSubset=on&timeStride=1";
+ private Properties paramInfo;
+ private final String TMP_FILE_PATH;
+ private final List<String> accumulatedParams = Arrays.asList("Q0", "RR");
+ private final SimpleDateFormat pathFormat;
+ private final SimpleDateFormat fileTimeStampFormat;
+
+
+ private final WeatherUtil weatherUtil;
+
+ public MetNoThreddsDataParser()
+ {
+ paramInfo = new Properties();
+ try (InputStream in = this.getClass().getResourceAsStream("parametersources.properties")) {
+ paramInfo.load(in);
+ }
+ catch(IOException ex)
+ {
+ }
+ this.TMP_FILE_PATH = System.getProperty("no.nibio.vips.logic.weather.METNOTHREDDS_TMP_FILE_PATH");
+ this.weatherUtil = new WeatherUtil();
+ this.pathFormat = new SimpleDateFormat("yyyy/MM/dd/");
+ this.pathFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ this.fileTimeStampFormat = new SimpleDateFormat("yyyyMMdd'T'00'Z'");
+ this.fileTimeStampFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ }
+
+ public List<PointWeatherObservationList> getGridData(Geometry geometry, Date dateFrom, Date dateTo, List<String> VIPSWeatherParameters)
+ {
+ Set<String> metParamNames = VIPSWeatherParameters.stream().map(
+ param -> paramInfo.getProperty(param)
+ ).collect(Collectors.toSet());
+
+
+ List<Date> archiveDates = getArchiveDates(dateFrom, dateTo);
+ Collections.sort(archiveDates);
+ //new GeometryFactory().
+ Polygon gridBounds = (Polygon) geometry.getEnvelope();
+ System.out.println("geometryType=" + gridBounds.getGeometryType());
+ System.out.println("gridBounds=" + gridBounds);
+
+ Map<Coordinate, Map<Long, WeatherObservation>> tempRetVal = new HashMap<>();
+ archiveDates.stream().forEach(archiveDate -> {
+ File tmp = null;
+ NetcdfFile ncFile = null;
+ try
+ {
+ URL threddsURL = new URL(NCSS_URL + MEPS25_PATH +
+ this.pathFormat.format(archiveDate) +
+ MEPS25_FILEBASENAME +
+ this.fileTimeStampFormat.format(archiveDate) +
+ NETCDF_EXT + STANDARD_PARAMS +
+ "&north=" + this.getNorth(gridBounds) + "&west=" + this.getWest(gridBounds) + "&east=" + this.getEast(gridBounds) + "&south=" + this.getSouth(gridBounds) +
+ "&var=" + String.join(",", metParamNames) +
+ "&addLatLon=true");
+ System.out.println("URL: " + threddsURL.toString());
+ Long timestamp = new Date().getTime();
+ tmp = new File(this.TMP_FILE_PATH + timestamp + ".nc");
+ FileUtils.copyURLToFile(threddsURL, tmp);
+ ncFile = NetcdfFile.open(tmp.getAbsolutePath());
+
+ Variable time = ncFile.findVariable("time");
+ Array times = time.read();
+ Date[] timeMeasured = new Date[times.getShape()[0]];
+ for(int i=0; i<timeMeasured.length;i++)
+ {
+ timeMeasured[i] = new Date(times.getLong(i) * 1000);
+ }
+ // The points are in a matrix. Get the dims first
+ ArrayDouble.D2 longitude = (ArrayDouble.D2) ncFile.findVariable("lon").read();
+ ArrayDouble.D2 latitude = (ArrayDouble.D2) ncFile.findVariable("lat").read();
+ int[] latLonShape = longitude.getShape(); // Assuming they're identical!
+ Coordinate[][] coords = new Coordinate[latLonShape[0]][latLonShape[1]];
+ for(int y=0;y<latLonShape[0];y++)
+ {
+ for(int x=0;x<latLonShape[1];x++)
+ {
+ double lat = latitude.get(y, x);
+ double lon = longitude.get(y,x);
+ Coordinate c = new Coordinate(lon, lat);
+ coords[y][x] = c;
+ //System.out.println("lat/lon=" + lat + "/" + lon);
+ }
+ }
+ for(String metParamName:metParamNames)
+ {
+ String VIPSParamName = this.findVIPSParamNameFromMetParamName(metParamName);
+
+ Variable var = ncFile.findVariable(metParamName);
+ // All variables have the dims [time, height, y, x]. All
+ // variables represent only one height, and can therefore be reduced
+ // from 4 to three dimensions
+ // Assuming all weather parameters are of type "float"
+
+ ArrayFloat.D3 varArray = (ArrayFloat.D3) var.read().reduce();
+ // We always skip the two first timestamps, since the model needs
+ // a couple of hours to spin up
+ for(int i=2;i<timeMeasured.length;i++)
+ {
+ if(timeMeasured[i].compareTo(dateFrom) < 0 || timeMeasured[i].after(dateTo))
+ {
+ continue;
+ }
+ for(int y=0;y<latLonShape[0];y++)
+ {
+ for(int x=0;x<latLonShape[1];x++)
+ {
+ Map<Long, WeatherObservation> obsMapForPoint = tempRetVal.get(coords[y][x]);
+ if(obsMapForPoint == null)
+ {
+ obsMapForPoint = new HashMap<>();
+ tempRetVal.put(coords[y][x], obsMapForPoint);
+ }
+ Double value = new Double(varArray.get(i, y, x));
+ // For accumulated parameters, substract previous value in array
+ if(i>0 && this.accumulatedParams.contains(VIPSParamName))
+ {
+ value = value - new Double(varArray.get(i-1, y, x));
+ }
+ WeatherObservation newObs = new WeatherObservation(
+ timeMeasured[i],
+ VIPSParamName,
+ WeatherObservation.LOG_INTERVAL_ID_1H,
+ this.getConvertedValue(value, VIPSParamName)
+ );
+
+ obsMapForPoint.put(newObs.getValiditySignature(), newObs);
+ }
+ }
+
+ }
+ }
+
+ }
+ catch(IOException ioe)
+ {
+
+ }
+ finally {
+ if(ncFile != null)
+ {
+ try
+ {
+ ncFile.close();
+ }catch(IOException ex) {}
+ }
+ if(tmp != null)
+ {
+ tmp.delete();
+ }
+ }
+ });
+ List<PointWeatherObservationList> retVal = new ArrayList<>();
+ for(Coordinate c : tempRetVal.keySet())
+ {
+ Map<Long, WeatherObservation> hashedObsForPoint = tempRetVal.get(c);
+ List<WeatherObservation> obsList = new ArrayList(hashedObsForPoint.values());
+ Collections.sort(obsList);
+ PointWeatherObservationList obsForPoint = new PointWeatherObservationList(c,obsList);
+ retVal.add(obsForPoint);
+ }
+ return retVal;
+ }
+
+ public List<WeatherObservation> getPointData(Double longitude, Double latitude, Date dateFrom, Date dateTo, List<String> VIPSWeatherParameters)
+ {
+ Set<String> metParamNames = VIPSWeatherParameters.stream().map(
+ param -> paramInfo.getProperty(param)
+ ).collect(Collectors.toSet());
+
+ List<Date> archiveDates = getArchiveDates(dateFrom, dateTo);
+ Collections.sort(archiveDates);
+
+ Map<Long, WeatherObservation> retVal = new HashMap<>();
+ archiveDates.stream().forEach(archiveDate -> {
+ File tmp = null;
+ NetcdfFile ncFile = null;
+ try
+ {
+ URL threddsURL = new URL(NCSS_URL + MEPS25_PATH +
+ this.pathFormat.format(archiveDate) +
+ MEPS25_FILEBASENAME +
+ this.fileTimeStampFormat.format(archiveDate) +
+ NETCDF_EXT + STANDARD_PARAMS + "&longitude=" + longitude + "&latitude=" + latitude
+ + "&var=" + String.join(",", metParamNames));
+ System.out.println("URL: " + threddsURL.toString());
+ Long timestamp = new Date().getTime();
+ tmp = new File(this.TMP_FILE_PATH + timestamp + ".nc");
+ FileUtils.copyURLToFile(threddsURL, tmp);
+ ncFile = NetcdfFile.open(tmp.getAbsolutePath());
+
+ // Set up the array of timestamps
+ // Unfortunately, the time value is a Double
+ // Dimensions of the time array are [station, timestamp]. Since theres
+ // Only one station, we can reduce the array dims. The reduce() method
+ // Simply removes all dims with a length of 1. Handy!
+ ArrayDouble.D1 time = (ArrayDouble.D1) ncFile.findVariable("time").read().reduce();
+ int[] timeDims = time.getShape();
+ // Get the length of the timestamp dimension
+ Date[] timeMeasured = new Date[timeDims[0]];
+ // Iterate, create parallel array with Dates
+ for(int i=0;i<timeDims[0];i++)
+ {
+ timeMeasured[i] = new Date(Math.round(time.get(i) * 1000));
+ }
+
+ // The position variables have 1 dimension and one element. Simple!
+ SimpleWGS84Coordinate coordinate = new SimpleWGS84Coordinate(
+ ((ArrayDouble.D1) ncFile.findVariable("latitude").read()).get(0),
+ ((ArrayDouble.D1) ncFile.findVariable("longitude").read()).get(0)
+ );
+ // Can't use stream API here because the ncfile would need to be
+ // final and therefore not closeable in a try-finally statement
+ for(String metParamName:metParamNames)
+ {
+ String VIPSParamName = this.findVIPSParamNameFromMetParamName(metParamName);
+
+ Variable var = ncFile.findVariable(metParamName);
+
+ // The variables have two different dims. The variables that
+ // are height dependent (e.g. temperature) have [station, value, height]
+ // and the height independent (e.g. precipitation_amount) have
+ // [station, value]. Either way, all dims except value have a length
+ // of 1 and can be reduced.
+ // Assuming all weather parameters are of type "float"
+ try
+ {
+ ArrayFloat.D1 varArray = (ArrayFloat.D1) var.read().reduce();
+ // We always skip the two first timestamps, since the model needs
+ // a couple of hours to spin up
+ for(int i=2;i<timeMeasured.length;i++)
+ {
+ if(timeMeasured[i].compareTo(dateFrom) < 0 || timeMeasured[i].after(dateTo))
+ {
+ continue;
+ }
+ Double value = new Double(varArray.get(i));
+ // For accumulated parameters, substract previous value in array
+ if(i>0 && this.accumulatedParams.contains(VIPSParamName))
+ {
+ value = value - new Double(varArray.get(i-1));
+ }
+ WeatherObservation newObs = new WeatherObservation(
+ timeMeasured[i],
+ VIPSParamName,
+ WeatherObservation.LOG_INTERVAL_ID_1H,
+ this.getConvertedValue(value, VIPSParamName)
+ );
+ retVal.put(newObs.getValiditySignature(), newObs);
+ }
+ }catch(IOException ioe)
+ {
+ ioe.printStackTrace();
+ }
+ }
+
+
+ }
+ catch(IOException ioe)
+ {
+
+ }
+ finally {
+ if(ncFile != null)
+ {
+ try
+ {
+ ncFile.close();
+ }catch(IOException ex) {}
+ }
+ if(tmp != null)
+ {
+ tmp.delete();
+ }
+ }
+ });
+ return new ArrayList(retVal.values());
+ }
+
+ public String findVIPSParamNameFromMetParamName(String metParamName)
+ {
+ // Need to treat the Properties object as a map and search for a key that
+ // maps to the given met param name
+ Optional<Entry<Object,Object>> opt = this.paramInfo.entrySet().stream().filter(propEntry -> propEntry.getValue().equals(metParamName)).findFirst();
+ return opt.isPresent() ? (String) opt.get().getKey() : null;
+ }
+
+ /**
+ * Performing the correct transformations between Met params and VIPS params
+ * @param value
+ * @param VIPSParamName
+ * @return
+ */
+ public double getConvertedValue(Double value, String VIPSParamName)
+ {
+ switch(VIPSParamName){
+ case WeatherElements.TEMPERATURE_MEAN:
+ case WeatherElements.TEMPERATURE_MAXIMUM:
+ case WeatherElements.TEMPERATURE_MINIMUM:
+ return this.weatherUtil.getCelciusFromKelvin(value);
+ case WeatherElements.RELATIVE_HUMIDITY_MEAN:
+ case WeatherElements.RELATIVE_HUMIDITY_INSTANTANEOUS:
+ return this.getPercentFromFraction(value);
+ case WeatherElements.PRECIPITATION:
+ return roundToDecimals(value, 1);
+ case WeatherElements.GLOBAL_RADIATION:
+ return Math.round(value/3600);
+ default:
+ return value;
+ }
+ }
+
+
+
+ public double getPercentFromFraction(Double value)
+ {
+ return value * 100;
+ }
+
+ public double roundToDecimals(Double value, Integer decimals)
+ {
+ return Math.round(value * Math.pow(10.0, decimals)) / Math.pow(10.0, decimals);
+ }
+
+ /**
+ * Provide the list of date paths in which we can find files that contain
+ * data for the given period
+ * @param dateFrom
+ * @param dateTo
+ * @return
+ */
+ public List<Date> getArchiveDates(Date dateFrom, Date dateTo) {
+ Date today = new Date();
+ dateTo = dateTo.after(today) ? today : dateTo;
+ if(dateFrom.after(dateTo))
+ {
+ return null;
+ }
+ TimeZone UTC = TimeZone.getTimeZone("UTC");
+
+ // If the date is set at earlier than xx:02:00 UTC
+ // Going one day back in time from dateFrom, since we skip the two
+ // first runs of each model
+ Calendar cal = Calendar.getInstance(UTC);
+ cal.setTime(dateFrom);
+ if(cal.get(Calendar.HOUR_OF_DAY) < 2)
+ {
+ cal.add(Calendar.DATE, -1);
+ dateFrom = cal.getTime();
+ }
+
+ // Adjusting to UTC midnight so that the last date (dateTo) is not
+ // overstepped
+ dateFrom = this.weatherUtil.normalizeToExactDate(dateFrom, UTC);
+ List<Date> retVal = new ArrayList<>();
+ while(dateFrom.before(dateTo))
+ {
+ retVal.add(dateFrom);
+ cal.setTime(dateFrom);
+ cal.set(Calendar.HOUR_OF_DAY, 0);
+ cal.add(Calendar.DATE, 1);
+ dateFrom = cal.getTime();
+ }
+
+ return retVal;
+ }
+
+ public double getNorth(Polygon envelope)
+ {
+ return envelope.getCoordinates()[2].y;
+ }
+
+ public double getSouth(Polygon envelope)
+ {
+ return envelope.getCoordinates()[0].y;
+ }
+
+ public double getWest(Polygon envelope)
+ {
+ return envelope.getCoordinates()[0].x;
+ }
+
+ public double getEast(Polygon envelope)
+ {
+ return envelope.getCoordinates()[2].x;
+ }
+}
diff --git a/src/main/resources/no/nibio/vips/util/weather/metnothredds/parametersources.properties b/src/main/resources/no/nibio/vips/util/weather/metnothredds/parametersources.properties
new file mode 100644
index 00000000..5707d828
--- /dev/null
+++ b/src/main/resources/no/nibio/vips/util/weather/metnothredds/parametersources.properties
@@ -0,0 +1,22 @@
+# Copyright (c) 2017 NIBIO <http://www.nibio.no/>.
+#
+# This file is part of VIPSLogic.
+# VIPSLogic is free software: you can redistribute it and/or modify
+# it under the terms of the NIBIO Open Source License as published by
+# NIBIO, either version 1 of the License, or (at your option) any
+# later version.
+#
+# VIPSLogic is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# NIBIO Open Source License for more details.
+#
+# You should have received a copy of the NIBIO Open Source License
+# along with VIPSLogic. If not, see <http://www.nibio.no/licenses/>.
+#
+TM=air_temperature_2m
+TX=air_temperature_max
+TN=air_temperature_min
+UM=relative_humidity_2m
+RR=precipitation_amount_acc
+Q0=integral_of_surface_net_downward_shortwave_flux_wrt_time
\ No newline at end of file
diff --git a/src/test/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParserTest.java b/src/test/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParserTest.java
new file mode 100644
index 00000000..a44f8045
--- /dev/null
+++ b/src/test/java/no/nibio/vips/util/weather/metnothredds/MetNoThreddsDataParserTest.java
@@ -0,0 +1,166 @@
+/*
+ * Copyright (c) 2017 NIBIO <http://www.nibio.no/>.
+ *
+ * This file is part of VIPSLogic.
+ * VIPSLogic is free software: you can redistribute it and/or modify
+ * it under the terms of the NIBIO Open Source License as published by
+ * NIBIO, either version 1 of the License, or (at your option) any
+ * later version.
+ *
+ * VIPSLogic is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * NIBIO Open Source License for more details.
+ *
+ * You should have received a copy of the NIBIO Open Source License
+ * along with VIPSLogic. If not, see <http://www.nibio.no/licenses/>.
+ *
+ */
+package no.nibio.vips.util.weather.metnothredds;
+
+import com.vividsolutions.jts.geom.Coordinate;
+import com.vividsolutions.jts.geom.GeometryFactory;
+import com.vividsolutions.jts.geom.Polygon;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
+import no.nibio.vips.entity.PointWeatherObservationList;
+import no.nibio.vips.entity.WeatherObservation;
+import no.nibio.vips.util.WeatherUtil;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author treinar
+ */
+public class MetNoThreddsDataParserTest {
+
+ public MetNoThreddsDataParserTest() {
+ }
+
+ @BeforeClass
+ public static void setUpClass() {
+ }
+
+ @AfterClass
+ public static void tearDownClass() {
+ }
+
+ @Before
+ public void setUp() {
+ }
+
+ @After
+ public void tearDown() {
+ }
+
+ //@Test
+ public void testGetGridData()
+ {
+ System.out.println("testGetGridData");
+ Coordinate[] coords = new Coordinate[5];
+ coords[0] = new Coordinate(10.31,59.91);
+ coords[1] = new Coordinate(11.07,59.91);
+ coords[2] = new Coordinate(11.07,59.52);
+ coords[3] = new Coordinate(10.31,59.52);
+ coords[4] = new Coordinate(10.31,59.91);
+ GeometryFactory gFac = new GeometryFactory();
+ Polygon pol = gFac.createPolygon(coords);
+
+ List<String> weatherParameters = Arrays.asList("TM","RR", "Q0", "TX","TN","UM");
+ MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
+
+ Date start = new Date();
+ Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("Europe/Oslo"));
+ cal.setTime(new Date());
+ //cal.add(Calendar.DATE, -2);
+ cal.set(Calendar.HOUR_OF_DAY, 5);
+ Date dateFrom = new WeatherUtil().normalizeToExactHour(cal.getTime(), TimeZone.getDefault());
+ cal.add(Calendar.DATE, 5);
+ Date dateTo = cal.getTime();
+
+
+ List<PointWeatherObservationList> result = instance.getGridData(pol, dateFrom, dateTo, weatherParameters);
+ assertNotNull(result);
+ Long timeSpent = new Date().getTime() - start.getTime();
+ System.out.println("Time spent=" + (new Double(timeSpent)/1000) + " seconds");
+ result.stream().forEach(mp->System.out.println(mp));
+ }
+
+ /**
+ * Test of getPointData method, of class MetNoThreddsDataParser.
+ */
+ @Test
+ public void testGetPointData() {
+ System.out.println("getPointData");
+
+ Double longitude = 10.7946;
+ Double latitude = 59.6652;
+
+ List<String> weatherParameters = Arrays.asList("TM","RR", "Q0", "TX","TN","UM");
+ MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
+
+ Date start = new Date();
+ Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("Europe/Oslo"));
+ cal.setTime(new Date());
+ cal.add(Calendar.DATE, -20);
+ cal.set(Calendar.HOUR_OF_DAY, 1);
+ Date dateFrom = new WeatherUtil().normalizeToExactHour(cal.getTime(), TimeZone.getDefault());
+ cal.add(Calendar.DATE, 20);
+ Date dateTo = cal.getTime();
+
+
+ List<WeatherObservation> result = instance.getPointData(longitude, latitude, dateFrom, dateTo, weatherParameters);
+ Long timeSpent = new Date().getTime() - start.getTime();
+ System.out.println("Time spent=" + (new Double(timeSpent)/1000) + " seconds");
+
+ Collections.sort(result);
+ Long hoursIncluded = 1 + ((result.get(result.size()-1).getTimeMeasured().getTime() - result.get(0).getTimeMeasured().getTime()) / 3600000);
+ long expResult = weatherParameters.size() * hoursIncluded;
+ result.stream().forEach(obs->System.out.println(obs));
+ assertEquals(expResult, result.size());
+ assertTrue(result.size() > 10);
+
+ }
+
+ @Test
+ public void testFindVIPSParamNameFromMetParamName()
+ {
+ System.out.println("testFindVIPSParamNameFromMetParamName");
+ MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
+ String expResult = "TM";
+ String result = instance.findVIPSParamNameFromMetParamName("air_temperature_2m");
+ assertEquals(expResult, result);
+ result = instance.findVIPSParamNameFromMetParamName("foo_bar_1000m");
+ assertNull(result);
+ }
+
+ @Test
+ public void testGetArchiveDates()
+ {
+ System.out.println("testGetArchiveDates");
+ MetNoThreddsDataParser instance = new MetNoThreddsDataParser();
+ Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("Europe/Oslo"));
+ cal.setTime(new Date());
+ cal.add(Calendar.DATE, -2);
+ cal.set(Calendar.HOUR_OF_DAY, 0);
+ Date dateFrom = new WeatherUtil().normalizeToExactHour(cal.getTime(), TimeZone.getDefault());
+ cal.add(Calendar.DATE, 5);
+ List<Date> result = instance.getArchiveDates(dateFrom, cal.getTime());
+ SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd HHZ");
+ f.setTimeZone(TimeZone.getTimeZone("UTC"));
+ result.stream().forEach(r->System.out.println(f.format(r)));
+ assertEquals(4,result.size());
+
+ }
+
+}
--
GitLab