Built aggregation algorithms for weather data

pom.xml

@@ -57,6 +57,15 @@
           </tags>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>2.3.2</version>
+        <configuration>
+          <source>1.7</source>
+          <target>1.7</target>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 </project>

src/main/java/no/bioforsk/vips/core/service/ManagerResource.java → src/main/java/no/bioforsk/vips/coremanager/service/ManagerResource.java

-package no.bioforsk.vips.core.service;
+package no.bioforsk.vips.coremanager.service;
 
 import java.util.Map;
 import javax.ws.rs.Consumes;
@@ -9,7 +9,7 @@ import javax.ws.rs.Produces;
 import javax.ws.rs.core.Response;
 
 /**
- * Interface for the ModelResource. Implemented by ManagerResourceImpl
+ * Interface for the ManagerResource. Implemented by ManagerResourceImpl
  * in VIPSCoreManager and also as client in VIPSBatch
  * @copyright 2013 <a href="http://www.bioforsk.no/">Bioforsk</a>
  * @author Tor-Einar Skog <tor-einar.skog@bioforsk.no>

src/main/java/no/bioforsk/vips/entity/WeatherObservation.java

@@ -9,6 +9,10 @@ import java.util.Date;
  * @author  Tor-Einar Skog <tor-einar.skog@bioforsk.no>
  */
 public class WeatherObservation implements Comparable{
+    
+    public final static Integer LOG_INTERVAL_ID_1H = 1;
+    public final static Integer LOG_INTERVAL_ID_1D = 2;
+
    
     private String elementMeasurementTypeId;
     private Integer logIntervalId;

src/main/java/no/bioforsk/vips/util/InvalidAggregationTypeException.java

+package no.bioforsk.vips.util;
+
+/**
+ * @copyright 2013 <a href="http://www.bioforsk.no/">Bioforsk</a>
+ * @author Tor-Einar Skog <tor-einar.skog@bioforsk.no>
+ */
+class InvalidAggregationTypeException extends Exception {
+
+    public InvalidAggregationTypeException(String string) {
+    }
+
+}

src/main/java/no/bioforsk/vips/util/WeatherObservationListException.java

+package no.bioforsk.vips.util;
+
+/**
+ * @copyright 2013 <a href="http://www.bioforsk.no/">Bioforsk</a>
+ * @author Tor-Einar Skog <tor-einar.skog@bioforsk.no>
+ */
+public class WeatherObservationListException extends Exception {
+
+    public WeatherObservationListException(String msg)
+    {
+        super(msg);
+    }
+}

src/main/java/no/bioforsk/vips/util/WeatherUtil.java

 package no.bioforsk.vips.util;
 
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import no.bioforsk.vips.entity.WeatherObservation;
+
 /**
  * Weather related utility methods
  * @copyright 2013 <a href="http://www.bioforsk.no/">Bioforsk</a>
@@ -7,6 +19,12 @@ package no.bioforsk.vips.util;
  */
 public class WeatherUtil {
     
+    public final static int AGGREGATION_TYPE_AVERAGE = 1;
+    public final static int AGGREGATION_TYPE_SUM = 2;
+    public final static int AGGREGATION_TYPE_MINIMUM = 3;
+    public final static int AGGREGATION_TYPE_MAXIMUM = 4;
+
+    
     /**
      * Calculates Water Vapor Deficiency in Pa
      * @param temperature
@@ -42,5 +60,133 @@ public class WeatherUtil {
         return 0.61078 * Math.exp(17.269 * temperature / (temperature + 237.3));
     }
     
+    /**
+     * 
+     * @param observation hourly Values (logInterval 1h)
+     * @return aggregated daily values (logInterval 24h/1d)
+     */
+    public List<WeatherObservation> getAggregatedDailyValues(
+            List<WeatherObservation> observations, 
+            TimeZone timeZone, 
+            Integer minimumObservationsPerDay, 
+            Integer typeOfAggregation) 
+            throws WeatherObservationListException,
+            InvalidAggregationTypeException
+    {
+        // First we organize the hourly values into one bucket per day
+        Map<Date,Map> dateBucket = new HashMap<>();
+        Calendar cal = Calendar.getInstance(timeZone);
+        String expectedParameter = observations.get(0).getElementMeasurementTypeId();
+        for(WeatherObservation observation:observations)
+        {
+            if(!observation.getElementMeasurementTypeId().equals(expectedParameter))
+            {
+                throw new WeatherObservationListException("Found multiple parameters: " + observation.getElementMeasurementTypeId() + " and " + expectedParameter);
+            }
+            Date theDate = normalizeToExactDate(observation.getTimeMeasured(), timeZone);
+            Map<Date, Double> hourValuesForDate = dateBucket.get(theDate);
+            if(hourValuesForDate == null)
+            {
+                hourValuesForDate = new HashMap<>();
+                dateBucket.put(theDate, hourValuesForDate);
+            }
+            
+            // Check for double entries
+            Double possibleDuplicate = hourValuesForDate.get(observation.getTimeMeasured());
+            if(possibleDuplicate != null)
+            {
+                throw new WeatherObservationListException(
+                        "Found duplicate weatherObservations for parameter " + 
+                        observation.getElementMeasurementTypeId() + " at time " +
+                        observation.getTimeMeasured()
+                );
+            }
+            hourValuesForDate.put(observation.getTimeMeasured(), observation.getValue());
+        }
+        
+        // Then we iterate the buckets, do the aggregation and create return values
+        List<WeatherObservation> aggregatedObservations = new ArrayList<>();
+        WeatherObservation templateObservation = observations.get(0);
+        Double aggregateValue;
+        for(Date aDay:dateBucket.keySet())
+        {
+            //System.out.println("date=" + aDay);
+            Map hourValuesForADay = dateBucket.get(aDay);
+            if(hourValuesForADay.size() < minimumObservationsPerDay)
+            {
+                throw new WeatherObservationListException(
+                        "Too few observations to aggregate for parameter " +
+                        " at date " + aDay +". Found " + hourValuesForADay.size() +
+                        ", expected minimum " + minimumObservationsPerDay
+                        );
+            }
+            switch(typeOfAggregation){
+                case WeatherUtil.AGGREGATION_TYPE_AVERAGE:
+                    aggregateValue = getAverage(hourValuesForADay.values()); break;
+                case WeatherUtil.AGGREGATION_TYPE_SUM:
+                    aggregateValue = getSum(hourValuesForADay.values()); break;
+                case WeatherUtil.AGGREGATION_TYPE_MINIMUM:
+                    aggregateValue = getMinimum(hourValuesForADay.values()); break;
+                case WeatherUtil.AGGREGATION_TYPE_MAXIMUM:
+                    aggregateValue = getMaximum(hourValuesForADay.values()); break;
+                default:
+                    throw new InvalidAggregationTypeException(
+                            "No aggregation method with id= " + typeOfAggregation  + " exists."
+                            );
+            }
+            WeatherObservation aggregatedObservation = new WeatherObservation();
+            aggregatedObservation.setElementMeasurementTypeId(templateObservation.getElementMeasurementTypeId());
+            aggregatedObservation.setLogIntervalId(WeatherObservation.LOG_INTERVAL_ID_1D);
+            aggregatedObservation.setTimeMeasured(aDay);
+            aggregatedObservation.setValue(aggregateValue);
+            aggregatedObservations.add(aggregatedObservation);
+        }
+        return aggregatedObservations;
+    }
     
+    private Double getAverage(Collection<Double> values)
+    {
+        return this.getSum(values)/values.size();
+    }
+
+    private Double getSum(Collection<Double> values)
+    {
+       Double sum = 0d;
+        for(Double value:values)
+        {
+            sum += value;
+        }
+        return sum;
+    }
+    
+    private Double getMinimum(Collection<Double> values)
+    {
+        Double minimum = null;
+        for(Double value:values)
+        {
+            minimum = minimum == null ? value : Math.min(minimum, value);
+        }
+        return minimum;
+    }
+    
+    private Double getMaximum(Collection<Double> values)
+    {
+        Double maximum = null;
+        for(Double value:values)
+        {
+            maximum = maximum == null ? value : Math.max(maximum, value);
+        }
+        return maximum;
+    }
+    
+    public Date normalizeToExactDate(Date timeStamp, TimeZone timeZone)
+    {
+        Calendar cal = Calendar.getInstance(timeZone);
+        cal.setTime(timeStamp);
+        cal.set(Calendar.HOUR_OF_DAY, 0);
+        cal.set(Calendar.MINUTE, 0);
+        cal.set(Calendar.SECOND,0);
+        cal.set(Calendar.MILLISECOND,0);
+        return cal.getTime();
+    }
 }

src/test/java/no/bioforsk/vips/util/WeatherUtilTest.java

+/*
+ * To change this template, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package no.bioforsk.vips.util;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.fail;
+import junit.framework.TestCase;
+import no.bioforsk.vips.entity.WeatherObservation;
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.JsonParser;
+import org.codehaus.jackson.map.MappingJsonFactory;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.type.TypeReference;
+
+/**
+ *
+ * @author treinar
+ */
+public class WeatherUtilTest extends TestCase {
+    
+    public WeatherUtilTest(String testName) {
+        super(testName);
+    }
+    
+    @Override
+    protected void setUp() throws Exception {
+        super.setUp();
+    }
+    
+    @Override
+    protected void tearDown() throws Exception {
+        super.tearDown();
+    }
+
+    /**
+     * Test of getWVD method, of class WeatherUtil.
+     */
+    /*
+    public void testGetWVD() {
+        System.out.println("getWVD");
+        double temperature = 0.0;
+        double relativeHumidity = 0.0;
+        WeatherUtil instance = new WeatherUtil();
+        double expResult = 0.0;
+        double result = instance.getWVD(temperature, relativeHumidity);
+        assertEquals(expResult, result, 0.0);
+        // TODO review the generated test code and remove the default call to fail.
+        fail("The test case is a prototype.");
+    }*/
+
+    /**
+     * Test of getPartialPressure method, of class WeatherUtil.
+     */
+    /*
+    public void testGetPartialPressure() {
+        System.out.println("getPartialPressure");
+        double saturationPressure = 0.0;
+        double relativeHumidity = 0.0;
+        WeatherUtil instance = new WeatherUtil();
+        double expResult = 0.0;
+        double result = instance.getPartialPressure(saturationPressure, relativeHumidity);
+        assertEquals(expResult, result, 0.0);
+        // TODO review the generated test code and remove the default call to fail.
+        fail("The test case is a prototype.");
+    }*/
+
+    /**
+     * Test of getSaturationPressure method, of class WeatherUtil.
+     */
+    /*
+    public void testGetSaturationPressure() {
+        System.out.println("getSaturationPressure");
+        double temperature = 0.0;
+        WeatherUtil instance = new WeatherUtil();
+        double expResult = 0.0;
+        double result = instance.getSaturationPressure(temperature);
+        assertEquals(expResult, result, 0.0);
+        // TODO review the generated test code and remove the default call to fail.
+        fail("The test case is a prototype.");
+    }*/
+
+    /**
+     * Test of getAggregatedDailyValues method, of class WeatherUtil.
+     */
+    public void testGetAggregatedDailyValues() {
+        System.out.println("getAggregatedDailyValues");
+        List<WeatherObservation> allObservations = this.getObservations();
+        List<WeatherObservation> TM = new ArrayList<>();
+        for(WeatherObservation obs:allObservations)
+        {
+            if(obs.getElementMeasurementTypeId().equals("TM"))
+                TM.add(obs);
+        }
+        WeatherUtil instance = new WeatherUtil();
+        List expResult = null;
+        Double expSum = 143.091;
+        Double expAvg = 5.962125;
+        Double expMin = -0.191;
+        Double expMax = 12.71;
+        TimeZone timeZone = TimeZone.getDefault();
+        //System.out.println("TimeZone=" + timeZone.getDisplayName());
+        List<WeatherObservation> avgResults = null;
+        List<WeatherObservation> sumResults = null;
+        List<WeatherObservation> minResults = null;
+        List<WeatherObservation> maxResults = null;
+        try {
+            avgResults = instance.getAggregatedDailyValues(TM, timeZone,23,WeatherUtil.AGGREGATION_TYPE_AVERAGE);
+            Collections.sort(avgResults);
+            sumResults = instance.getAggregatedDailyValues(TM, timeZone,23,WeatherUtil.AGGREGATION_TYPE_SUM);
+            Collections.sort(sumResults);
+            minResults = instance.getAggregatedDailyValues(TM, timeZone,23,WeatherUtil.AGGREGATION_TYPE_MINIMUM);
+            Collections.sort(minResults);
+            maxResults = instance.getAggregatedDailyValues(TM, timeZone,23,WeatherUtil.AGGREGATION_TYPE_MAXIMUM);
+            Collections.sort(maxResults);
+            
+        } catch (WeatherObservationListException | InvalidAggregationTypeException ex) {
+            fail(ex.getMessage());
+        }
+        assertEquals(expAvg, avgResults.get(0).getValue());
+        assertEquals(expSum, sumResults.get(0).getValue());
+        assertEquals(expMin, minResults.get(0).getValue());
+        assertEquals(expMax, maxResults.get(0).getValue());
+    }
+
+    /**
+     * Test of normalizeToExactDate method, of class WeatherUtil.
+     *
+    public void testNormalizeToExactDate() {
+        System.out.println("normalizeToExactDate");
+        Date timeStamp = null;
+        TimeZone timeZone = null;
+        WeatherUtil instance = new WeatherUtil();
+        Date expResult = null;
+        Date result = instance.normalizeToExactDate(timeStamp, timeZone);
+        assertEquals(expResult, result);
+        // TODO review the generated test code and remove the default call to fail.
+        fail("The test case is a prototype.");
+    }*/
+    
+    private List<WeatherObservation> getObservations()
+    {
+        try
+        {
+        BufferedInputStream inputStream = new BufferedInputStream(this.getClass().getResourceAsStream("/weatherData.json"));
+            JsonFactory f = new MappingJsonFactory();
+            JsonParser jp = f.createJsonParser(inputStream);
+            JsonNode all = jp.readValueAsTree();
+            List<WeatherObservation> observations = new ArrayList<>();
+            ObjectMapper mapper = new ObjectMapper();
+
+            Date firstDate = null;
+            Date lastDate = null;
+            if(all.isArray())
+            {
+                for(JsonNode node : all){
+                    Date timeMeasured = (Date)mapper.convertValue(node.get("timeMeasured").getTextValue(), new TypeReference<Date>(){});
+                    if(firstDate == null || firstDate.compareTo(timeMeasured) > 0)
+                    {
+                        firstDate = timeMeasured;
+                    }
+                    if(lastDate == null || lastDate.compareTo(timeMeasured) < 0)
+                    {
+                        lastDate = timeMeasured;
+                    }
+                    //System.out.println(node.toString());
+                    WeatherObservation observation = new WeatherObservation();
+                    observation.setTimeMeasured(timeMeasured);
+                    observation.setLogIntervalId(node.get("logIntervalId").getIntValue());
+                    observation.setElementMeasurementTypeId(node.get("elementMeasurementTypeId").getTextValue());
+                    observation.setValue(node.get("value").getDoubleValue());
+                    observations.add(observation);
+                }
+
+            }
+            else
+            {
+                fail("Data input from file is not a JSON array");
+            }
+            return observations;
+        }
+        catch(IOException ex)
+        {
+            return null;
+        }
+    }
+}