Added stuff for getting results from local data files

src/test/java/no/nibio/vips/model/naerstadmodel/NaerstadModelTest.java

@@ -26,10 +26,15 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.MappingJsonFactory;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Calendar;
 import java.util.Date;
+import java.util.HashSet;
 import java.util.List;
 import java.util.TimeZone;
 import static junit.framework.Assert.fail;
@@ -40,6 +45,9 @@ import no.nibio.vips.entity.WeatherObservation;
 import no.nibio.vips.model.ConfigValidationException;
 import no.nibio.vips.model.ModelExcecutionException;
 import no.nibio.vips.util.JSONUtil;
+import no.nibio.vips.util.WeatherElements;
+import no.nibio.vips.util.WeatherObservationListException;
+import no.nibio.vips.util.WeatherUtil;
 
 
 /**
@@ -61,6 +69,41 @@ public class NaerstadModelTest extends TestCase {
     protected void tearDown() throws Exception {
         super.tearDown();
     }
+    
+    /**
+     * This is not a test!!
+     */
+    public void donttestMladenData() {
+        try {
+            System.out.println("testMladenData");
+            NaerstadModel instance;
+            
+            List<Result> results;
+            ObjectMapper mapper = new ObjectMapper();
+            for(int year=2006;year<=2016;year++)
+            {
+                // Bad weather data files
+                if(year == 2006 || year == 2009 || year == 2013)
+                {
+                    continue;
+                }
+                System.out.println("testMladenData for " + year);
+                instance = new NaerstadModel();
+                instance.setConfiguration(this.getConfiguration(new File("/home/treinar/prosjekter/vips/projects/2016_irland/maynooth_" + year + ".json")));
+                FileOutputStream out = new FileOutputStream(new File("/home/treinar/prosjekter/vips/projects/2016_irland/results_maynooth_" +  year + ".json"));
+                results = instance.getResult();
+                mapper.writeValue(out, results);
+                /*for(Result result:results)
+                {
+                    System.out.println(result.toString());
+                }*/
+            }
+            
+           
+        } catch (ConfigValidationException | ModelExcecutionException | IOException ex) {
+                fail("Exception: " + ex.getMessage());
+        }
+    }
 
     /**
      * Test of getResult method, of class NaerstadModel.
@@ -227,6 +270,59 @@ public class NaerstadModelTest extends TestCase {
             return null;
         } 
     }
+    
+    private ModelConfiguration getConfiguration(File dataFile)
+    {
+        try {
+            ModelConfiguration config = new ModelConfiguration();
+            config.setModelId("NAERSTADMO");
+            BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(dataFile));
+            JsonFactory f = new MappingJsonFactory();
+            JsonParser jp = f.createParser(inputStream);
+            JsonNode all = jp.readValueAsTree();
+            List<WeatherObservation> observations = new ArrayList<>();
+            ObjectMapper mapper = new ObjectMapper();
+
+            Date firstDate = null;
+            Date lastDate = null;
+            if(all.isArray())
+            {
+                for(JsonNode node : all){
+                    Date timeMeasured = (Date)mapper.convertValue(node.get("timeMeasured").asText(), new TypeReference<Date>(){});
+                    if(firstDate == null || firstDate.compareTo(timeMeasured) > 0)
+                    {
+                        firstDate = timeMeasured;
+                    }
+                    if(lastDate == null || lastDate.compareTo(timeMeasured) < 0)
+                    {
+                        lastDate = timeMeasured;
+                    }
+                    //System.out.println(node.toString());
+                    WeatherObservation observation = new WeatherObservation();
+                    observation.setTimeMeasured(timeMeasured);
+                    observation.setLogIntervalId(node.get("logIntervalId").asInt());
+                    observation.setElementMeasurementTypeId(node.get("elementMeasurementTypeId").asText());
+                    observation.setValue(node.get("value").asDouble());
+                    observations.add(observation);
+                }
+
+            }
+            else
+            {
+                fail("Data input from file is not a JSON array");
+            }
+            config.setConfigParameter("calculationStart", firstDate);
+            config.setConfigParameter("calculationEnd", lastDate);
+            observations = this.validateAndSanitizeObservations(observations, firstDate);
+            config.setConfigParameter("observations", observations);
+            
+            
+            return config;
+        } catch (IOException | WeatherObservationListException | ConfigValidationException ex) {
+            ex.printStackTrace();
+            return null;
+        } 
+    }
 
     /**
      * Test of getTSWH method, of class NaerstadModel.
@@ -327,4 +423,142 @@ public class NaerstadModelTest extends TestCase {
         assertEquals(expResult, result);
 
     }
+    
+    
+    /**
+     * Attempts to fix problems with weather data sets
+     * @param observations
+     * @return
+     * @throws WeatherObservationListException
+     * @throws ConfigValidationException 
+     */
+    private List<WeatherObservation> validateAndSanitizeObservations(List<WeatherObservation> observations, Date firstTimestamp) throws WeatherObservationListException, ConfigValidationException {
+        
+        
+        WeatherUtil wUtil = new WeatherUtil();
+        
+        // First we truncate observations at end of the period, so that all
+        // parameter series end at the same time
+        // Then we fix holes
+        List<WeatherObservation> fixedObservations = wUtil.fixHourlyValuesForParameters(
+                wUtil.truncateToLastCommonObservation(
+                    wUtil.filterWeatherObservationsByParameter(
+                            observations, 
+                            new HashSet(Arrays.asList("TM","RR","UM","Q0"))
+                    )
+                ), 
+                new HashSet(Arrays.asList("TM","RR","UM","Q0")), 
+                firstTimestamp, 
+                null
+        );
+        
+        // Now we need to validate and possibly try to fix the weather data
+        List<WeatherObservation> TM = new ArrayList<>();
+        List<WeatherObservation> RR = new ArrayList<>();
+        List<WeatherObservation> UM = new ArrayList<>();
+        List<WeatherObservation> Q0 = new ArrayList<>();
+        
+        for(WeatherObservation o:fixedObservations)
+        {
+            switch(o.getElementMeasurementTypeId())
+            {
+                case WeatherElements.TEMPERATURE_MEAN:
+                    TM.add(o); 
+                    break;
+                case WeatherElements.PRECIPITATION:
+                    RR.add(o); 
+                    break;
+                case WeatherElements.RELATIVE_HUMIDITY_MEAN:
+                    UM.add(o); 
+                    break;
+                case WeatherElements.GLOBAL_RADIATION:
+                    Q0.add(o);
+                    break;
+                default:
+                    // Let it pass in silence 
+                    break;
+            }
+        }
+        
+        // BT (Leaf wetness) may be optionally calculated
+        // FM2 (Mean wind speed at 2m above ground)
+        List<WeatherObservation> BT = new ArrayList<>();
+        List<WeatherObservation> FM2 = new ArrayList<>();
+        for(WeatherObservation o:observations)
+        {
+            switch(o.getElementMeasurementTypeId())
+            {
+                case WeatherElements.LEAF_WETNESS:
+                    BT.add(o); 
+                    break;
+                
+                case WeatherElements.WIND_SPEED_2M:
+                    FM2.add(o);
+                    break;
+                default:
+                    // Let it pass in silence 
+                    break;
+            }
+        }
+                
+        // Problems with weather observations
+        //System.out.println("BT=" + BT.size() + " [First=," + BT.get(0).getTimeMeasured() + " last=" + BT.get(BT.size()-1).getTimeMeasured() + "]");
+        BT = wUtil.checkForAndFixHourlyTimeSeriesHoles(BT);
+        
+        // Unequal length of lists
+        if  (        
+                RR.size() != TM.size()
+                || BT.size() != TM.size()
+                || RR.size() != TM.size()
+            )
+        {
+            UM = wUtil.checkForAndFixHourlyTimeSeriesHoles(UM);
+            // Fallback if missing leaf wetness: If we have relative humidity. leaf wetness may be calculated
+            if(BT.size() != TM.size() && UM.size() == TM.size())
+            {
+                //Collections.sort(BT);
+                //System.out.println("BT=" + BT.size() + " [First=," + BT.get(0).getTimeMeasured() + " last=" + BT.get(BT.size()-1).getTimeMeasured() + "]");
+                FM2 = wUtil.checkForAndFixHourlyTimeSeriesHoles(FM2);
+                BT = wUtil.calculateLeafWetnessHourSeriesBestEffort(BT,TM, RR, Q0, FM2, UM);
+                
+                if(BT.size() != TM.size())
+                {
+                    // Last attempt, trying to fix minor problems:
+                    fixedObservations.addAll(BT);
+                    fixedObservations = wUtil.fixHourlyValuesForParameters(
+                            fixedObservations, 
+                            new HashSet(Arrays.asList("TM","RR","UM","Q0","BT")), 
+                            firstTimestamp, 
+                            null
+                    );
+                    BT = wUtil.filterWeatherObservationsByParameter(fixedObservations, new HashSet(Arrays.asList("BT")));
+                            
+                    /* 
+                    // DEBUG
+                    Collections.sort(BT);Collections.sort(TM);
+                            
+                    System.out.println("BT=" + BT.size() + " [First=," + BT.get(0).getTimeMeasured() + " last=" + BT.get(BT.size()-1).getTimeMeasured() 
+                            + "], TM=" + TM.size()+ " [First=" + TM.get(0).getTimeMeasured() + ", last=" + TM.get(TM.size()-1).getTimeMeasured() + "]");
+                    */
+                    if(BT.size() != TM.size())
+                    {
+                        throw new ConfigValidationException("Missing leaf wetness data. Also, attempting to calculate leaf wetness from other weather parameters failed.");
+                    }
+                }
+            }
+            else
+            {
+                throw new ConfigValidationException("Incorrect number of weather data. TM.size() = " + TM.size() + ", BT.size()=" + BT.size() +  ", RR.size()=" + RR.size() );
+            }
+        }
+        List<WeatherObservation> retVal = new ArrayList<>();
+        retVal.addAll(TM);
+        retVal.addAll(RR);
+        retVal.addAll(BT);
+        retVal.addAll(UM);
+        retVal.addAll(Q0);
+        retVal = wUtil.truncateToLastCommonObservation(retVal);
+        return retVal;
+    }
+
 }