diff --git a/src/test/java/no/nibio/vips/model/naerstadmodel/NaerstadModelTest.java b/src/test/java/no/nibio/vips/model/naerstadmodel/NaerstadModelTest.java index aa8f4296a60def81e31127c78a22a74bde1c271b..320d257e2afd68345be0c68c05012c61eb99b9ba 100644 --- a/src/test/java/no/nibio/vips/model/naerstadmodel/NaerstadModelTest.java +++ b/src/test/java/no/nibio/vips/model/naerstadmodel/NaerstadModelTest.java @@ -26,10 +26,15 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.MappingJsonFactory; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Calendar; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.TimeZone; import static junit.framework.Assert.fail; @@ -40,6 +45,9 @@ import no.nibio.vips.entity.WeatherObservation; import no.nibio.vips.model.ConfigValidationException; import no.nibio.vips.model.ModelExcecutionException; import no.nibio.vips.util.JSONUtil; +import no.nibio.vips.util.WeatherElements; +import no.nibio.vips.util.WeatherObservationListException; +import no.nibio.vips.util.WeatherUtil; /** @@ -61,6 +69,41 @@ public class NaerstadModelTest extends TestCase { protected void tearDown() throws Exception { super.tearDown(); } + + /** + * This is not a test!! + */ + public void donttestMladenData() { + try { + System.out.println("testMladenData"); + NaerstadModel instance; + + List<Result> results; + ObjectMapper mapper = new ObjectMapper(); + for(int year=2006;year<=2016;year++) + { + // Bad weather data files + if(year == 2006 || year == 2009 || year == 2013) + { + continue; + } + System.out.println("testMladenData for " + year); + instance = new NaerstadModel(); + instance.setConfiguration(this.getConfiguration(new File("/home/treinar/prosjekter/vips/projects/2016_irland/maynooth_" + year + ".json"))); + FileOutputStream out = new FileOutputStream(new File("/home/treinar/prosjekter/vips/projects/2016_irland/results_maynooth_" + year + ".json")); + results = instance.getResult(); + mapper.writeValue(out, results); + /*for(Result result:results) + { + System.out.println(result.toString()); + }*/ + } + + + } catch (ConfigValidationException | ModelExcecutionException | IOException ex) { + fail("Exception: " + ex.getMessage()); + } + } /** * Test of getResult method, of class NaerstadModel. @@ -227,6 +270,59 @@ public class NaerstadModelTest extends TestCase { return null; } } + + private ModelConfiguration getConfiguration(File dataFile) + { + try { + ModelConfiguration config = new ModelConfiguration(); + config.setModelId("NAERSTADMO"); + BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(dataFile)); + JsonFactory f = new MappingJsonFactory(); + JsonParser jp = f.createParser(inputStream); + JsonNode all = jp.readValueAsTree(); + List<WeatherObservation> observations = new ArrayList<>(); + ObjectMapper mapper = new ObjectMapper(); + + Date firstDate = null; + Date lastDate = null; + if(all.isArray()) + { + for(JsonNode node : all){ + Date timeMeasured = (Date)mapper.convertValue(node.get("timeMeasured").asText(), new TypeReference<Date>(){}); + if(firstDate == null || firstDate.compareTo(timeMeasured) > 0) + { + firstDate = timeMeasured; + } + if(lastDate == null || lastDate.compareTo(timeMeasured) < 0) + { + lastDate = timeMeasured; + } + //System.out.println(node.toString()); + WeatherObservation observation = new WeatherObservation(); + observation.setTimeMeasured(timeMeasured); + observation.setLogIntervalId(node.get("logIntervalId").asInt()); + observation.setElementMeasurementTypeId(node.get("elementMeasurementTypeId").asText()); + observation.setValue(node.get("value").asDouble()); + observations.add(observation); + } + + } + else + { + fail("Data input from file is not a JSON array"); + } + config.setConfigParameter("calculationStart", firstDate); + config.setConfigParameter("calculationEnd", lastDate); + observations = this.validateAndSanitizeObservations(observations, firstDate); + config.setConfigParameter("observations", observations); + + + return config; + } catch (IOException | WeatherObservationListException | ConfigValidationException ex) { + ex.printStackTrace(); + return null; + } + } /** * Test of getTSWH method, of class NaerstadModel. @@ -327,4 +423,142 @@ public class NaerstadModelTest extends TestCase { assertEquals(expResult, result); } + + + /** + * Attempts to fix problems with weather data sets + * @param observations + * @return + * @throws WeatherObservationListException + * @throws ConfigValidationException + */ + private List<WeatherObservation> validateAndSanitizeObservations(List<WeatherObservation> observations, Date firstTimestamp) throws WeatherObservationListException, ConfigValidationException { + + + WeatherUtil wUtil = new WeatherUtil(); + + // First we truncate observations at end of the period, so that all + // parameter series end at the same time + // Then we fix holes + List<WeatherObservation> fixedObservations = wUtil.fixHourlyValuesForParameters( + wUtil.truncateToLastCommonObservation( + wUtil.filterWeatherObservationsByParameter( + observations, + new HashSet(Arrays.asList("TM","RR","UM","Q0")) + ) + ), + new HashSet(Arrays.asList("TM","RR","UM","Q0")), + firstTimestamp, + null + ); + + // Now we need to validate and possibly try to fix the weather data + List<WeatherObservation> TM = new ArrayList<>(); + List<WeatherObservation> RR = new ArrayList<>(); + List<WeatherObservation> UM = new ArrayList<>(); + List<WeatherObservation> Q0 = new ArrayList<>(); + + for(WeatherObservation o:fixedObservations) + { + switch(o.getElementMeasurementTypeId()) + { + case WeatherElements.TEMPERATURE_MEAN: + TM.add(o); + break; + case WeatherElements.PRECIPITATION: + RR.add(o); + break; + case WeatherElements.RELATIVE_HUMIDITY_MEAN: + UM.add(o); + break; + case WeatherElements.GLOBAL_RADIATION: + Q0.add(o); + break; + default: + // Let it pass in silence + break; + } + } + + // BT (Leaf wetness) may be optionally calculated + // FM2 (Mean wind speed at 2m above ground) + List<WeatherObservation> BT = new ArrayList<>(); + List<WeatherObservation> FM2 = new ArrayList<>(); + for(WeatherObservation o:observations) + { + switch(o.getElementMeasurementTypeId()) + { + case WeatherElements.LEAF_WETNESS: + BT.add(o); + break; + + case WeatherElements.WIND_SPEED_2M: + FM2.add(o); + break; + default: + // Let it pass in silence + break; + } + } + + // Problems with weather observations + //System.out.println("BT=" + BT.size() + " [First=," + BT.get(0).getTimeMeasured() + " last=" + BT.get(BT.size()-1).getTimeMeasured() + "]"); + BT = wUtil.checkForAndFixHourlyTimeSeriesHoles(BT); + + // Unequal length of lists + if ( + RR.size() != TM.size() + || BT.size() != TM.size() + || RR.size() != TM.size() + ) + { + UM = wUtil.checkForAndFixHourlyTimeSeriesHoles(UM); + // Fallback if missing leaf wetness: If we have relative humidity. leaf wetness may be calculated + if(BT.size() != TM.size() && UM.size() == TM.size()) + { + //Collections.sort(BT); + //System.out.println("BT=" + BT.size() + " [First=," + BT.get(0).getTimeMeasured() + " last=" + BT.get(BT.size()-1).getTimeMeasured() + "]"); + FM2 = wUtil.checkForAndFixHourlyTimeSeriesHoles(FM2); + BT = wUtil.calculateLeafWetnessHourSeriesBestEffort(BT,TM, RR, Q0, FM2, UM); + + if(BT.size() != TM.size()) + { + // Last attempt, trying to fix minor problems: + fixedObservations.addAll(BT); + fixedObservations = wUtil.fixHourlyValuesForParameters( + fixedObservations, + new HashSet(Arrays.asList("TM","RR","UM","Q0","BT")), + firstTimestamp, + null + ); + BT = wUtil.filterWeatherObservationsByParameter(fixedObservations, new HashSet(Arrays.asList("BT"))); + + /* + // DEBUG + Collections.sort(BT);Collections.sort(TM); + + System.out.println("BT=" + BT.size() + " [First=," + BT.get(0).getTimeMeasured() + " last=" + BT.get(BT.size()-1).getTimeMeasured() + + "], TM=" + TM.size()+ " [First=" + TM.get(0).getTimeMeasured() + ", last=" + TM.get(TM.size()-1).getTimeMeasured() + "]"); + */ + if(BT.size() != TM.size()) + { + throw new ConfigValidationException("Missing leaf wetness data. Also, attempting to calculate leaf wetness from other weather parameters failed."); + } + } + } + else + { + throw new ConfigValidationException("Incorrect number of weather data. TM.size() = " + TM.size() + ", BT.size()=" + BT.size() + ", RR.size()=" + RR.size() ); + } + } + List<WeatherObservation> retVal = new ArrayList<>(); + retVal.addAll(TM); + retVal.addAll(RR); + retVal.addAll(BT); + retVal.addAll(UM); + retVal.addAll(Q0); + retVal = wUtil.truncateToLastCommonObservation(retVal); + return retVal; + } + }