Skip to content
Snippets Groups Projects
Commit 889096f1 authored by Tor-Einar Skog's avatar Tor-Einar Skog
Browse files

Major timeZone awareness change and fix of weather data treatment in NaerstadModelPreprocessor

parent 49f91914
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 173 additions and 22 deletions
src/main/java/no/bioforsk/vips/logic/controller/servlet/ForecastConfigurationController.java
+ 6
− 0
View file @ 889096f1
......@@ -29,6 +29,7 @@ import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.servlet.ServletException;
......@@ -43,6 +44,7 @@ import no.bioforsk.vips.logic.entity.Organization;
import no.bioforsk.vips.logic.entity.VipsLogicRole;
import no.bioforsk.vips.logic.entity.VipsLogicUser;
import no.bioforsk.vips.logic.i18n.SessionLocaleUtil;
import no.bioforsk.vips.logic.util.Globals;
import no.bioforsk.vips.logic.util.SessionControllerGetter;
import no.bioforsk.vips.logic.util.SystemTime;
import no.bioforsk.vips.util.ArrayUtil;
......@@ -197,6 +199,8 @@ public class ForecastConfigurationController extends HttpServlet {
request.setAttribute("vipsLogicUsers", em.createNamedQuery("VipsLogicUser.findByOrganizationId").setParameter("organizationId", user.getOrganizationId()).getResultList());
}
request.setAttribute("forecastConfiguration", forecastConfiguration);
request.getSession().setAttribute("availableTimeZones", Globals.availableTimeZones);
request.getSession().setAttribute("defaultTimeZoneId", user.getOrganizationId().getDefaultTimeZone());
// All organisms used for parent organism list
request.setAttribute("allCrops", em.createNamedQuery("Organism.findAllCrops").getResultList());
request.setAttribute("allPests", em.createNamedQuery("Organism.findAllPests").getResultList());
......@@ -275,6 +279,8 @@ public class ForecastConfigurationController extends HttpServlet {
request.setAttribute("vipsLogicUsers", em.createNamedQuery("VipsLogicUser.findAll").getResultList());
request.setAttribute("dateStart_dateFormat", formFields.get("dateStart").getDateFormat());
request.setAttribute("dateEnd_dateFormat", formFields.get("dateEnd").getDateFormat());
request.getSession().setAttribute("availableTimeZones", Globals.availableTimeZones);
request.getSession().setAttribute("defaultTimeZoneId", user.getOrganizationId().getDefaultTimeZone());
request.setAttribute("modelInformations", em.createNamedQuery("ModelInformation.findAll").getResultList());
request.setAttribute("forecastConfiguration", forecastConfiguration);
request.getRequestDispatcher("/forecastConfigurationForm.ftl").forward(request, response);
......
src/main/java/no/bioforsk/vips/logic/controller/servlet/PointOfInterestController.java
+ 5
− 5
View file @ 889096f1
......@@ -25,7 +25,6 @@ import com.vividsolutions.jts.geom.Point;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.servlet.ServletException;
......@@ -47,6 +46,7 @@ import no.bioforsk.vips.logic.util.SessionControllerGetter;
import no.bioforsk.vips.util.ExceptionUtil;
import no.bioforsk.vips.util.ServletUtil;
import no.bioforsk.vips.logic.entity.WeatherForecastProvider;
import no.bioforsk.vips.logic.util.Globals;
import no.bioforsk.web.forms.FormField;
import no.bioforsk.web.forms.FormValidation;
import no.bioforsk.web.forms.FormValidationException;
......@@ -157,7 +157,7 @@ public class PointOfInterestController extends HttpServlet {
{
request.getSession().setAttribute("weatherStation", weatherStation);
}
request.getSession().setAttribute("availableTimeZones", TimeZone.getAvailableIDs());
request.getSession().setAttribute("availableTimeZones", Globals.availableTimeZones);
request.getSession().setAttribute("defaultTimeZoneId", user.getOrganizationId().getDefaultTimeZone());
request.setAttribute("defaultMapCenter",user.getOrganizationId().getDefaultMapCenter());
request.setAttribute("defaultMapZoom", user.getOrganizationId().getDefaultMapZoom());
......@@ -178,7 +178,7 @@ public class PointOfInterestController extends HttpServlet {
request.setAttribute("defaultMapCenter",user.getOrganizationId().getDefaultMapCenter());
request.setAttribute("defaultMapZoom", user.getOrganizationId().getDefaultMapZoom());
request.getSession().setAttribute("dataSources", SessionControllerGetter.getPointOfInterestBean().getWeatherStationDataSources());
request.getSession().setAttribute("availableTimeZones", TimeZone.getAvailableIDs());
request.getSession().setAttribute("availableTimeZones", Globals.availableTimeZones);
request.getSession().setAttribute("defaultTimeZoneId", user.getOrganizationId().getDefaultTimeZone());
request.getSession().setAttribute("availableCountries", em.createNamedQuery("Country.findAll").getResultList());
request.getSession().setAttribute("defaultCountryCode", user.getOrganizationId().getCountryCode().getCountryCode());
......@@ -217,7 +217,7 @@ public class PointOfInterestController extends HttpServlet {
request.setAttribute("defaultMapCenter",user.getOrganizationId().getDefaultMapCenter());
request.setAttribute("defaultMapZoom", user.getOrganizationId().getDefaultMapZoom());
request.getSession().setAttribute("dataSources", SessionControllerGetter.getPointOfInterestBean().getWeatherStationDataSources());
request.getSession().setAttribute("availableTimeZones", TimeZone.getAvailableIDs());
request.getSession().setAttribute("availableTimeZones", Globals.availableTimeZones);
request.getSession().setAttribute("defaultTimeZoneId", user.getOrganizationId().getDefaultTimeZone());
request.getSession().setAttribute("availableCountries", em.createNamedQuery("Country.findAll").getResultList());
request.getSession().setAttribute("defaultCountryCode", user.getOrganizationId().getCountryCode().getCountryCode());
......@@ -320,7 +320,7 @@ public class PointOfInterestController extends HttpServlet {
request.setAttribute("defaultMapCenter",user.getOrganizationId().getDefaultMapCenter());
request.setAttribute("defaultMapZoom", user.getOrganizationId().getDefaultMapZoom());
request.getSession().setAttribute("dataSources", SessionControllerGetter.getPointOfInterestBean().getWeatherStationDataSources());
request.getSession().setAttribute("availableTimeZones", TimeZone.getAvailableIDs());
request.getSession().setAttribute("availableTimeZones", Globals.availableTimeZones);
request.getSession().setAttribute("defaultTimeZoneId", user.getOrganizationId().getDefaultTimeZone());
request.getSession().setAttribute("availableCountries", em.createNamedQuery("Country.findAll").getResultList());
request.getSession().setAttribute("defaultCountryCode", user.getOrganizationId().getCountryCode().getCountryCode());
......
src/main/java/no/bioforsk/vips/logic/controller/session/ForecastBean.java
+ 2
− 0
View file @ 889096f1
......@@ -376,6 +376,8 @@ public class ForecastBean {
forecastConfiguration.setLocationPointOfInterestId(locationPoi);
PointOfInterest weatherStationPoi = em.find(PointOfInterestWeatherStation.class, formFields.get("weatherStationPointOfInterestId").getValueAsInteger());
forecastConfiguration.setWeatherStationPointOfInterestId(weatherStationPoi);
String timeZone = formFields.get("timeZone").getWebValue();
forecastConfiguration.setTimeZone(timeZone);
forecastConfiguration.setDateStart(formFields.get("dateStart").getValueAsDate());
forecastConfiguration.setDateEnd(formFields.get("dateEnd").getValueAsDate());
VipsLogicUser forecastConfigurationUser = em.find(VipsLogicUser.class, formFields.get("vipsLogicUserId").getValueAsInteger());
......
src/main/java/no/bioforsk/vips/logic/entity/ForecastConfiguration.java
+ 77
− 0
View file @ 889096f1
......@@ -42,7 +42,9 @@ import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.List;
import java.util.TimeZone;
import javax.persistence.Transient;
import no.bioforsk.vips.util.WeatherUtil;
/**
* @copyright 2014-2015 <a href="http://www.bioforsk.no/">Bioforsk</a>
......@@ -86,6 +88,9 @@ public class ForecastConfiguration implements Serializable, Comparable {
@Column(name = "date_end")
@Temporal(TemporalType.DATE)
private Date dateEnd;
@Size(max = 255)
@Column(name = "time_zone")
private String timeZone;
@JsonIgnore
@JoinColumn(name = "vips_logic_user_id", referencedColumnName = "user_id")
@ManyToOne(optional = false)
......@@ -103,11 +108,15 @@ public class ForecastConfiguration implements Serializable, Comparable {
@ManyToOne
private Organism pestOrganismId;
@Transient
private WeatherUtil weatherUtil;
// Relations connected manually
@Transient
private List<ForecastSummary> forecastSummaries;
public ForecastConfiguration() {
}
public ForecastConfiguration(Long forecastConfigurationId) {
......@@ -291,4 +300,72 @@ public class ForecastConfiguration implements Serializable, Comparable {
public void setPestOrganismId(Organism pestOrganismId) {
this.pestOrganismId = pestOrganismId;
}
@Transient
private WeatherUtil getWeatherUtil()
{
if(this.weatherUtil == null)
{
this.weatherUtil = new WeatherUtil();
}
return this.weatherUtil;
}
/**
* @return the timeZone
*/
public String getTimeZone() {
return timeZone;
}
/**
* @param timeZone the timeZone to set
*/
public void setTimeZone(String timeZone) {
this.timeZone = timeZone;
}
@Transient
public Date getDateStartInTimeZone()
{
return this.getDateInTimeZone(this.getDateStart());
}
@Transient
public Date getDateEndInTimeZone()
{
return this.getDateInTimeZone(this.getDateEnd());
}
@Transient
public Date getDateInTimeZone(Date theDate)
{
return this.getWeatherUtil().changeDateTimeZone(
theDate,
TimeZone.getDefault(),
this.getTimeZone() != null ? TimeZone.getTimeZone(this.getTimeZone()) : TimeZone.getDefault()
);
}
/*
@Transient
public String getISOFormattedDateStart()
{
TimeZone timeZone1 = this.getTimeZone() != null ? TimeZone.getTimeZone(this.getTimeZone())
: TimeZone.getDefault();
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
format.setTimeZone(timeZone1);
return format.format(this.getDateStart());
}
@Transient
public String getISOFormattedDateEnd()
{
TimeZone timeZone1 = this.getTimeZone() != null ? TimeZone.getTimeZone(this.getTimeZone())
: TimeZone.getDefault();
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
format.setTimeZone(timeZone1);
return format.format(this.getDateEnd());
}*/
}
src/main/java/no/bioforsk/vips/logic/scheduling/model/preprocessor/NaerstadModelPreprocessor.java
+ 50
− 17
View file @ 889096f1
......@@ -24,6 +24,7 @@ import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
......@@ -75,7 +76,7 @@ public class NaerstadModelPreprocessor extends ModelRunPreprocessor{
cal.setTime(SystemTime.getSystemTime());
cal.add(Calendar.DATE, 3);
Date nearFuture = cal.getTime();
Date dateEndWeatherData = configuration.getDateEnd().compareTo(nearFuture) < 0 ? configuration.getDateEnd() : nearFuture;
Date dateEndWeatherData = configuration.getDateEndInTimeZone().compareTo(nearFuture) < 0 ? configuration.getDateEndInTimeZone(): nearFuture;
List<WeatherObservation> observations = getWeatherObservations(
weatherStation.getDataFetchUri(),
......@@ -88,12 +89,13 @@ public class NaerstadModelPreprocessor extends ModelRunPreprocessor{
WeatherElements.GLOBAL_RADIATION,
WeatherElements.WIND_SPEED_2M
},
configuration.getDateStart(),
configuration.getDateStartInTimeZone(),
dateEndWeatherData,
timeZone);
try {
observations = validateAndSanitizeObservations(observations, configuration.getDateStart());
observations = validateAndSanitizeObservations(observations, configuration.getDateStartInTimeZone());
//System.out.println(configuration.getDateStartInTimeZone());
} catch (ConfigValidationException | WeatherObservationListException ex) {
ex.printStackTrace();
throw new PreprocessorException(ex.getMessage());
......@@ -133,10 +135,20 @@ public class NaerstadModelPreprocessor extends ModelRunPreprocessor{
*/
private List<WeatherObservation> validateAndSanitizeObservations(List<WeatherObservation> observations, Date firstTimestamp) throws WeatherObservationListException, ConfigValidationException {
WeatherUtil wUtil = new WeatherUtil();
// First we truncate observations at end of the period, so that all
// parameter series end at the same time
// Then we fix holes
List<WeatherObservation> fixedObservations = wUtil.fixHourlyValuesForParameters(
observations,
new HashSet(Arrays.asList("TM","RR","UM")),
wUtil.truncateToLastCommonObservation(
wUtil.filterWeatherObservationsByParameter(
observations,
new HashSet(Arrays.asList("TM","RR","UM","Q0"))
)
),
new HashSet(Arrays.asList("TM","RR","UM","Q0")),
firstTimestamp,
null
);
......@@ -145,8 +157,9 @@ public class NaerstadModelPreprocessor extends ModelRunPreprocessor{
List<WeatherObservation> TM = new ArrayList<>();
List<WeatherObservation> RR = new ArrayList<>();
List<WeatherObservation> UM = new ArrayList<>();
List<WeatherObservation> Q0 = new ArrayList<>();
for(WeatherObservation o:observations)
for(WeatherObservation o:fixedObservations)
{
switch(o.getElementMeasurementTypeId())
{
......@@ -159,14 +172,18 @@ public class NaerstadModelPreprocessor extends ModelRunPreprocessor{
case WeatherElements.RELATIVE_HUMIDITY_MEAN:
UM.add(o);
break;
case WeatherElements.GLOBAL_RADIATION:
Q0.add(o);
break;
default:
// Let it pass in silence
break;
}
}
// BT (Leaf wetness) may be optionally calculated
// FM2 (Mean wind speed at 2m above ground)
List<WeatherObservation> BT = new ArrayList<>();
List<WeatherObservation> Q0 = new ArrayList<>();
List<WeatherObservation> FM2 = new ArrayList<>();
for(WeatherObservation o:observations)
{
......@@ -175,9 +192,7 @@ public class NaerstadModelPreprocessor extends ModelRunPreprocessor{
case WeatherElements.LEAF_WETNESS:
BT.add(o);
break;
case WeatherElements.GLOBAL_RADIATION:
Q0.add(o);
break;
case WeatherElements.WIND_SPEED_2M:
FM2.add(o);
break;
......@@ -186,12 +201,9 @@ public class NaerstadModelPreprocessor extends ModelRunPreprocessor{
break;
}
}
// Problems with weather observations
TM = wUtil.checkForAndFixHourlyTimeSeriesHoles(TM);
RR = wUtil.checkForAndFixHourlyTimeSeriesHoles(RR);
//System.out.println("BT=" + BT.size() + " [First=," + BT.get(0).getTimeMeasured() + " last=" + BT.get(BT.size()-1).getTimeMeasured() + "]");
BT = wUtil.checkForAndFixHourlyTimeSeriesHoles(BT);
// Unequal length of lists
......@@ -205,13 +217,34 @@ public class NaerstadModelPreprocessor extends ModelRunPreprocessor{
// Fallback if missing leaf wetness: If we have relative humidity. leaf wetness may be calculated
if(BT.size() != TM.size() && UM.size() == TM.size())
{
Q0 = wUtil.checkForAndFixHourlyTimeSeriesHoles(Q0);
//Collections.sort(BT);
//System.out.println("BT=" + BT.size() + " [First=," + BT.get(0).getTimeMeasured() + " last=" + BT.get(BT.size()-1).getTimeMeasured() + "]");
FM2 = wUtil.checkForAndFixHourlyTimeSeriesHoles(FM2);
BT = wUtil.calculateLeafWetnessHourSeriesBestEffort(BT,TM, RR, Q0, FM2, UM);
if(BT.size() != TM.size())
{
throw new ConfigValidationException("Missing leaf wetness data. Also, attempting to calculate leaf wetness from other weather parameters failed.");
// Last attempt, trying to fix minor problems:
fixedObservations.addAll(BT);
fixedObservations = wUtil.fixHourlyValuesForParameters(
fixedObservations,
new HashSet(Arrays.asList("TM","RR","UM","Q0","BT")),
firstTimestamp,
null
);
BT = wUtil.filterWeatherObservationsByParameter(fixedObservations, new HashSet(Arrays.asList("BT")));
/*
// DEBUG
Collections.sort(BT);Collections.sort(TM);
System.out.println("BT=" + BT.size() + " [First=," + BT.get(0).getTimeMeasured() + " last=" + BT.get(BT.size()-1).getTimeMeasured()
+ "], TM=" + TM.size()+ " [First=" + TM.get(0).getTimeMeasured() + ", last=" + TM.get(TM.size()-1).getTimeMeasured() + "]");
*/
if(BT.size() != TM.size())
{
throw new ConfigValidationException("Missing leaf wetness data. Also, attempting to calculate leaf wetness from other weather parameters failed.");
}
}
}
else
......
src/main/java/no/bioforsk/vips/logic/util/Globals.java
+ 9
− 0
View file @ 889096f1
......@@ -66,4 +66,13 @@ public class Globals {
public static String defaultTimestampFormat = "yyyy-MM-dd HH:mm:ssXXX";
public static String defaultDateFormat = "yyyy-MM-dd";
public static String[] availableTimeZones = {
"Etc/GMT-1",
"Europe/Oslo",
"Europe/Sarajevo",
"Europe/Stockholm",
"Europe/Helsinki",
"Europe/Sofia"
};
}
src/main/java/no/bioforsk/web/forms/FormField.java
+ 6
− 0
View file @ 889096f1
......@@ -126,6 +126,12 @@ public class FormField {
}
}
@JsonIgnore
public Date getValueAsDate(String timeZone)
{
return this.getValueAsDate(TimeZone.getTimeZone(timeZone));
}
@JsonIgnore
public Date getValueAsTimestamp()
{
......
src/main/webapp/formdefinitions/forecastConfigurationForm.json
+ 6
− 0
View file @ 889096f1
......@@ -74,6 +74,12 @@
"name" : "dateEnd",
"dataType" : "DATE",
"required" : true
},
{
"name" : "timeZone",
"dataType" : "STRING",
"fieldType" : "SELECT_SINGLE",
"required" : true
}
],
......
src/main/webapp/templates/forecastConfigurationForm.ftl
+ 12
− 0
View file @ 889096f1
......@@ -129,6 +129,18 @@
<input type="date" class="form-control" name="dateEnd" placeholder="${i18nBundle.dateEnd}" value="${(forecastConfiguration.dateEnd)!""}" onblur="validateField(this);" />
<span class="help-block" id="${formId}_dateEnd_validation"></span>
</div>
<div class="form-group">
<label for="timeZone">${i18nBundle.timeZone}</label>
<select class="form-control" name="timeZone" onblur="validateField(this);">
<#list availableTimeZones as timeZoneId>
<option value="${timeZoneId}"<#if
(forecastConfiguration.timeZone?has_content && timeZoneId == forecastConfiguration.timeZone)
|| (!forecastConfiguration.timeZone?has_content && timeZoneId == defaultTimeZoneId)
> selected="selected"</#if>>${timeZoneId}</option>
</#list>
</select>
<span class="help-block" id="${formId}_timeZone_validation"></span>
</div>
<div class="form-group">
<label for="vipsLogicUserId">${i18nBundle.vipsLogicUserId}</label>
<select class="form-control" name="vipsLogicUserId" onblur="validateField(this);">
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment