-
Tor-Einar Skog authoredTor-Einar Skog authored
models.py 26.02 KiB
# coding: utf-8
#
# Copyright (c) 2014 NIBIO <http://www.nibio.no/>.
#
# This file is part of VIPSWeb.
# VIPSWeb is free software: you can redistribute it and/or modify
# it under the terms of the NIBIO Open Source License as published by
# NIBIO, either version 1 of the License, or (at your option) any
# later version.
#
# VIPSWeb is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# NIBIO Open Source License for more details.
#
# You should have received a copy of the NIBIO Open Source License
# along with VIPSWeb. If not, see <http://www.nibio.no/licenses/>.
#
import math
import requests
import json
from decimal import Decimal, InvalidOperation
from django.utils.translation import ugettext as _
from django.conf import settings
from django.db import models
from django.utils import translation
from django.core.exceptions import ValidationError
from organisms.models import Organism
from common import util
# Create your models here.
"""
Represents a single result from the running of a forecasting model
"""
class ForecastResult:
def __init__(self,forecast_result_id,valid_time_start,warning_status,all_values):
self.forecast_result_id = forecast_result_id
#self.valid_time_start = datetime.fromtimestamp(valid_time_start/1000)
self.valid_time_start = util.parse_json_timestamp(valid_time_start)
self.warning_status = warning_status
self.all_values = all_values
"""
Getting information from persistence layer
Currently this is a REST service
"""
@staticmethod
def get_forecast_results(forecast_configuration_id, user_uuid, latest_days=30):
forecastResults = []
for item in ForecastResult.get_forecast_results_as_json(forecast_configuration_id, user_uuid, latest_days):
forecastResults.append(ForecastResult.get_instance_from_dict(item))
return forecastResults
@staticmethod
def get_forecast_results_as_json(forecast_configuration_id, user_uuid, latest_days=30):
auth_param = ""
if user_uuid != None:
auth_param = "?userUUID=%s" % user_uuid
requestResult = requests.get("%s://%s/rest/forecastresults/%s/%s%s" % (settings.VIPSLOGIC_PROTOCOL, settings.VIPSLOGIC_SERVER_NAME,forecast_configuration_id, latest_days, auth_param))
#print requestResult.json()
return requestResult.json()
@staticmethod
def get_instance_from_dict(theDict):
return ForecastResult(theDict["forecastResultId"],theDict["validTimeStart"],theDict["warningStatus"],theDict["allValues"])
"""
Creates plot bands for the warning statuses
"""
@staticmethod
def get_forecast_warning_statuses_highcharts(forecast_results):
warning_statuses = {
0: {"name":"No forecast available", "color":"#C9C9C9"},
1: {"name":"Missing data", "color":"#4DA0FE"},
2: {"name":"No risk", "color":"#6FC49A"},
3: {"name":"Medium risk","color":"#FEDA4D"},
4: {"name":"High risk","color":"#F0564D"}
}
plot_bands = []
previous_forecast_result = None
plot_band = None
band_offset = 0
if len(forecast_results) >=2:
band_offset = (util.get_unix_timestamp(forecast_results[1].valid_time_start) - util.get_unix_timestamp(forecast_results[0].valid_time_start)) / 2
for forecast_result in forecast_results:
if previous_forecast_result == None or previous_forecast_result.warning_status != forecast_result.warning_status:
# Add the previous plot band
if plot_band != None:
plot_band["to"] = util.get_unix_timestamp(forecast_result.valid_time_start) - band_offset
plot_bands.append(plot_band)
plot_band = {
"color": warning_statuses[forecast_result.warning_status]["color"],
"from" : util.get_unix_timestamp(forecast_result.valid_time_start) - band_offset
}
previous_forecast_result = forecast_result
# We probably have a rouge plot band that needs to be finished and appended
if plot_band != None and plot_band.get("to", None) == None:
plot_band["to"] = util.get_unix_timestamp(forecast_results[-1].valid_time_start) + band_offset
plot_bands.append(plot_band)
return plot_bands
@staticmethod
def get_forecast_results_highcharts(forecast_results, forecast_id, user_uuid):
#print forecast_results
# We must narrow this down to only the most important ones
# First: Get modelId
forecast_configuration = ForecastConfiguration.get_forecast_configuration(forecast_id, user_uuid)
if forecast_configuration == None:
return None
#print "Result 2: %s seconds from start" % (time.time() - start)
model_graph_result_parameters = ModelGraphParameter.objects.filter(model_id =forecast_configuration.model_id)
#print "Result 3: %s seconds from start" % (time.time() - start)
highcharts_dict = {}
timeseries = {}
y_axis_list = []
# First: Check if model has a graph configuration
# If not: display all result parameters (default view)
# Setting up all parallel time series
for model_graph_result_parameter in model_graph_result_parameters:
result_parameter = model_graph_result_parameter.resultparameter
result_parameter_local_set = ResultParameterLocal.objects.filter(result_parameter_id=result_parameter.id)
if len(result_parameter_local_set) > 0:
for language_code in [translation.get_language(), settings.LANGUAGE_CODE,'en']:
for result_parameter_local in result_parameter_local_set:
if result_parameter_local.language_code == language_code:
result_parameter.set_local_name(result_parameter_local.local_name)
result_parameter_timeseries = {}
result_parameter_timeseries["name"] = result_parameter.get_local_name
result_parameter_timeseries["type"] = model_graph_result_parameter.highcharts_type.name
result_parameter_timeseries["data"] = []
#TODO: Dynamic connection of parameters and axis
result_parameter_timeseries["yAxis"] = get_y_axis_for_parameter(result_parameter, y_axis_list)
timeseries[result_parameter.getNamespaceKey()] = result_parameter_timeseries
if model_graph_result_parameter.color_hexcode != None:
if model_graph_result_parameter.color_hexcode[0] != "#":
model_graph_result_parameter.color_hexcode = "#" + model_graph_result_parameter.color_hexcode
timeseries[result_parameter.getNamespaceKey()]["color"] = model_graph_result_parameter.color_hexcode
for forecast_result in forecast_results:
for model_graph_result_parameter in model_graph_result_parameters:
result_parameter = model_graph_result_parameter.resultparameter
the_value = forecast_result.all_values.get(result_parameter.getNamespaceKey(), "")
if the_value != "" and timeseries.get(result_parameter.getNamespaceKey(), None) != None:
try:
data_point = [util.get_unix_timestamp(forecast_result.valid_time_start),Decimal(the_value)]
timeseries[result_parameter.getNamespaceKey()]["data"].append(data_point)
except InvalidOperation:
continue
highcharts_dict["series"] = timeseries
highcharts_dict["yAxis"] = y_axis_list
the_response = json.dumps(highcharts_dict,default=default_decimal)
return the_response
class ForecastConfiguration:
def __init__(self,
forecast_configuration_id,
model_id, date_start,
date_end,
location_point_of_interest,
weather_station_point_of_interest,
user_id = None,
crop_organism = None,
pest_organism = None):
self.forecast_configuration_id = forecast_configuration_id
self.model_id = model_id
self.date_start = date_start
self.date_end = date_end
self.location_point_of_interest = location_point_of_interest#PointOfInterest(location_point_of_interest)
self.weather_station_point_of_interest = weather_station_point_of_interest
self.user_id = user_id
self.crop_organism = crop_organism
self.pest_organism = pest_organism
self.forecast_summaries = []
def set_model_local_name(self, model_local_name):
self.model_local_name = model_local_name
def set_model_local_warning_status_interpretation(self, warning_status_interpretation):
self.model_local_warning_status_interpretation = warning_status_interpretation
def set_forecast_summaries(self, forecast_summaries):
self.forecast_summaries = forecast_summaries
"""
Getting information from persistence layer
Currently this is a REST service
"""
@staticmethod
def get_forecast_configurations(crop_organism_ids):
forecasts = []
model_local_names = {}
for item in ForecastConfiguration.get_forecast_configurations_as_json(None):
forecast = ForecastConfiguration.get_instance_from_dict(item)
# Get the name of the model in the current locale
if model_local_names.get(forecast.model_id, None) == None:
model_local_names[forecast.model_id] = Model.get_local_name_for_model(forecast.model_id)
forecast.set_model_local_name(model_local_names[forecast.model_id])
# Append forecast to list
forecasts.append(forecast)
return forecasts
@staticmethod
def get_forecast_configurations_as_json(crop_organism_ids):
return ForecastConfiguration.get_forecast_configurations_from_vipslogic(crop_organism_ids).json()
# Returns a result object
@staticmethod
def get_forecast_configurations_from_vipslogic(crop_organism_ids, season):
crop_organism_id_paramstring = ""
if crop_organism_ids != None:
for crop_organism_id in crop_organism_ids:
crop_organism_id_paramstring += "&cropOrganismId=%s" % crop_organism_id
print "%s://%s/rest/organizationforecastconfigurations/%s?from=%s-01-01&to=%s-12-31%s" % (
settings.VIPSLOGIC_PROTOCOL,
settings.VIPSLOGIC_SERVER_NAME,
settings.VIPS_ORGANIZATION_ID,
season,
season,
crop_organism_id_paramstring
)
request_result = requests.get("%s://%s/rest/organizationforecastconfigurations/%s?from=%s-01-01&to=%s-12-31%s" % (
settings.VIPSLOGIC_PROTOCOL,
settings.VIPSLOGIC_SERVER_NAME,
settings.VIPS_ORGANIZATION_ID,
season,
season,
crop_organism_id_paramstring
)
)
return request_result
@staticmethod
def get_private_forecast_configurations(user_uuid):
if user_uuid == None:
return []
forecasts = []
request_result = requests.get("%s://%s/rest/forecastconfigurations/private/%s" % (settings.VIPSLOGIC_PROTOCOL, settings.VIPSLOGIC_SERVER_NAME, user_uuid))
if request_result.status_code != 200:
return None
for item in request_result.json():
forecasts.append(ForecastConfiguration.get_instance_from_dict(item))
return forecasts
@staticmethod
def get_forecast_configuration(forecast_configuration_id, user_uuid):
forecast_configuration = ForecastConfiguration.get_instance_from_dict(ForecastConfiguration.get_forecast_configuration_as_json(forecast_configuration_id, user_uuid))
if forecast_configuration == None:
return None
forecast_configuration.set_model_local_name(Model.get_local_name_for_model(forecast_configuration.model_id))
forecast_configuration.set_model_local_warning_status_interpretation(Model.get_local_warning_status_interpretation_for_model(forecast_configuration.model_id))
return forecast_configuration
@staticmethod
def get_forecast_configuration_as_json(forecast_configuration_id, user_uuid):
auth_param = ""
if user_uuid != None:
auth_param = "?userUUID=%s" % user_uuid
requestResult = requests.get("%s://%s/rest/forecastconfigurations/%s%s" % (settings.VIPSLOGIC_PROTOCOL, settings.VIPSLOGIC_SERVER_NAME, forecast_configuration_id, auth_param))
try:
return requestResult.json()
except ValueError:
return None
@staticmethod
def get_forecast_configurations_with_summaries(crop_organism_ids):
forecasts = []
model_local_names = {}
for item in ForecastConfiguration.get_forecast_configurations_with_summaries_as_json(None):
forecast = ForecastConfiguration.get_instance_from_dict(item)
forecast.set_forecast_summaries(item.get("forecastSummaries", []))
# Get the name of the model in the current locale
if model_local_names.get(forecast.model_id, None) == None:
model_local_names[forecast.model_id] = Model.get_local_name_for_model(forecast.model_id)
forecast.set_model_local_name(model_local_names[forecast.model_id])
# Append forecast to list
forecasts.append(forecast)
return forecasts
@staticmethod
def get_forecast_configurations_with_summaries_as_json(crop_organism_ids):
"""
crop_organism_id_paramstring = ""
if crop_organism_ids != None:
for crop_organism_id in crop_organism_ids:
crop_organism_id_paramstring += "&cropOrganismId=%s" % crop_organism_id
"""
request_result = requests.get("%s://%s/rest/forecastconfigurationsummaries/%s" % (
settings.VIPSLOGIC_PROTOCOL,
settings.VIPSLOGIC_SERVER_NAME,
settings.VIPS_ORGANIZATION_ID
)
)
return request_result.json()
@staticmethod
def get_instance_from_dict(theDict):
if theDict == None:
return None
instance = ForecastConfiguration(
theDict["forecastConfigurationId"],
theDict["modelId"],
theDict["dateStart"],
theDict["dateEnd"],
PointOfInterest(theDict["locationPointOfInterestId"]),
PointOfInterest(theDict["weatherStationPointOfInterestId"]),
theDict.get("vipsLogicUserId",None),
Organism.get_instance_from_dict(theDict.get("cropOrganismId", None)),
Organism.get_instance_from_dict(theDict.get("pestOrganismId", None))
)
instance.set_model_local_name(Model.get_local_name_for_model(instance.model_id))
return instance
"""
A geographical point. Could be i.e. a weatherstation or a field. Fetched from VIPSLogic backend (REST)
"""
class PointOfInterest:
def __init__(self, dictionary):
self.point_of_interest_id = dictionary.get("pointOfInterestId", None)
self.name = dictionary.get("name", None)
self.time_zone = dictionary.get("timeZone",None)
# The location is served as a GEOJson FeatureCollection
geo_json_str = dictionary.get("geoJSON", None)
if geo_json_str is not None:
geo_json = json.loads(geo_json_str)
coordinate = geo_json["features"][0]["geometry"]["coordinates"]
self.longitude = coordinate[0]
self.latitude = coordinate[1]
self.altitude = coordinate[2]
else: # Keeping legacy compatibility
self.longitude = dictionary.get("longitude", None)
self.latitude = dictionary.get("latitude", None)
self.altitude = dictionary.get("altitude", None)
countrycode_dict = dictionary.get("countryCode", None)
if countrycode_dict != None:
self.countryCode = countrycode_dict.get("countryCode", None)
point_of_interest_type_dict = dictionary.get("pointOfInterestType", None)
if point_of_interest_type_dict != None:
self.point_of_interest_type_id = point_of_interest_type_dict.get("pointOfInterestTypeId", None)
self.point_of_interest_type_name = point_of_interest_type_dict.get("defaultName", None)
## Weather station specific stuff
self.weather_station_remote_id = dictionary.get("weatherStationRemoteId", None)
self.data_fetch_uri = dictionary.get("dataFetchUri", None)
weather_station_data_source_dict = dictionary.get("weatherStationDataSourceId", None)
if weather_station_data_source_dict != None:
self.weather_station_data_source_id = weather_station_data_source_dict.get("weatherStationDataSourceId", None)
self.weather_station_data_source_name = weather_station_data_source_dict.get("name", None)
self.weather_station_data_source_default_description = weather_station_data_source_dict.get("defaultDescription", None)
self.weather_station_data_source_uri = weather_station_data_source_dict.get("uri", None)
self.weather_station_data_source_uri_expression = weather_station_data_source_dict.get("datafetchUriExpression", None)
"""
Calculates distance between self and other poi in kilometers
Method found here: <a href="http://www.movable-type.co.uk/scripts/latlong.html">herfra</a>. Description as follows:
This uses the "haversine" formula to calculate the great-circle distance between two points ? that is, the shortest distance over the earth?s surface ? giving an ?as-the-crow-flies? distance between the points (ignoring any hills, of course!).
Haversine formula:
a = sin²(Δlat/2) + cos(lat1).cos(lat2).sin²(Δlong/2)
c = 2*atan2(a, (1-a))
d = R*c
where R is earth's radius (mean radius = 6,371km);
Note that angles need to be in radians to pass to trig functions!
"""
def distance_to(self, point_of_interest):
R = 6371 # Earth's radius in km
dLat = math.radians(point_of_interest.latitude - self.latitude)
dLon = math.radians(point_of_interest.longitude - self.longitude)
lat1 = math.radians(self.latitude)
lat2 = math.radians(point_of_interest.latitude)
a = math.sin(dLat/2) * math.sin(dLat/2) + math.sin(dLon/2) * math.sin(dLon/2) * math.cos(lat1) * math.cos(lat2)
c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a))
d = R * c
return d
"""
Information about the forecasting model. Fetched from VIPSCoreManager backend (REST)
"""
class Model:
def __init__(self, model_id, local_name, description,warning_status_interpretation, usage,sample_config):
self.model_id=model_id
self.local_name=local_name
self.description=description
self.warning_status_interpretation=warning_status_interpretation
self.usage=usage
self.sample_config=sample_config
"""
Fetch model from backend.
TODO: Maybe there should be a method in Core/CoreManager that returns
all properties as JSON?
"""
@staticmethod
def get_model(model_id):
cur_language = translation.get_language()
local_name = Model.get_local_name_for_model(model_id)
description = requests.get("http://%s/models/%s/description/%s" % (settings.VIPSCOREMANAGER_SERVER_NAME, model_id, cur_language), verify=settings.VIPSCOREMANAGER_SSLCERT_PATH).text
warning_status_interpretation = requests.get("http://%s/models/%s/warningstatusinterpretation/%s" % (settings.VIPSCOREMANAGER_SERVER_NAME, model_id, cur_language), verify=settings.VIPSCOREMANAGER_SSLCERT_PATH).text
usage = requests.get("http://%s/models/%s/usage/%s" % (settings.VIPSCOREMANAGER_SERVER_NAME, model_id, cur_language), verify=settings.VIPSCOREMANAGER_SSLCERT_PATH).text
sample_config = requests.get("http://%s/models/%s/sampleconfig" % (settings.VIPSCOREMANAGER_SERVER_NAME, model_id), verify=settings.VIPSCOREMANAGER_SSLCERT_PATH).text
return Model(model_id, local_name, description, warning_status_interpretation, usage, sample_config)
@staticmethod
def get_local_name_for_model(model_id):
cur_language = translation.get_language()
return requests.get("http://%s/models/%s/name/%s" % (settings.VIPSCOREMANAGER_SERVER_NAME, model_id, cur_language), verify=settings.VIPSCOREMANAGER_SSLCERT_PATH).text
@staticmethod
def get_local_warning_status_interpretation_for_model(model_id):
cur_language = translation.get_language()
return requests.get("http://%s/models/%s/warningstatusinterpretation/%s" % (settings.VIPSCOREMANAGER_SERVER_NAME, model_id, cur_language), verify=settings.VIPSCOREMANAGER_SSLCERT_PATH).text
@staticmethod
def get_models_local_names():
cur_language = translation.get_language()
model_ids = requests.get("http://%s/models/json/%s" % (settings.VIPSCOREMANAGER_SERVER_NAME,cur_language), verify=settings.VIPSCOREMANAGER_SSLCERT_PATH)
#print model_ids.text
return model_ids
@staticmethod
def get_models_local_names_as_json():
return Model.get_models_local_names().json()
# Util method for graph definition
def get_y_axis_for_parameter(result_parameter, axis_list):
# Get the ResultParameter from database
# If not found, fallback to "Number" as measurement unit
result = ResultParameter.objects.filter(id=result_parameter.id)
measurement_unit = {}
if len(result) == 1:
parameter = result[0]
measurement_unit = parameter.measurementunit
else:
measurement_unit = MeasurementUnit.objects.filter(name="Number")[0]
# Does it exist in axis_list?
for index,axis in enumerate(axis_list):
if(axis["title"] == _(measurement_unit.name)):
return index
# If not, add to axis_list
y_axis = {"title":_(measurement_unit.name), "abbr":measurement_unit.abbr,"color":measurement_unit.color_hexcode,"opposite":measurement_unit.opposite}
axis_list.append(y_axis)
# Return axis index
return len(axis_list) - 1
def default_decimal(obj):
if isinstance(obj, Decimal):
return float(obj)
raise TypeError
"""
STUFF IN LOCAL DATABASE
"""
class MeasurementUnit(models.Model):
name = models.CharField(max_length=200)
abbr = models.CharField(max_length=50)
color_hexcode = models.CharField(max_length=7)
opposite = models.BooleanField(default=False)
def __unicode__(self):
return self.name
class Meta:
ordering = ['name']
class ResultParameter(models.Model):
namespace = models.CharField(max_length=50)
key = models.CharField(max_length=50)
name = models.CharField(max_length=200)
description = models.TextField()
measurementunit = models.ForeignKey(MeasurementUnit)
local_name = None
def getNamespaceKey(self):
return self.namespace + "." + self.key
def __unicode__(self):
return self.namespace + "." + self.key
class Meta:
ordering = ['key']
def set_local_name(self, local_name):
self.local_name = local_name
@property
def get_local_name(self):
if self.local_name != None:
return self.local_name
else:
return self.name
class ResultParameterLocal(models.Model):
result_parameter = models.ForeignKey(ResultParameter)
local_name = models.CharField(max_length=200)
language_code = models.CharField(max_length=2, verbose_name=_("Language code"))
def validate_model_id_length(value):
if len(value) != 10:
raise ValidationError(u'%s must be 10 characters' % (value))
class HighChartsType(models.Model):
name = models.CharField(max_length=63)
def __unicode__(self):
return self.name
class ModelGraphParameter(models.Model):
model_id = models.CharField(max_length=10, validators=[validate_model_id_length])
resultparameter = models.ForeignKey(ResultParameter)
color_hexcode = models.CharField(max_length=6)
highcharts_type = models.ForeignKey(HighChartsType)
def __unicode__(self):
return self.model_id + "/" + self.resultparameter.getNamespaceKey()