From a23e9cf8fafe45c84ce4a2f791acfc650fab6171 Mon Sep 17 00:00:00 2001 From: Tor-Einar Skog <tor-einar.skog@nibio.no> Date: Thu, 9 Mar 2023 15:14:48 +0100 Subject: [PATCH] VIPSLogic-2023.1 Supports multiple VIPSCore resources. [VIPSUTV-383] --- pom.xml | 2 +- .../controller/session/ForecastBean.java | 161 +++++++++++------- .../vips/logic/entity/ModelInformation.java | 23 ++- .../vips/logic/entity/VipsCoreInstance.java | 119 +++++++++++++ .../vips/logic/scheduling/TaskResult.java | 43 +++++ .../tasks/UpdateModelInformationTask.java | 9 +- .../db/migration/V11__vipscore_instance.sql | 30 ++++ .../YrWeatherForecastProviderTest.java | 3 +- 8 files changed, 317 insertions(+), 73 deletions(-) create mode 100755 src/main/java/no/nibio/vips/logic/entity/VipsCoreInstance.java create mode 100644 src/main/java/no/nibio/vips/logic/scheduling/TaskResult.java create mode 100644 src/main/resources/db/migration/V11__vipscore_instance.sql diff --git a/pom.xml b/pom.xml index 737e4527..f0953827 100755 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ <groupId>no.nibio.vips.</groupId> <artifactId>VIPSLogic</artifactId> <packaging>war</packaging> - <version>2022.1</version> + <version>2023.1</version> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> diff --git a/src/main/java/no/nibio/vips/logic/controller/session/ForecastBean.java b/src/main/java/no/nibio/vips/logic/controller/session/ForecastBean.java index 215e0815..038b51e8 100755 --- a/src/main/java/no/nibio/vips/logic/controller/session/ForecastBean.java +++ b/src/main/java/no/nibio/vips/logic/controller/session/ForecastBean.java @@ -51,6 +51,7 @@ import javax.persistence.EntityManager; import javax.persistence.NoResultException; import javax.persistence.PersistenceContext; import javax.persistence.Query; +import javax.ws.rs.ProcessingException; import javax.ws.rs.client.Client; import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.client.WebTarget; @@ -70,7 +71,9 @@ import no.nibio.vips.logic.entity.Organism; import no.nibio.vips.logic.entity.Organization; import no.nibio.vips.logic.entity.PointOfInterest; import no.nibio.vips.logic.entity.PointOfInterestWeatherStation; +import no.nibio.vips.logic.entity.VipsCoreInstance; import no.nibio.vips.logic.entity.VipsLogicUser; +import no.nibio.vips.logic.scheduling.TaskResult; import no.nibio.vips.logic.scheduling.model.ModelRunPreprocessor; import no.nibio.vips.logic.scheduling.model.ModelRunPreprocessorFactory; import no.nibio.vips.logic.scheduling.model.PreprocessorException; @@ -81,6 +84,7 @@ import no.nibio.vips.util.WeatherUtil; import no.nibio.web.forms.FormField; import org.apache.commons.lang.StringUtils; import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; +import org.slf4j.LoggerFactory; /** * @copyright 2013-2022 <a href="http://www.nibio.no/">NIBIO</a> @@ -89,6 +93,8 @@ import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; @Stateless public class ForecastBean { + private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(ForecastBean.class); + @PersistenceContext(unitName="VIPSLogic-PU") EntityManager em; @@ -463,61 +469,84 @@ public class ForecastBean { * Requests all info about models currently available in VIPSCoreManager * Stores in local db for easy access. */ - public void updateModelInformation() + public TaskResult updateModelInformation() { - - Client client = ClientBuilder.newClient(); - WebTarget target = client.target(System.getProperty("no.nibio.vips.logic.VIPSCOREMANAGER_URL")); - ResteasyWebTarget rTarget = (ResteasyWebTarget) target; - ManagerResource resource = rTarget.proxy(ManagerResource.class); - // Get all model Ids from Core Manager - Response resp = resource.printModelListJSON(); - - for(JsonNode modelIdItem: resp.readEntity(JsonNode.class).findValues("modelId")) - { - - String modelId = modelIdItem.asText(); - - - // We get the corresponding modelInformation entry - ModelInformation modelInformation = em.find(ModelInformation.class, modelId); - if(modelInformation == null) - { - modelInformation = new ModelInformation(modelId); - em.persist(modelInformation); - modelInformation.setDateFirstRegistered(new Date()); + TaskResult taskResult = new TaskResult(); + String jobReport = ""; + // Iterate all the VIPSCore instances + List<VipsCoreInstance> vipsCoreInstances = em.createNamedQuery("VipsCoreInstance.findAll").getResultList(); + Integer instancesCompleted = 0; + for(VipsCoreInstance vipsCoreInstance:vipsCoreInstances) { + LOGGER.debug("Attempting to connect to " + vipsCoreInstance.getUri() ); + try { + Client client = ClientBuilder.newClient(); + WebTarget target = client.target(vipsCoreInstance.getUri()); + ResteasyWebTarget rTarget = (ResteasyWebTarget) target; + ManagerResource resource = rTarget.proxy(ManagerResource.class); + + // Get all model Ids from Core Manager + Response resp = resource.printModelListJSON(); + LOGGER.debug(vipsCoreInstance.getUri() + " returned status code " + String.valueOf(resp.getStatus())); + if(resp.getStatus() != 200) { + jobReport += vipsCoreInstance.getUri() + " returned status code " + String.valueOf(resp.getStatus()) + "\n"; + continue; } - - // Retrieve and store information - - Response r = resource.printModelName(modelId); - modelInformation.setDefaultName(r.readEntity(String.class)); - //r.close(); - - r = resource.printModelDescription(modelId); - modelInformation.setDefaultDescription(r.readEntity(String.class)); - //r.close(); - - r = resource.printModelLicense(modelId); - modelInformation.setLicense(r.readEntity(String.class)); - //r.close(); - - r = resource.printModelCopyright(modelId); - modelInformation.setCopyrightHolder(r.readEntity(String.class)); - //r.close(); - - r = resource.printModelUsage(modelId); - modelInformation.setUsage(r.readEntity(String.class)); - //r.close(); - - r = resource.printModelSampleConfig(modelId); - modelInformation.setSampleConfig(r.readEntity(String.class)); - //r.close(); - - modelInformation.setDateLastRegistered(new Date()); + + for (JsonNode modelIdItem : resp.readEntity(JsonNode.class).findValues("modelId")) { + + String modelId = modelIdItem.asText(); + + + // We get the corresponding modelInformation entry + ModelInformation modelInformation = em.find(ModelInformation.class, modelId); + if (modelInformation == null) { + modelInformation = new ModelInformation(modelId); + em.persist(modelInformation); + modelInformation.setDateFirstRegistered(new Date()); + } + + modelInformation.setVipsCoreInstanceId(vipsCoreInstance); + + // Retrieve and store information + Response r = resource.printModelName(modelId); + modelInformation.setDefaultName(r.readEntity(String.class)); + //r.close(); + + r = resource.printModelDescription(modelId); + modelInformation.setDefaultDescription(r.readEntity(String.class)); + //r.close(); + + r = resource.printModelLicense(modelId); + modelInformation.setLicense(r.readEntity(String.class)); + //r.close(); + + r = resource.printModelCopyright(modelId); + modelInformation.setCopyrightHolder(r.readEntity(String.class)); + //r.close(); + + r = resource.printModelUsage(modelId); + modelInformation.setUsage(r.readEntity(String.class)); + //r.close(); + + r = resource.printModelSampleConfig(modelId); + modelInformation.setSampleConfig(r.readEntity(String.class)); + //r.close(); + + modelInformation.setDateLastRegistered(new Date()); + } + resp.close(); + client.close(); + instancesCompleted++; } - resp.close(); - client.close(); + catch(ProcessingException ex) + { + jobReport += ex.getMessage() + "\n"; + } + + } + taskResult.setCompleteness(instancesCompleted / Double.valueOf(vipsCoreInstances.size())); + taskResult.setMessage(jobReport); + return taskResult; } /** @@ -672,10 +701,16 @@ public class ForecastBean { ModelConfiguration config = preprocessor.getModelConfiguration(forecastConfiguration); ModelRunRequest request = new ModelRunRequest(config); Map<String,String> loginInfo = new HashMap<>(); + // VIPSLogic logs in on behalf of client - loginInfo.put("username",System.getProperty("no.nibio.vips.logic.CORE_BATCH_USERNAME")); + ModelInformation modelInformation = em.find(ModelInformation.class, config.getModelId()); + if(modelInformation.getVipsCoreInstanceId() == null) + { + throw new RunModelException("ERROR: Model " + modelInformation.getDefaultName() + "(" + config.getModelId() + ") is not connected to a VIPSCoreInstance. Please check your server configuration."); + } + loginInfo.put("username",modelInformation.getVipsCoreInstanceId().getUsername()); //loginInfo.put("username","wrongusername"); - loginInfo.put("password",System.getProperty("no.nibio.vips.logic.CORE_BATCH_PASSWORD")); + loginInfo.put("password",modelInformation.getVipsCoreInstanceId().getPassword()); request.setLoginInfo(loginInfo); // We tell which client this is (the db Id in VIPSCoreManager) Integer VIPSCoreUserId = forecastConfiguration.getVipsLogicUserId().getVipsCoreUserIdWithFallback(); @@ -700,7 +735,8 @@ public class ForecastBean { { ex.printStackTrace(); }*/ - Response resp = this.getManagerResource().runModel(config.getModelId(), request); + + Response resp = this.getManagerResource(modelInformation).runModel(config.getModelId(), request); if(resp.getStatus() == Response.Status.OK.getStatusCode()) { //System.out.println(resp.readEntity(String.class)); @@ -724,17 +760,22 @@ public class ForecastBean { public List<Result> runForecast(ModelConfiguration config, Integer VIPSCoreUserId) throws RunModelException { + ModelInformation modelInformation = em.find(ModelInformation.class, config.getModelId()); + if(modelInformation.getVipsCoreInstanceId() == null) + { + throw new RunModelException("ERROR: Model " + modelInformation.getDefaultName() + "(" + config.getModelId() + ") is not connected to a VIPSCoreInstance. Please check your server configuration."); + } ModelRunRequest request = new ModelRunRequest(config); Map<String,String> loginInfo = new HashMap<>(); // VIPSLogic logs in on behalf of client - loginInfo.put("username",System.getProperty("no.nibio.vips.logic.CORE_BATCH_USERNAME")); + loginInfo.put("username",modelInformation.getVipsCoreInstanceId().getUsername()); //loginInfo.put("username","wrongusername"); - loginInfo.put("password",System.getProperty("no.nibio.vips.logic.CORE_BATCH_PASSWORD")); + loginInfo.put("password",modelInformation.getVipsCoreInstanceId().getPassword()); request.setLoginInfo(loginInfo); //System.out.println("VIPSCoreUserId = " + VIPSCoreUserId + ", name=" + forecastConfiguration.getVipsLogicUserId().getLastName()); request.setVipsCoreUserId(VIPSCoreUserId); //System.out.println("RunModel for wsId" + forecastConfiguration.getWeatherStationPointOfInterestId()); - Response resp = this.getManagerResource().runModel(config.getModelId(), request); + Response resp = this.getManagerResource(modelInformation).runModel(config.getModelId(), request); if(resp.getStatus() == Response.Status.OK.getStatusCode()) { @@ -751,10 +792,10 @@ public class ForecastBean { * Get the interface for REST resources in VIPSCoreManager * @return */ - private ManagerResource getManagerResource() + private ManagerResource getManagerResource(ModelInformation modelInformation) { Client client = ClientBuilder.newClient(); - WebTarget target = client.target(System.getProperty("no.nibio.vips.logic.VIPSCOREMANAGER_URL")); + WebTarget target = client.target(modelInformation.getVipsCoreInstanceId().getUri()); ResteasyWebTarget rTarget = (ResteasyWebTarget) target; ManagerResource resource = rTarget.proxy(ManagerResource.class); return resource; diff --git a/src/main/java/no/nibio/vips/logic/entity/ModelInformation.java b/src/main/java/no/nibio/vips/logic/entity/ModelInformation.java index 54859b14..93d0dda5 100755 --- a/src/main/java/no/nibio/vips/logic/entity/ModelInformation.java +++ b/src/main/java/no/nibio/vips/logic/entity/ModelInformation.java @@ -21,15 +21,7 @@ package no.nibio.vips.logic.entity; import java.io.Serializable; import java.util.Date; -import javax.persistence.Basic; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.persistence.NamedQueries; -import javax.persistence.NamedQuery; -import javax.persistence.Table; -import javax.persistence.Temporal; -import javax.persistence.TemporalType; +import javax.persistence.*; import javax.validation.constraints.NotNull; import javax.validation.constraints.Size; import javax.xml.bind.annotation.XmlRootElement; @@ -60,6 +52,12 @@ public class ModelInformation implements Serializable { @Size(min = 1, max = 10) @Column(name = "model_id") private String modelId; + + + @JoinColumn(name = "vipscore_instance_id", referencedColumnName = "vipscore_instance_id") + @ManyToOne + private VipsCoreInstance vipsCoreInstanceId; + @Column(name = "date_first_registered") @Temporal(TemporalType.DATE) private Date dateFirstRegistered; @@ -189,4 +187,11 @@ public class ModelInformation implements Serializable { return "no.nibio.vips.logic.entity.ModelInformation[ modelId=" + modelId + " ]"; } + public VipsCoreInstance getVipsCoreInstanceId() { + return vipsCoreInstanceId; + } + + public void setVipsCoreInstanceId(VipsCoreInstance vipsCoreInstanceId) { + this.vipsCoreInstanceId = vipsCoreInstanceId; + } } diff --git a/src/main/java/no/nibio/vips/logic/entity/VipsCoreInstance.java b/src/main/java/no/nibio/vips/logic/entity/VipsCoreInstance.java new file mode 100755 index 00000000..986f95bb --- /dev/null +++ b/src/main/java/no/nibio/vips/logic/entity/VipsCoreInstance.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2014 NIBIO <http://www.nibio.no/>. + * + * This file is part of VIPSLogic. + * VIPSLogic is free software: you can redistribute it and/or modify + * it under the terms of the NIBIO Open Source License as published by + * NIBIO, either version 1 of the License, or (at your option) any + * later version. + * + * VIPSLogic is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * NIBIO Open Source License for more details. + * + * You should have received a copy of the NIBIO Open Source License + * along with VIPSLogic. If not, see <http://www.nibio.no/licenses/>. + * + */ + +package no.nibio.vips.logic.entity; + +import java.io.Serializable; +import javax.persistence.*; +import javax.validation.constraints.Size; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * @copyright 2023 <a href="http://www.nibio.no/">NIBIO</a> + * @author Tor-Einar Skog <tor-einar.skog@nibio.no> + */ +@Entity +@Table(name = "vipscore_instance") +@NamedQueries({ + @NamedQuery(name = "VipsCoreInstance.findAll", query = "SELECT v FROM VipsCoreInstance v") +}) +@XmlRootElement +public class VipsCoreInstance implements Serializable { + private static final long serialVersionUID = 1L; + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Basic(optional = false) + @Column(name = "vipscore_instance_id") + private Integer vipscoreInstanceId; + @Size(max = 255) + @Column(name = "uri") + private String uri; + @Size(max = 255) + @Column(name = "username") + private String username; + @Size(max = 255) + @Column(name = "password") + private String password; + + + public VipsCoreInstance() { + } + + public VipsCoreInstance(Integer vipscoreInstanceId) { + this.setVipscoreInstanceId(vipscoreInstanceId); + } + + + @Override + public int hashCode() { + int hash = 0; + hash += (getVipscoreInstanceId() != null ? getVipscoreInstanceId().hashCode() : 0); + return hash; + } + + @Override + public boolean equals(Object object) { + // TODO: Warning - this method won't work in the case the id fields are not set + if (!(object instanceof VipsCoreInstance)) { + return false; + } + VipsCoreInstance other = (VipsCoreInstance) object; + if ((this.getVipscoreInstanceId() == null && other.getVipscoreInstanceId() != null) || (this.getVipscoreInstanceId() != null && !this.getVipscoreInstanceId().equals(other.getVipscoreInstanceId()))) { + return false; + } + return true; + } + + @Override + public String toString() { + return "no.nibio.vips.logic.entity.VIPSCoreInstance[ vipsCoreInstanceId=" + vipscoreInstanceId + " ]"; + } + + public Integer getVipscoreInstanceId() { + return vipscoreInstanceId; + } + + public void setVipscoreInstanceId(Integer vipscoreInstanceId) { + this.vipscoreInstanceId = vipscoreInstanceId; + } + + public String getUri() { + return uri; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } +} diff --git a/src/main/java/no/nibio/vips/logic/scheduling/TaskResult.java b/src/main/java/no/nibio/vips/logic/scheduling/TaskResult.java new file mode 100644 index 00000000..b7037caa --- /dev/null +++ b/src/main/java/no/nibio/vips/logic/scheduling/TaskResult.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023 NIBIO <http://www.nibio.no/>. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + * + */ +package no.nibio.vips.logic.scheduling; + + +/** + * A util entity for reporting back to a cron4j Task + */ +public class TaskResult { + private Double completeness; + private String message; + + public Double getCompleteness() { + return completeness; + } + + public void setCompleteness(Double completeness) { + this.completeness = completeness; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } +} diff --git a/src/main/java/no/nibio/vips/logic/scheduling/tasks/UpdateModelInformationTask.java b/src/main/java/no/nibio/vips/logic/scheduling/tasks/UpdateModelInformationTask.java index 8211bfc5..7c486865 100755 --- a/src/main/java/no/nibio/vips/logic/scheduling/tasks/UpdateModelInformationTask.java +++ b/src/main/java/no/nibio/vips/logic/scheduling/tasks/UpdateModelInformationTask.java @@ -23,6 +23,7 @@ import it.sauronsoftware.cron4j.TaskExecutionContext; import javax.ejb.EJB; import no.nibio.vips.logic.controller.session.ForecastBean; import no.nibio.vips.logic.controller.session.SessionControllerGetter; +import no.nibio.vips.logic.scheduling.TaskResult; import no.nibio.vips.logic.scheduling.VipsLogicTask; /** @@ -45,8 +46,12 @@ public class UpdateModelInformationTask extends VipsLogicTask{ @Override public void execute(TaskExecutionContext tec) throws RuntimeException { tec.setCompleteness(0d); - SessionControllerGetter.getForecastBean().updateModelInformation(); - tec.setCompleteness(1d); + TaskResult taskResult = SessionControllerGetter.getForecastBean().updateModelInformation(); + tec.setCompleteness(taskResult.getCompleteness()); + tec.setStatusMessage(taskResult.getMessage()); + if(taskResult.getCompleteness() < 1.0){ + throw new RuntimeException(); + } } @Override diff --git a/src/main/resources/db/migration/V11__vipscore_instance.sql b/src/main/resources/db/migration/V11__vipscore_instance.sql new file mode 100644 index 00000000..9181f0ac --- /dev/null +++ b/src/main/resources/db/migration/V11__vipscore_instance.sql @@ -0,0 +1,30 @@ +-- We are refactoring VIPS to support multiple instances of VIPSCore, +-- allowing for e.g. implementations in different programming languages, +-- such as VIPSCore-Python (https://gitlab.nibio.no/VIPS/VIPSCore-Python) +-- The entity VipsCoreInstance has been added +-- After this migration, remember to add at least one VIPSCoreInstance in the db +-- And you can safely delete these system-properties from your config: +-- * no.nibio.vips.coremanager.VIPSCORE_URL +-- * no.nibio.vips.logic.CORE_BATCH_USERNAME +-- * no.nibio.vips.logic.CORE_BATCH_PASSWORD + +--ALTER TABLE public.model_information +--DROP COLUMN vipscore_instance_id; + +--DROP TABLE public.vipscore_instance; + +CREATE TABLE public.vipscore_instance +( + vipscore_instance_id SERIAL PRIMARY KEY, + uri VARCHAR(255), + username VARCHAR(255), + password VARCHAR(255) +); + +ALTER TABLE public.model_information + ADD COLUMN vipscore_instance_id INTEGER REFERENCES public.vipscore_instance(vipscore_instance_id) DEFAULT NULL; + +ALTER TABLE if exists public.vipscore_instance + OWNER TO vipslogic; + + diff --git a/src/test/java/no/nibio/vips/util/weather/YrWeatherForecastProviderTest.java b/src/test/java/no/nibio/vips/util/weather/YrWeatherForecastProviderTest.java index 811158e8..4b8061d4 100755 --- a/src/test/java/no/nibio/vips/util/weather/YrWeatherForecastProviderTest.java +++ b/src/test/java/no/nibio/vips/util/weather/YrWeatherForecastProviderTest.java @@ -93,11 +93,12 @@ public class YrWeatherForecastProviderTest { result = instance.getWeatherForecasts(weatherStation); assertNotNull(result); Collections.sort(result); + /* for(WeatherObservation obs:result) { if(obs.getElementMeasurementTypeId().equals("TM")) System.out.println(obs.toString()); - } + }*/ WeatherUtil wUtil = new WeatherUtil(); //wUtil.checkForAndFixHourlyTimeSeriesHoles(result); -- GitLab