VIPSLogic-2023.1 Supports multiple VIPSCore resources. [VIPSUTV-383]

pom.xml

@@ -5,7 +5,7 @@
     <groupId>no.nibio.vips.</groupId>
     <artifactId>VIPSLogic</artifactId>
     <packaging>war</packaging>
-    <version>2022.1</version>
+    <version>2023.1</version>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

src/main/java/no/nibio/vips/logic/controller/session/ForecastBean.java

@@ -51,6 +51,7 @@ import javax.persistence.EntityManager;
 import javax.persistence.NoResultException;
 import javax.persistence.PersistenceContext;
 import javax.persistence.Query;
+import javax.ws.rs.ProcessingException;
 import javax.ws.rs.client.Client;
 import javax.ws.rs.client.ClientBuilder;
 import javax.ws.rs.client.WebTarget;
@@ -70,7 +71,9 @@ import no.nibio.vips.logic.entity.Organism;
 import no.nibio.vips.logic.entity.Organization;
 import no.nibio.vips.logic.entity.PointOfInterest;
 import no.nibio.vips.logic.entity.PointOfInterestWeatherStation;
+import no.nibio.vips.logic.entity.VipsCoreInstance;
 import no.nibio.vips.logic.entity.VipsLogicUser;
+import no.nibio.vips.logic.scheduling.TaskResult;
 import no.nibio.vips.logic.scheduling.model.ModelRunPreprocessor;
 import no.nibio.vips.logic.scheduling.model.ModelRunPreprocessorFactory;
 import no.nibio.vips.logic.scheduling.model.PreprocessorException;
@@ -81,6 +84,7 @@ import no.nibio.vips.util.WeatherUtil;
 import no.nibio.web.forms.FormField;
 import org.apache.commons.lang.StringUtils;
 import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget;
+import org.slf4j.LoggerFactory;
 
 /**
  * @copyright 2013-2022 <a href="http://www.nibio.no/">NIBIO</a>
@@ -89,6 +93,8 @@ import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget;
 @Stateless
 public class ForecastBean {
 
+    private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(ForecastBean.class);
+
     @PersistenceContext(unitName="VIPSLogic-PU")
     EntityManager em;
     
@@ -463,61 +469,84 @@ public class ForecastBean {
      * Requests all info about models currently available in VIPSCoreManager
      * Stores in local db for easy access. 
      */
-    public void updateModelInformation()
+    public TaskResult updateModelInformation()
     {
-    	
-	    	Client client = ClientBuilder.newClient();
-	        WebTarget target = client.target(System.getProperty("no.nibio.vips.logic.VIPSCOREMANAGER_URL"));
-	        ResteasyWebTarget rTarget = (ResteasyWebTarget) target;
-	        ManagerResource resource = rTarget.proxy(ManagerResource.class);
-            // Get all model Ids from Core Manager
-            Response resp = resource.printModelListJSON();
-            
-            for(JsonNode modelIdItem: resp.readEntity(JsonNode.class).findValues("modelId"))
-            {
-            	
-                String modelId = modelIdItem.asText();
-                
-                
-                // We get the corresponding modelInformation entry 
-                ModelInformation modelInformation = em.find(ModelInformation.class, modelId);
-                if(modelInformation == null)
-                {
-                    modelInformation = new ModelInformation(modelId);
-                    em.persist(modelInformation);
-                    modelInformation.setDateFirstRegistered(new Date());
+        TaskResult taskResult = new TaskResult();
+        String jobReport = "";
+    	// Iterate all the VIPSCore instances
+        List<VipsCoreInstance> vipsCoreInstances = em.createNamedQuery("VipsCoreInstance.findAll").getResultList();
+        Integer instancesCompleted = 0;
+        for(VipsCoreInstance vipsCoreInstance:vipsCoreInstances) {
+            LOGGER.debug("Attempting to connect to " + vipsCoreInstance.getUri() );
+            try {
+                Client client = ClientBuilder.newClient();
+                WebTarget target = client.target(vipsCoreInstance.getUri());
+                ResteasyWebTarget rTarget = (ResteasyWebTarget) target;
+                ManagerResource resource = rTarget.proxy(ManagerResource.class);
+
+                // Get all model Ids from Core Manager
+                Response resp = resource.printModelListJSON();
+                LOGGER.debug(vipsCoreInstance.getUri() + " returned status code " + String.valueOf(resp.getStatus()));
+                if(resp.getStatus() != 200) {
+                    jobReport += vipsCoreInstance.getUri() + " returned status code " + String.valueOf(resp.getStatus()) + "\n";
+                    continue;
                 }
-                
-                // Retrieve and store information
-                
-                Response r = resource.printModelName(modelId);
-                modelInformation.setDefaultName(r.readEntity(String.class));
-                //r.close();
-                
-                r = resource.printModelDescription(modelId);
-                modelInformation.setDefaultDescription(r.readEntity(String.class));
-                //r.close();
-                
-                r = resource.printModelLicense(modelId);
-                modelInformation.setLicense(r.readEntity(String.class));
-                //r.close();
-                
-                r = resource.printModelCopyright(modelId);
-                modelInformation.setCopyrightHolder(r.readEntity(String.class));
-                //r.close();
-                
-                r = resource.printModelUsage(modelId);
-                modelInformation.setUsage(r.readEntity(String.class));
-                //r.close();
-                
-                r = resource.printModelSampleConfig(modelId);
-                modelInformation.setSampleConfig(r.readEntity(String.class));
-                //r.close();
-                
-                modelInformation.setDateLastRegistered(new Date());
+
+                for (JsonNode modelIdItem : resp.readEntity(JsonNode.class).findValues("modelId")) {
+
+                    String modelId = modelIdItem.asText();
+
+
+                    // We get the corresponding modelInformation entry
+                    ModelInformation modelInformation = em.find(ModelInformation.class, modelId);
+                    if (modelInformation == null) {
+                        modelInformation = new ModelInformation(modelId);
+                        em.persist(modelInformation);
+                        modelInformation.setDateFirstRegistered(new Date());
+                    }
+
+                    modelInformation.setVipsCoreInstanceId(vipsCoreInstance);
+
+                    // Retrieve and store information
+                    Response r = resource.printModelName(modelId);
+                    modelInformation.setDefaultName(r.readEntity(String.class));
+                    //r.close();
+
+                    r = resource.printModelDescription(modelId);
+                    modelInformation.setDefaultDescription(r.readEntity(String.class));
+                    //r.close();
+
+                    r = resource.printModelLicense(modelId);
+                    modelInformation.setLicense(r.readEntity(String.class));
+                    //r.close();
+
+                    r = resource.printModelCopyright(modelId);
+                    modelInformation.setCopyrightHolder(r.readEntity(String.class));
+                    //r.close();
+
+                    r = resource.printModelUsage(modelId);
+                    modelInformation.setUsage(r.readEntity(String.class));
+                    //r.close();
+
+                    r = resource.printModelSampleConfig(modelId);
+                    modelInformation.setSampleConfig(r.readEntity(String.class));
+                    //r.close();
+
+                    modelInformation.setDateLastRegistered(new Date());
+                }
+                resp.close();
+                client.close();
+                instancesCompleted++;
             }
-            resp.close();
-            client.close();
+            catch(ProcessingException ex)
+            {
+                jobReport += ex.getMessage() + "\n";
+            }
+
+        }
+        taskResult.setCompleteness(instancesCompleted / Double.valueOf(vipsCoreInstances.size()));
+        taskResult.setMessage(jobReport);
+        return taskResult;
     }
     
     /**
@@ -672,10 +701,16 @@ public class ForecastBean {
                 ModelConfiguration config = preprocessor.getModelConfiguration(forecastConfiguration);
                 ModelRunRequest request = new ModelRunRequest(config);
                 Map<String,String> loginInfo = new HashMap<>();
+
                 // VIPSLogic logs in on behalf of client
-                loginInfo.put("username",System.getProperty("no.nibio.vips.logic.CORE_BATCH_USERNAME"));
+                ModelInformation modelInformation = em.find(ModelInformation.class, config.getModelId());
+                if(modelInformation.getVipsCoreInstanceId() == null)
+                {
+                    throw new RunModelException("ERROR: Model " + modelInformation.getDefaultName() + "(" + config.getModelId() + ") is not connected to a VIPSCoreInstance. Please check your server configuration.");
+                }
+                loginInfo.put("username",modelInformation.getVipsCoreInstanceId().getUsername());
                 //loginInfo.put("username","wrongusername");
-                loginInfo.put("password",System.getProperty("no.nibio.vips.logic.CORE_BATCH_PASSWORD"));
+                loginInfo.put("password",modelInformation.getVipsCoreInstanceId().getPassword());
                 request.setLoginInfo(loginInfo);
                 // We tell which client this is (the db Id in VIPSCoreManager)
                 Integer VIPSCoreUserId = forecastConfiguration.getVipsLogicUserId().getVipsCoreUserIdWithFallback();
@@ -700,7 +735,8 @@ public class ForecastBean {
                 {
                     ex.printStackTrace();
                 }*/
-                    Response resp = this.getManagerResource().runModel(config.getModelId(), request);         
+
+                    Response resp = this.getManagerResource(modelInformation).runModel(config.getModelId(), request);
                     if(resp.getStatus() == Response.Status.OK.getStatusCode())
                     {
                         //System.out.println(resp.readEntity(String.class));
@@ -724,17 +760,22 @@ public class ForecastBean {
     
     public List<Result> runForecast(ModelConfiguration config, Integer VIPSCoreUserId) throws RunModelException
     {
+        ModelInformation modelInformation = em.find(ModelInformation.class, config.getModelId());
+        if(modelInformation.getVipsCoreInstanceId() == null)
+        {
+            throw new RunModelException("ERROR: Model " + modelInformation.getDefaultName() + "(" + config.getModelId() + ") is not connected to a VIPSCoreInstance. Please check your server configuration.");
+        }
         ModelRunRequest request = new ModelRunRequest(config);
         Map<String,String> loginInfo = new HashMap<>();
         // VIPSLogic logs in on behalf of client
-        loginInfo.put("username",System.getProperty("no.nibio.vips.logic.CORE_BATCH_USERNAME"));
+        loginInfo.put("username",modelInformation.getVipsCoreInstanceId().getUsername());
         //loginInfo.put("username","wrongusername");
-        loginInfo.put("password",System.getProperty("no.nibio.vips.logic.CORE_BATCH_PASSWORD"));
+        loginInfo.put("password",modelInformation.getVipsCoreInstanceId().getPassword());
         request.setLoginInfo(loginInfo);
         //System.out.println("VIPSCoreUserId = " + VIPSCoreUserId + ", name=" + forecastConfiguration.getVipsLogicUserId().getLastName());
         request.setVipsCoreUserId(VIPSCoreUserId);
         //System.out.println("RunModel for wsId" + forecastConfiguration.getWeatherStationPointOfInterestId());
-        Response resp = this.getManagerResource().runModel(config.getModelId(), request);
+        Response resp = this.getManagerResource(modelInformation).runModel(config.getModelId(), request);
 
         if(resp.getStatus() == Response.Status.OK.getStatusCode())
         {
@@ -751,10 +792,10 @@ public class ForecastBean {
      * Get the interface for REST resources in VIPSCoreManager
      * @return 
      */
-    private ManagerResource getManagerResource()
+    private ManagerResource getManagerResource(ModelInformation modelInformation)
     {
         Client client = ClientBuilder.newClient();
-        WebTarget target = client.target(System.getProperty("no.nibio.vips.logic.VIPSCOREMANAGER_URL"));
+        WebTarget target = client.target(modelInformation.getVipsCoreInstanceId().getUri());
         ResteasyWebTarget rTarget = (ResteasyWebTarget) target;
         ManagerResource resource = rTarget.proxy(ManagerResource.class);
         return resource;

src/main/java/no/nibio/vips/logic/entity/ModelInformation.java

@@ -21,15 +21,7 @@ package no.nibio.vips.logic.entity;
 
 import java.io.Serializable;
 import java.util.Date;
-import javax.persistence.Basic;
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.Id;
-import javax.persistence.NamedQueries;
-import javax.persistence.NamedQuery;
-import javax.persistence.Table;
-import javax.persistence.Temporal;
-import javax.persistence.TemporalType;
+import javax.persistence.*;
 import javax.validation.constraints.NotNull;
 import javax.validation.constraints.Size;
 import javax.xml.bind.annotation.XmlRootElement;
@@ -60,6 +52,12 @@ public class ModelInformation implements Serializable {
     @Size(min = 1, max = 10)
     @Column(name = "model_id")
     private String modelId;
+
+
+    @JoinColumn(name = "vipscore_instance_id", referencedColumnName = "vipscore_instance_id")
+    @ManyToOne
+    private VipsCoreInstance vipsCoreInstanceId;
+
     @Column(name = "date_first_registered")
     @Temporal(TemporalType.DATE)
     private Date dateFirstRegistered;
@@ -189,4 +187,11 @@ public class ModelInformation implements Serializable {
         return "no.nibio.vips.logic.entity.ModelInformation[ modelId=" + modelId + " ]";
     }
 
+    public VipsCoreInstance getVipsCoreInstanceId() {
+        return vipsCoreInstanceId;
+    }
+
+    public void setVipsCoreInstanceId(VipsCoreInstance vipsCoreInstanceId) {
+        this.vipsCoreInstanceId = vipsCoreInstanceId;
+    }
 }

src/main/java/no/nibio/vips/logic/entity/VipsCoreInstance.java

+/*
+ * Copyright (c) 2014 NIBIO <http://www.nibio.no/>. 
+ * 
+ * This file is part of VIPSLogic.
+ * VIPSLogic is free software: you can redistribute it and/or modify
+ * it under the terms of the NIBIO Open Source License as published by 
+ * NIBIO, either version 1 of the License, or (at your option) any
+ * later version.
+ * 
+ * VIPSLogic is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * NIBIO Open Source License for more details.
+ * 
+ * You should have received a copy of the NIBIO Open Source License
+ * along with VIPSLogic.  If not, see <http://www.nibio.no/licenses/>.
+ * 
+ */
+
+package no.nibio.vips.logic.entity;
+
+import java.io.Serializable;
+import javax.persistence.*;
+import javax.validation.constraints.Size;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * @copyright 2023 <a href="http://www.nibio.no/">NIBIO</a>
+ * @author Tor-Einar Skog <tor-einar.skog@nibio.no>
+ */
+@Entity
+@Table(name = "vipscore_instance")
+@NamedQueries({
+        @NamedQuery(name = "VipsCoreInstance.findAll", query = "SELECT v FROM VipsCoreInstance v")
+})
+@XmlRootElement
+public class VipsCoreInstance implements Serializable {
+    private static final long serialVersionUID = 1L;
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    @Basic(optional = false)
+    @Column(name = "vipscore_instance_id")
+    private Integer vipscoreInstanceId;
+    @Size(max = 255)
+    @Column(name = "uri")
+    private String uri;
+    @Size(max = 255)
+    @Column(name = "username")
+    private String username;
+    @Size(max = 255)
+    @Column(name = "password")
+    private String password;
+
+
+    public VipsCoreInstance() {
+    }
+
+    public VipsCoreInstance(Integer vipscoreInstanceId) {
+        this.setVipscoreInstanceId(vipscoreInstanceId);
+    }
+
+
+    @Override
+    public int hashCode() {
+        int hash = 0;
+        hash += (getVipscoreInstanceId() != null ? getVipscoreInstanceId().hashCode() : 0);
+        return hash;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        // TODO: Warning - this method won't work in the case the id fields are not set
+        if (!(object instanceof VipsCoreInstance)) {
+            return false;
+        }
+        VipsCoreInstance other = (VipsCoreInstance) object;
+        if ((this.getVipscoreInstanceId() == null && other.getVipscoreInstanceId() != null) || (this.getVipscoreInstanceId() != null && !this.getVipscoreInstanceId().equals(other.getVipscoreInstanceId()))) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public String toString() {
+        return "no.nibio.vips.logic.entity.VIPSCoreInstance[ vipsCoreInstanceId=" + vipscoreInstanceId + " ]";
+    }
+
+    public Integer getVipscoreInstanceId() {
+        return vipscoreInstanceId;
+    }
+
+    public void setVipscoreInstanceId(Integer vipscoreInstanceId) {
+        this.vipscoreInstanceId = vipscoreInstanceId;
+    }
+
+    public String getUri() {
+        return uri;
+    }
+
+    public void setUri(String uri) {
+        this.uri = uri;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+}

src/main/java/no/nibio/vips/logic/scheduling/TaskResult.java

+/*
+ * Copyright (c) 2023 NIBIO <http://www.nibio.no/>.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+package no.nibio.vips.logic.scheduling;
+
+
+/**
+ * A util entity for reporting back to a cron4j Task
+ */
+public class TaskResult  {
+    private Double completeness;
+    private String message;
+
+    public Double getCompleteness() {
+        return completeness;
+    }
+
+    public void setCompleteness(Double completeness) {
+        this.completeness = completeness;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public void setMessage(String message) {
+        this.message = message;
+    }
+}

src/main/java/no/nibio/vips/logic/scheduling/tasks/UpdateModelInformationTask.java

@@ -23,6 +23,7 @@ import it.sauronsoftware.cron4j.TaskExecutionContext;
 import javax.ejb.EJB;
 import no.nibio.vips.logic.controller.session.ForecastBean;
 import no.nibio.vips.logic.controller.session.SessionControllerGetter;
+import no.nibio.vips.logic.scheduling.TaskResult;
 import no.nibio.vips.logic.scheduling.VipsLogicTask;
 
 /**
@@ -45,8 +46,12 @@ public class UpdateModelInformationTask extends VipsLogicTask{
     @Override
     public void execute(TaskExecutionContext tec) throws RuntimeException {
         tec.setCompleteness(0d);
-        SessionControllerGetter.getForecastBean().updateModelInformation();
-        tec.setCompleteness(1d);
+        TaskResult taskResult = SessionControllerGetter.getForecastBean().updateModelInformation();
+        tec.setCompleteness(taskResult.getCompleteness());
+        tec.setStatusMessage(taskResult.getMessage());
+        if(taskResult.getCompleteness() < 1.0){
+            throw new RuntimeException();
+        }
     }
     
     @Override

src/main/resources/db/migration/V11__vipscore_instance.sql

+-- We are refactoring VIPS to support multiple instances of VIPSCore,
+-- allowing for e.g. implementations in different programming languages,
+-- such as VIPSCore-Python (https://gitlab.nibio.no/VIPS/VIPSCore-Python)
+-- The entity VipsCoreInstance has been added
+-- After this migration, remember to add at least one VIPSCoreInstance in the db
+-- And you can safely delete these system-properties from your config:
+-- * no.nibio.vips.coremanager.VIPSCORE_URL
+-- * no.nibio.vips.logic.CORE_BATCH_USERNAME
+-- * no.nibio.vips.logic.CORE_BATCH_PASSWORD
+
+--ALTER TABLE public.model_information
+--DROP COLUMN vipscore_instance_id;
+
+--DROP TABLE public.vipscore_instance;
+
+CREATE TABLE public.vipscore_instance
+(
+    vipscore_instance_id SERIAL PRIMARY KEY,
+    uri VARCHAR(255),
+    username VARCHAR(255),
+    password VARCHAR(255)
+);
+
+ALTER TABLE public.model_information
+    ADD COLUMN vipscore_instance_id INTEGER REFERENCES public.vipscore_instance(vipscore_instance_id) DEFAULT NULL;
+
+ALTER TABLE if exists public.vipscore_instance
+    OWNER TO vipslogic;
+
+

src/test/java/no/nibio/vips/util/weather/YrWeatherForecastProviderTest.java

@@ -93,11 +93,12 @@ public class YrWeatherForecastProviderTest {
             result = instance.getWeatherForecasts(weatherStation);
             assertNotNull(result);
             Collections.sort(result);
+            /*
             for(WeatherObservation obs:result)
             {
                 if(obs.getElementMeasurementTypeId().equals("TM"))
                 System.out.println(obs.toString());
-            }
+            }*/
             
             WeatherUtil wUtil = new WeatherUtil();
             //wUtil.checkForAndFixHourlyTimeSeriesHoles(result);