Skip to content
Snippets Groups Projects
Commit 93448ae4 authored by Lene Wasskog's avatar Lene Wasskog
Browse files

Merge branch 'main' into release

parents 692ede5d 57852f91
Branches
Tags
No related merge requests found
Pipeline #2540 failed
......@@ -3,6 +3,7 @@ stages:
- build
- test
- deploy
- upload
variables:
COMMON_CONFIG_REMOTE: https://root:$CONFIG_ACCESS_TOKEN@$CI_SERVER_HOST/VIPS/vips-common-config.git
......@@ -14,6 +15,10 @@ variables:
MAVEN_OPTS: "-Dmaven.repo.local=.m2/repository -Dmaven.artifact.threads=10"
MAIN_BRANCH: "main"
RELEASE_BRANCH: "release"
CLI_PATH: "/disks/data01/wildfly/wildfly-25.0.1.Final/bin"
TMP_PATH: "/home/deployer/gitlab_tmp"
DEPLOYMENT_PATH: "/home/wildfly/deployments"
ARCHIVE_PATH: "/home/wildfly/archive"
cache:
paths:
......@@ -28,38 +33,127 @@ before_script:
- cp $SETTINGS_XML ../$SETTINGS_XML
- cd ..
- rm -rf $COMMON_CONFIG_LOCAL
- apt-get update -y && apt-get install -y python
- 'python build_pom_with_models.py'
- apt-get update -y && apt-get install -y python && apt-get install -y rsync openssh-client
- "python build_pom_with_models.py"
build:
stage: build
script:
- './mvnw $MAVEN_POM_WITH_MODELS $MAVEN_CLI_OPTS $MAVEN_OPTS package -DskipTests'
- "./mvnw $MAVEN_POM_WITH_MODELS $MAVEN_CLI_OPTS $MAVEN_OPTS package -DskipTests"
tags:
- vips-java
artifacts:
paths:
- target/*.war
rules:
- if: $CI_COMMIT_REF_NAME == $MAIN_BRANCH
- if: $CI_COMMIT_REF_NAME == $RELEASE_BRANCH
test:
stage: test
script:
- './mvnw $MAVEN_POM_WITH_MODELS $MAVEN_CLI_OPTS $MAVEN_OPTS test'
- "./mvnw $MAVEN_POM_WITH_MODELS $MAVEN_CLI_OPTS $MAVEN_OPTS test"
tags:
- vips-java
rules:
- if: $CI_COMMIT_REF_NAME == $MAIN_BRANCH
- if: $CI_COMMIT_REF_NAME == $RELEASE_BRANCH
deploy-snapshot:
deploy-to-staging:
stage: deploy
script:
# Authentication setup
- mkdir -p ~/.ssh
- echo "$SSH_PRIVATE_KEY" >> ~/.ssh/id_dsa
- chmod 600 ~/.ssh/id_dsa
- echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
# Find name of war file in target folder (built previously in build stage), exit if none exists
- WAR_FILE=$(find target -maxdepth 1 -type f -name "*.war" | sort -n | tail -1 | xargs basename)
- if [ -z "$WAR_FILE" ]; then echo "No war file found in target directory"; exit 1; fi
# Create temporary directory on server, copy war file and deployment script there
- echo "Create temporary directory on the server"
- ssh $SERVER_USER@$SERVER_IP "mkdir -p $TMP_PATH"
- scp target/$WAR_FILE $SERVER_USER@$SERVER_IP:$TMP_PATH
- echo "deploy $TMP_PATH/$WAR_FILE --force" > deploy.cli
- scp deploy.cli $SERVER_USER@$SERVER_IP:$TMP_PATH
# Find name of currently deployed application. Undeploy.
- CURRENTLY_DEPLOYED=$(ssh $SERVER_USER@$SERVER_IP "$CLI_PATH/jboss-cli.sh --user=$WILDFLY_ADMIN_USERNAME --password=$WILDFLY_ADMIN_PASSWORD --connect --commands=ls\ deployment | grep VIPSCore-.*war")
- echo $CURRENTLY_DEPLOYED
- ssh $SERVER_USER@$SERVER_IP "$CLI_PATH/jboss-cli.sh --user=$WILDFLY_ADMIN_USERNAME --password=$WILDFLY_ADMIN_PASSWORD --connect --command='undeploy $CURRENTLY_DEPLOYED'"
# Deploy war file using JBoss CLI and credentials given in CI/CD variables
- echo "Deploy $WAR_FILE using JBoss CLI"
- ssh $SERVER_USER@$SERVER_IP "/disks/data01/wildfly/wildfly-25.0.1.Final/bin/jboss-cli.sh --user=$WILDFLY_ADMIN_USERNAME --password=$WILDFLY_ADMIN_PASSWORD --connect --file=$TMP_PATH/deploy.cli" > deployment.log 2>&1
# Restart should not be necessary
- echo "Restart WildFly"
- ssh $SERVER_USER@$SERVER_IP "sudo systemctl restart wildfly"
# Delete temporary directory on server
- echo "Delete temporary directory"
- ssh $SERVER_USER@$SERVER_IP "rm -rf $TMP_PATH"
after_script:
- echo "Deployment complete"
tags:
- vips-java
rules:
- if: $CI_COMMIT_REF_NAME == $MAIN_BRANCH
environment:
name: staging
url: https://vipscore02test.nibio.no
upload-snapshot:
stage: upload
script:
- export VERSION=$(./mvnw $MAVEN_POM_WITH_MODELS $MAVEN_CLI_OPTS $MAVEN_OPTS --batch-mode --no-transfer-progress --non-recursive help:evaluate -Dexpression=project.version | grep -v "\[.*")
- if ! [[ $VERSION =~ .*SNAPSHOT ]]; then
echo "Version '$VERSION' is not SNAPSHOT"; exit 1;
echo "Version '$VERSION' is not SNAPSHOT"; exit 1;
fi
- './mvnw $MAVEN_POM_WITH_MODELS $MAVEN_CLI_OPTS $MAVEN_OPTS deploy -DskipTests'
- "./mvnw $MAVEN_POM_WITH_MODELS $MAVEN_CLI_OPTS $MAVEN_OPTS deploy -DskipTests"
tags:
- vips-java
rules:
- if: '$CI_COMMIT_REF_NAME == $MAIN_BRANCH'
- if: $CI_COMMIT_REF_NAME == $MAIN_BRANCH
deploy-release:
deploy-to-production:
stage: deploy
script:
# Authentication setup
- mkdir -p ~/.ssh
- echo "$SSH_PRIVATE_KEY" >> ~/.ssh/id_dsa
- chmod 600 ~/.ssh/id_dsa
- echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
# Find name of war file in target folder (built previously in build stage), exit if none exists
- WAR_FILE=$(find target -maxdepth 1 -type f -name "*.war" | sort -n | tail -1 | xargs basename)
- if [ -z "$WAR_FILE" ]; then echo "No war file found in target directory"; exit 1; fi
# Create temporary directory on server, copy war file and deployment script there
- echo "Create temporary directory on the server"
- ssh $SERVER_USER@$SERVER_IP "mkdir -p $TMP_PATH"
- scp target/$WAR_FILE $SERVER_USER@$SERVER_IP:$TMP_PATH
- echo "deploy $TMP_PATH/$WAR_FILE --force" > deploy.cli
- scp deploy.cli $SERVER_USER@$SERVER_IP:$TMP_PATH
# Find name of currently deployed application. Undeploy.
- CURRENTLY_DEPLOYED=$(ssh $SERVER_USER@$SERVER_IP "$CLI_PATH/jboss-cli.sh --user=$WILDFLY_ADMIN_USERNAME --password=$WILDFLY_ADMIN_PASSWORD --connect --commands=ls\ deployment | grep VIPSCore-.*war")
- echo $CURRENTLY_DEPLOYED
- ssh $SERVER_USER@$SERVER_IP "$CLI_PATH/jboss-cli.sh --user=$WILDFLY_ADMIN_USERNAME --password=$WILDFLY_ADMIN_PASSWORD --connect --command='undeploy $CURRENTLY_DEPLOYED'"
# Deploy war file using JBoss CLI and credentials given in CI/CD variables
- echo "Deploy $WAR_FILE using JBoss CLI"
- ssh $SERVER_USER@$SERVER_IP "/disks/data01/wildfly/wildfly-25.0.1.Final/bin/jboss-cli.sh --user=$WILDFLY_ADMIN_USERNAME --password=$WILDFLY_ADMIN_PASSWORD --connect --file=$TMP_PATH/deploy.cli" > deployment.log 2>&1
# Restart should not be necessary
- echo "Restart WildFly"
- ssh $SERVER_USER@$SERVER_IP "sudo systemctl restart wildfly"
# Delete temporary directory on server
- echo "Delete temporary directory"
- ssh $SERVER_USER@$SERVER_IP "rm -rf $TMP_PATH"
after_script:
- echo "Deployment complete"
tags:
- vips-java
rules:
- if: $CI_COMMIT_REF_NAME == $RELEASE_BRANCH
environment:
name: production
url: https://vipscore02.nibio.no
upload-and-tag-release:
stage: upload
script:
# Use access token given by CI/CD variable ACCESS_TOKEN to authenticate
- &authenticate
......@@ -75,10 +169,9 @@ deploy-release:
- git stash pop
# Set release version and deploy. Commit, tag and push.
- &release_and_tag
# Change version in pom.xml - build new pom_with_models.xml
- './mvnw $MAVEN_CLI_OPTS $MAVEN_OPTS validate -DremoveSnapshot'
- 'python build_pom_with_models.py'
- &release_and_tag # Change version in pom.xml - build new pom_with_models.xml
- "./mvnw $MAVEN_CLI_OPTS $MAVEN_OPTS validate -DremoveSnapshot"
- "python build_pom_with_models.py"
- export RELEASE_VERSION=$(./mvnw $MAVEN_POM_WITH_MODELS $MAVEN_CLI_OPTS $MAVEN_OPTS --batch-mode --no-transfer-progress --non-recursive help:evaluate -Dexpression=project.version | grep -v "\[.*")
- echo "New version in release '$RELEASE_VERSION'"
- export RELEASE_TAG="v${RELEASE_VERSION}"
......@@ -86,9 +179,9 @@ deploy-release:
- export EXISTING_TAGS=$(git ls-remote --tags origin | cut -f 2 | grep "refs/tags/$RELEASE_TAG$")
- echo "Existing tags '$EXISTING_TAGS'"
- if [ -n "$EXISTING_TAGS" ]; then
echo "Tag $RELEASE_TAG already exists"; exit 1;
echo "Tag $RELEASE_TAG already exists"; exit 1;
fi
- './mvnw $MAVEN_POM_WITH_MODELS $MAVEN_CLI_OPTS $MAVEN_OPTS deploy -DskipTests'
- "./mvnw $MAVEN_POM_WITH_MODELS $MAVEN_CLI_OPTS $MAVEN_OPTS deploy -DskipTests"
- git add pom.xml
- git commit -m "[ci skip] Set release version ${RELEASE_VERSION}"
- git tag -a $RELEASE_TAG -m "Tag release ${RELEASE_TAG}"
......@@ -106,8 +199,8 @@ deploy-release:
- &merge_and_bump
- git merge -m "[ci skip] Merge branch '${RELEASE_BRANCH}' into ${MAIN_BRANCH}" --no-ff $RELEASE_BRANCH
# Change version in pom.xml - build new pom_with_models.xml
- './mvnw $MAVEN_CLI_OPTS $MAVEN_OPTS validate -DbumpPatch'
- 'python build_pom_with_models.py'
- "./mvnw $MAVEN_CLI_OPTS $MAVEN_OPTS validate -DbumpPatch"
- "python build_pom_with_models.py"
- export SNAPSHOT_VERSION=$(./mvnw $MAVEN_POM_WITH_MODELS $MAVEN_CLI_OPTS $MAVEN_OPTS --batch-mode --no-transfer-progress --non-recursive help:evaluate -Dexpression=project.version | grep -v "\[.*")
- echo "New version in main '$SNAPSHOT_VERSION'"
- git add pom.xml
......@@ -116,4 +209,4 @@ deploy-release:
tags:
- vips-java
rules:
- if: '$CI_COMMIT_REF_NAME == $RELEASE_BRANCH'
\ No newline at end of file
- if: $CI_COMMIT_REF_NAME == $RELEASE_BRANCH
# VIPSCore
This is the server component that hosts the models. It is accessible via
VIPSCoreManager over http/REST. Models that are on the component's classpath
are autodetected, indexed and made available through the REST service. The service is used by
This is the server component that hosts the models. It is accessible via VIPSCoreManager over http/REST. Models that are on the component's classpath
are autodetected, indexed and made available through the REST service. The service is used by
[VIPSLogic](https://gitlab.nibio.no/VIPS/VIPSLogic) when running models. It can also be contacted directly by any client.
VIPSCore is a part of the [VIPS platform for automatic pest prediction](https://gitlab.nibio.no/VIPS/documentation)
## VIPSCoreManager
[VIPSCoreManager](https://gitlab.nibio.no/VIPS/VIPSCoreManager) is a layer in front of VIPSCore that handles authentication of clients and logging of usage. All services available in VIPSLogic are transparently accessible via VIPSCoreManager. Add this layer if you want to limit and log usage, and also if you want to set up e.g. scaling of VIPSCore instances.
![How VIPSCore and VIPSCoreManager fits into the whole VIPS architecture](./docs/illustrations/VIPS_whole_architecture.png "All VIPS system parts")
......@@ -14,14 +15,19 @@ VIPSCore is a part of the [VIPS platform for automatic pest prediction](https://
## Build and deploy
Clone [the code](https://gitlab.nibio.no/VIPS/VIPSCore):
```
$ git clone git@gitlab.nibio.no:VIPS/VIPSCore.git
```
The project includes [maven-wrapper](https://maven.apache.org/wrapper/), which means that you do not need Maven installed locally. Build using the wrapper script with the following command
The project includes [maven-wrapper](https://maven.apache.org/wrapper/), which means that you do not need Maven installed locally. Build using the wrapper script with the following command
```
$ ./mvnw clean install
```
or on Windows
```
$ mvnw.cmd clean install
```
......@@ -30,17 +36,18 @@ $ mvnw.cmd clean install
Download and install [WildFly](https://www.wildfly.org/) >= 25.0.1
Deploy the build from this project in Wildfly.
Deploy the build from this project in Wildfly.
Wildfly should run on Java >= 11
### Adding models
Models are listed as dependencies in the POM and built into the WARfile. In order to make it possible
to configure your own set of models to include, we have separated the model dependencies out in a
separate file. The default is `models.xml`, and contains the models that VIPS in Norway are using
currently.
to configure your own set of models to include, we have separated the model dependencies out in a
separate file. The default is `models.xml`, and contains the models that VIPS in Norway are using
currently.
To build the package with the default set of models, run
To build the package with the default set of models, run
```bash
$ build_with_models.py
......@@ -53,7 +60,9 @@ $ build_with_models.py my_models.xml
```
## Implement a model
See [implement_model.md](./docs/implement_model.md)
## Create client for a model
See [create_client.md](./docs/create_client.md)
......@@ -92,7 +92,7 @@
<dependency>
<groupId>no.nibio.vips.model</groupId>
<artifactId>RoughageNutritionModel</artifactId>
<version>1.1.1</version>
<version>1.1.4</version>
</dependency>
<dependency>
<groupId>no.nibio.vips.model</groupId>
......
......@@ -9,7 +9,7 @@
<groupId>no.nibio</groupId>
<artifactId>VIPSCore</artifactId>
<packaging>war</packaging>
<version>2.0.1</version>
<version>2.0.3-SNAPSHOT</version>
<name>VIPSCore</name>
<url>http://maven.apache.org</url>
......@@ -66,7 +66,7 @@
<dependency>
<groupId>no.nibio.vips</groupId>
<artifactId>VIPSCommon</artifactId>
<version>2.0.0</version>
<version>2.0.1</version>
</dependency>
<dependency>
<groupId>javax</groupId>
......@@ -119,7 +119,6 @@
<groupId>commons-math</groupId>
<artifactId>commons-math</artifactId>
<version>1.2</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-net</groupId>
......@@ -131,7 +130,6 @@
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.15</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.googlecode.json-simple</groupId>
......
Source diff could not be displayed: it is too large. Options to address this: view the blob.
/*
* Copyright (c) 2018 NIBIO <http://www.nibio.no/>.
*
* This file is part of VIPSCore.
* VIPSCore is free software: you can redistribute it and/or modify
* it under the terms of the NIBIO Open Source License as published by
* NIBIO, either version 1 of the License, or (at your option) any
* later version.
*
* VIPSCore is distributed in the hope that it will be useful,
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* NIBIO Open Source License for more details.
*
* You should have received a copy of the NIBIO Open Source License
* along with VIPSCore. If not, see <http://www.nibio.no/licenses/>.
*
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package no.nibio.vips.core.config;
......
/*
* Copyright (c) 2014 NIBIO <http://www.nibio.no/>.
*
* This file is part of VIPSCore.
* VIPSCore is free software: you can redistribute it and/or modify
* it under the terms of the NIBIO Open Source License as published by
* NIBIO, either version 1 of the License, or (at your option) any
* later version.
*
* VIPSCore is distributed in the hope that it will be useful,
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* NIBIO Open Source License for more details.
*
* You should have received a copy of the NIBIO Open Source License
* along with VIPSCore. If not, see <http://www.nibio.no/licenses/>.
*
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package no.nibio.vips.core.service;
......
/*
* Copyright (c) 2014 NIBIO <http://www.nibio.no/>.
*
* This file is part of VIPSCore.
* VIPSCore is free software: you can redistribute it and/or modify
* it under the terms of the NIBIO Open Source License as published by
* NIBIO, either version 1 of the License, or (at your option) any
* later version.
*
* VIPSCore is distributed in the hope that it will be useful,
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* NIBIO Open Source License for more details.
*
* You should have received a copy of the NIBIO Open Source License
* along with VIPSCore. If not, see <http://www.nibio.no/licenses/>.
*
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package no.nibio.vips.core.startup;
......
/*
* Copyright (c) 2014 NIBIO <http://www.nibio.no/>.
*
* This file is part of VIPSCore.
* VIPSCore is free software: you can redistribute it and/or modify
* it under the terms of the NIBIO Open Source License as published by
* NIBIO, either version 1 of the License, or (at your option) any
* later version.
*
* VIPSCore is distributed in the hope that it will be useful,
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* NIBIO Open Source License for more details.
*
* You should have received a copy of the NIBIO Open Source License
* along with VIPSCore. If not, see <http://www.nibio.no/licenses/>.
*
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package no.nibio.vips.model.factory;
......
/*
* Copyright (c) 2014 NIBIO <http://www.nibio.no/>.
*
* This file is part of VIPSCore.
* VIPSCore is free software: you can redistribute it and/or modify
* it under the terms of the NIBIO Open Source License as published by
* NIBIO, either version 1 of the License, or (at your option) any
* later version.
*
* VIPSCore is distributed in the hope that it will be useful,
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* NIBIO Open Source License for more details.
*
* You should have received a copy of the NIBIO Open Source License
* along with VIPSCore. If not, see <http://www.nibio.no/licenses/>.
*
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package no.nibio.vips.model.factory;
......@@ -139,10 +138,10 @@ public class ModelFactory {
reflections = new Reflections(topDomain);
subTypes.addAll(reflections.getSubTypesOf(Model.class));
}*/
boolean verboseClassGraph = System.getProperty("no.nibio.vips.core.VERBOSE_CLASSGRAPH") != null && System.getProperty("no.nibio.vips.core.VERBOSE_CLASSGRAPH").equals("true");
try (ScanResult scanResult = // Assign scanResult in try-with-resources
new ClassGraph() // Create a new ClassGraph instance
.verbose() // If you want to enable logging to stderr
.verbose(verboseClassGraph) // If you want to enable logging to stderr
.enableClassInfo() // Scan classes, methods, fields, annotations
//.acceptPackages("com.xyz") // Scan com.xyz and subpackages
.scan()) { // Perform the scan and return a ScanResult
......
/*
* Copyright (c) 2015 NIBIO <http://www.nibio.no/>.
*
* This file is part of VIPSCore.
* VIPSCore is free software: you can redistribute it and/or modify
* it under the terms of the NIBIO Open Source License as published by
* NIBIO, either version 1 of the License, or (at your option) any
* later version.
*
* VIPSCore is distributed in the hope that it will be useful,
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* NIBIO Open Source License for more details.
*
* You should have received a copy of the NIBIO Open Source License
* along with VIPSCore. If not, see <http://www.nibio.no/licenses/>.
*
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package no.nibio.vips.model.factory;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment