Skip to content

Commit

Permalink
version update to 3.6.0 (#219)
Browse files Browse the repository at this point in the history
* upgrade springboot 2.6.6 and misc dependencies (#200)

* cannot start runs on prod #204 --  added config params to enable rabbitmq consumers to reconnect when the connection breaks.

* Develop with loggers (#208)

* Added logger statements

* Added logger statements

* research and investigation

Co-authored-by: UmmulkiramR <[email protected]>

* removed some logging (#211)

Co-authored-by: UmmulkiramR <[email protected]>

* updated nextflow version for #196 (#212)

Co-authored-by: UmmulkiramR <[email protected]>

* Nextflow 22 10 7 plugin fix (#213)

* changed nextflow library version to use the forked copy that contains the plugins fix

* changed repository name

* add github token for maven packages

---------

Co-authored-by: UmmulkiramR <[email protected]>
Co-authored-by: Anders Richardsson <[email protected]>

* readme update (#216)

* updated readme with instructions on how to use the forked version of the nextflow library

* Update README.md

Co-authored-by: Anders Richardsson <[email protected]>

* Update README.md

Co-authored-by: Anders Richardsson <[email protected]>

* Update README.md

Co-authored-by: Anders Richardsson <[email protected]>

* Update README.md

Co-authored-by: Anders Richardsson <[email protected]>

* Update README.md

Co-authored-by: Anders Richardsson <[email protected]>

* Update README.md

Co-authored-by: Anders Richardsson <[email protected]>

* added review suggestions.

---------

Co-authored-by: UmmulkiramR <[email protected]>
Co-authored-by: Anders Richardsson <[email protected]>

* Jenkins migration (#217)

* Update Jenkinsfile

* docker file update for adding server.xml for maven

---------

Co-authored-by: Yelizar Alturmessov <[email protected]>
Co-authored-by: UmmulkiramR <[email protected]>

* test command edit (#218)

Co-authored-by: UmmulkiramR <[email protected]>

* update to 3.6.0
contains:
- nextflow version change with the forked library
- Jenkins file update for cumulus build pipelines.

---------

Co-authored-by: jaserud <[email protected]>
Co-authored-by: UmmulkiramR <[email protected]>
Co-authored-by: Anders Richardsson <[email protected]>
Co-authored-by: Anders Richardsson <[email protected]>
Co-authored-by: Yelizar Alturmessov <[email protected]>
  • Loading branch information
6 people authored Dec 14, 2023
1 parent 71e22be commit 9c333c3
Show file tree
Hide file tree
Showing 11 changed files with 83 additions and 152 deletions.
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
FROM adoptopenjdk/openjdk11:jdk-11.0.6_10-alpine-slim as builder
WORKDIR /usr/src/app
ADD . .
RUN ./mvnw clean package -DskipTests
RUN ./mvnw clean package -DskipTests -s custom-settings.xml

#############################
# Server
Expand Down
145 changes: 9 additions & 136 deletions Jenkinsfile
Original file line number Diff line number Diff line change
@@ -1,136 +1,9 @@
def dockerRepo = "ghcr.io/icgc-argo/workflow-management"
def gitHubRepo = "icgc-argo/workflow-management"
def commit = "UNKNOWN"
def version = "UNKNOWN"

pipeline {
agent {
kubernetes {
label 'wf-management'
yaml """
apiVersion: v1
kind: Pod
spec:
containers:
- name: jdk
tty: true
image: openjdk:11
env:
- name: DOCKER_HOST
value: tcp://localhost:2375
- name: dind-daemon
image: docker:18.06-dind
securityContext:
privileged: true
runAsUser: 0
volumeMounts:
- name: docker-graph-storage
mountPath: /var/lib/docker
- name: helm
image: alpine/helm:2.12.3
command:
- cat
tty: true
- name: docker
image: docker:18-git
tty: true
env:
- name: DOCKER_HOST
value: tcp://localhost:2375
- name: HOME
value: /home/jenkins/agent
securityContext:
runAsUser: 1000
volumes:
- name: docker-graph-storage
emptyDir: {}
"""
}
}
stages {
stage('Prepare') {
steps {
script {
commit = sh(returnStdout: true, script: 'git describe --always').trim()
}
script {
version = readMavenPom().getVersion()
}
}
}
stage('Test') {
steps {
container('jdk') {
sh "./mvnw test"
}
}
}
stage('Build & Publish Develop') {
when {
branch "develop"
}
steps {
container('docker') {
withCredentials([usernamePassword(credentialsId:'argoContainers', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
sh 'docker login ghcr.io -u $USERNAME -p $PASSWORD'
}

// DNS error if --network is default
sh "docker build --network=host . -t ${dockerRepo}:edge -t ${dockerRepo}:${version}-${commit}"

sh "docker push ${dockerRepo}:${version}-${commit}"
sh "docker push ${dockerRepo}:edge"
}
}
}

stage('deploy to rdpc-collab-dev') {
when {
branch "develop"
}
steps {
build(job: "/provision/update-app-version", parameters: [
[$class: 'StringParameterValue', name: 'RDPC_ENV', value: 'dev' ],
[$class: 'StringParameterValue', name: 'TARGET_RELEASE', value: 'management'],
[$class: 'StringParameterValue', name: 'NEW_APP_VERSION', value: "${version}-${commit}" ]
])
}
}

stage('Release & Tag') {
when {
branch "master"
}
steps {
container('docker') {
withCredentials([usernamePassword(credentialsId: 'argoGithub', passwordVariable: 'GIT_PASSWORD', usernameVariable: 'GIT_USERNAME')]) {
sh "git tag ${version}"
sh "git push https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${gitHubRepo} --tags"
}

withCredentials([usernamePassword(credentialsId:'argoContainers', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
sh 'docker login ghcr.io -u $USERNAME -p $PASSWORD'
}

// DNS error if --network is default
sh "docker build --network=host . -t ${dockerRepo}:latest -t ${dockerRepo}:${version}"

sh "docker push ${dockerRepo}:${version}"
sh "docker push ${dockerRepo}:latest"
}
}
}
stage('deploy to rdpc-collab-qa') {
when {
branch "master"
}
steps {
build(job: "/provision/update-app-version", parameters: [
[$class: 'StringParameterValue', name: 'RDPC_ENV', value: 'qa' ],
[$class: 'StringParameterValue', name: 'TARGET_RELEASE', value: 'management'],
[$class: 'StringParameterValue', name: 'NEW_APP_VERSION', value: "${version}" ]
])
}
}
}
}
@Library(value='jenkins-pipeline-library@master', changelog=false) _
pipelineRDPCWorkflowManagement(
buildImage: "openjdk:11",
dockerRegistry: "ghcr.io",
dockerRepo: "icgc-argo/workflow-management",
gitRepo: "icgc-argo/workflow-management",
testCommand: "./mvnw test",
helmRelease: "management"
)
19 changes: 19 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -156,3 +156,22 @@ docker run ghcr.io/icgc-argo/workflow-management
```bash
mvn clean test
```


##### Using a forked version of the Nextflow library
The nextflow library is to be pulled temporarily from https://maven.pkg.github.com/icgc-argo/nextflow instead of nextflow.io.
To do this:
- create a new config file (maven's settings.xml) in rdpc jenkins which is needed during build and deployment. (Dashboard -> manage jenkins -> manage Files -> Config Files)

- a Github "personal access token" (PAT) is required for the creation of the config file. This Github token is to be added in rdpc jenkins (Dashboard -> Manage Jenkins -> Credentails -> click on System -> Global Credentials -> Add credentials)

- the jenkinsfile has to be updated to use the id of the config file (maven's settings.xml) created in rdpc jenkins
e.g. `configFileProvider([configFile(fileId: '11c739e4-8ac5-4fd3-983a-c20bd29846ef', variable: 'MAVEN_SETTINGS_PATH')])`

- the contents of the Maven's settings.xml to be copied to a path ./custom-settings.xml for use inside the dockerfile
`sh 'cp $MAVEN_SETTINGS_PATH ./custom-settings.xml'`

- the docker build command should then use the contents of the custom-settings.xml file during packaging
`RUN ./mvnw clean package -DskipTests -s custom-settings.xml`


32 changes: 26 additions & 6 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.5.4</version>
<version>2.6.6</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>org.icgc.argo</groupId>
<artifactId>workflow-management</artifactId>
<version>3.5.0</version>
<version>3.6.0</version>
<name>workflow-management</name>
<description>ARGO Workflow Management</description>

Expand All @@ -29,11 +29,25 @@
<id>jcenter</id>
<url>https://jcenter.bintray.com/</url>
</repository>
<repository>
<id>github</id>
<url>https://maven.pkg.github.com/icgc-argo/nextflow</url>
<snapshots>
<enabled>true</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>

<properties>
<java.version>11</java.version>
<reactor-rabbitmq-streams.version>0.0.9</reactor-rabbitmq-streams.version>
<spring-cloud-stream-kafka.version>3.2.2</spring-cloud-stream-kafka.version>
<spring-security-oauth2-autoconfigure.version>2.6.6</spring-security-oauth2-autoconfigure.version>
<kubernetes-client.version>4.7.0</kubernetes-client.version>
<jackson.version>2.13.2</jackson.version>
</properties>

<dependencies>
Expand Down Expand Up @@ -62,7 +76,7 @@
<dependency>
<groupId>io.nextflow</groupId>
<artifactId>nextflow</artifactId>
<version>21.04.3</version>
<version>22.10.7-SNAPSHOT-PluginsFix</version>
</dependency>

<!-- Health Check -->
Expand All @@ -74,14 +88,20 @@
<dependency>
<groupId>io.fabric8</groupId>
<artifactId>kubernetes-client</artifactId>
<version>4.7.0</version>
<version>${kubernetes-client.version}</version>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>${jackson.version}</version>
</dependency>

<!-- Spring-boot security dependencies -->
<dependency>
<groupId>org.springframework.security.oauth.boot</groupId>
<artifactId>spring-security-oauth2-autoconfigure</artifactId>
<version>2.5.2</version>
<version>${spring-security-oauth2-autoconfigure.version}</version>
</dependency>

<!-- Graphql -->
Expand Down Expand Up @@ -132,7 +152,7 @@
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-stream-kafka</artifactId>
<version>3.1.3</version>
<version>${spring-cloud-stream-kafka.version}</version>
</dependency>
</dependencies>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ private Run checkActiveRunAndUpdate(Run knownRun, RunState inputState) {

// check if this is a valid state transition
val nextStateOpt = nextState(currentState, inputState);
log.debug("nextStateOpt: {}",nextStateOpt);
if (nextStateOpt.isEmpty()) {
return null;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public Flux<Transaction<WfMgmtRunMsg>> apply(
// WeblogEvents only change run state in gatekeeper service, not other params
val allowedMsg =
service.checkWithExistingAndUpdateStateOnly(msg.getRunId(), msg.getState());

log.debug("getWeblogInputMsgsTransformer:allowedMsg: {}",allowedMsg);
if (allowedMsg.isEmpty()) {
tx.reject();
log.debug("WeblogConsumer - Gatekeeper Rejected: {}, YOU SHALL NOT PASS!", msg);
Expand All @@ -84,7 +84,7 @@ public Flux<Transaction<WfMgmtRunMsg>> apply(
(tx, sink) -> {
val msg = tx.get();
val allowedMsg = service.checkWfMgmtRunMsgAndUpdate(msg);

log.debug("getGateKeeperInputMsgTransformer:allowedMsg: {}",allowedMsg);
if (allowedMsg.isEmpty()) {
tx.reject();
log.debug(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ private Disposable createGatekeeperProducer() {
return Mono.just(tx);
})
.onErrorContinue(handleError());

log.debug("GateKeeperProducer Sending to: {}",producerTopicExchangeName);
return createTransProducerStream(rabbit, producerTopicExchangeName)
.send(processedFlux)
.onErrorContinue(handleError())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ public void sendNextflowEventAsync(NextflowMetadata metadata, NextflowEvent even
// This msg has to emulate a next flow event for it to be processed by workflow-relay.
// The actual value of runId should be metadata.workflow.sessionId but that is always
// null since it doesn't exist. But runId is NonNull in wf-relay, so set it to a default.
log.debug("sending Nextflow event");
val msg =
WorkflowEvent.builder()
.runId("NO-SESSION-ID")
Expand Down Expand Up @@ -103,6 +104,8 @@ public Mono<Boolean> sendWfMgmtEvent(WfManagementEvent event) {
}

private Mono<Boolean> sendHttpMessage(Object jsonReadyObject) {
log.debug("sending event message to: {}",endpoint);
log.debug("jsonReadyObject: {}",toJsonString(jsonReadyObject));
return WebClient.create(endpoint)
.post()
.contentType(MediaType.APPLICATION_JSON)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ private Disposable createWfMgmtRunMsgForExecuteConsumer() {
}

private Mono<Boolean> consumeMessageAndExecuteInitializeOrCancel(Transaction<WfMgmtRunMsg> tx) {
log.debug("Message received from: {} {} {}",topicExchangeName,queueName,topicRoutingKeys);
val msg = tx.get();
log.debug("WfMgmtRunMsg received: {}", msg);

Expand Down Expand Up @@ -114,6 +115,8 @@ private Mono<Boolean> rejectAndWeblogTx(Throwable t, Transaction<WfMgmtRunMsg> t

log.error("Error occurred", t);
log.error("WES SYSTEM_ERROR msg: {}", msg);
log.debug("Error occurred", t);
log.debug("WES SYSTEM_ERROR msg: {}", msg);
tx.reject();

return webLogEventSender
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ public NextflowService(
this.secretProvider = secretProvider;
this.webLogSender = webLogSender;
this.workflowRunK8sClient = createWorkflowRunK8sClient();
log.debug("WorkflowRunK8sClient created");
this.scheduler = Schedulers.newElastic("nextflow-service");
}

Expand Down Expand Up @@ -114,8 +115,9 @@ private Function<Throwable, Throwable> toRuntimeException(String methodName, Str

@SneakyThrows
private String startRun(RunParams params) {
log.debug("startRun");
val cmd = createCmd(createLauncher(), params);

log.debug("command created:cmd {}",cmd);
val driver = createDriver(cmd);
driver.run(params.getWorkflowUrl(), Collections.emptyList());
val exitStatus = driver.shutdown();
Expand All @@ -124,6 +126,7 @@ private String startRun(RunParams params) {

// Build required objects for monitoring THIS run.
val workflowMetadata = new NextflowWorkflowMetadata(cmd, driver, params);
log.debug("workflowMetadata: {}", workflowMetadata);
val meta = new NextflowMetadata(workflowMetadata, params.getWorkflowParams());
val monitor =
new NextflowWorkflowMonitor(
Expand All @@ -132,7 +135,7 @@ private String startRun(RunParams params) {
// Schedule a workflow monitor to watch over our nextflow pod and make sure
// that we report an error to our web-log service if it fails to run.
scheduler.schedule(monitor, config.getMonitor().getSleepInterval(), TimeUnit.MILLISECONDS);

log.debug("workflow scheduled");
return cmd.getRunName();
} else {
throw new NextflowRunException(
Expand Down Expand Up @@ -161,10 +164,12 @@ private DefaultKubernetesClient createWorkflowRunK8sClient() {
log.error(e.getMessage(), e);
throw new RuntimeException(e.getLocalizedMessage());
}

}

@SneakyThrows
private String cancelRun(@NonNull String runId) {
log.debug("cancelling run");
val state = getPhase(runId);

if (state.equals(KubernetesPhase.FAILED)) {
Expand Down Expand Up @@ -223,6 +228,7 @@ private String handleFailedPod(String podName) {
}

private Launcher createLauncher() throws ReflectionUtilsException {

// Add a launcher to the mix
val launcherParams = new HashMap<String, Object>();
val cliOptions = new CliOptions();
Expand Down
Loading

0 comments on commit 9c333c3

Please sign in to comment.