TASK-6445 - Implement OpenCGA Workflows with NextFlow

.github/workflows/develop.yml

@@ -32,5 +32,5 @@ jobs:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
     needs: build
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,workflow,init
     secrets: inherit

.github/workflows/manual-deploy-docker.yml

@@ -55,5 +55,5 @@ jobs:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
     needs: build
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag ${{ inputs.tag }}
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,workflow,init --tag ${{ inputs.tag }}
     secrets: inherit

.github/workflows/pull-request-merged.yml

@@ -19,6 +19,6 @@ jobs:
     uses: opencb/java-common-libs/.github/workflows/delete-docker-hub-workflow.yml@develop
     needs: build
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py delete --images base --tag ${{ github.head_ref }}
+      cli: python3 ./build/cloud/docker/docker-build.py delete --images base,workflow --tag ${{ github.head_ref }}
     secrets: inherit
 

.github/workflows/release.yml

@@ -19,7 +19,7 @@ jobs:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
     needs: build-hdp
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag "${{ needs.build-hdp.outputs.version }}-hdp3.1"
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,workflow,init --tag "${{ needs.build-hdp.outputs.version }}-hdp3.1"
       build_folder: build-folder
     secrets: inherit
 

.github/workflows/task.yml

@@ -25,5 +25,5 @@ jobs:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
     needs: test
     with:
-      cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag ${{ github.ref_name }}
+      cli: python3 ./build/cloud/docker/docker-build.py push --images base,workflow --tag ${{ github.ref_name }}
     secrets: inherit

opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/qc/AlignmentQcAnalysis.java

@@ -35,6 +35,7 @@
 import org.opencb.opencga.core.models.file.FileQualityControl;
 import org.opencb.opencga.core.models.file.FileUpdateParams;
 import org.opencb.opencga.core.models.job.Job;
+import org.opencb.opencga.core.models.job.JobType;
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.opencb.opencga.core.tools.annotations.Tool;
 import org.opencb.opencga.core.tools.annotations.ToolParams;
@@ -119,7 +120,7 @@ protected void run() throws ToolException {
                             .toParams(new ObjectMap(ParamConstants.STUDY_PARAM, study));
 
                     OpenCGAResult<Job> flagStatsJobResult = catalogManager.getJobManager()
-                            .submit(study, AlignmentFlagStatsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by "
+                            .submit(study, JobType.NATIVE, AlignmentFlagStatsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by "
                                     + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), getJobId(), null,
                                     false, token);
                     flagStatsJobId = flagStatsJobResult.first().getId();
@@ -137,7 +138,7 @@ protected void run() throws ToolException {
                             .toParams(new ObjectMap(ParamConstants.STUDY_PARAM, study));
 
                     OpenCGAResult<Job> statsJobResult = catalogManager.getJobManager()
-                            .submit(study, AlignmentStatsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by "
+                            .submit(study, JobType.NATIVE, AlignmentStatsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by "
                                     + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), getJobId(), null,
                                     false, token);
                     statsJobId = statsJobResult.first().getId();
@@ -154,7 +155,7 @@ protected void run() throws ToolException {
                             .toParams(new ObjectMap(ParamConstants.STUDY_PARAM, study));
 
                     OpenCGAResult<Job> fastQcMetricsJobResult = catalogManager.getJobManager()
-                            .submit(study, AlignmentFastQcMetricsAnalysis.ID, Enums.Priority.MEDIUM, params, null,
+                            .submit(study, JobType.NATIVE, AlignmentFastQcMetricsAnalysis.ID, Enums.Priority.MEDIUM, params, null,
                                     "Job generated by " + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(),
                                     getJobId(), null, false, token);
                     fastQcMetricsJobId = fastQcMetricsJobResult.first().getId();

opencga-analysis/src/main/java/org/opencb/opencga/analysis/customTool/CustomToolBuilder.java

@@ -0,0 +1,128 @@
+package org.opencb.opencga.analysis.customTool;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.StopWatch;
+import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy;
+import org.opencb.opencga.core.common.TimeUtils;
+import org.opencb.opencga.core.exceptions.ToolException;
+import org.opencb.opencga.core.models.common.Enums;
+import org.opencb.opencga.core.models.job.JobToolBuildParams;
+import org.opencb.opencga.core.tools.annotations.Tool;
+import org.opencb.opencga.core.tools.annotations.ToolParams;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+
+@Tool(id = CustomToolBuilder.ID, resource = Enums.Resource.JOB, description = CustomToolBuilder.DESCRIPTION)
+public class CustomToolBuilder extends OpenCgaToolScopeStudy {
+
+    public static final String ID = "custom-tool-builder";
+    public static final String DESCRIPTION = "Build an external bioinformatic tool from a GitHub repository URL.";
+
+    private static final String PYTHON_SCRIPT_NAME = "custom-tool-docker-build.py";
+
+    @ToolParams
+    protected JobToolBuildParams toolBuildParams = new JobToolBuildParams();
+
+    private final static Logger logger = LoggerFactory.getLogger(CustomToolBuilder.class);
+
+    @Override
+    protected void check() throws Exception {
+        super.check();
+
+        // Check any condition
+        if (toolBuildParams == null) {
+            throw new ToolException("Missing toolBuildParams");
+        }
+        if (StringUtils.isEmpty(toolBuildParams.getGitRepository())) {
+            throw new ToolException("Missing Git repository URL");
+        }
+        if (toolBuildParams.getDocker() == null
+                || StringUtils.isEmpty(toolBuildParams.getDocker().getOrganisation())
+                || StringUtils.isEmpty(toolBuildParams.getDocker().getName())
+                || StringUtils.isEmpty(toolBuildParams.getDocker().getTag())
+                || StringUtils.isEmpty(toolBuildParams.getDocker().getUser())
+                || StringUtils.isEmpty(toolBuildParams.getDocker().getPassword())
+        ) {
+            throw new ToolException("Missing Docker parameters, please provide the organisation, name, tag, user and password");
+        }
+        logger.debug("Checking finished, all seems fine");
+    }
+
+    @Override
+    protected void run() throws Exception {
+        // Build CLI params
+        StringBuilder cliBuilder = new StringBuilder();
+
+        // 1. Get the path to the custom-tool-docker-builder script
+        Path customToolDockerBuilderPath = this.getOpencgaHome()
+                .resolve("cloud")
+                .resolve("docker")
+                .resolve("custom-tool-docker-builder")
+                .resolve(PYTHON_SCRIPT_NAME);
+        logger.info("Executing custom-tool-docker-builder script: {}", customToolDockerBuilderPath);
+
+        // 2. Prepare basic CLI params
+        cliBuilder.append("python3").append(" ")
+                .append(customToolDockerBuilderPath).append(" ")
+                .append("push").append(" ")
+                .append("-t").append(" ").append(toolBuildParams.getGitRepository()).append(" ")
+                .append("-o").append(" ").append(toolBuildParams.getDocker().getOrganisation()).append(" ")
+                .append("-n").append(" ").append(toolBuildParams.getDocker().getName()).append(" ")
+                .append("-v").append(" ").append(toolBuildParams.getDocker().getTag()).append(" ")
+                .append("-u").append(" ").append(toolBuildParams.getDocker().getUser()).append(" ")
+                .append("-p").append(" ").append(toolBuildParams.getDocker().getPassword()).append(" ");
+
+        // 3. Check if 'apt-get' is provided
+        if (StringUtils.isNotEmpty(toolBuildParams.getAptGet())) {
+            cliBuilder.append("--apt-get").append(" ").append(toolBuildParams.getAptGet()).append(" ");
+        }
+
+        // 4. Check if 'installR' is provided
+        if (toolBuildParams.getInstallR() != null && toolBuildParams.getInstallR()) {
+            cliBuilder.append("--install-r").append(" ");
+        }
+
+        // 5. Execute the script
+        logger.info("CLI params: {}", cliBuilder);
+        // 5.1 Split the CLI params into a list
+        List<String> cliArgs = Arrays.asList(StringUtils.split(cliBuilder.toString(), " "));
+
+        // 5.2 Statr the stop watch
+        StopWatch stopWatch = StopWatch.createStarted();
+
+        // 5.3 Execute Python CLI
+        ProcessBuilder processBuilder = new ProcessBuilder(cliArgs);
+
+        // 5.4 Set the working directory of the process
+        processBuilder.directory(getOutDir().toFile());
+
+        // 5.5 Start execution
+        logger.info("Executing: {}", cliBuilder);
+        Process p;
+        try {
+            p = processBuilder.start();
+            BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream()));
+            BufferedReader error = new BufferedReader(new InputStreamReader(p.getErrorStream()));
+            String line;
+            while ((line = input.readLine()) != null) {
+                logger.info("{}", line);
+            }
+            while ((line = error.readLine()) != null) {
+                logger.error("{} ", line);
+            }
+            p.waitFor();
+            input.close();
+        } catch (IOException | InterruptedException e) {
+            throw new RuntimeException("Error executing cli: " + e.getMessage(), e);
+        }
+        logger.info("Execution time: {}", TimeUtils.durationToString(stopWatch));
+    }
+
+}

opencga-analysis/src/main/java/org/opencb/opencga/analysis/customTool/CustomToolExecutor.java

@@ -0,0 +1,82 @@
+package org.opencb.opencga.analysis.customTool;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.StopWatch;
+import org.opencb.opencga.analysis.tools.OpenCgaDockerToolScopeStudy;
+import org.opencb.opencga.core.common.TimeUtils;
+import org.opencb.opencga.core.exceptions.ToolException;
+import org.opencb.opencga.core.models.common.Enums;
+import org.opencb.opencga.core.models.job.JobRunDockerParams;
+import org.opencb.opencga.core.models.job.JobRunParams;
+import org.opencb.opencga.core.models.job.ToolInfoExecutor;
+import org.opencb.opencga.core.tools.annotations.Tool;
+import org.opencb.opencga.core.tools.annotations.ToolParams;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+@Tool(id = CustomToolExecutor.ID, resource = Enums.Resource.JOB, description = CustomToolExecutor.DESCRIPTION)
+public class CustomToolExecutor extends OpenCgaDockerToolScopeStudy {
+
+    public final static String ID = "custom-tool";
+    public static final String DESCRIPTION = "Execute an analysis from a custom binary.";
+
+    @ToolParams
+    protected JobRunParams runParams = new JobRunParams();
+
+    private String cliParams;
+    private String dockerImage;
+
+    private final static Logger logger = LoggerFactory.getLogger(CustomToolExecutor.class);
+
+    @Override
+    protected void check() throws Exception {
+        super.check();
+
+        // Check any condition
+        if (runParams == null) {
+            throw new ToolException("Missing runParams");
+        }
+        if (StringUtils.isEmpty(runParams.getCommandLine())) {
+            throw new ToolException("Missing commandLine");
+        }
+        if (runParams.getDocker() == null || StringUtils.isEmpty(runParams.getDocker().getId())) {
+            runParams.setDocker(new JobRunDockerParams("opencb/opencga-ext-tools", "3.2.1", null));
+        }
+        if (!runParams.getDocker().getId().contains("/")) {
+            throw new ToolException("Missing repository organization. Format for the docker image should be 'organization/image'");
+        }
+        this.dockerImage = runParams.getDocker().getId();
+        if (StringUtils.isNotEmpty(runParams.getDocker().getTag())) {
+            this.dockerImage += ":" + runParams.getDocker().getTag();
+        }
+
+        // Update job tags and attributes
+        ToolInfoExecutor toolInfoExecutor = new ToolInfoExecutor(runParams.getDocker().getId(), runParams.getDocker().getTag());
+        List<String> tags = new LinkedList<>();
+        tags.add(ID);
+        tags.add(this.dockerImage);
+        updateJobInformation(tags, toolInfoExecutor);
+
+        StringBuilder cliParamsBuilder = new StringBuilder();
+        processInputParams(runParams.getCommandLine(), cliParamsBuilder);
+        this.cliParams = cliParamsBuilder.toString();
+    }
+
+    @Override
+    protected void run() throws Exception {
+        StopWatch stopWatch = StopWatch.createStarted();
+
+        Map<String, String> dockerParams = new HashMap<>();
+        dockerParams.put("-e", "OPENCGA_TOKEN=" + getExpiringToken());
+        String cmdline = runDocker(dockerImage, null, cliParams, dockerParams);
+
+        logger.info("Docker command line: " + cmdline);
+        logger.info("Execution time: " + TimeUtils.durationToString(stopWatch));
+    }
+
+}

opencga-analysis/src/main/java/org/opencb/opencga/analysis/file/FileLinkTask.java

@@ -9,6 +9,7 @@
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.file.*;
 import org.opencb.opencga.core.models.job.Job;
+import org.opencb.opencga.core.models.job.JobType;
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.opencb.opencga.core.tools.annotations.Tool;
 import org.opencb.opencga.core.tools.annotations.ToolParams;
@@ -68,7 +69,7 @@ protected void run() throws Exception {
             Map<String, Object> params = new PostLinkToolParams(postLinkFileIds, null)
                     .toParams(new ObjectMap(ParamConstants.STUDY_PARAM, study));
             Job postLinkJob = catalogManager.getJobManager()
-                    .submit(getStudy(), PostLinkSampleAssociation.ID, Enums.Priority.MEDIUM, params, null,
+                    .submit(getStudy(), JobType.NATIVE, PostLinkSampleAssociation.ID, Enums.Priority.MEDIUM, params, null,
                             "Job generated by " + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(),
                             getJobId(), null, false, getToken()).first();
             logger.info("Submit post-link job : " + postLinkJob.getId());

opencga-analysis/src/main/java/org/opencb/opencga/analysis/sample/qc/SampleQcAnalysis.java

@@ -39,6 +39,7 @@
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.file.File;
 import org.opencb.opencga.core.models.job.Job;
+import org.opencb.opencga.core.models.job.JobType;
 import org.opencb.opencga.core.models.sample.Sample;
 import org.opencb.opencga.core.models.sample.SampleQualityControl;
 import org.opencb.opencga.core.models.sample.SampleUpdateParams;
@@ -224,7 +225,7 @@ protected void run() throws ToolException {
                             .toParams(new ObjectMap(ParamConstants.STUDY_PARAM, getStudy()));
 
                     OpenCGAResult<Job> variantStatsJobResult = catalogManager.getJobManager()
-                            .submit(study, SampleVariantStatsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by "
+                            .submit(study, JobType.NATIVE, SampleVariantStatsAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by "
                                     + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), getJobId(), null,
                                     false, token);
                     variantStatsJobId = variantStatsJobResult.first().getId();
@@ -267,7 +268,7 @@ protected void run() throws ToolException {
                             .toParams(new ObjectMap(ParamConstants.STUDY_PARAM, getStudy()));
 
                     OpenCGAResult<Job> signatureJobResult = catalogManager.getJobManager()
-                            .submit(getStudy(), MutationalSignatureAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by "
+                            .submit(getStudy(), JobType.NATIVE, MutationalSignatureAnalysis.ID, Enums.Priority.MEDIUM, params, null, "Job generated by "
                                     + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(), getJobId(), null,
                                     false, token);
                     signatureJobId = signatureJobResult.first().getId();
@@ -289,7 +290,7 @@ protected void run() throws ToolException {
                             .toParams(new ObjectMap(ParamConstants.STUDY_PARAM, getStudy()));
 
                     OpenCGAResult<Job> genomePlotJobResult = catalogManager.getJobManager()
-                            .submit(getStudy(), GenomePlotAnalysis.ID, Enums.Priority.MEDIUM, params, null,
+                            .submit(getStudy(), JobType.NATIVE, GenomePlotAnalysis.ID, Enums.Priority.MEDIUM, params, null,
                                     "Job generated by " + getId() + " - " + getJobId(), Collections.emptyList(), Collections.emptyList(),
                                     getJobId(), null, false, token);
                     genomePlotJobId = genomePlotJobResult.first().getId();