Merge branch 'develop' into TASK-6219

.github/workflows/manual-deploy-python-notebook.yml

@@ -0,0 +1,59 @@
+name: Reusable deploy Docker python-notebook
+run-name: 'Run deploy Docker python-notebook ${{ inputs.branch }} by @${{ github.actor }}'
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: "The branch, tag or SHA of the source code to build docker."
+        type: string
+        required: true
+      tag:
+        description: "The tag for the new docker."
+        type: string
+        required: true
+      hadoop:
+        type: string
+        description: 'Hadoop flavour. Any of: [hdp3.1, hdi5.1, emr6.1, emr6.13]'
+        required: false
+        default: "hdp3.1"
+      pyopencga_version:
+        type: string
+        description: 'PyOpenCGA version.'
+        required: true
+
+jobs:
+  build:
+    name: Build Java app
+    runs-on: ubuntu-22.04
+    outputs:
+      version: ${{ steps.get_project_version.outputs.version }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: '10'
+          ref: "${{ inputs.branch }}"
+      - name: Set up JDK 8
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'maven'
+      - name: Install dependencies branches
+        run: |
+          if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
+            chmod +x ./.github/workflows/scripts/get_same_branch.sh
+            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} ${{ inputs.hadoop }}
+          fi
+      - name: Maven Build (skip tests)
+        run: mvn -T 2 clean install -DskipTests
+      - uses: actions/upload-artifact@v4
+        with:
+          name: build-folder
+          path: build
+
+  deploy-docker-python-notebook:
+    uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
+    needs: build
+    with:
+      cli: python3 ./build/cloud/docker/docker-build.py push --images python-notebook --tag ${{ inputs.tag }} --docker-build-args "-e VERSION=${{ inputs.pyopencga_version }}"
+    secrets: inherit

.github/workflows/pull-request-approved.yml

@@ -1,19 +1,41 @@
 name: Pull request approve workflow
+run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref }} -> ${{ github.event.pull_request.base.ref }} by @${{ github.actor }}'
 
 on:
   pull_request_review:
-    types: [submitted]
+    types: [ submitted ]
 
 jobs:
-  build:
-    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
-    with:
-      maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+  calculate-xetabase-branch:
+    name: Calculate Xetabase branch
+    runs-on: ubuntu-22.04
+    outputs:
+      xetabase_branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }}
+    steps:
+      - name: Clone project
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: '10'
+          ## This is important to avoid the error in the next step: "fatal: repository 'https://github.com/zetta-genomics/opencga-enterprise.git/' not found"
+          persist-credentials: false
+      - id: get_xetabase_branch
+        name: "Get current branch for Xetabase from target branch"
+        run: |
+          chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh
+          echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}"
+          echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}"
+          xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }})
+          echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY}
+          echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT
+        env:
+          ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
 
   test:
-    name: "Run all tests before merging, ie. short, medium and long tests."
-    uses: ./.github/workflows/test-analysis.yml
-    needs: build
+    name: "Run all tests before merging"
+    needs: calculate-xetabase-branch
+    uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@develop
     with:
-      test_profile: runShortTests,runMediumTests,runLongTests
+      branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }}
+      task: ${{ github.event.pull_request.head.ref }}
     secrets: inherit
+

.github/workflows/scripts/get-xetabase-branch.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# Function to calculate the corresponding branch of Xetabase project
+get_xetabase_branch() {
+  # Input parameter (branch name)
+  input_branch="$1"
+
+  # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it
+  if [[ $input_branch == TASK* ]]; then
+    if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then
+      echo $input_branch;
+      return 0;
+    fi
+  fi
+
+  # Check if the branch name is "develop" in that case return the same branch name
+  if [[ "$input_branch" == "develop" ]]; then
+    echo "develop"
+    return 0
+  fi
+
+  # Check if the branch name starts with "release-" and follows the patterns "release-a.x.x" or "release-a.b.x"
+  if [[ "$input_branch" =~ ^release-([0-9]+)\.x\.x$ ]] || [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.x$ ]]; then
+    # Extract the MAJOR part of the branch name
+    MAJOR=${BASH_REMATCH[1]}
+    # Calculate the XETABASE_MAJOR by subtracting 1 from MAJOR of opencga
+    XETABASE_MAJOR=$((MAJOR - 1))
+    # Check if the XETABASE_MAJOR is negative
+    if (( XETABASE_MAJOR < 0 )); then
+      echo "Error: 'MAJOR' digit after subtraction results in a negative number."
+      return 1
+    fi
+    # Construct and echo the new branch name
+    echo "release-$XETABASE_MAJOR.${input_branch#release-$MAJOR.}"
+    return 0
+  fi
+
+  # If the branch name does not match any of the expected patterns
+  echo "Error: The branch name is not correct."
+  return 1
+}
+
+# Check if the script receives exactly one argument
+if [ "$#" -ne 1 ]; then
+  echo "Usage: $0 <branch-name>"
+  exit 1
+fi
+
+# Call the function with the input branch name
+get_xetabase_branch "$1"

.github/workflows/test-analysis.yml

@@ -90,7 +90,7 @@ jobs:
       - name: Maven build
         run: mvn -B clean install -DskipTests -P ${{ inputs.hadoop }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
       - name: Run Junit tests
-        run: mvn -B verify surefire-report:report --fail-never -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
+        run: mvn -B verify surefire-report:report --fail-never -Dsurefire.testFailureIgnore=true -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
       - name: Publish Test Report on GitHub
         uses: scacap/action-surefire-report@v1
         env:

README.md

@@ -6,19 +6,13 @@ OpenCGA constitutes the big data analysis component of [OpenCB](http://www.openc
 ### Documentation
 You can find OpenCGA documentation and tutorials at: https://github.com/opencb/opencga/wiki.
 
-### Issues Tracking
-You can report bugs or request new features at [GitHub issue tracking](https://github.com/opencb/opencga/issues).
+### Feedback and Feature Requests
+Found a bug or have an idea for a new feature? Let us know at zettagenomics.com/academic.
 
 ### Release Notes and Roadmap
-Releases notes are available at [GitHub releases](https://github.com/opencb/opencga/releases).
-
-Roadmap is available at [GitHub milestones](https://github.com/opencb/opencga/milestones). You can report bugs or request new features at [GitHub issue tracking](https://github.com/opencb/opencga/issues).
-
-### Versioning
-OpenCGA is versioned following the rules from [Semantic versioning](https://semver.org/).
+Releases notes are available at https://zettagenomics.com/release-notes/
 
 ### Maintainers
-We recommend to contact OpenCGA developers by writing to OpenCB mailing list [email protected]. Current main developers and maintainers are:
 * Ignacio Medina ([email protected]) (_Founder and Project Leader_)
 * Jacobo Coll ([email protected])
 * Pedro Furio ([email protected])
@@ -33,7 +27,7 @@ We recommend to contact OpenCGA developers by writing to OpenCB mailing list ope
 * Franscisco Salavert ([email protected])
 
 ##### Contributing
-OpenCGA is an open-source and collaborative project. We appreciate any help and feedback from users, you can contribute in many different ways such as simple bug reporting and feature request. Dependending on your skills you are more than welcome to develop client tools, new features or even fixing bugs.
+OpenCGA is an open-source and collaborative project. We appreciate any help and feedback from users, you can contribute in many different ways such as simple bug reporting and feature request. Dependending on your skills you are more than welcome to develop client tools, new features, or even fix bugs.
 
 
 # How to build

opencga-analysis/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>3.2.0-SNAPSHOT</version>
+        <version>4.0.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

opencga-analysis/src/main/java/org/opencb/opencga/analysis/ConfigurationUtils.java

@@ -16,9 +16,13 @@
 
 package org.opencb.opencga.analysis;
 
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.opencb.commons.utils.FileUtils;
+import org.opencb.opencga.core.config.AnalysisTool;
 import org.opencb.opencga.core.config.Configuration;
 import org.opencb.opencga.core.config.storage.StorageConfiguration;
+import org.opencb.opencga.core.exceptions.ToolException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -28,10 +32,15 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
 
 public class ConfigurationUtils {
     private static Logger logger = LoggerFactory.getLogger(ConfigurationUtils.class);
 
+    private ConfigurationUtils() {
+        throw new IllegalStateException("Utility class");
+    }
     /**
      * This method attempts to load general configuration from OpenCGA installation folder, if not exists then loads JAR configuration.yml.
      *
@@ -83,4 +92,33 @@ public static StorageConfiguration loadStorageConfiguration(String opencgaHome)
                     .load(StorageConfiguration.class.getClassLoader().getResourceAsStream("storage-configuration.yml"));
         }
     }
+
+    public static String getToolDefaultVersion(String toolId, Configuration configuration) throws ToolException {
+        List<AnalysisTool> tools = new ArrayList<>();
+        for (AnalysisTool tool : configuration.getAnalysis().getTools()) {
+            if (tool.getId().equals(toolId)) {
+                tools.add(tool);
+            }
+        }
+        if (CollectionUtils.isEmpty(tools)) {
+            throw new ToolException("Tool ID '" + toolId + "' missing in the configuration file");
+        }
+        if (tools.size() == 1) {
+            return tools.get(0).getVersion();
+        }
+        String defaultVersion = null;
+        for (AnalysisTool tool : tools) {
+            if (tool.isDefaultVersion()) {
+                if (!StringUtils.isEmpty(defaultVersion)) {
+                    throw new ToolException("More than one default version found for tool ID '" + toolId + "'");
+                } else {
+                    defaultVersion = tool.getVersion();
+                }
+            }
+        }
+        if (StringUtils.isEmpty(defaultVersion)) {
+            throw new ToolException("Multiple tools '" + toolId + "' were found, but none have the default version set to true");
+        }
+        return defaultVersion;
+    }
 }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java

@@ -23,7 +23,9 @@
 import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.models.alignment.CoverageIndexParams;
 import org.opencb.opencga.core.models.common.Enums;
+import org.opencb.opencga.core.models.common.InternalStatus;
 import org.opencb.opencga.core.models.file.File;
+import org.opencb.opencga.core.models.file.FileInternalCoverageIndex;
 import org.opencb.opencga.core.tools.annotations.Tool;
 import org.opencb.opencga.core.tools.annotations.ToolParams;
 
@@ -40,7 +42,7 @@
 public class AlignmentCoverageAnalysis extends OpenCgaToolScopeStudy {
 
     public static final String ID = "coverage-index-run";
-    public static final String DESCRIPTION = "Compute the coverage from a given alignment file, e.g., create a "
+    public static final String DESCRIPTION = "Compute the coverage from a given BAM alignment file, e.g., create a "
             + AlignmentConstants.BIGWIG_EXTENSION + " file from a " + AlignmentConstants.BAM_EXTENSION + " file";
 
     @ToolParams
@@ -64,37 +66,36 @@ protected void check() throws Exception {
 
         //  Checking BAM file ID
         try {
-            bamCatalogFile = catalogManager.getFileManager().get(getStudy(), coverageParams.getBamFileId(), QueryOptions.empty(),
+            bamCatalogFile = catalogManager.getFileManager().get(getStudy(), coverageParams.getFileId(), QueryOptions.empty(),
                     getToken()).first();
             if (bamCatalogFile == null) {
-                throw new ToolException("Could not find BAM file from ID '" + coverageParams.getBamFileId() + "'");
+                throw new ToolException("Could not find BAM file from ID '" + coverageParams.getFileId() + "'");
             }
         } catch (Exception e) {
-            throw new ToolException("Could not get BAM file from ID " + coverageParams.getBamFileId());
+            throw new ToolException("Could not get BAM file from ID " + coverageParams.getFileId());
         }
 
         // Check if the input file is .bam
         if (!bamCatalogFile.getName().endsWith(AlignmentConstants.BAM_EXTENSION)) {
-            throw new ToolException("Invalid input alignment file '" + coverageParams.getBamFileId() + "' (" + bamCatalogFile.getName()
+            throw new ToolException("Invalid input alignment file '" + coverageParams.getFileId() + "' (" + bamCatalogFile.getName()
                     + "): it must be in BAM format");
         }
 
         // Getting BAI file
-        String baiFileId = coverageParams.getBaiFileId();
+        String baiFileId;
+        try {
+            baiFileId = bamCatalogFile.getInternal().getAlignment().getIndex().getFileId();
+        } catch (Exception e) {
+            throw new ToolException("Could not get internal alignment index file Id from BAM file ID '" + bamCatalogFile.getId() + "'");
+        }
         if (StringUtils.isEmpty(baiFileId)) {
-            // BAI file ID was not provided, looking for it
-            logger.info("BAI file ID was not provided, getting it from the internal alignment index of the BAM file ID {}",
-                    bamCatalogFile.getId());
-            try {
-                baiFileId = bamCatalogFile.getInternal().getAlignment().getIndex().getFileId();
-            } catch (Exception e) {
-                throw new ToolException("Could not get internal alignment index file Id from BAM file ID '" + bamCatalogFile.getId());
-            }
+            throw new ToolException("Could not find the alignment index file for the BAM file ID '" + bamCatalogFile.getId() + "'. Please,"
+                    + " create the alignment index file before computing the coverage");
         }
         try {
             baiCatalogFile = catalogManager.getFileManager().get(getStudy(), baiFileId, QueryOptions.empty(), getToken()).first();
             if (baiCatalogFile == null) {
-                throw new ToolException("Could not find BAI file from ID '" + coverageParams.getBaiFileId() + "'");
+                throw new ToolException("Could not find BAI file from ID '" + baiFileId + "'");
             }
         } catch (Exception e) {
             throw new ToolException("Could not get BAI file from file ID " + baiFileId);
@@ -115,6 +116,18 @@ protected void check() throws Exception {
             coverageParams.setWindowSize(Integer.parseInt(COVERAGE_WINDOW_SIZE_DEFAULT));
             logger.info("Window size is set to {}", coverageParams.getWindowSize());
         }
+
+        // Check overwrite
+        String bwFileId;
+        try {
+            bwFileId = bamCatalogFile.getInternal().getAlignment().getCoverage().getFileId();
+        } catch (Exception e) {
+            bwFileId = null;
+        }
+        if (StringUtils.isNotEmpty(bwFileId) && !coverageParams.isOverwrite()) {
+            throw new ToolException("Coverage file ID '" + bwFileId + "' already exists for file ID '" + bamCatalogFile.getId()
+                    + "'. To overwrite the BIGWIG file use the flag --overwrite");
+        }
     }
 
     @Override
@@ -192,7 +205,12 @@ protected void run() throws Exception {
             }
 
             // Link generated BIGWIG file and update samples info
-            AlignmentAnalysisUtils.linkAndUpdate(bamCatalogFile, bwPath, getJobId(), study, catalogManager, token);
+            File bwCatalogFile = AlignmentAnalysisUtils.linkAndUpdate(bamCatalogFile, bwPath, getJobId(), study, catalogManager, token);
+
+            // Update BAM file internal in order to set the coverage index (BIGWIG)
+            FileInternalCoverageIndex fileCoverageIndex = new FileInternalCoverageIndex(new InternalStatus(InternalStatus.READY),
+                    bwCatalogFile.getId(), "deeptools bamCoverage", coverageParams.getWindowSize());
+            catalogManager.getFileManager().updateFileInternalCoverageIndex(study, bamCatalogFile, fileCoverageIndex, token);
         });
     }
 }