storage: Delete opencga-storage-hadoop-deps. Fix profiles. #TASK-5240

opencb · Jan 29, 2024 · 4371bf2 · 4371bf2
1 parent de30db8
commit 4371bf2
Show file tree

Hide file tree

Showing 56 changed files with 898 additions and 4,514 deletions.
diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
@@ -11,7 +11,7 @@ jobs:
   build:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
     with:
-      maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6'
+      maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
 
   test:
     uses: ./.github/workflows/test-analysis.yml
@@ -24,7 +24,7 @@ jobs:
     uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
     needs: test
     with:
-      maven_opts: -P storage-hadoop,hdp3.1 -Dopencga.war.name=opencga
+      maven_opts: -P hdp3.1 -Dopencga.war.name=opencga
     secrets: inherit
 
   deploy-docker:

diff --git a/.github/workflows/docker-aws-emr.yml b/.github/workflows/docker-aws-emr.yml
@@ -10,7 +10,7 @@ jobs:
   build:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
     with:
-      maven_opts: -P storage-hadoop,emr6.1 -Dopencga.war.name=opencga
+      maven_opts: -Pemr6.1 -Dopencga.war.name=opencga
 
   deploy-docker:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop

diff --git a/.github/workflows/pull-request-merge.yml b/.github/workflows/pull-request-merge.yml
@@ -13,7 +13,7 @@ jobs:
   build:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
     with:
-      maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6'
+      maven_opts: -P hdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
 
   delete-docker:
     uses: opencb/java-common-libs/.github/workflows/delete-docker-hub-workflow.yml@develop

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -10,13 +10,13 @@ jobs:
   build:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
     with:
-      maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6'
+      maven_opts: -P hdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
 
   deploy-maven:
     uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
     needs: build
     with:
-      maven_opts: -P storage-hadoop,hdp3.1 -Dopencga.war.name=opencga
+      maven_opts: -P hdp3.1 -Dopencga.war.name=opencga
     secrets: inherit
 
   deploy-docker:

diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml
@@ -12,7 +12,7 @@ jobs:
   build:
     uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
     with:
-      maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6'
+      maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
 
   test:
     uses: ./.github/workflows/test-analysis.yml

diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml
@@ -41,7 +41,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-        run: mvn -T 2 clean install -P storage-hadoop,hdp3.1,${{ inputs.test_profile }} -DskipTests -Dcheckstyle.skip org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_opencga
+        run: mvn -T 2 clean install -P hdp3.1,${{ inputs.test_profile }} -DskipTests -Dcheckstyle.skip org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_opencga
 
   test:
     name: Execute JUnit and Jacoco tests
@@ -72,7 +72,7 @@ jobs:
           mongodb-version: 6.0
           mongodb-replica-set: rs-test
       - name: Run Junit tests
-        run: mvn -B verify surefire-report:report --fail-never -P storage-hadoop,hdp3.1,${{ inputs.test_profile }} -Dcheckstyle.skip -Popencga-storage-hadoop-deps -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' ${{ inputs.mvn_opts }}
+        run: mvn -B verify surefire-report:report --fail-never -P hdp3.1,${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
       - name: Publish Test Report
         uses: scacap/action-surefire-report@v1
         env:

diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml
@@ -71,26 +71,33 @@
         </dependency>
         <dependency>
             <groupId>org.opencb.opencga</groupId>
-            <artifactId>${opencga-storage-hadoop-deps.artifactId}</artifactId>
-            <classifier>shaded</classifier>
+            <artifactId>opencga-storage-hadoop-${opencga-hadoop-shaded.id}</artifactId>
+            <version>${project.parent.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>org.opencb.opencga</groupId>
-            <artifactId>${opencga-storage-hadoop-deps.artifactId}</artifactId>
-            <version>${project.parent.version}</version>
-            <classifier>tests</classifier>
+            <groupId>org.opencb.opencga.hadoop.thirdparty</groupId>
+            <artifactId>${opencga-hadoop-shaded.artifactId}</artifactId>
+            <version>${opencga.hadoop.thirdparty.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.opencb.opencga.hadoop.thirdparty</groupId>
+            <artifactId>${opencga-hadoop-shaded.artifactId}</artifactId>
+            <version>${opencga.hadoop.thirdparty.version}</version>
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
 
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.opencb.opencga</groupId>
@@ -105,7 +112,6 @@
         <dependency>
             <groupId>org.opencb.commons</groupId>
             <artifactId>commons-datastore-mongodb</artifactId>
-            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.opencb.commons</groupId>
@@ -159,10 +165,6 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
         </dependency>
-        <dependency>
-            <groupId>commons-lang</groupId>
-            <artifactId>commons-lang</artifactId>
-        </dependency>
         <dependency>
             <groupId>commons-io</groupId>
             <artifactId>commons-io</artifactId>
@@ -232,10 +234,11 @@
         <dependency>
             <groupId>org.apache.solr</groupId>
             <artifactId>solr-test-framework</artifactId>
+            <scope>test</scope>
         </dependency>
 
-        <!-- TODO Review -->
         <dependency>
+            <!-- Need dependency for running hadoop tests. -->
             <groupId>javax.servlet</groupId>
             <artifactId>javax.servlet-api</artifactId>
             <scope>test</scope>
@@ -271,12 +274,17 @@
                             <!-- This plugin is having a weird issue with biodata libraries. It fails if biodata is declared before. -->
                             <ignoredUsedUndeclaredDependencies>
                                 <ignoredUsedUndeclaredDependency>
-                                    com.google.code.findbugs:jsr305:jar:3.0.2
+                                    com.google.code.findbugs:jsr305
                                 </ignoredUsedUndeclaredDependency>
                             </ignoredUsedUndeclaredDependencies>
                             <ignoredUnusedDeclaredDependencies>
                                 <ignoredUnusedDeclaredDependency>*</ignoredUnusedDeclaredDependency>
                             </ignoredUnusedDeclaredDependencies>
+                            <ignoredNonTestScopedDependencies>
+                                <ignoredNonTestScopedDependency>
+                                    org.opencb.commons:commons-datastore-mongodb
+                                </ignoredNonTestScopedDependency>
+                            </ignoredNonTestScopedDependencies>
                         </configuration>
                     </execution>
                 </executions>

diff --git a/...in/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserClinicalVariantCreator.java b/...in/java/org/opencb/opencga/analysis/clinical/exomiser/ExomiserClinicalVariantCreator.java
@@ -1,7 +1,7 @@
 package org.opencb.opencga.analysis.clinical.exomiser;
 
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.opencb.biodata.models.clinical.ClinicalAcmg;
 import org.opencb.biodata.models.clinical.ClinicalDiscussion;
 import org.opencb.biodata.models.clinical.ClinicalProperty;

diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/family/qc/IBDComputation.java
@@ -16,12 +16,10 @@
 
 package org.opencb.opencga.analysis.family.qc;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.math3.ode.nonstiff.RungeKuttaFieldIntegrator;
+import org.apache.commons.lang3.StringUtils;
 import org.opencb.biodata.models.clinical.qc.RelatednessReport;
 import org.opencb.biodata.models.clinical.qc.RelatednessScore;
 import org.opencb.biodata.models.variant.avro.VariantType;
@@ -34,19 +32,16 @@
 import org.opencb.opencga.analysis.variant.relatedness.RelatednessAnalysis;
 import org.opencb.opencga.analysis.wrappers.plink.PlinkWrapperAnalysisExecutor;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
-import org.opencb.opencga.core.common.JacksonUtils;
 import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.models.family.Family;
 import org.opencb.opencga.core.models.individual.Individual;
 import org.opencb.opencga.core.models.sample.Sample;
 import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
 import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
-import org.opencb.opencga.storage.core.variant.query.VariantQueryUtils;
 
 import java.io.*;
 import java.net.URL;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.*;
 import java.util.stream.Collectors;
 

diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolFactory.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/ToolFactory.java
@@ -93,7 +93,7 @@ static Collection<URL> getUrls() {
         Collection<URL> urls = new LinkedList<>();
         for (URL url : ClasspathHelper.forPackage("org.opencb.opencga")) {
             String name = url.getPath().substring(url.getPath().lastIndexOf('/') + 1);
-            if (name.isEmpty() || (name.contains("opencga") && !name.contains("opencga-storage-hadoop-deps"))) {
+            if (name.isEmpty() || (name.contains("opencga") && !name.contains("opencga-hadoop-shaded"))) {
                 urls.add(url);
             }
         }

diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/family/FamilyAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/family/FamilyAnalysisTest.java
@@ -1,6 +1,6 @@
 package org.opencb.opencga.analysis.family;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.junit.*;
 import org.junit.rules.ExpectedException;
 import org.opencb.biodata.models.clinical.Disorder;

diff --git a/opencga-app/app/cloud/docker/docker-build.py b/opencga-app/app/cloud/docker/docker-build.py
@@ -88,7 +88,7 @@ def build():
 
 
 def tag_latest(image):
-    if "hdp" in tag or "dev" in tag:
+    if "hdi" in tag or "emr" in tag or "dev" in tag or "SNAPSHOT" in tag or "TASK" in tag:
         print("Don't use tag " + tag + " as latest")
         return
     if server:
@@ -160,7 +160,7 @@ def delete():
     build_folder = str(Path(__file__).resolve().parents[2])
 
 # 2. Set docker tag to default value if not set
-if args.tag is not None:
+if args.tag is not None and not args.tag == "":
     tag = args.tag
 else:
     # Read OpenCGA version from git.properties

diff --git a/opencga-app/app/misc/scripts/opencga-storage-hadoop.sh b/opencga-app/app/misc/scripts/opencga-storage-hadoop.sh