Merge pull request #9754 from IQSS/develop

merge v5.14 into master

.env

@@ -0,0 +1,4 @@
+APP_IMAGE=gdcc/dataverse:unstable
+POSTGRES_VERSION=13
+DATAVERSE_DB_USER=dataverse
+SOLR_VERSION=8.11.1

.github/workflows/container_app_pr.yml

@@ -0,0 +1,96 @@
+---
+name: Preview Application Container Image
+
+on:
+    # We only run the push commands if we are asked to by an issue comment with the correct command.
+    # This workflow is always taken from the default branch and runs in repo context with access to secrets.
+    repository_dispatch:
+        types: [ push-image-command ]
+
+env:
+    IMAGE_TAG: unstable
+    BASE_IMAGE_TAG: unstable
+    PLATFORMS: "linux/amd64,linux/arm64"
+
+jobs:
+    deploy:
+        name: "Package & Push"
+        runs-on: ubuntu-latest
+        # Only run in upstream repo - avoid unnecessary runs in forks
+        if: ${{ github.repository_owner == 'IQSS' }}
+        steps:
+            # Checkout the pull request code as when merged
+            - uses: actions/checkout@v3
+              with:
+                  ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge'
+            - uses: actions/setup-java@v3
+              with:
+                  java-version: "11"
+                  distribution: 'adopt'
+            - uses: actions/cache@v3
+              with:
+                  path: ~/.m2
+                  key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+                  restore-keys: ${{ runner.os }}-m2
+
+            # Note: Accessing, pushing tags etc. to GHCR will only succeed in upstream because secrets.
+            - name: Login to Github Container Registry
+              uses: docker/login-action@v2
+              with:
+                  registry: ghcr.io
+                  username: ${{ secrets.GHCR_USERNAME }}
+                  password: ${{ secrets.GHCR_TOKEN }}
+
+            - name: Set up QEMU for multi-arch builds
+              uses: docker/setup-qemu-action@v2
+
+            # Get the image tag from either the command or default to branch name (Not used for now)
+            #-   name: Get the target tag name
+            #    id: vars
+            #    run: |
+            #        tag=${{ github.event.client_payload.slash_command.args.named.tag }}
+            #        if [[ -z "$tag" ]]; then tag=$(echo "${{ github.event.client_payload.pull_request.head.ref }}" | tr '\\/_:&+,;#*' '-'); fi
+            #        echo "IMAGE_TAG=$tag" >> $GITHUB_ENV
+
+            # Set image tag to branch name of the PR
+            - name: Set image tag to branch name
+              run: |
+                  echo "IMAGE_TAG=$(echo "${{ github.event.client_payload.pull_request.head.ref }}" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV
+
+            # Necessary to split as otherwise the submodules are not available (deploy skips install)
+            - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
+              run: >
+                  mvn -B -f modules/dataverse-parent
+                  -P ct -pl edu.harvard.iq:dataverse -am
+                  install
+            - name: Deploy multi-arch application and configbaker container image
+              run: >
+                  mvn 
+                  -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }}
+                  -Ddocker.registry=ghcr.io -Ddocker.platforms=${{ env.PLATFORMS }}
+                  -Pct deploy
+
+            - uses: marocchino/sticky-pull-request-comment@v2
+              with:
+                  header: registry-push
+                  hide_and_recreate: true
+                  hide_classify: "OUTDATED"
+                  number: ${{ github.event.client_payload.pull_request.number }}
+                  message: |
+                      :package: Pushed preview images as
+                      ```
+                      ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}
+                      ```
+                      ```
+                      ghcr.io/gdcc/configbaker:${{ env.IMAGE_TAG }}
+                      ```
+                      :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container). Use by referencing with full name as printed above, mind the registry name.
+
+            # Leave a note when things have gone sideways
+            - uses: peter-evans/create-or-update-comment@v3
+              if: ${{ failure() }}
+              with:
+                  issue-number: ${{ github.event.client_payload.pull_request.number }}
+                  body: >
+                      :package: Could not push preview images :disappointed:.
+                      See [log](https://github.com/IQSS/dataverse/actions/runs/${{ github.run_id }}) for details.

.github/workflows/container_app_push.yml

@@ -0,0 +1,167 @@
+---
+name: Application Container Image
+
+on:
+    # We are deliberately *not* running on push events here to avoid double runs.
+    # Instead, push events will trigger from the base image and maven unit tests via workflow_call.
+    workflow_call:
+    pull_request:
+        branches:
+            - develop
+            - master
+        paths:
+            - 'src/main/docker/**'
+            - 'modules/container-configbaker/**'
+            - '.github/workflows/container_app_push.yml'
+
+env:
+    IMAGE_TAG: unstable
+    BASE_IMAGE_TAG: unstable
+    REGISTRY: "" # Empty means default to Docker Hub
+    PLATFORMS: "linux/amd64,linux/arm64"
+    MASTER_BRANCH_TAG: alpha
+
+jobs:
+    build:
+        name: "Build & Test"
+        runs-on: ubuntu-latest
+        permissions:
+            contents: read
+            packages: write
+            pull-requests: write
+        # Only run in upstream repo - avoid unnecessary runs in forks
+        if: ${{ github.repository_owner == 'IQSS' }}
+
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v3
+
+            - name: Set up JDK 11
+              uses: actions/setup-java@v3
+              with:
+                  java-version: "11"
+                  distribution: temurin
+                  cache: maven
+
+            - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
+              run: >
+                  mvn -B -f modules/dataverse-parent
+                  -P ct -pl edu.harvard.iq:dataverse -am
+                  install
+
+            # TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false")
+
+    hub-description:
+        needs: build
+        name: Push image descriptions to Docker Hub
+        # Run this when triggered via push or schedule as reused workflow from base / maven unit tests.
+        # Excluding PRs here means we will have no trouble with secrets access. Also avoid runs in forks.
+        if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }}
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v3
+            - uses: peter-evans/dockerhub-description@v3
+              with:
+                  username: ${{ secrets.DOCKERHUB_USERNAME }}
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+                  repository: gdcc/dataverse
+                  short-description: "Dataverse Application Container Image providing the executable"
+                  readme-filepath: ./src/main/docker/README.md
+            - uses: peter-evans/dockerhub-description@v3
+              with:
+                  username: ${{ secrets.DOCKERHUB_USERNAME }}
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+                  repository: gdcc/configbaker
+                  short-description: "Dataverse Config Baker Container Image providing setup tooling and more"
+                  readme-filepath: ./modules/container-configbaker/README.md
+
+    # Note: Accessing, pushing tags etc. to DockerHub or GHCR will only succeed in upstream because secrets.
+    # We check for them here and subsequent jobs can rely on this to decide if they shall run.
+    check-secrets:
+        needs: build
+        name: Check for Secrets Availability
+        runs-on: ubuntu-latest
+        outputs:
+            available: ${{ steps.secret-check.outputs.available }}
+        steps:
+            - id: secret-check
+              # perform secret check & put boolean result as an output
+              shell: bash
+              run: |
+                  if [ "${{ secrets.DOCKERHUB_TOKEN }}" != '' ]; then
+                      echo "available=true" >> $GITHUB_OUTPUT;
+                  else
+                      echo "available=false" >> $GITHUB_OUTPUT;
+                  fi
+
+    deploy:
+        needs: check-secrets
+        name: "Package & Publish"
+        runs-on: ubuntu-latest
+        # Only run this job if we have access to secrets. This is true for events like push/schedule which run in
+        # context of main repo, but for PRs only true if coming from the main repo! Forks have no secret access.
+        if: needs.check-secrets.outputs.available == 'true'
+        steps:
+            - uses: actions/checkout@v3
+            - uses: actions/setup-java@v3
+              with:
+                  java-version: "11"
+                  distribution: temurin
+
+            # Depending on context, we push to different targets. Login accordingly.
+            - if: ${{ github.event_name != 'pull_request' }}
+              name: Log in to Docker Hub registry
+              uses: docker/login-action@v2
+              with:
+                  username: ${{ secrets.DOCKERHUB_USERNAME }}
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+            - if: ${{ github.event_name == 'pull_request' }}
+              name: Login to Github Container Registry
+              uses: docker/login-action@v2
+              with:
+                  registry: ghcr.io
+                  username: ${{ secrets.GHCR_USERNAME }}
+                  password: ${{ secrets.GHCR_TOKEN }}
+
+            - name: Set up QEMU for multi-arch builds
+              uses: docker/setup-qemu-action@v2
+
+            - name: Re-set image tag based on branch (if master)
+              if: ${{ github.ref_name == 'master' }}
+              run: |
+                  echo "IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV
+                  echo "BASE_IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV
+            - name: Re-set image tag and container registry when on PR
+              if: ${{ github.event_name == 'pull_request' }}
+              run: |
+                  echo "IMAGE_TAG=$(echo "$GITHUB_HEAD_REF" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV
+                  echo "REGISTRY='-Ddocker.registry=ghcr.io'" >> $GITHUB_ENV
+
+            # Necessary to split as otherwise the submodules are not available (deploy skips install)
+            - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
+              run: >
+                  mvn -B -f modules/dataverse-parent
+                  -P ct -pl edu.harvard.iq:dataverse -am
+                  install
+            - name: Deploy multi-arch application and configbaker container image
+              run: >
+                  mvn 
+                  -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }}
+                  ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }}
+                  -P ct deploy
+
+            - uses: marocchino/sticky-pull-request-comment@v2
+              if: ${{ github.event_name == 'pull_request' }}
+              with:
+                  header: registry-push
+                  hide_and_recreate: true
+                  hide_classify: "OUTDATED"
+                  message: |
+                      :package: Pushed preview images as
+                      ```
+                      ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}
+                      ```
+                      ```
+                      ghcr.io/gdcc/configbaker:${{ env.IMAGE_TAG }}
+                      ```
+                      :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container). Use by referencing with full name as printed above, mind the registry name.

.github/workflows/container_base_push.yml

@@ -1,5 +1,5 @@
 ---
-name: Container Base Module
+name: Base Container Image
 
 on:
     push:
@@ -18,9 +18,12 @@ on:
             - 'modules/container-base/**'
             - 'modules/dataverse-parent/pom.xml'
             - '.github/workflows/container_base_push.yml'
+    schedule:
+        - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC
 
 env:
     IMAGE_TAG: unstable
+    PLATFORMS: linux/amd64,linux/arm64
 
 jobs:
     build:
@@ -79,7 +82,18 @@ jobs:
               uses: docker/setup-qemu-action@v2
             - name: Re-set image tag based on branch
               if: ${{ github.ref_name == 'master' }}
-              run: echo "IMAGE_TAG=stable"
+              run: echo "IMAGE_TAG=alpha" >> $GITHUB_ENV
             - if: ${{ github.event_name != 'pull_request' }}
               name: Deploy multi-arch base container image to Docker Hub
-              run: mvn -f modules/container-base -Pct deploy -Dbase.image.tag=${{ env.IMAGE_TAG }}
+              run: mvn -f modules/container-base -Pct deploy -Dbase.image.tag=${{ env.IMAGE_TAG }} -Ddocker.platforms=${{ env.PLATFORMS }}
+    push-app-img:
+        name: "Rebase & Publish App Image"
+        permissions:
+            contents: read
+            packages: write
+            pull-requests: write
+        needs: build
+        # We do not release a new base image for pull requests, so do not trigger.
+        if: ${{ github.event_name != 'pull_request' }}
+        uses: ./.github/workflows/container_app_push.yml
+        secrets: inherit

.github/workflows/deploy_beta_testing.yml

@@ -0,0 +1,80 @@
+name: 'Deploy to Beta Testing'
+
+on:
+  push:
+    branches:
+      - develop
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    environment: beta-testing
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: '11'
+
+      - name: Build application war
+        run: mvn package
+
+      - name: Get war file name
+        working-directory: target
+        run: echo "war_file=$(ls *.war | head -1)">> $GITHUB_ENV
+
+      - name: Upload war artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: built-app
+          path: ./target/${{ env.war_file }}
+
+  deploy-to-payara:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: beta-testing
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Download war artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: built-app
+          path: ./
+
+      - name: Get war file name
+        run: echo "war_file=$(ls *.war | head -1)">> $GITHUB_ENV
+
+      - name: Copy war file to remote instance
+        uses: appleboy/scp-action@master
+        with:
+          host: ${{ secrets.PAYARA_INSTANCE_HOST }}
+          username: ${{ secrets.PAYARA_INSTANCE_USERNAME }}
+          key: ${{ secrets.PAYARA_INSTANCE_SSH_PRIVATE_KEY }}
+          source: './${{ env.war_file }}'
+          target: '/home/${{ secrets.PAYARA_INSTANCE_USERNAME }}'
+          overwrite: true
+
+      - name: Execute payara war deployment remotely
+        uses: appleboy/[email protected]
+        env:
+          INPUT_WAR_FILE: ${{ env.war_file }}
+        with:
+          host: ${{ secrets.PAYARA_INSTANCE_HOST }}
+          username: ${{ secrets.PAYARA_INSTANCE_USERNAME }}
+          key: ${{ secrets.PAYARA_INSTANCE_SSH_PRIVATE_KEY }}
+          envs: INPUT_WAR_FILE
+          script: |
+            APPLICATION_NAME=dataverse-backend
+            ASADMIN='/usr/local/payara5/bin/asadmin --user admin'
+            $ASADMIN undeploy $APPLICATION_NAME
+            $ASADMIN stop-domain
+            rm -rf /usr/local/payara5/glassfish/domains/domain1/generated
+            rm -rf /usr/local/payara5/glassfish/domains/domain1/osgi-cache
+            $ASADMIN start-domain
+            $ASADMIN deploy --name $APPLICATION_NAME $INPUT_WAR_FILE
+            $ASADMIN stop-domain
+            $ASADMIN start-domain