Remove apptainer (cumc#1071)

* remove apptainer * remove apptainer * Update README --------- Co-authored-by: danielnachun <[email protected]>
Chunmingl · Sep 25, 2024 · 8ea326e · 8ea326e
1 parent 960f50d
commit 8ea326e
Show file tree

Hide file tree

Showing 5 changed files with 179 additions and 177 deletions.
diff --git a/.github/workflows/container_build.yml b/.github/workflows/container_build.yml
@@ -83,34 +83,34 @@ jobs:
  tags: ${{ steps.context_basename.outputs.basename }}_docker:latest
  outputs: type=docker,dest=/tmp/${{ steps.context_basename.outputs.basename }}_docker.tar
 
- apptainer_container:
- needs: setup
- if: needs.setup.outputs.all_changed_files_count > 0
- runs-on: ubuntu-latest
- strategy:
- fail-fast: false
- matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
-
- steps:
- - name: Checkout pull request branch
- uses: actions/checkout@v3
- with:
- fetch-depth: 0
-
- - name: Setup micromamba
- uses: mamba-org/setup-micromamba@v1
- with:
- environment-name: containers
- create-args: apptainer
- generate-run-shell: true
-
- - name: Get basename
- id: context_basename
- run: echo "basename=$(basename ${{ matrix.context }})" >> "$GITHUB_OUTPUT"
-
- - name: Build apptainer containers
- shell: micromamba-shell {0}
- run: |
- pushd ${{ matrix.context}}; \
- apptainer build /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif ${{ github.workspace }}/.github/apptainer.def; \
- popd
+ # apptainer_container:
+ #  needs: setup
+ #  if: needs.setup.outputs.all_changed_files_count > 0
+ #  runs-on: ubuntu-latest
+ #  strategy:
+ #  fail-fast: false
+ #  matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
+ #
+ #  steps:
+ #  - name: Checkout pull request branch
+ #  uses: actions/checkout@v3
+ #  with:
+ #  fetch-depth: 0
+ #
+ #  - name: Setup micromamba
+ #  uses: mamba-org/setup-micromamba@v1
+ #  with:
+ #  environment-name: containers
+ #  create-args: apptainer
+ #  generate-run-shell: true
+ #
+ #  - name: Get basename
+ #  id: context_basename
+ #  run: echo "basename=$(basename ${{ matrix.context }})" >> "$GITHUB_OUTPUT"
+ #
+ #  - name: Build apptainer containers
+ #  shell: micromamba-shell {0}
+ #  run: |
+ #  pushd ${{ matrix.context}}; \
+ #  apptainer build /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif ${{ github.workspace }}/.github/apptainer.def; \
+ #  popd
diff --git a/.github/workflows/container_upload.yml b/.github/workflows/container_upload.yml
@@ -88,79 +88,79 @@ jobs:
  quay.io/${{ vars.QUAYIO_USERNAME }}/${{ steps.context_basename.outputs.basename }}_docker:latest
  docker.io/${{ vars.DOCKERIO_USERNAME }}/${{ steps.context_basename.outputs.basename }}_docker:latest
 
- push_apptainer_containers:
- needs: setup
- if: needs.setup.outputs.any_changed == 'true'
- runs-on: ubuntu-latest
- strategy:
- fail-fast: false
- matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
- permissions: 
- contents: read
- packages: write
-
- steps:
- - name: Checkout pull request branch
- uses: actions/checkout@v3
- with:
- fetch-depth: 0
-
- - name: Setup micromamba
- uses: mamba-org/setup-micromamba@v1
- with:
- environment-name: containers
- create-args: apptainer
- generate-run-shell: true
-
- - name: Get basename
- id: context_basename
- run: echo "basename=$(basename ${{ matrix.context }})" >> "$GITHUB_OUTPUT"
-
- - name: Build apptainer containers
- shell: micromamba-shell {0}
- run: |
- pushd ${{ matrix.context}}; \
- apptainer build /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif ${{ github.workspace }}/.github/apptainer.def; \
- popd
-
- - name: Log in to ghcr.io with apptainer
- shell: micromamba-shell {0}
- run: |
- apptainer remote login --username ${{ github.repository_owner }} \
- --password ${{ secrets.GITHUB_TOKEN }} \
- docker://ghcr.io/${{ github.repository_owner }}
-
- - name: Push apptainer containers to ghcr.io
- shell: micromamba-shell {0}
- run: |
- apptainer push \
- /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif \
- oras://ghcr.io/${{ github.repository_owner }}/${{ steps.context_basename.outputs.basename }}_apptainer:latest
-
- - name: Log in to quay.io with apptainer
- shell: micromamba-shell {0}
- run: |
- apptainer remote login --username ${{ vars.QUAYIO_USERNAME }} \
- --password ${{ secrets.QUAYIO_TOKEN }} \
- docker://quay.io/${{ vars.QUAYIO_USERNAME }}
-
- - name: Push apptainer containers to quay.io
- shell: micromamba-shell {0}
- run: |
- apptainer push \
- /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif \
- oras://quay.io/${{ vars.QUAYIO_USERNAME }}/${{ steps.context_basename.outputs.basename }}_apptainer:latest 
-
- - name: Log in to docker.io with apptainer
- shell: micromamba-shell {0}
- run: |
- apptainer remote login --username ${{ vars.DOCKERIO_USERNAME }} \
- --password ${{ secrets.DOCKERIO_TOKEN }} \
- docker://docker.io
-
- - name: Push apptainer containers to docker.io
- shell: micromamba-shell {0}
- run: |
- apptainer push \
- /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif \
- oras://docker.io/${{ vars.DOCKERIO_USERNAME }}/${{ steps.context_basename.outputs.basename }}_apptainer:latest
+ # push_apptainer_containers:
+ #  needs: setup
+ #  if: needs.setup.outputs.any_changed == 'true'
+ #  runs-on: ubuntu-latest
+ #  strategy:
+ #  fail-fast: false
+ #  matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
+ #  permissions: 
+ #  contents: read
+ #  packages: write
+ #
+ #  steps:
+ #  - name: Checkout pull request branch
+ #  uses: actions/checkout@v3
+ #  with:
+ #  fetch-depth: 0
+ #
+ #  - name: Setup micromamba
+ #  uses: mamba-org/setup-micromamba@v1
+ #  with:
+ #  environment-name: containers
+ #  create-args: apptainer
+ #  generate-run-shell: true
+ #
+ #  - name: Get basename
+ #  id: context_basename
+ #  run: echo "basename=$(basename ${{ matrix.context }})" >> "$GITHUB_OUTPUT"
+ #
+ #  - name: Build apptainer containers
+ #  shell: micromamba-shell {0}
+ #  run: |
+ #  pushd ${{ matrix.context}}; \
+ #  apptainer build /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif ${{ github.workspace }}/.github/apptainer.def; \
+ #  popd
+ #
+ #  - name: Log in to ghcr.io with apptainer
+ #  shell: micromamba-shell {0}
+ #  run: |
+ #  apptainer remote login --username ${{ github.repository_owner }} \
+ #  --password ${{ secrets.GITHUB_TOKEN }} \
+ #  docker://ghcr.io/${{ github.repository_owner }}
+ #
+ #  - name: Push apptainer containers to ghcr.io
+ #  shell: micromamba-shell {0}
+ #  run: |
+ #  apptainer push \
+ #  /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif \
+ #  oras://ghcr.io/${{ github.repository_owner }}/${{ steps.context_basename.outputs.basename }}_apptainer:latest
+ #
+ #  - name: Log in to quay.io with apptainer
+ #  shell: micromamba-shell {0}
+ #  run: |
+ #  apptainer remote login --username ${{ vars.QUAYIO_USERNAME }} \
+ #  --password ${{ secrets.QUAYIO_TOKEN }} \
+ #  docker://quay.io/${{ vars.QUAYIO_USERNAME }}
+ #
+ #  - name: Push apptainer containers to quay.io
+ #  shell: micromamba-shell {0}
+ #  run: |
+ #  apptainer push \
+ #  /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif \
+ #  oras://quay.io/${{ vars.QUAYIO_USERNAME }}/${{ steps.context_basename.outputs.basename }}_apptainer:latest 
+ #
+ #  - name: Log in to docker.io with apptainer
+ #  shell: micromamba-shell {0}
+ #  run: |
+ #  apptainer remote login --username ${{ vars.DOCKERIO_USERNAME }} \
+ #  --password ${{ secrets.DOCKERIO_TOKEN }} \
+ #  docker://docker.io
+ #
+ #  - name: Push apptainer containers to docker.io
+ #  shell: micromamba-shell {0}
+ #  run: |
+ #  apptainer push \
+ #  /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif \
+ #  oras://docker.io/${{ vars.DOCKERIO_USERNAME }}/${{ steps.context_basename.outputs.basename }}_apptainer:latest
diff --git a/.github/workflows/dispatch_container_upload.yml b/.github/workflows/dispatch_container_upload.yml
@@ -56,71 +56,71 @@ jobs:
  quay.io/${{ vars.QUAYIO_USERNAME }}/${{ github.event.inputs.container_name }}_docker:latest
  docker.io/${{ vars.DOCKERIO_USERNAME }}/${{ github.event.inputs.container_name }}_docker:latest
 
- push_apptainer_containers:
- runs-on: ubuntu-latest
- strategy:
- fail-fast: false
- permissions: 
- contents: read
- packages: write
-
- steps:
- - uses: actions/checkout@v4
- with:
- fetch-depth: 0
-
- - name: Setup micromamba
- uses: mamba-org/setup-micromamba@v1
- with:
- environment-name: containers
- create-args: apptainer
- generate-run-shell: true
-
- - name: Build apptainer containers
- shell: micromamba-shell {0}
- run: |
- pushd container/${{ github.event.inputs.container_name }}; \
- apptainer build /tmp/${{ github.event.inputs.container_name }}_apptainer.sif ${{ github.workspace }}/.github/apptainer.def; \
- popd
-
- - name: Log in to ghcr.io with apptainer
- shell: micromamba-shell {0}
- run: |
- apptainer remote login --username ${{ github.repository_owner }} \
- --password ${{ secrets.GITHUB_TOKEN }} \
- docker://ghcr.io/${{ github.repository_owner }}
-
- - name: Push apptainer containers to ghcr.io
- shell: micromamba-shell {0}
- run: |
- apptainer push \
- /tmp/${{ github.event.inputs.container_name }}_apptainer.sif \
- oras://ghcr.io/${{ github.repository_owner }}/${{ github.event.inputs.container_name }}_apptainer:latest
-
- - name: Log in to quay.io with apptainer
- shell: micromamba-shell {0}
- run: |
- apptainer remote login --username ${{ vars.QUAYIO_USERNAME }} \
- --password ${{ secrets.QUAYIO_TOKEN }} \
- docker://quay.io/${{ vars.QUAYIO_USERNAME }}
-
- - name: Push apptainer containers to quay.io
- shell: micromamba-shell {0}
- run: |
- apptainer push \
- /tmp/${{ github.event.inputs.container_name }}_apptainer.sif \
- oras://quay.io/${{ vars.QUAYIO_USERNAME }}/${{ github.event.inputs.container_name }}_apptainer:latest 
-
- - name: Log in to docker.io with apptainer
- shell: micromamba-shell {0}
- run: |
- apptainer remote login --username ${{ vars.DOCKERIO_USERNAME }} \
- --password ${{ secrets.DOCKERIO_TOKEN }} \
- docker://docker.io
-
- - name: Push apptainer containers to docker.io
- shell: micromamba-shell {0}
- run: |
- apptainer push \
- /tmp/${{ github.event.inputs.container_name }}_apptainer.sif \
- oras://docker.io/${{ vars.DOCKERIO_USERNAME }}/${{ github.event.inputs.container_name }}_apptainer:latest
+ # push_apptainer_containers:
+ #  runs-on: ubuntu-latest
+ #  strategy:
+ #  fail-fast: false
+ #  permissions: 
+ #  contents: read
+ #  packages: write
+ #
+ #  steps:
+ #  - uses: actions/checkout@v4
+ #  with:
+ #  fetch-depth: 0
+ #
+ #  - name: Setup micromamba
+ #  uses: mamba-org/setup-micromamba@v1
+ #  with:
+ #  environment-name: containers
+ #  create-args: apptainer
+ #  generate-run-shell: true
+ #
+ #  - name: Build apptainer containers
+ #  shell: micromamba-shell {0}
+ #  run: |
+ #  pushd container/${{ github.event.inputs.container_name }}; \
+ #  apptainer build /tmp/${{ github.event.inputs.container_name }}_apptainer.sif ${{ github.workspace }}/.github/apptainer.def; \
+ #  popd
+ #
+ #  - name: Log in to ghcr.io with apptainer
+ #  shell: micromamba-shell {0}
+ #  run: |
+ #  apptainer remote login --username ${{ github.repository_owner }} \
+ #  --password ${{ secrets.GITHUB_TOKEN }} \
+ #  docker://ghcr.io/${{ github.repository_owner }}
+ #
+ #  - name: Push apptainer containers to ghcr.io
+ #  shell: micromamba-shell {0}
+ #  run: |
+ #  apptainer push \
+ #  /tmp/${{ github.event.inputs.container_name }}_apptainer.sif \
+ #  oras://ghcr.io/${{ github.repository_owner }}/${{ github.event.inputs.container_name }}_apptainer:latest
+ #
+ #  - name: Log in to quay.io with apptainer
+ #  shell: micromamba-shell {0}
+ #  run: |
+ #  apptainer remote login --username ${{ vars.QUAYIO_USERNAME }} \
+ #  --password ${{ secrets.QUAYIO_TOKEN }} \
+ #  docker://quay.io/${{ vars.QUAYIO_USERNAME }}
+ #
+ #  - name: Push apptainer containers to quay.io
+ #  shell: micromamba-shell {0}
+ #  run: |
+ #  apptainer push \
+ #  /tmp/${{ github.event.inputs.container_name }}_apptainer.sif \
+ #  oras://quay.io/${{ vars.QUAYIO_USERNAME }}/${{ github.event.inputs.container_name }}_apptainer:latest 
+ #
+ #  - name: Log in to docker.io with apptainer
+ #  shell: micromamba-shell {0}
+ #  run: |
+ #  apptainer remote login --username ${{ vars.DOCKERIO_USERNAME }} \
+ #  --password ${{ secrets.DOCKERIO_TOKEN }} \
+ #  docker://docker.io
+ #
+ #  - name: Push apptainer containers to docker.io
+ #  shell: micromamba-shell {0}
+ #  run: |
+ #  apptainer push \
+ #  /tmp/${{ github.event.inputs.container_name }}_apptainer.sif \
+ #  oras://docker.io/${{ vars.DOCKERIO_USERNAME }}/${{ github.event.inputs.container_name }}_apptainer:latest
diff --git a/container/README.md b/container/README.md
@@ -1,7 +1,8 @@
 # Container provisioning pipeline
 Containers in the xQTL pipeline are built from conda environments and pushed to container registry using GitHub Actions. We use the [micromamba container](https://hub.docker.com/r/mambaorg/micromamba) as our base image and install a Conda environment into the container. Conda packages are only used from [conda-forge](https://anaconda.org/conda-forge), [bioconda](https://anaconda.org/bioconda), and the [personal channel](https://anaconda.org/dnachun) of contributor @danielnachun.
 
-Note: the minimum required version of Singularity for our containers is **3.6.0**. This release made backwards incompatibile changes to the container signature format that mean any containers built with Singularity 3.6.0 or newer will not work on 3.5.3 or older. We use the latest version of Apptainer (renamed from Singularity) to build and sign our containers. 
+~~Note: the minimum required version of Singularity for our containers is **3.6.0**. This release made backwards incompatibile changes to the container signature format that mean any containers built with Singularity 3.6.0 or newer will not work on 3.5.3 or older. We use the latest version of Apptainer (renamed from Singularity) to build and sign our containers.~~
+Support for apptainer is now dropped as we are migrating our software stack to `pixi` with Conda packages. Docker images should only be used for batch MemVerge jobs, and this will only continue until a containerless solution is found for this system.
 
 ## Pipeline description
 The GitHub Actions are located in the [.github/workflows](https://github.com/cumc/xqtl-protocol/tree/main/.github/workflows) folder, while the Conda environments and the CSV table used to generate those environments are found in this folder. The environments for a single container are each in their own folder e.g. `bioinfo/bioinfo.yml` – currently each container only has a single environment in it, but our pipeline allows for multiple environments in the same container should the need arise. We build containers for both Docker and Singularity and push the containers to the registries at [ghcr.io](https://ghcr.io), [quay.io](https://quay.io), and [docker.io](https://docker.io).

diff --git a/container/readme/README.md b/container/readme/README.md
@@ -1,7 +1,8 @@
 # Container provisioning pipeline
 Containers in the xQTL pipeline are built from conda environments and pushed to container registry using GitHub Actions. We use the [micromamba container](https://hub.docker.com/r/mambaorg/micromamba) as our base image and install a Conda environment into the container. Conda packages are only used from [conda-forge](https://anaconda.org/conda-forge), [bioconda](https://anaconda.org/bioconda), and the [personal channel](https://anaconda.org/dnachun) of contributor @danielnachun.
 
-Note: the minimum required version of Singularity for our containers is **3.6.0**. This release made backwards incompatibile changes to the container signature format that mean any containers built with Singularity 3.6.0 or newer will not work on 3.5.3 or older. We use the latest version of Apptainer (renamed from Singularity) to build and sign our containers. 
+~~Note: the minimum required version of Singularity for our containers is **3.6.0**. This release made backwards incompatibile changes to the container signature format that mean any containers built with Singularity 3.6.0 or newer will not work on 3.5.3 or older. We use the latest version of Apptainer (renamed from Singularity) to build and sign our containers.~~
+Support for apptainer is now dropped as we are migrating our software stack to `pixi` with Conda packages. Docker images should only be used for batch MemVerge jobs, and this will only continue until a containerless solution is found for this system.
 
 ## Pipeline description
 The GitHub Actions are located in the [.github/workflows](https://github.com/cumc/xqtl-protocol/tree/main/.github/workflows) folder, while the Conda environments and the CSV table used to generate those environments are found in this folder. The environments for a single container are each in their own folder e.g. `bioinfo/bioinfo.yml` – currently each container only has a single environment in it, but our pipeline allows for multiple environments in the same container should the need arise. We build containers for both Docker and Singularity and push the containers to the registries at [ghcr.io](https://ghcr.io), [quay.io](https://quay.io), and [docker.io](https://docker.io).