feat: simplify unify containers #101
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Build all container images. | |
| # | |
| # Most of the work is done in generic_build.bash, so see that file for details. | |
| name: Build images | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - '.circleci/**' | |
| - 'docs/**' | |
| - 'test/**' | |
| env: | |
| BIOCONDA_UTILS_VERSION: ${{ github.event.release && github.event.release.tag_name || github.head_ref || github.ref_name }} | |
| jobs: | |
| build-base-debian: | |
| name: Build base-debian | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| TAG_EXISTS_base-debian: ${{ steps.base-debian.outputs.TAG_EXISTS_base-debian }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install qemu dependency | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y qemu-user-static | |
| - name: Build base-debian | |
| id: base-debian | |
| run: | | |
| source images/versions.sh | |
| if [ $(tag_exists $BASE_DEBIAN_IMAGE_NAME $BASE_TAG) ]; then | |
| echo "TAG_EXISTS_base-debian=true" >> $GITHUB_OUTPUT | |
| else | |
| cd images && bash build.sh base-glibc-debian-bash | |
| fi | |
| - name: push to ghcr | |
| if: '${{ ! steps.base-debian.outputs.TAG_EXISTS_base-debian }}' | |
| run: | | |
| echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| source images/versions.sh | |
| push_to_ghcr $BASE_DEBIAN_IMAGE_NAME $BASE_TAG | |
| build-base-busybox: | |
| name: Build base-busybox | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| TAG_EXISTS_base-busybox: ${{ steps.base-busybox.outputs.TAG_EXISTS_base-busybox }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install qemu dependency | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y qemu-user-static | |
| - name: Build base-busybox | |
| id: base-busybox | |
| run: | | |
| source images/versions.sh | |
| if [ $(tag_exists $BASE_BUSYBOX_IMAGE_NAME $BASE_TAG) ]; then | |
| echo "TAG_EXISTS_base-busybox=true" >> $GITHUB_OUTPUT | |
| else | |
| cd images && bash build.sh base-glibc-busybox-bash | |
| fi | |
| - name: push to ghcr | |
| if: '${{ ! steps.base-busybox.outputs.TAG_EXISTS_base-busybox }}' | |
| run: | | |
| echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| source images/versions.sh | |
| push_to_ghcr $BASE_BUSYBOX_IMAGE_NAME $BASE_TAG | |
| build-build-env: | |
| name: Build build-env | |
| outputs: | |
| TAG_EXISTS_build-env: ${{ steps.build-env.outputs.TAG_EXISTS_build-env }} | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install qemu dependency | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y qemu-user-static | |
| - name: Build build-env | |
| id: build-env | |
| run: | | |
| source images/versions.sh | |
| if [ $(tag_exists $BUILD_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG) ]; then | |
| echo "TAG_EXISTS_build-env=true" >> $GITHUB_OUTPUT | |
| else | |
| cd images && bash build.sh bioconda-utils-build-env-cos7 | |
| fi | |
| - name: push to ghcr | |
| if: '${{ ! steps.build-env.outputs.TAG_EXISTS_build-env }}' | |
| run: | | |
| echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| source images/versions.sh | |
| push_to_ghcr $BUILD_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG | |
| build-create-env: | |
| name: Build create-env | |
| needs: [build-build-env, build-base-busybox] | |
| outputs: | |
| TAG_EXISTS_create-env: ${{ steps.create-env.outputs.TAG_EXISTS_create-env }} | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install qemu dependency | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y qemu-user-static | |
| - name: Build create-env | |
| id: create-env | |
| run: | | |
| source images/versions.sh | |
| echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| if [ $(tag_exists $CREATE_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG) ]; then | |
| echo "TAG_EXISTS_create-env=true" >> $GITHUB_OUTPUT | |
| else | |
| cd images && bash build.sh create-env | |
| fi | |
| - name: push to ghcr | |
| if: '${{ ! steps.create-env.outputs.TAG_EXISTS_create-env }}' | |
| run: | | |
| echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| source images/versions.sh | |
| push_to_ghcr $CREATE_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG | |
| # END OF BUILDING IMAGES | |
| # ---------------------------------------------------------------------- | |
| # START TESTING | |
| test: | |
| name: test bioconda-utils with images | |
| runs-on: ubuntu-20.04 | |
| needs: [build-base-debian, build-base-busybox, build-build-env, build-create-env] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # Clone bioconda-recipes to use as part of the tests. | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| repository: bioconda/bioconda-recipes | |
| path: recipes | |
| - name: set path | |
| run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH | |
| - name: Install bioconda-utils | |
| run: | | |
| export BIOCONDA_DISABLE_BUILD_PREP=1 | |
| wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{common,install-and-set-up-conda,configure-conda}.sh | |
| bash install-and-set-up-conda.sh | |
| eval "$(conda shell.bash hook)" | |
| mamba create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt | |
| conda activate bioconda | |
| python setup.py install | |
| - name: test | |
| run: | | |
| eval "$(conda shell.bash hook)" | |
| conda activate bioconda | |
| source images/versions.sh | |
| # Figure out which registry to use for each image, based on what was built. | |
| [ ${{ needs.build-build-env.outputs.TAG_EXISTS_build-env }} ] && BUILD_ENV_REGISTRY='quay.io/bioconda' || BUILD_ENV_REGISTRY="ghcr.io/bioconda" | |
| [ ${{ needs.build-create-env.outputs.TAG_EXISTS_create-env }} ] && CREATE_ENV_REGISTRY='quay.io/bioconda' || CREATE_ENV_REGISTRY="ghcr.io/bioconda" | |
| [ ${{ needs.build-base-busybox.outputs.TAG_EXISTS_base_busybox }} ] && DEST_BASE_IMAGE_REGISTRY='quay.io/bioconda' || DEST_BASE_REGISTRY="ghcr.io/bioconda" | |
| # Tell mulled-build which image to use | |
| export DEST_BASE_IMAGE="${DEST_BASE_IMAGE_REGISTRY}/${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG}" | |
| # Build a package with containers. | |
| cd recipes | |
| bioconda-utils build \ | |
| --docker-base-image "${BUILD_ENV_REGISTRY}/${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" \ | |
| --mulled-conda-image "${CREATE_ENV_REGISTRY}/${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" \ | |
| --packages seqtk \ | |
| --docker \ | |
| --mulled-test \ | |
| --force | |
| # END TESTING | |
| # ------------------------------------------------------------------------ | |
| # START PUSHING IMAGES | |
| # For these push steps, a repository must first exist on quay.io/bioconda | |
| # AND that repository must also be configured to allow write access for the | |
| # appropriate service account. This must be done by a user with admin | |
| # access to quay.io/bioconda. | |
| # | |
| # generic_build.bash reported whether the tag exists to the log; that was | |
| # added to GITHUB_OUTPUT, those outputs are exposed to the jobs, and | |
| # those jobs are dependencies of this job. So now we can use those | |
| # outputs to determine if we should upload. | |
| # | |
| # Note that "latest" is built by generic_build.bash as well, and we're | |
| # including it here in the upload. | |
| push: | |
| name: push images | |
| runs-on: ubuntu-20.04 | |
| needs: [build-base-debian, build-base-busybox, build-build-env, build-create-env, test] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: push base-debian | |
| if: ${{ ! needs.base-debian.outputs.TAG_EXISTS_base-debian }} | |
| run: | | |
| echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin | |
| source images/versions.sh | |
| move_from_ghcr_to_quay ${BASE_DEBIAN_IMAGE_NAME} ${BASE_TAG} | |
| - name: push base-busybox | |
| if: ${{ ! needs.base-busybox.outputs.TAG_EXISTS_base-busybox }} | |
| run: | | |
| echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin | |
| source images/versions.sh | |
| move_from_ghcr_to_quay ${BASE_BUSYBOX_IMAGE_NAME} ${BASE_TAG} | |
| - name: push create-env | |
| if: ${{ ! needs.create-env.outputs.TAG_EXISTS_create-env }} | |
| run: | | |
| echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin | |
| source images/versions.sh | |
| move_from_ghcr_to_quay ${CREATE_ENV_IMAGE_NAME} ${BIOCONDA_IMAGE_TAG} | |
| - name: push build-env | |
| if: ${{ ! needs.build-env.outputs.TAG_EXISTS_build-env }} | |
| run: | | |
| echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin | |
| source images/versions.sh | |
| move_from_ghcr_to_quay ${BUILD_ENV_IMAGE_NAME} ${BIOCONDA_IMAGE_TAG} |