feat: simplify unify containers #85
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Build all container images. | |
| # | |
| # Most of the work is done in generic_build.bash, so see that file for details. | |
| name: Build images | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - '.circleci/**' | |
| - 'docs/**' | |
| - 'test/**' | |
| env: | |
| BIOCONDA_UTILS_VERSION: ${{ github.event.release && github.event.release.tag_name || github.head_ref || github.ref_name }} | |
| jobs: | |
| build-base-debian: | |
| name: Build base-debian | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| TAG_EXISTS_base-debian: ${{ steps.base-debian.outputs.TAG_EXISTS_base-debian }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install qemu dependency | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y qemu-user-static | |
| - name: Build base-debian | |
| id: base-debian | |
| run: | | |
| source images/versions.sh | |
| if [ $(tag_exists $BASE_DEBIAN_IMAGE_NAME $BASE_TAG) ]; then | |
| echo "TAG_EXISTS_base-debian=true" >> $GITHUB_OUTPUT | |
| else | |
| cd images && bash build.sh base-glibc-debian-bash | |
| fi | |
| - name: push to ghcr | |
| if: '${{ ! steps.base-debian.outputs.TAG_EXISTS_base-debian }}' | |
| run: | | |
| echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| source images/versions.sh | |
| push_to_ghcr $BASE_DEBIAN_IMAGE_NAME $BASE_TAG | |
| push_to_ghcr $BASE_DEBIAN_IMAGE_NAME latest | |
| build-base-busybox: | |
| name: Build base-busybox | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| TAG_EXISTS_base-busybox: ${{ steps.base-busybox.outputs.TAG_EXISTS_base-busybox }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install qemu dependency | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y qemu-user-static | |
| - name: Build base-busybox | |
| id: base-busybox | |
| run: | | |
| source images/versions.sh | |
| if [ $(tag_exists $BASE_BUSYBOX_IMAGE_NAME $BASE_TAG) ]; then | |
| echo "TAG_EXISTS_base-busybox=true" >> $GITHUB_OUTPUT | |
| else | |
| cd images && bash build.sh base-glibc-busybox-bash | |
| fi | |
| - name: push to ghcr | |
| if: '${{ ! steps.base-busybox.outputs.TAG_EXISTS_base-busybox }}' | |
| run: | | |
| echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| source images/versions.sh | |
| push_to_ghcr $BASE_BUSYBOX_IMAGE_NAME $BASE_TAG | |
| push_to_ghcr $BASE_BUSYBOX_IMAGE_NAME latest | |
| build-build-env: | |
| name: Build build-env | |
| outputs: | |
| TAG_EXISTS_build-env: ${{ steps.build-env.outputs.TAG_EXISTS_build-env }} | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install qemu dependency | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y qemu-user-static | |
| - name: Build build-env | |
| id: build-env | |
| run: | | |
| source images/versions.sh | |
| if [ $(tag_exists $BUILD_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG) ]; then | |
| echo "TAG_EXISTS_build-env=true" >> $GITHUB_OUTPUT | |
| else | |
| cd images && bash build.sh bioconda-utils-build-env-cos7 | |
| fi | |
| - name: push to ghcr | |
| if: '${{ ! steps.build-env.outputs.TAG_EXISTS_build-env }}' | |
| run: | | |
| echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| source images/versions.sh | |
| push_to_ghcr $BUILD_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG | |
| push_to_ghcr $BUILD_ENV_IMAGE_NAME latest | |
| build-create-env: | |
| name: Build create-env | |
| needs: [build-build-env, build-base-busybox] | |
| outputs: | |
| TAG_EXISTS_create-env: ${{ steps.create-env.outputs.TAG_EXISTS_create-env }} | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install qemu dependency | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y qemu-user-static | |
| - name: Build create-env | |
| id: create-env | |
| run: | | |
| source images/versions.sh | |
| if [ $(tag_exists $CREATE_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG) ]; then | |
| echo "TAG_EXISTS_create-env=true" >> $GITHUB_OUTPUT | |
| else | |
| cd images && bash build.sh create-env | |
| fi | |
| - name: push to ghcr | |
| if: '${{ ! steps.create-env.outputs.TAG_EXISTS_create-env }}' | |
| run: | | |
| echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| source images/versions.sh | |
| push_to_ghcr $CREATE_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG | |
| push_to_ghcr $CREATE_ENV_IMAGE_NAME latest | |
| # END OF BUILDING IMAGES | |
| # ---------------------------------------------------------------------- | |
| # START TESTING | |
| test: | |
| if: false | |
| name: test bioconda-utils with images | |
| runs-on: ubuntu-20.04 | |
| needs: [build-base-debian, build-base-busybox, build-build-env, build-create-env] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # Clone bioconda-recipes to use as part of the tests. | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| repository: bioconda/bioconda-recipes | |
| path: recipes | |
| - name: set path | |
| run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH | |
| - name: Install bioconda-utils | |
| run: | | |
| export BIOCONDA_DISABLE_BUILD_PREP=1 | |
| wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{common,install-and-set-up-conda,configure-conda}.sh | |
| bash install-and-set-up-conda.sh | |
| eval "$(conda shell.bash hook)" | |
| mamba create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt | |
| conda activate bioconda | |
| python setup.py install | |
| - name: test | |
| run: | | |
| BIOCONDA_UTILS_VERSION='${{ needs.build-others.outputs.BIOCONDA_UTILS_TAG }}' | |
| # bioconda-utils uses docker, so log in to ghcr.io with docker. | |
| echo '${{ secrets.GITHUB_TOKEN }}' | docker login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| # we also want to use podman to push to quay.io, but we need the images | |
| # locally to this runner to do so, hence also logging in with podman. | |
| echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| # Decide, for each image, whether it was just built as part of this run | |
| # (in which case it would have been just uploaded to ghcr.io) or | |
| # otherwise pull from quay.io. | |
| # | |
| # If ghcr.io, then also pull the image with podman so it will be | |
| # available to upload to quay.io in subsequent steps. We do this even | |
| # for base-debian, even if it's not used for the test. | |
| if [ ${{ ! needs.build-base-debian.outputs.TAG_EXISTS_base-debian }} ]; then | |
| podman pull "ghcr.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:${BASE_TAG}" | |
| podman pull "ghcr.io/bioconda/${BASE_DEBIAN_IMAGE_NAME}:latest" | |
| fi | |
| if [ ${{ needs.build-others.outputs.TAG_EXISTS_base-busybox }} ]; then | |
| DEST_BASE_IMAGE_REGISTRY='quay.io/bioconda' | |
| else | |
| DEST_BASE_IMAGE_REGISTRY="ghcr.io/bioconda" | |
| podman pull "${DEST_BASE_IMAGE_REGISTRY}/${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG}" | |
| podman pull "${DEST_BASE_IMAGE_REGISTRY}/${BASE_BUSYBOX_IMAGE_NAME}:latest" | |
| fi | |
| if [ ${{ needs.build-others.outputs.TAG_EXISTS_build-env }} ]; then | |
| BUILD_ENV_REGISTRY='quay.io/bioconda' | |
| else | |
| BUILD_ENV_REGISTRY="ghcr.io/bioconda" | |
| podman pull "${BUILD_ENV_REGISTRY}/${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_UTILS_VERSION}-base${BASE_TAG}" | |
| podman pull "${BUILD_ENV_REGISTRY}/${BUILD_ENV_IMAGE_NAME}:latest" | |
| fi | |
| if [ ${{ needs.build-others.outputs.TAG_EXISTS_create-env }} ]; then | |
| CREATE_ENV_REGISTRY='quay.io/bioconda' | |
| else | |
| CREATE_ENV_REGISTRY="ghcr.io/bioconda" | |
| podman pull "${CREATE_ENV_REGISTRY}/${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_UTILS_VERSION}-base${BASE_TAG}" | |
| podman pull "${CREATE_ENV_REGISTRY}/${CREATE_ENV_IMAGE_NAME}:latest" | |
| fi | |
| cd recipes | |
| # Run a test build, specifying the exact images to use. | |
| eval "$(conda shell.bash hook)" | |
| conda activate bioconda | |
| # Used to tell mulled-build which image to use | |
| export DEST_BASE_IMAGE="${DEST_BASE_IMAGE_REGISTRY}/${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG}" | |
| # Build a package with containers. | |
| bioconda-utils build \ | |
| --docker-base-image "${BUILD_ENV_REGISTRY}/${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_UTILS_VERSION}-base${BASE_TAG}" \ | |
| --mulled-conda-image "${CREATE_ENV_REGISTRY}/${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_UTILS_VERSION}-base${BASE_TAG}" \ | |
| --packages seqtk \ | |
| --docker \ | |
| --mulled-test \ | |
| --force | |
| # END TESTING | |
| # ------------------------------------------------------------------------ | |
| # START PUSHING IMAGES | |
| # For these push steps, a repository must first exist on quay.io/bioconda | |
| # AND that repository must also be configured to allow write access for the | |
| # appropriate service account. This must be done by a user with admin | |
| # access to quay.io/bioconda. | |
| # | |
| # generic_build.bash reported whether the tag exists to the log; that was | |
| # added to GITHUB_OUTPUT, those outputs are exposed to the jobs, and | |
| # those jobs are dependencies of this job. So now we can use those | |
| # outputs to determine if we should upload. | |
| # | |
| # Note that "latest" is built by generic_build.bash as well, and we're | |
| # including it here in the upload. | |
| - name: Push base-debian | |
| id: push-base-debian | |
| uses: redhat-actions/push-to-registry@v2 | |
| if: ${{ ! needs.base-debian.outputs.TAG_EXISTS_base-debian }} | |
| with: | |
| image: ${{ env.BASE_DEBIAN_IMAGE_NAME }} | |
| tags: latest ${{ env.BASE_TAG }} | |
| registry: quay.io/bioconda | |
| username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} | |
| password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} | |
| - name: Push base-busybox | |
| id: push-base-busybox | |
| uses: redhat-actions/push-to-registry@v2 | |
| if: ${{ ! needs.build-others.outputs.TAG_EXISTS_base-busybox }} | |
| with: | |
| image: ${{ env.BASE_BUSYBOX_IMAGE_NAME }} | |
| tags: latest ${{ env.BASE_TAG }} | |
| registry: quay.io/bioconda | |
| username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} | |
| password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} | |
| - name: Push build-env | |
| id: push-build-env | |
| uses: redhat-actions/push-to-registry@v2 | |
| if: ${{ ! needs.build-others.outputs.TAG_EXISTS_build-env }} | |
| with: | |
| image: ${{ env.BUILD_ENV_IMAGE_NAME }} | |
| tags: latest ${{ needs.build-others.outputs.BIOCONDA_UTILS_TAG }}-base${{ env.BASE_TAG }} | |
| registry: quay.io/bioconda | |
| username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} | |
| password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} | |
| - name: Push create-env | |
| id: push-create-env | |
| uses: redhat-actions/push-to-registry@v2 | |
| if: ${{ ! needs.build-others.outputs.TAG_EXISTS_create-env }} | |
| with: | |
| image: ${{ env.CREATE_ENV_IMAGE_NAME }} | |
| tags: latest ${{ needs.build-others.outputs.BIOCONDA_UTILS_TAG }}-base${{ env.BASE_TAG }} | |
| registry: quay.io/bioconda | |
| username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} | |
| password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} |