From 9f1e99886239f2087bcd5347adf66d8d79af7137 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Wed, 20 Mar 2024 14:33:42 -0400 Subject: [PATCH 01/53] example recipe for osx-arm64 --- recipes/bioawk/meta.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/recipes/bioawk/meta.yaml b/recipes/bioawk/meta.yaml index c5dd0a636caf2..ea6a9dee20bf7 100644 --- a/recipes/bioawk/meta.yaml +++ b/recipes/bioawk/meta.yaml @@ -7,7 +7,7 @@ source: sha256: 5cbef3f39b085daba45510ff450afcf943cfdfdd483a546c8a509d3075ff51b5 build: - number: 10 + number: 11 run_exports: - {{ pin_subpackage("bioawk", max_pin="x.x.x") }} @@ -34,3 +34,4 @@ about: extra: additional-platforms: - linux-aarch64 + - osx-arm64 From d99ac26ae4c06b14374605fdc5c9113b71e00240 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Wed, 20 Mar 2024 14:48:52 -0400 Subject: [PATCH 02/53] test arm builds on macOS-14 --- .github/workflows/PR-arm.yml | 88 ++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 .github/workflows/PR-arm.yml diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml new file mode 100644 index 0000000000000..e39281637f18c --- /dev/null +++ b/.github/workflows/PR-arm.yml @@ -0,0 +1,88 @@ +name: PR (osx-arm64) +on: + push: + branches: + - osx-arm-builds + +jobs: + build-osx-arm: + name: OSX-ARM64 Tests + runs-on: macOS-14 # M1 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # TODO: bail if there's no osx-arm64 recipes + # - run: + # name: Check for Additional Platforms + # command: ./.circleci/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" + + - name: set path + run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH + + - name: Fetch conda install script + run: | + wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh + + - name: Restore cache + id: cache + uses: actions/cache@v3 + with: + path: /opt/mambaforge + key: ${{ runner.os }}--bulk--${{ hashFiles('install-and-set-up-conda.sh', 'common.sh', 'configure-conda.sh') }} + + - name: Set up bioconda-utils + if: steps.cache.outputs.cache-hit != 'true' + run: bash install-and-set-up-conda.sh + + # This script can be used to reconfigure conda to use the right channel setup. + # This has to be done after the cache is restored, because + # the channel setup is not cached as it resides in the home directory. + # We could use a system-wide (and therefore cached) channel setup, + # but mamba does not support that at the time of implementation + # (it ignores settings made with --system). + - name: Configure conda + run: bash configure-conda.sh + + - name: Build and Test + env: + # Mimic circleci + OSTYPE: "darwin" + CI: "true" + run: | + set -e + eval "$(conda shell.bash hook)" + conda activate bioconda + + # Clean up lingering build artifacts + rm -f /opt/mambaforge/envs/bioconda/conda-bld/osx-arm64/*.tar.bz2 + + # The SDK isn't actually cached, so reinstall it + run_conda_forge_build_setup + + if [ -z "$GITHUB_BASE_REF" ] ; then + export GITHUB_BASE_REF="master" + fi + git fetch origin "$GITHUB_BASE_REF" + bioconda-utils build recipes config.yml \ + --git-range origin/"$GITHUB_BASE_REF" HEAD + + - name: Prepare artifacts + run: | + ( + rm -rf /tmp/artifacts + mkdir -p /tmp/artifacts/packages + cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0 + find -name .cache | xargs rm -rf || true + for n in index.html channeldata.json osx-arm64; do + cp -rv $n /tmp/artifacts/packages || true + done + ) || true + + - name: Archive packages + uses: actions/upload-artifact@v4 + with: + name: osx-arm64-packages + path: | + /tmp/artifacts/packages From 0f256cc2565bbc904e0b06965d9d3dc9b2d997ef Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 25 Mar 2024 17:13:43 -0400 Subject: [PATCH 03/53] add arch flag --- recipes/bioawk/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes/bioawk/build.sh b/recipes/bioawk/build.sh index 81160129b8cda..1a1478c9d0dea 100644 --- a/recipes/bioawk/build.sh +++ b/recipes/bioawk/build.sh @@ -1,5 +1,6 @@ #!/bin/bash export LIBRARY_PATH="$PREFIX/lib" +export ARCHFLAGS="-arch $(uname -m)" make CC=$CC CFLAGS="-g -Wall -O2 -I$PREFIX/include -L$PREFIX/lib" From c9c2f70eefa435b461dd335b9f46000d910b45b3 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 26 Mar 2024 09:54:31 -0400 Subject: [PATCH 04/53] try setting a higher sdk version --- recipes/bioawk/conda_build_config.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 recipes/bioawk/conda_build_config.yaml diff --git a/recipes/bioawk/conda_build_config.yaml b/recipes/bioawk/conda_build_config.yaml new file mode 100644 index 0000000000000..6b6abfd001f78 --- /dev/null +++ b/recipes/bioawk/conda_build_config.yaml @@ -0,0 +1,5 @@ +MACOSX_DEPLOYMENT_TARGET: # [osx and arm64] + - "14.0" # [osx and arm64] +MACOSX_SDK_VERSION: # [osx and arm64] + - "14.0" # [osx and arm64] + From 6860ca9d06da9f605d49a5404e3728094e08c137 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 26 Mar 2024 10:10:02 -0400 Subject: [PATCH 05/53] check sdk versions --- recipes/bioawk/build.sh | 5 +++++ recipes/bioawk/conda_build_config.yaml | 9 ++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/recipes/bioawk/build.sh b/recipes/bioawk/build.sh index 1a1478c9d0dea..3618445b7e75a 100644 --- a/recipes/bioawk/build.sh +++ b/recipes/bioawk/build.sh @@ -6,3 +6,8 @@ make CC=$CC CFLAGS="-g -Wall -O2 -I$PREFIX/include -L$PREFIX/lib" mkdir -p $PREFIX/bin cp bioawk $PREFIX/bin + +echo "MACOSX_SDK_VERSION: " +echo $MACOSX_SDK_VERSION +echo "MACOSX_DEPLOYMENT_TARGET: " +echo $MACOSX_DEPLOYMENT_TARGET \ No newline at end of file diff --git a/recipes/bioawk/conda_build_config.yaml b/recipes/bioawk/conda_build_config.yaml index 6b6abfd001f78..bc9b4a54b1099 100644 --- a/recipes/bioawk/conda_build_config.yaml +++ b/recipes/bioawk/conda_build_config.yaml @@ -1,5 +1,4 @@ -MACOSX_DEPLOYMENT_TARGET: # [osx and arm64] - - "14.0" # [osx and arm64] -MACOSX_SDK_VERSION: # [osx and arm64] - - "14.0" # [osx and arm64] - +MACOSX_DEPLOYMENT_TARGET: + - "14.0" +MACOSX_SDK_VERSION: + - "14.0" From c0d311b812bb9c13d38637b328d1826b05b1f116 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 26 Mar 2024 10:31:29 -0400 Subject: [PATCH 06/53] set sdk earlier --- .github/workflows/PR-arm.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index e39281637f18c..eb8cbfdb36c96 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -65,6 +65,10 @@ jobs: export GITHUB_BASE_REF="master" fi git fetch origin "$GITHUB_BASE_REF" + + export MACOSX_DEPLOYMENT_TARGET=10.12 + export MACOSX_SDK_VERSION=14.0 + bioconda-utils build recipes config.yml \ --git-range origin/"$GITHUB_BASE_REF" HEAD From 6f341bca3c56cb316b873e56898dd317e70b8c82 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 26 Mar 2024 10:44:28 -0400 Subject: [PATCH 07/53] move logging --- recipes/bioawk/build.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/recipes/bioawk/build.sh b/recipes/bioawk/build.sh index 3618445b7e75a..ae071d37b21d0 100644 --- a/recipes/bioawk/build.sh +++ b/recipes/bioawk/build.sh @@ -2,12 +2,12 @@ export LIBRARY_PATH="$PREFIX/lib" export ARCHFLAGS="-arch $(uname -m)" +echo "MACOSX_SDK_VERSION: " +echo $MACOSX_SDK_VERSION +echo "MACOSX_DEPLOYMENT_TARGET: " +echo $MACOSX_DEPLOYMENT_TARGET + make CC=$CC CFLAGS="-g -Wall -O2 -I$PREFIX/include -L$PREFIX/lib" mkdir -p $PREFIX/bin cp bioawk $PREFIX/bin - -echo "MACOSX_SDK_VERSION: " -echo $MACOSX_SDK_VERSION -echo "MACOSX_DEPLOYMENT_TARGET: " -echo $MACOSX_DEPLOYMENT_TARGET \ No newline at end of file From c9b46b52edd96b5af77ff2167f0cb0dc4b2e09fb Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 26 Mar 2024 10:55:28 -0400 Subject: [PATCH 08/53] set deployment target --- recipes/bioawk/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes/bioawk/build.sh b/recipes/bioawk/build.sh index ae071d37b21d0..1486d5b962e1e 100644 --- a/recipes/bioawk/build.sh +++ b/recipes/bioawk/build.sh @@ -1,6 +1,7 @@ #!/bin/bash export LIBRARY_PATH="$PREFIX/lib" export ARCHFLAGS="-arch $(uname -m)" +export MACOSX_DEPLOYMENT_TARGET=14.0 echo "MACOSX_SDK_VERSION: " echo $MACOSX_SDK_VERSION From 636bb8b5d9a035bf79661a919c31bc5ce84da951 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 26 Mar 2024 11:07:55 -0400 Subject: [PATCH 09/53] move var setting --- .github/workflows/PR-arm.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index eb8cbfdb36c96..d610553604024 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -55,6 +55,9 @@ jobs: eval "$(conda shell.bash hook)" conda activate bioconda + export MACOSX_DEPLOYMENT_TARGET=14.0 + export MACOSX_SDK_VERSION=14.0 + # Clean up lingering build artifacts rm -f /opt/mambaforge/envs/bioconda/conda-bld/osx-arm64/*.tar.bz2 @@ -66,9 +69,6 @@ jobs: fi git fetch origin "$GITHUB_BASE_REF" - export MACOSX_DEPLOYMENT_TARGET=10.12 - export MACOSX_SDK_VERSION=14.0 - bioconda-utils build recipes config.yml \ --git-range origin/"$GITHUB_BASE_REF" HEAD From eeac233e68ffb91395309fe4104b4b6d2a014d5c Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 26 Mar 2024 11:37:42 -0400 Subject: [PATCH 10/53] how old can we go? --- .github/workflows/PR-arm.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index d610553604024..73418a6b091e5 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -55,8 +55,7 @@ jobs: eval "$(conda shell.bash hook)" conda activate bioconda - export MACOSX_DEPLOYMENT_TARGET=14.0 - export MACOSX_SDK_VERSION=14.0 + export MACOSX_DEPLOYMENT_TARGET=10.15 # Clean up lingering build artifacts rm -f /opt/mambaforge/envs/bioconda/conda-bld/osx-arm64/*.tar.bz2 From 1d7c601218c926eee94f664c650a94cc81c26e62 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 26 Mar 2024 11:49:02 -0400 Subject: [PATCH 11/53] try 11.0 sdk --- .github/workflows/PR-arm.yml | 3 ++- recipes/bioawk/build.sh | 7 ------- recipes/bioawk/conda_build_config.yaml | 4 ---- 3 files changed, 2 insertions(+), 12 deletions(-) delete mode 100644 recipes/bioawk/conda_build_config.yaml diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index 73418a6b091e5..330b1fc21af0a 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -55,7 +55,8 @@ jobs: eval "$(conda shell.bash hook)" conda activate bioconda - export MACOSX_DEPLOYMENT_TARGET=10.15 + # Min arm64 support + export MACOSX_DEPLOYMENT_TARGET=11.0 # Clean up lingering build artifacts rm -f /opt/mambaforge/envs/bioconda/conda-bld/osx-arm64/*.tar.bz2 diff --git a/recipes/bioawk/build.sh b/recipes/bioawk/build.sh index 1486d5b962e1e..81160129b8cda 100644 --- a/recipes/bioawk/build.sh +++ b/recipes/bioawk/build.sh @@ -1,12 +1,5 @@ #!/bin/bash export LIBRARY_PATH="$PREFIX/lib" -export ARCHFLAGS="-arch $(uname -m)" -export MACOSX_DEPLOYMENT_TARGET=14.0 - -echo "MACOSX_SDK_VERSION: " -echo $MACOSX_SDK_VERSION -echo "MACOSX_DEPLOYMENT_TARGET: " -echo $MACOSX_DEPLOYMENT_TARGET make CC=$CC CFLAGS="-g -Wall -O2 -I$PREFIX/include -L$PREFIX/lib" diff --git a/recipes/bioawk/conda_build_config.yaml b/recipes/bioawk/conda_build_config.yaml deleted file mode 100644 index bc9b4a54b1099..0000000000000 --- a/recipes/bioawk/conda_build_config.yaml +++ /dev/null @@ -1,4 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: - - "14.0" -MACOSX_SDK_VERSION: - - "14.0" From 8df4439aceb30b94845b420a8ef2454544c3491b Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 26 Mar 2024 14:46:51 -0400 Subject: [PATCH 12/53] trigger on PR --- .github/workflows/PR-arm.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index 330b1fc21af0a..f377cd68d5e11 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -1,13 +1,17 @@ name: PR (osx-arm64) on: - push: - branches: - - osx-arm-builds - + pull_request: + types: [opened, synchronize, reopened] +concurrency: + group: build-${{ github.event.pull_request.number || github.head_ref }} + cancel-in-progress: true jobs: build-osx-arm: name: OSX-ARM64 Tests runs-on: macOS-14 # M1 + strategy: + fail-fast: true + max-parallel: 13 steps: - uses: actions/checkout@v3 with: @@ -55,7 +59,7 @@ jobs: eval "$(conda shell.bash hook)" conda activate bioconda - # Min arm64 support + # Use SDK with arm64 support export MACOSX_DEPLOYMENT_TARGET=11.0 # Clean up lingering build artifacts @@ -70,6 +74,7 @@ jobs: git fetch origin "$GITHUB_BASE_REF" bioconda-utils build recipes config.yml \ + --lint \ --git-range origin/"$GITHUB_BASE_REF" HEAD - name: Prepare artifacts From c1c8812a839bd95e13433c9afd87c64620b64415 Mon Sep 17 00:00:00 2001 From: "Alicia A. Evans" <108547992+aliciaaevans@users.noreply.github.com> Date: Wed, 27 Mar 2024 18:05:34 -0400 Subject: [PATCH 13/53] Apply suggestions from code review Co-authored-by: Martin Grigorov --- .github/workflows/PR-arm.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index f377cd68d5e11..eb466976a798e 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -13,7 +13,7 @@ jobs: fail-fast: true max-parallel: 13 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -31,7 +31,7 @@ jobs: - name: Restore cache id: cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /opt/mambaforge key: ${{ runner.os }}--bulk--${{ hashFiles('install-and-set-up-conda.sh', 'common.sh', 'configure-conda.sh') }} From d982386d5630be1d12816d9ec9c283ffe73ee82d Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Wed, 27 Mar 2024 18:07:23 -0400 Subject: [PATCH 14/53] zip a level up to match other ci --- .github/workflows/PR-arm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index eb466976a798e..a2d6c3b6e8c8e 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -94,4 +94,4 @@ jobs: with: name: osx-arm64-packages path: | - /tmp/artifacts/packages + /tmp/artifacts From b6f8ebff2e08d1e72a931032eaaecb00068cdba0 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Fri, 29 Mar 2024 18:09:11 -0400 Subject: [PATCH 15/53] must include noarch --- .github/workflows/PR-arm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index a2d6c3b6e8c8e..cba5561d2a0d7 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -84,7 +84,7 @@ jobs: mkdir -p /tmp/artifacts/packages cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0 find -name .cache | xargs rm -rf || true - for n in index.html channeldata.json osx-arm64; do + for n in index.html channeldata.json osx-arm64 noarch; do cp -rv $n /tmp/artifacts/packages || true done ) || true From ec3d5a250e9f45bd4c610d013c643089499f53f5 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Apr 2024 17:03:44 -0400 Subject: [PATCH 16/53] remove sdk workaround --- .github/workflows/PR-arm.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index cba5561d2a0d7..5dfad362431da 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -59,9 +59,6 @@ jobs: eval "$(conda shell.bash hook)" conda activate bioconda - # Use SDK with arm64 support - export MACOSX_DEPLOYMENT_TARGET=11.0 - # Clean up lingering build artifacts rm -f /opt/mambaforge/envs/bioconda/conda-bld/osx-arm64/*.tar.bz2 From a624f82e7d21af4c0a9e462c558a3e4ce343018d Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Apr 2024 17:54:19 -0400 Subject: [PATCH 17/53] log deployment target --- .github/workflows/PR-arm.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index 5dfad362431da..aac7bd9d9f016 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -59,6 +59,10 @@ jobs: eval "$(conda shell.bash hook)" conda activate bioconda + # Use SDK with arm64 support + echo "MACOSX_DEPLOYMENT_TARGET (before): $MACOSX_DEPLOYMENT_TARGET" + export MACOSX_DEPLOYMENT_TARGET=11.0 + # Clean up lingering build artifacts rm -f /opt/mambaforge/envs/bioconda/conda-bld/osx-arm64/*.tar.bz2 From dc4525c7467f40697def3a89b7ada4c02d5610e4 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Apr 2024 17:54:40 -0400 Subject: [PATCH 18/53] use master cache --- .github/workflows/PR-arm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index aac7bd9d9f016..e0bde1e50e41b 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -34,7 +34,7 @@ jobs: uses: actions/cache@v4 with: path: /opt/mambaforge - key: ${{ runner.os }}--bulk--${{ hashFiles('install-and-set-up-conda.sh', 'common.sh', 'configure-conda.sh') }} + key: ${{ runner.os }}--master--${{ hashFiles('install-and-set-up-conda.sh', 'common.sh', 'configure-conda.sh') }} - name: Set up bioconda-utils if: steps.cache.outputs.cache-hit != 'true' From aa72299334dba7953c85f94cc5a6048a7ecac1d5 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 2 Apr 2024 10:33:01 -0400 Subject: [PATCH 19/53] case for job name --- .github/workflows/PR-arm.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index e0bde1e50e41b..9569293d4e8fa 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -7,7 +7,7 @@ concurrency: cancel-in-progress: true jobs: build-osx-arm: - name: OSX-ARM64 Tests + name: osx-arm64 Tests runs-on: macOS-14 # M1 strategy: fail-fast: true @@ -60,7 +60,6 @@ jobs: conda activate bioconda # Use SDK with arm64 support - echo "MACOSX_DEPLOYMENT_TARGET (before): $MACOSX_DEPLOYMENT_TARGET" export MACOSX_DEPLOYMENT_TARGET=11.0 # Clean up lingering build artifacts From 6f6fb5d9d5e609c067b60077042a4919859496d7 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 2 Apr 2024 10:33:32 -0400 Subject: [PATCH 20/53] initial merge wf (set up for testing) --- .github/workflows/master-arm.yml | 79 ++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 .github/workflows/master-arm.yml diff --git a/.github/workflows/master-arm.yml b/.github/workflows/master-arm.yml new file mode 100644 index 0000000000000..2f3c93bd52356 --- /dev/null +++ b/.github/workflows/master-arm.yml @@ -0,0 +1,79 @@ +name: Upload +on: + push: + branches: + - target-for-arm-osx +jobs: + + build-upload-osx-arm: + name: osx-arm64 Upload + if: github.repository == 'bioconda/bioconda-recipes' + runs-on: macOS-14 # M1 + strategy: + fail-fast: false + max-parallel: 4 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # TODO: bail if there's no osx-arm64 recipes + # - run: + # name: Check for Additional Platforms + # command: ./.circleci/check-for-additional-platforms.sh "${CIRCLE_SHA1}~1 ${CIRCLE_SHA1}" "build_and_upload" + + - name: set path + run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH + + - name: Fetch conda install script + run: | + wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh + + - name: Restore cache + id: cache + uses: actions/cache@v4 + with: + path: /opt/mambaforge + key: ${{ runner.os }}--master--${{ hashFiles('install-and-set-up-conda.sh', 'common.sh', 'configure-conda.sh') }} + + - name: Set up bioconda-utils + if: steps.cache.outputs.cache-hit != 'true' + run: bash install-and-set-up-conda.sh + + # This script can be used to reconfigure conda to use the right channel setup. + # This has to be done after the cache is restored, because + # the channel setup is not cached as it resides in the home directory. + # We could use a system-wide (and therefore cached) channel setup, + # but mamba does not support that at the time of implementation + # (it ignores settings made with --system). + - name: Configure conda + run: bash configure-conda.sh + + - name: Build and Upload + env: + QUAY_LOGIN: ${{ secrets.QUAY_LOGIN }} + QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_OAUTH_TOKEN }} + ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + INVOLUCRO_AUTH: ${{ secrets.INVOLUCRO_AUTH }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Mimic circleci + OSTYPE: "darwin" + CI: "true" + run: | + set -e + eval "$(conda shell.bash hook)" + conda activate bioconda + + # Use SDK with arm64 support + export MACOSX_DEPLOYMENT_TARGET=11.0 + + # The SDK isn't actually cached, so reinstall it + run_conda_forge_build_setup + + bioconda-utils handle-merged-pr recipes config.yml \ + --repo bioconda/bioconda-recipes \ + --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} \ + --fallback build \ + --artifact-source github-actions \ + --dry-run + # TODO: remove dry-run after testing \ No newline at end of file From 93f21d007df3747258637fbb1d16beb773a51bab Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Thu, 4 Apr 2024 12:40:05 -0400 Subject: [PATCH 21/53] updates from testing --- .github/check-for-additional-platforms.sh | 47 +++++++++++++++++++++++ .github/workflows/PR-arm.yml | 32 ++++++--------- .github/workflows/master-arm.yml | 43 ++++++++------------- 3 files changed, 75 insertions(+), 47 deletions(-) create mode 100755 .github/check-for-additional-platforms.sh diff --git a/.github/check-for-additional-platforms.sh b/.github/check-for-additional-platforms.sh new file mode 100755 index 0000000000000..26bf85a38e7a8 --- /dev/null +++ b/.github/check-for-additional-platforms.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Check to see if any changed recipes have specified the key +# extra:additional-platforms, and if so, if they match the platform of the +# currently-running machine. + +# arguments +git_range=$1 +job_name=$2 + +# Download ARM version of yq +yq_platform=$(uname) +yq_arch=$(uname -m) +[[ $yq_arch = "aarch64" ]] && yq_arch="arm64" +mkdir -p ${HOME}/bin +wget https://github.com/mikefarah/yq/releases/latest/download/yq_${yq_platform}_${yq_arch} -O ${HOME}/bin/yq +chmod +x ${HOME}/bin/yq + +# Find recipes changed from this merge +files=`git diff --name-only --diff-filter AMR ${git_range} | grep -E 'meta.yaml$' ` +build=0 + +for file in $files; do + echo $file + # To create a properly-formatted yaml that yq can parse, comment out jinja2 + # variable setting with {% %} and remove variable use with {{ }}. + additional_platforms=$(cat $file \ + | sed -E 's/(.*)\{%(.*)%\}(.*)/# \1\2\3/g' \ + | tr -d '{{' | tr -d '}}' \ + | ${HOME}/bin/yq '.extra.additional-platforms[]') + # Check if any additional platforms match this job + for additional_platform in $additional_platforms; do + if [ "${GITHUB_JOB}" = "${job_name}-${additional_platform}" ] + then + echo "Found at least one recipe for ${job_name}-${additional_platform}" + build=1 + break + fi + done +done + +# If no changed recipes apply to this platform, skip remaining steps +if [[ build -lt 1 ]] +then + echo "No recipes using this platform, skipping rest of job." + echo "skip_build=true" >> $GITHUB_OUTPUT +fi diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index 9569293d4e8fa..2b9fcc09ec841 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -6,8 +6,8 @@ concurrency: group: build-${{ github.event.pull_request.number || github.head_ref }} cancel-in-progress: true jobs: - build-osx-arm: - name: osx-arm64 Tests + build_and_test-osx-arm64: + name: build_and_test-osx-arm64 runs-on: macOS-14 # M1 strategy: fail-fast: true @@ -17,39 +17,29 @@ jobs: with: fetch-depth: 0 - # TODO: bail if there's no osx-arm64 recipes - # - run: - # name: Check for Additional Platforms - # command: ./.circleci/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" + # bail if there's no osx-arm64 recipes + - name: Check for Additional Platforms + id: additional_platforms + run: ./.github/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" - name: set path run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH - name: Fetch conda install script + if: steps.additional_platforms.outputs.skip_build != 'true' run: | wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - name: Restore cache - id: cache - uses: actions/cache@v4 - with: - path: /opt/mambaforge - key: ${{ runner.os }}--master--${{ hashFiles('install-and-set-up-conda.sh', 'common.sh', 'configure-conda.sh') }} - - name: Set up bioconda-utils - if: steps.cache.outputs.cache-hit != 'true' + if: steps.additional_platforms.outputs.skip_build != 'true' run: bash install-and-set-up-conda.sh - # This script can be used to reconfigure conda to use the right channel setup. - # This has to be done after the cache is restored, because - # the channel setup is not cached as it resides in the home directory. - # We could use a system-wide (and therefore cached) channel setup, - # but mamba does not support that at the time of implementation - # (it ignores settings made with --system). - name: Configure conda + if: steps.additional_platforms.outputs.skip_build != 'true' run: bash configure-conda.sh - name: Build and Test + if: steps.additional_platforms.outputs.skip_build != 'true' env: # Mimic circleci OSTYPE: "darwin" @@ -78,6 +68,7 @@ jobs: --git-range origin/"$GITHUB_BASE_REF" HEAD - name: Prepare artifacts + if: steps.additional_platforms.outputs.skip_build != 'true' run: | ( rm -rf /tmp/artifacts @@ -90,6 +81,7 @@ jobs: ) || true - name: Archive packages + if: steps.additional_platforms.outputs.skip_build != 'true' uses: actions/upload-artifact@v4 with: name: osx-arm64-packages diff --git a/.github/workflows/master-arm.yml b/.github/workflows/master-arm.yml index 2f3c93bd52356..985ae063103de 100644 --- a/.github/workflows/master-arm.yml +++ b/.github/workflows/master-arm.yml @@ -1,12 +1,12 @@ -name: Upload +name: Upload (osx-arm64) on: push: branches: - - target-for-arm-osx -jobs: + - master - build-upload-osx-arm: - name: osx-arm64 Upload +jobs: + build_and_upload-osx-arm64: + name: build_and_upload-osx-arm64 if: github.repository == 'bioconda/bioconda-recipes' runs-on: macOS-14 # M1 strategy: @@ -17,39 +17,30 @@ jobs: with: fetch-depth: 0 - # TODO: bail if there's no osx-arm64 recipes - # - run: - # name: Check for Additional Platforms - # command: ./.circleci/check-for-additional-platforms.sh "${CIRCLE_SHA1}~1 ${CIRCLE_SHA1}" "build_and_upload" + # bail if there's no osx-arm64 recipes + - name: Check for Additional Platforms + id: additional_platforms + run: ./.github/check-for-additional-platforms.sh "${GITHUB_SHA}~1 ${GITHUB_SHA}" "build_and_upload" - name: set path - run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH + run: | + echo "/opt/mambaforge/bin" >> $GITHUB_PATH - name: Fetch conda install script + if: steps.additional_platforms.outputs.skip_build != 'true' run: | wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - name: Restore cache - id: cache - uses: actions/cache@v4 - with: - path: /opt/mambaforge - key: ${{ runner.os }}--master--${{ hashFiles('install-and-set-up-conda.sh', 'common.sh', 'configure-conda.sh') }} - - name: Set up bioconda-utils - if: steps.cache.outputs.cache-hit != 'true' + if: steps.additional_platforms.outputs.skip_build != 'true' run: bash install-and-set-up-conda.sh - # This script can be used to reconfigure conda to use the right channel setup. - # This has to be done after the cache is restored, because - # the channel setup is not cached as it resides in the home directory. - # We could use a system-wide (and therefore cached) channel setup, - # but mamba does not support that at the time of implementation - # (it ignores settings made with --system). - name: Configure conda + if: steps.additional_platforms.outputs.skip_build != 'true' run: bash configure-conda.sh - name: Build and Upload + if: steps.additional_platforms.outputs.skip_build != 'true' env: QUAY_LOGIN: ${{ secrets.QUAY_LOGIN }} QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_OAUTH_TOKEN }} @@ -74,6 +65,4 @@ jobs: --repo bioconda/bioconda-recipes \ --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} \ --fallback build \ - --artifact-source github-actions \ - --dry-run - # TODO: remove dry-run after testing \ No newline at end of file + --artifact-source github-actions From b64a61630b0d346979ade3f4efa5cf5e7560f4fa Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Jul 2024 12:11:23 -0400 Subject: [PATCH 22/53] source common.sh instead of directly setting env vars --- .github/workflows/PR-arm.yml | 2 +- .github/workflows/master-arm.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index 2b9fcc09ec841..0fc01eabb08eb 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -50,7 +50,7 @@ jobs: conda activate bioconda # Use SDK with arm64 support - export MACOSX_DEPLOYMENT_TARGET=11.0 + source common.sh # Clean up lingering build artifacts rm -f /opt/mambaforge/envs/bioconda/conda-bld/osx-arm64/*.tar.bz2 diff --git a/.github/workflows/master-arm.yml b/.github/workflows/master-arm.yml index 985ae063103de..f818a412e9a73 100644 --- a/.github/workflows/master-arm.yml +++ b/.github/workflows/master-arm.yml @@ -56,7 +56,7 @@ jobs: conda activate bioconda # Use SDK with arm64 support - export MACOSX_DEPLOYMENT_TARGET=11.0 + source common.sh # The SDK isn't actually cached, so reinstall it run_conda_forge_build_setup From 65d07ccda32968f9ba950d6cbe38c4fde558ea67 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Jul 2024 12:26:01 -0400 Subject: [PATCH 23/53] enable osx-arm64 on CircleCI --- .circleci/config.yml | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c4a3117dd6022..b869ca091960e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,10 +2,9 @@ version: 2.1 executors: - # osx-arm64: - # macos: - # xcode: 14.2.0 # indicate your selected version of Xcode - # resource_class: macos.m1.large.gen1 + osx-arm64: + macos: + resource_class: macos.m1.medium.gen1 linux-aarch64: machine: image: ubuntu-2204:current @@ -51,6 +50,10 @@ jobs: # a basic unit of work in a run source /opt/mambaforge/etc/profile.d/conda.sh source /opt/mambaforge/etc/profile.d/mamba.sh mamba activate bioconda + + # Use SDK with arm64 support + source common.sh + bioconda-utils build recipes config.yml \ --lint --docker --mulled-test \ --docker-base-image "quay.io/bioconda/bioconda-utils-build-env-cos7-$(arch):${BIOCONDA_UTILS_TAG#v}" \ @@ -121,6 +124,10 @@ jobs: # a basic unit of work in a run source /opt/mambaforge/etc/profile.d/conda.sh source /opt/mambaforge/etc/profile.d/mamba.sh mamba activate bioconda + + # Use SDK with arm64 support + source common.sh + bioconda-utils handle-merged-pr recipes config.yml \ --repo bioconda/bioconda-recipes \ --git-range ${CIRCLE_SHA1}~1 ${CIRCLE_SHA1} \ @@ -188,6 +195,10 @@ jobs: # a basic unit of work in a run source /opt/mambaforge/etc/profile.d/conda.sh source /opt/mambaforge/etc/profile.d/mamba.sh mamba activate bioconda + + # Use SDK with arm64 support + source common.sh + echo '============' conda info --all conda config --show-sources @@ -214,7 +225,7 @@ workflows: matrix: parameters: os: - #- osx-arm64 + - osx-arm64 - linux-aarch64 build and upload (ARM): @@ -226,7 +237,7 @@ workflows: matrix: parameters: os: - #- osx-arm64 + - osx-arm64 - linux-aarch64 @@ -239,6 +250,6 @@ workflows: matrix: parameters: os: - #- osx-arm64 + - osx-arm64 - linux-aarch64 runner: [0, 1, 2, 3, 4, 5] From 2f29babccc3236aa3c5b3caabe5d5c368f2b6132 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Jul 2024 14:05:25 -0400 Subject: [PATCH 24/53] xcode version (match github actions) --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b869ca091960e..821159dbca7c7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,6 +4,7 @@ version: 2.1 executors: osx-arm64: macos: + xcode: 15.0.0 resource_class: macos.m1.medium.gen1 linux-aarch64: machine: From bbae926742655e5b4b6271a3069cd334b051b648 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Jul 2024 14:58:13 -0400 Subject: [PATCH 25/53] debugging --- .github/workflows/PR-arm.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index 0fc01eabb08eb..adfcbfba01319 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -48,6 +48,7 @@ jobs: set -e eval "$(conda shell.bash hook)" conda activate bioconda + echo '============1' # Use SDK with arm64 support source common.sh @@ -55,17 +56,21 @@ jobs: # Clean up lingering build artifacts rm -f /opt/mambaforge/envs/bioconda/conda-bld/osx-arm64/*.tar.bz2 + echo '============2' # The SDK isn't actually cached, so reinstall it run_conda_forge_build_setup + echo '============3' if [ -z "$GITHUB_BASE_REF" ] ; then export GITHUB_BASE_REF="master" fi git fetch origin "$GITHUB_BASE_REF" + echo '============4' bioconda-utils build recipes config.yml \ --lint \ --git-range origin/"$GITHUB_BASE_REF" HEAD + echo '============5' - name: Prepare artifacts if: steps.additional_platforms.outputs.skip_build != 'true' From 9fbffa17e5991e50ddb55bb1e42132bced0b06b6 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Jul 2024 14:58:24 -0400 Subject: [PATCH 26/53] no docker for osx --- .circleci/config.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 821159dbca7c7..ce6f8b917c0ad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,13 +52,15 @@ jobs: # a basic unit of work in a run source /opt/mambaforge/etc/profile.d/mamba.sh mamba activate bioconda - # Use SDK with arm64 support - source common.sh - - bioconda-utils build recipes config.yml \ - --lint --docker --mulled-test \ - --docker-base-image "quay.io/bioconda/bioconda-utils-build-env-cos7-$(arch):${BIOCONDA_UTILS_TAG#v}" \ - --git-range origin/master HEAD + if [[ "$platform" == "Darwin" ]]; then + bioconda-utils build recipes config.yml \ + --lint --git-range origin/master HEAD + else + bioconda-utils build recipes config.yml \ + --lint --docker --mulled-test \ + --docker-base-image "quay.io/bioconda/bioconda-utils-build-env-cos7-$(arch):${BIOCONDA_UTILS_TAG#v}" \ + --git-range origin/master HEAD + fi - run: name: Prepare artifacts From 7e5f85ced1185be0e4e9b6564c0bb708b4068bd0 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Jul 2024 16:27:04 -0400 Subject: [PATCH 27/53] match bulk setup --- .github/workflows/PR-arm.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index adfcbfba01319..ea22b280b26ff 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -48,16 +48,8 @@ jobs: set -e eval "$(conda shell.bash hook)" conda activate bioconda - echo '============1' - # Use SDK with arm64 support source common.sh - - # Clean up lingering build artifacts - rm -f /opt/mambaforge/envs/bioconda/conda-bld/osx-arm64/*.tar.bz2 - - echo '============2' - # The SDK isn't actually cached, so reinstall it run_conda_forge_build_setup echo '============3' From 32a703217dad870867a3511b8cab8abeb8672fc3 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Jul 2024 16:31:32 -0400 Subject: [PATCH 28/53] comment out run_conda_forge_build_setup --- .github/workflows/PR-arm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index ea22b280b26ff..f3c86b9bc405e 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -50,7 +50,7 @@ jobs: conda activate bioconda source common.sh - run_conda_forge_build_setup + # run_conda_forge_build_setup echo '============3' if [ -z "$GITHUB_BASE_REF" ] ; then From fbd98a13711b0e9fc980a63a0bd4157bfe615611 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Jul 2024 17:25:29 -0400 Subject: [PATCH 29/53] Little more generic test --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ce6f8b917c0ad..0334a8da42fc7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,14 +52,14 @@ jobs: # a basic unit of work in a run source /opt/mambaforge/etc/profile.d/mamba.sh mamba activate bioconda - if [[ "$platform" == "Darwin" ]]; then - bioconda-utils build recipes config.yml \ - --lint --git-range origin/master HEAD - else + if command -V docker >/dev/null; then bioconda-utils build recipes config.yml \ --lint --docker --mulled-test \ --docker-base-image "quay.io/bioconda/bioconda-utils-build-env-cos7-$(arch):${BIOCONDA_UTILS_TAG#v}" \ --git-range origin/master HEAD + else + bioconda-utils build recipes config.yml \ + --lint --git-range origin/master HEAD fi - run: From 674ade565e560730896e0806d943fb75582fc368 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Jul 2024 17:25:42 -0400 Subject: [PATCH 30/53] longer build test --- recipes/bamtools/meta.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/recipes/bamtools/meta.yaml b/recipes/bamtools/meta.yaml index 803497b6c9a4b..af163092eab03 100644 --- a/recipes/bamtools/meta.yaml +++ b/recipes/bamtools/meta.yaml @@ -12,7 +12,7 @@ source: # - 0001-Const-qualify-all-functors-operator-member-functions.patch build: - number: 3 + number: 4 run_exports: - {{ pin_subpackage(name, max_pin='x.x') }} @@ -37,5 +37,6 @@ about: extra: additional-platforms: - linux-aarch64 + - osx-arm64 identifiers: - biotools:bamtools From cd6df20c9df8030d342e39d9e508d581adaa31fc Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Jul 2024 17:26:40 -0400 Subject: [PATCH 31/53] clean up --- .github/workflows/PR-arm.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index f3c86b9bc405e..c08276587fba7 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -48,21 +48,15 @@ jobs: set -e eval "$(conda shell.bash hook)" conda activate bioconda - source common.sh - # run_conda_forge_build_setup - echo '============3' if [ -z "$GITHUB_BASE_REF" ] ; then export GITHUB_BASE_REF="master" fi git fetch origin "$GITHUB_BASE_REF" - echo '============4' bioconda-utils build recipes config.yml \ - --lint \ - --git-range origin/"$GITHUB_BASE_REF" HEAD - echo '============5' + --lint --git-range origin/"$GITHUB_BASE_REF" HEAD - name: Prepare artifacts if: steps.additional_platforms.outputs.skip_build != 'true' From dda42dc2596c30b75f3b61f5c155b2882e8b1694 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Jul 2024 17:27:09 -0400 Subject: [PATCH 32/53] remove setup --- .github/workflows/master-arm.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/master-arm.yml b/.github/workflows/master-arm.yml index f818a412e9a73..c627dfd6d634a 100644 --- a/.github/workflows/master-arm.yml +++ b/.github/workflows/master-arm.yml @@ -54,13 +54,8 @@ jobs: set -e eval "$(conda shell.bash hook)" conda activate bioconda - - # Use SDK with arm64 support source common.sh - # The SDK isn't actually cached, so reinstall it - run_conda_forge_build_setup - bioconda-utils handle-merged-pr recipes config.yml \ --repo bioconda/bioconda-recipes \ --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} \ From 8ed90a0e296053be4c5ddbc1a8718124c58b4101 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Jul 2024 17:52:18 -0400 Subject: [PATCH 33/53] simplify and add osx-arm64 --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0334a8da42fc7..871c3f575e374 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,10 +47,10 @@ jobs: # a basic unit of work in a run - run: name: Build and test command: | - . common.sh - source /opt/mambaforge/etc/profile.d/conda.sh - source /opt/mambaforge/etc/profile.d/mamba.sh - mamba activate bioconda + set -e + eval "$(conda shell.bash hook)" + conda activate bioconda + source common.sh if command -V docker >/dev/null; then bioconda-utils build recipes config.yml \ @@ -69,7 +69,7 @@ jobs: # a basic unit of work in a run mkdir -p /tmp/artifacts/packages cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0 find -name .cache | xargs rm -rf || true - for n in index.html channeldata.json linux-aarch64 linux-64 osx-64 noarch; do + for n in index.html channeldata.json linux-aarch64 linux-64 osx-64 osx-arm64 noarch; do cp -rv $n /tmp/artifacts/packages || true done if command -V docker >/dev/null; then From 62e8f45556e553009a87de2312ee9ca417b2ab98 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Mon, 1 Jul 2024 18:04:56 -0400 Subject: [PATCH 34/53] revert --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 871c3f575e374..20c356ccfcbe5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,10 +47,10 @@ jobs: # a basic unit of work in a run - run: name: Build and test command: | - set -e - eval "$(conda shell.bash hook)" - conda activate bioconda - source common.sh + . common.sh + source /opt/mambaforge/etc/profile.d/conda.sh + source /opt/mambaforge/etc/profile.d/mamba.sh + mamba activate bioconda if command -V docker >/dev/null; then bioconda-utils build recipes config.yml \ From 4742b8d990ff6de20c6fb95d7e2adbad51d40fef Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 2 Jul 2024 10:44:43 -0400 Subject: [PATCH 35/53] try to match other CI --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 20c356ccfcbe5..871c3f575e374 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,10 +47,10 @@ jobs: # a basic unit of work in a run - run: name: Build and test command: | - . common.sh - source /opt/mambaforge/etc/profile.d/conda.sh - source /opt/mambaforge/etc/profile.d/mamba.sh - mamba activate bioconda + set -e + eval "$(conda shell.bash hook)" + conda activate bioconda + source common.sh if command -V docker >/dev/null; then bioconda-utils build recipes config.yml \ From c51ea5539beb85f9a8c48d2bfaafaf0f6c929672 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Fri, 5 Jul 2024 15:07:04 -0400 Subject: [PATCH 36/53] standardize CI configurations for additional platforms --- .circleci/config.yml | 34 ++++++++------ .github/check-for-additional-platforms.sh | 47 ------------------- .github/workflows/PR-arm.yml | 8 +++- .github/workflows/master-arm.yml | 8 +++- .../check-for-additional-platforms.sh | 9 ++-- 5 files changed, 38 insertions(+), 68 deletions(-) delete mode 100755 .github/check-for-additional-platforms.sh rename {.circleci => scripts}/check-for-additional-platforms.sh (87%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 871c3f575e374..ed31cdb6059e9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,13 @@ jobs: # a basic unit of work in a run - run: name: Check for Additional Platforms - command: ./.circleci/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" + command: | + result = $(../scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${CIRCLE_JOB}") + if [[ result != "build" ]] + then + echo "No recipes using this platform, skipping rest of job." + circleci-agent step halt + fi - run: name: Fetch bioconda install script @@ -99,7 +105,13 @@ jobs: # a basic unit of work in a run - run: name: Check for Additional Platforms - command: ./.circleci/check-for-additional-platforms.sh "${CIRCLE_SHA1}~1 ${CIRCLE_SHA1}" "build_and_upload" + command: | + result = $(../scripts/check-for-additional-platforms.sh "${CIRCLE_SHA1}~1 ${CIRCLE_SHA1}" "build_and_upload" "${CIRCLE_JOB}") + if [[ result != "build" ]] + then + echo "No recipes using this platform, skipping rest of job." + circleci-agent step halt + fi - run: name: Fetch bioconda install script @@ -124,11 +136,9 @@ jobs: # a basic unit of work in a run - run: name: Build and push command: | - source /opt/mambaforge/etc/profile.d/conda.sh - source /opt/mambaforge/etc/profile.d/mamba.sh - mamba activate bioconda - - # Use SDK with arm64 support + set -e + eval "$(conda shell.bash hook)" + conda activate bioconda source common.sh bioconda-utils handle-merged-pr recipes config.yml \ @@ -194,12 +204,8 @@ jobs: # a basic unit of work in a run name: Build and upload command: | set -e - . common.sh - source /opt/mambaforge/etc/profile.d/conda.sh - source /opt/mambaforge/etc/profile.d/mamba.sh - mamba activate bioconda - - # Use SDK with arm64 support + eval "$(conda shell.bash hook)" + conda activate bioconda source common.sh echo '============' @@ -253,6 +259,6 @@ workflows: matrix: parameters: os: - - osx-arm64 + # - osx-arm64 Bulk is on GitHub Actions - linux-aarch64 runner: [0, 1, 2, 3, 4, 5] diff --git a/.github/check-for-additional-platforms.sh b/.github/check-for-additional-platforms.sh deleted file mode 100755 index 26bf85a38e7a8..0000000000000 --- a/.github/check-for-additional-platforms.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# Check to see if any changed recipes have specified the key -# extra:additional-platforms, and if so, if they match the platform of the -# currently-running machine. - -# arguments -git_range=$1 -job_name=$2 - -# Download ARM version of yq -yq_platform=$(uname) -yq_arch=$(uname -m) -[[ $yq_arch = "aarch64" ]] && yq_arch="arm64" -mkdir -p ${HOME}/bin -wget https://github.com/mikefarah/yq/releases/latest/download/yq_${yq_platform}_${yq_arch} -O ${HOME}/bin/yq -chmod +x ${HOME}/bin/yq - -# Find recipes changed from this merge -files=`git diff --name-only --diff-filter AMR ${git_range} | grep -E 'meta.yaml$' ` -build=0 - -for file in $files; do - echo $file - # To create a properly-formatted yaml that yq can parse, comment out jinja2 - # variable setting with {% %} and remove variable use with {{ }}. - additional_platforms=$(cat $file \ - | sed -E 's/(.*)\{%(.*)%\}(.*)/# \1\2\3/g' \ - | tr -d '{{' | tr -d '}}' \ - | ${HOME}/bin/yq '.extra.additional-platforms[]') - # Check if any additional platforms match this job - for additional_platform in $additional_platforms; do - if [ "${GITHUB_JOB}" = "${job_name}-${additional_platform}" ] - then - echo "Found at least one recipe for ${job_name}-${additional_platform}" - build=1 - break - fi - done -done - -# If no changed recipes apply to this platform, skip remaining steps -if [[ build -lt 1 ]] -then - echo "No recipes using this platform, skipping rest of job." - echo "skip_build=true" >> $GITHUB_OUTPUT -fi diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index c08276587fba7..689049f59af28 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -20,7 +20,13 @@ jobs: # bail if there's no osx-arm64 recipes - name: Check for Additional Platforms id: additional_platforms - run: ./.github/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" + run: | + result = $(../scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${GITHUB_JOB}") + if [[ result != "build" ]] + then + echo "No recipes using this platform, skipping rest of job." + echo "skip_build=true" >> $GITHUB_OUTPUT + fi - name: set path run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH diff --git a/.github/workflows/master-arm.yml b/.github/workflows/master-arm.yml index c627dfd6d634a..d2e649e447722 100644 --- a/.github/workflows/master-arm.yml +++ b/.github/workflows/master-arm.yml @@ -20,7 +20,13 @@ jobs: # bail if there's no osx-arm64 recipes - name: Check for Additional Platforms id: additional_platforms - run: ./.github/check-for-additional-platforms.sh "${GITHUB_SHA}~1 ${GITHUB_SHA}" "build_and_upload" + run: | + result = $(../scripts/check-for-additional-platforms.sh "${GITHUB_SHA}~1 ${GITHUB_SHA}" "build_and_upload" "${GITHUB_JOB}") + if [[ result != "build" ]] + then + echo "No recipes using this platform, skipping rest of job." + echo "skip_build=true" >> $GITHUB_OUTPUT + fi - name: set path run: | diff --git a/.circleci/check-for-additional-platforms.sh b/scripts/check-for-additional-platforms.sh similarity index 87% rename from .circleci/check-for-additional-platforms.sh rename to scripts/check-for-additional-platforms.sh index 50641d2b63052..d088794f14422 100755 --- a/.circleci/check-for-additional-platforms.sh +++ b/scripts/check-for-additional-platforms.sh @@ -7,6 +7,7 @@ # arguments git_range=$1 job_name=$2 +current_job=$3 # Download ARM version of yq yq_platform=$(uname) @@ -21,7 +22,6 @@ files=`git diff --name-only --diff-filter AMR ${git_range} | grep -E 'meta.yaml$ build=0 for file in $files; do - echo $file # To create a properly-formatted yaml that yq can parse, comment out jinja2 # variable setting with {% %} and remove variable use with {{ }}. additional_platforms=$(cat $file \ @@ -37,7 +37,7 @@ for file in $files; do # Check if any additional platforms match this job for additional_platform in $additional_platforms; do - if [ "${CIRCLE_JOB}" = "${job_name}-${additional_platform}" ] + if [ "${current_job}" = "${job_name}-${additional_platform}" ] then build=1 break @@ -46,8 +46,7 @@ for file in $files; do done # If no changed recipes apply to this platform, skip remaining steps -if [[ build -lt 1 ]] +if [[ build -gt 0 ]] then - echo "No recipes using this platform, skipping rest of job." - circleci-agent step halt + echo "build" fi From 188e89d00c4be0567158173583c9acfb71abc9b6 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Fri, 5 Jul 2024 15:20:58 -0400 Subject: [PATCH 37/53] fix scripts path --- .circleci/config.yml | 4 ++-- .github/workflows/PR-arm.yml | 2 +- .github/workflows/master-arm.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ed31cdb6059e9..01cc16d7566a0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,7 @@ jobs: # a basic unit of work in a run - run: name: Check for Additional Platforms command: | - result = $(../scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${CIRCLE_JOB}") + result = $(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${CIRCLE_JOB}") if [[ result != "build" ]] then echo "No recipes using this platform, skipping rest of job." @@ -106,7 +106,7 @@ jobs: # a basic unit of work in a run - run: name: Check for Additional Platforms command: | - result = $(../scripts/check-for-additional-platforms.sh "${CIRCLE_SHA1}~1 ${CIRCLE_SHA1}" "build_and_upload" "${CIRCLE_JOB}") + result = $(./scripts/check-for-additional-platforms.sh "${CIRCLE_SHA1}~1 ${CIRCLE_SHA1}" "build_and_upload" "${CIRCLE_JOB}") if [[ result != "build" ]] then echo "No recipes using this platform, skipping rest of job." diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index 689049f59af28..431b6617f3d21 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -21,7 +21,7 @@ jobs: - name: Check for Additional Platforms id: additional_platforms run: | - result = $(../scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${GITHUB_JOB}") + result = $(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${GITHUB_JOB}") if [[ result != "build" ]] then echo "No recipes using this platform, skipping rest of job." diff --git a/.github/workflows/master-arm.yml b/.github/workflows/master-arm.yml index d2e649e447722..16b4dffc45af1 100644 --- a/.github/workflows/master-arm.yml +++ b/.github/workflows/master-arm.yml @@ -21,7 +21,7 @@ jobs: - name: Check for Additional Platforms id: additional_platforms run: | - result = $(../scripts/check-for-additional-platforms.sh "${GITHUB_SHA}~1 ${GITHUB_SHA}" "build_and_upload" "${GITHUB_JOB}") + result = $(./scripts/check-for-additional-platforms.sh "${GITHUB_SHA}~1 ${GITHUB_SHA}" "build_and_upload" "${GITHUB_JOB}") if [[ result != "build" ]] then echo "No recipes using this platform, skipping rest of job." From feda25ffc8f84b5b2b3c52f320f8d098f127b29e Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Fri, 5 Jul 2024 16:30:40 -0400 Subject: [PATCH 38/53] fix check --- .circleci/config.yml | 8 ++++---- .github/workflows/PR-arm.yml | 4 ++-- .github/workflows/master-arm.yml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 01cc16d7566a0..1d864cefc4685 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,8 +23,8 @@ jobs: # a basic unit of work in a run - run: name: Check for Additional Platforms command: | - result = $(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${CIRCLE_JOB}") - if [[ result != "build" ]] + result=$(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${CIRCLE_JOB}") + if [[ ${result} != "build" ]] then echo "No recipes using this platform, skipping rest of job." circleci-agent step halt @@ -106,8 +106,8 @@ jobs: # a basic unit of work in a run - run: name: Check for Additional Platforms command: | - result = $(./scripts/check-for-additional-platforms.sh "${CIRCLE_SHA1}~1 ${CIRCLE_SHA1}" "build_and_upload" "${CIRCLE_JOB}") - if [[ result != "build" ]] + result=$(./scripts/check-for-additional-platforms.sh "${CIRCLE_SHA1}~1 ${CIRCLE_SHA1}" "build_and_upload" "${CIRCLE_JOB}") + if [[ ${result} != "build" ]] then echo "No recipes using this platform, skipping rest of job." circleci-agent step halt diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index 431b6617f3d21..50999e3f4d8b9 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -21,8 +21,8 @@ jobs: - name: Check for Additional Platforms id: additional_platforms run: | - result = $(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${GITHUB_JOB}") - if [[ result != "build" ]] + result=$(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${GITHUB_JOB}") + if [[ ${result} != "build" ]] then echo "No recipes using this platform, skipping rest of job." echo "skip_build=true" >> $GITHUB_OUTPUT diff --git a/.github/workflows/master-arm.yml b/.github/workflows/master-arm.yml index 16b4dffc45af1..09844165447ef 100644 --- a/.github/workflows/master-arm.yml +++ b/.github/workflows/master-arm.yml @@ -21,8 +21,8 @@ jobs: - name: Check for Additional Platforms id: additional_platforms run: | - result = $(./scripts/check-for-additional-platforms.sh "${GITHUB_SHA}~1 ${GITHUB_SHA}" "build_and_upload" "${GITHUB_JOB}") - if [[ result != "build" ]] + result=$(./scripts/check-for-additional-platforms.sh "${GITHUB_SHA}~1 ${GITHUB_SHA}" "build_and_upload" "${GITHUB_JOB}") + if [[ ${result} != "build" ]] then echo "No recipes using this platform, skipping rest of job." echo "skip_build=true" >> $GITHUB_OUTPUT From 09cbbc9fd15a502c20484d689f57fbf741580004 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Fri, 5 Jul 2024 16:49:45 -0400 Subject: [PATCH 39/53] test skip --- recipes/bamtools/meta.yaml | 2 +- recipes/bioawk/meta.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/bamtools/meta.yaml b/recipes/bamtools/meta.yaml index af163092eab03..895819ef0471a 100644 --- a/recipes/bamtools/meta.yaml +++ b/recipes/bamtools/meta.yaml @@ -37,6 +37,6 @@ about: extra: additional-platforms: - linux-aarch64 - - osx-arm64 + #- osx-arm64 identifiers: - biotools:bamtools diff --git a/recipes/bioawk/meta.yaml b/recipes/bioawk/meta.yaml index ea6a9dee20bf7..f0f6cc9702b17 100644 --- a/recipes/bioawk/meta.yaml +++ b/recipes/bioawk/meta.yaml @@ -34,4 +34,4 @@ about: extra: additional-platforms: - linux-aarch64 - - osx-arm64 + #- osx-arm64 From 57fb5c47ef4f20af09e224969536c2fca0a185dd Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Fri, 5 Jul 2024 16:55:59 -0400 Subject: [PATCH 40/53] reenable test packages --- recipes/bamtools/meta.yaml | 2 +- recipes/bioawk/meta.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/bamtools/meta.yaml b/recipes/bamtools/meta.yaml index 895819ef0471a..af163092eab03 100644 --- a/recipes/bamtools/meta.yaml +++ b/recipes/bamtools/meta.yaml @@ -37,6 +37,6 @@ about: extra: additional-platforms: - linux-aarch64 - #- osx-arm64 + - osx-arm64 identifiers: - biotools:bamtools diff --git a/recipes/bioawk/meta.yaml b/recipes/bioawk/meta.yaml index f0f6cc9702b17..ea6a9dee20bf7 100644 --- a/recipes/bioawk/meta.yaml +++ b/recipes/bioawk/meta.yaml @@ -34,4 +34,4 @@ about: extra: additional-platforms: - linux-aarch64 - #- osx-arm64 + - osx-arm64 From a3cee7e9884c263d12a83f55d1bad0f684ec3c8f Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Fri, 5 Jul 2024 17:23:02 -0400 Subject: [PATCH 41/53] add osx-arm64 to bulk --- .github/workflows/Bulk.yml | 63 +++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Bulk.yml b/.github/workflows/Bulk.yml index ec689ce742449..3121021a10ab9 100644 --- a/.github/workflows/Bulk.yml +++ b/.github/workflows/Bulk.yml @@ -62,7 +62,7 @@ jobs: conda clean -y --all build-osx: - name: Bulk OSX Builds + name: Bulk OSX-64 Builds if: "contains(github.event.head_commit.message, '[ci run]')" runs-on: macos-13 strategy: @@ -121,3 +121,64 @@ jobs: --lint --anaconda-upload --record-build-failures --skiplist-leafs \ --exclude bioconda-repodata-patches conda clean -y --all + + build-osx-arm: + name: Bulk OSX-ARM64 Builds + if: "contains(github.event.head_commit.message, '[ci run]')" + runs-on: macOS-14 # M1 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + runner: [0, 1, 2, 3] + steps: + - uses: actions/checkout@v4 + with: + # checkout as BiocondaBot in order to have the permission to push fail logs + token: ${{secrets.BIOCONDA_BOT_REPO_TOKEN}} + + - name: set git user + run: | + git config user.name BiocondaBot + git config user.email BiocondaBot@users.noreply.github.com + + - name: set path + run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH + + - name: Fetch conda install script + run: | + wget https://raw.githubusercontent.com/bioconda/bioconda-common/bulk/{common,install-and-set-up-conda,configure-conda}.sh + + - name: Set up bioconda-utils + run: bash install-and-set-up-conda.sh + + - name: Configure conda + run: bash configure-conda.sh + + - name: Build and upload + env: + ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + INVOLUCRO_AUTH: ${{ secrets.INVOLUCRO_AUTH }} + QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_OAUTH_TOKEN }} + # Mimic circleci + OSTYPE: "darwin" + CI: "true" + run: | + set -e + eval "$(conda shell.bash hook)" + conda activate bioconda + + source common.sh + # run_conda_forge_build_setup causes errors on arm + + echo '============' + conda info --all + conda config --show-sources + conda activate bioconda + python -c 'import bioconda_utils.utils as u ; import pathlib as p ; print(*(f"{f}:\n{p.Path(f).read_text()}" for f in u.load_conda_build_config().exclusive_config_files), sep="\n")' + echo '============' + bioconda-utils build recipes config.yml \ + --worker-offset ${{ matrix.runner }} --n-workers 4 \ + --lint --anaconda-upload --record-build-failures --skiplist-leafs \ + --exclude bioconda-repodata-patches + conda clean -y --all From e794400e7ba623832946d85f67b2e3c024ef4bb4 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Fri, 5 Jul 2024 17:41:24 -0400 Subject: [PATCH 42/53] test with user friendly name --- .github/workflows/PR-arm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml index 50999e3f4d8b9..49e7b855dfecb 100644 --- a/.github/workflows/PR-arm.yml +++ b/.github/workflows/PR-arm.yml @@ -7,7 +7,7 @@ concurrency: cancel-in-progress: true jobs: build_and_test-osx-arm64: - name: build_and_test-osx-arm64 + name: Build and Test OSX-ARM64 runs-on: macOS-14 # M1 strategy: fail-fast: true From ef3150d633f3e77c715ed373e9f3822d1ffe55a7 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Fri, 5 Jul 2024 18:00:52 -0400 Subject: [PATCH 43/53] move arm jobs to existing GHA config --- .github/workflows/PR-arm.yml | 86 -------------------- .github/workflows/PR.yml | 130 +++++++++++++++++++++---------- .github/workflows/master-arm.yml | 69 ---------------- .github/workflows/master.yml | 97 ++++++++++++++++------- 4 files changed, 155 insertions(+), 227 deletions(-) delete mode 100644 .github/workflows/PR-arm.yml delete mode 100644 .github/workflows/master-arm.yml diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml deleted file mode 100644 index 49e7b855dfecb..0000000000000 --- a/.github/workflows/PR-arm.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: PR (osx-arm64) -on: - pull_request: - types: [opened, synchronize, reopened] -concurrency: - group: build-${{ github.event.pull_request.number || github.head_ref }} - cancel-in-progress: true -jobs: - build_and_test-osx-arm64: - name: Build and Test OSX-ARM64 - runs-on: macOS-14 # M1 - strategy: - fail-fast: true - max-parallel: 13 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # bail if there's no osx-arm64 recipes - - name: Check for Additional Platforms - id: additional_platforms - run: | - result=$(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${GITHUB_JOB}") - if [[ ${result} != "build" ]] - then - echo "No recipes using this platform, skipping rest of job." - echo "skip_build=true" >> $GITHUB_OUTPUT - fi - - - name: set path - run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH - - - name: Fetch conda install script - if: steps.additional_platforms.outputs.skip_build != 'true' - run: | - wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - - name: Set up bioconda-utils - if: steps.additional_platforms.outputs.skip_build != 'true' - run: bash install-and-set-up-conda.sh - - - name: Configure conda - if: steps.additional_platforms.outputs.skip_build != 'true' - run: bash configure-conda.sh - - - name: Build and Test - if: steps.additional_platforms.outputs.skip_build != 'true' - env: - # Mimic circleci - OSTYPE: "darwin" - CI: "true" - run: | - set -e - eval "$(conda shell.bash hook)" - conda activate bioconda - source common.sh - - if [ -z "$GITHUB_BASE_REF" ] ; then - export GITHUB_BASE_REF="master" - fi - git fetch origin "$GITHUB_BASE_REF" - - bioconda-utils build recipes config.yml \ - --lint --git-range origin/"$GITHUB_BASE_REF" HEAD - - - name: Prepare artifacts - if: steps.additional_platforms.outputs.skip_build != 'true' - run: | - ( - rm -rf /tmp/artifacts - mkdir -p /tmp/artifacts/packages - cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0 - find -name .cache | xargs rm -rf || true - for n in index.html channeldata.json osx-arm64 noarch; do - cp -rv $n /tmp/artifacts/packages || true - done - ) || true - - - name: Archive packages - if: steps.additional_platforms.outputs.skip_build != 'true' - uses: actions/upload-artifact@v4 - with: - name: osx-arm64-packages - path: | - /tmp/artifacts diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index 9418eba60943d..c7d5d3fce07e6 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -13,7 +13,7 @@ jobs: fail-fast: true max-parallel: 13 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -24,15 +24,7 @@ jobs: run: | wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - name: Restore cache - id: cache - uses: actions/cache@v3 - with: - path: /opt/mambaforge - key: ${{ runner.os }}--bulk--${{ hashFiles('**/install-and-set-up-conda.sh') }} - - name: Set up bioconda-utils - if: steps.cache.outputs.cache-hit != 'true' run: bash install-and-set-up-conda.sh - name: Configure conda @@ -67,7 +59,7 @@ jobs: max-parallel: 13 needs: lint steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -78,23 +70,10 @@ jobs: run: | wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - name: Restore cache - id: cache - uses: actions/cache@v3 - with: - path: /opt/mambaforge - key: ${{ runner.os }}--bulk--${{ hashFiles('**/install-and-set-up-conda.sh') }} - - name: Set up bioconda-utils - if: steps.cache.outputs.cache-hit != 'true' run: bash install-and-set-up-conda.sh # This script can be used to reconfigure conda to use the right channel setup. - # This has to be done after the cache is restored, because - # the channel setup is not cached as it resides in the home directory. - # We could use a system-wide (and therefore cached) channel setup, - # but mamba does not support that at the time of implementation - # (it ignores settings made with --system). - name: Configure conda run: bash configure-conda.sh @@ -151,28 +130,28 @@ jobs: ) || true - name: Archive packages - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: linux-packages path: | /tmp/artifacts/packages - name: Archive images - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: docker-images path: | /tmp/artifacts/images - build-osx: - name: OSX Tests + build-osx-64: + name: OSX-64 Tests runs-on: macos-13 strategy: fail-fast: true max-parallel: 4 needs: build-linux steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -183,23 +162,10 @@ jobs: run: | wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - name: Restore cache - id: cache - uses: actions/cache@v3 - with: - path: /opt/mambaforge - key: ${{ runner.os }}--bulk--${{ hashFiles('**/install-and-set-up-conda.sh') }} - - name: Set up bioconda-utils - if: steps.cache.outputs.cache-hit != 'true' run: bash install-and-set-up-conda.sh # This script can be used to reconfigure conda to use the right channel setup. - # This has to be done after the cache is restored, because - # the channel setup is not cached as it resides in the home directory. - # We could use a system-wide (and therefore cached) channel setup, - # but mamba does not support that at the time of implementation - # (it ignores settings made with --system). - name: Configure conda run: bash configure-conda.sh @@ -241,8 +207,88 @@ jobs: ) || true - name: Archive packages - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: osx-packages path: | /tmp/artifacts/packages + + build_and_test-osx-arm64: + name: OSX-ARM64 Tests + runs-on: macOS-14 # M1 + strategy: + fail-fast: true + max-parallel: 4 + needs: build-linux + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # bail if there's no osx-arm64 recipes + - name: Check for Additional Platforms + id: additional_platforms + run: | + result=$(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${GITHUB_JOB}") + if [[ ${result} != "build" ]] + then + echo "No recipes using this platform, skipping rest of job." + echo "skip_build=true" >> $GITHUB_OUTPUT + fi + + - name: set path + run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH + + - name: Fetch conda install script + if: steps.additional_platforms.outputs.skip_build != 'true' + run: | + wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh + + - name: Set up bioconda-utils + if: steps.additional_platforms.outputs.skip_build != 'true' + run: bash install-and-set-up-conda.sh + + - name: Configure conda + if: steps.additional_platforms.outputs.skip_build != 'true' + run: bash configure-conda.sh + + - name: Build and Test + if: steps.additional_platforms.outputs.skip_build != 'true' + env: + # Mimic circleci + OSTYPE: "darwin" + CI: "true" + run: | + set -e + eval "$(conda shell.bash hook)" + conda activate bioconda + source common.sh + + if [ -z "$GITHUB_BASE_REF" ] ; then + export GITHUB_BASE_REF="master" + fi + git fetch origin "$GITHUB_BASE_REF" + + bioconda-utils build recipes config.yml \ + --lint --git-range origin/"$GITHUB_BASE_REF" HEAD + + - name: Prepare artifacts + if: steps.additional_platforms.outputs.skip_build != 'true' + run: | + ( + rm -rf /tmp/artifacts + mkdir -p /tmp/artifacts/packages + cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0 + find -name .cache | xargs rm -rf || true + for n in index.html channeldata.json osx-arm64 noarch; do + cp -rv $n /tmp/artifacts/packages || true + done + ) || true + + - name: Archive packages + if: steps.additional_platforms.outputs.skip_build != 'true' + uses: actions/upload-artifact@v4 + with: + name: osx-arm64-packages + path: | + /tmp/artifacts diff --git a/.github/workflows/master-arm.yml b/.github/workflows/master-arm.yml deleted file mode 100644 index 09844165447ef..0000000000000 --- a/.github/workflows/master-arm.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Upload (osx-arm64) -on: - push: - branches: - - master - -jobs: - build_and_upload-osx-arm64: - name: build_and_upload-osx-arm64 - if: github.repository == 'bioconda/bioconda-recipes' - runs-on: macOS-14 # M1 - strategy: - fail-fast: false - max-parallel: 4 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # bail if there's no osx-arm64 recipes - - name: Check for Additional Platforms - id: additional_platforms - run: | - result=$(./scripts/check-for-additional-platforms.sh "${GITHUB_SHA}~1 ${GITHUB_SHA}" "build_and_upload" "${GITHUB_JOB}") - if [[ ${result} != "build" ]] - then - echo "No recipes using this platform, skipping rest of job." - echo "skip_build=true" >> $GITHUB_OUTPUT - fi - - - name: set path - run: | - echo "/opt/mambaforge/bin" >> $GITHUB_PATH - - - name: Fetch conda install script - if: steps.additional_platforms.outputs.skip_build != 'true' - run: | - wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - - name: Set up bioconda-utils - if: steps.additional_platforms.outputs.skip_build != 'true' - run: bash install-and-set-up-conda.sh - - - name: Configure conda - if: steps.additional_platforms.outputs.skip_build != 'true' - run: bash configure-conda.sh - - - name: Build and Upload - if: steps.additional_platforms.outputs.skip_build != 'true' - env: - QUAY_LOGIN: ${{ secrets.QUAY_LOGIN }} - QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_OAUTH_TOKEN }} - ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} - INVOLUCRO_AUTH: ${{ secrets.INVOLUCRO_AUTH }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Mimic circleci - OSTYPE: "darwin" - CI: "true" - run: | - set -e - eval "$(conda shell.bash hook)" - conda activate bioconda - source common.sh - - bioconda-utils handle-merged-pr recipes config.yml \ - --repo bioconda/bioconda-recipes \ - --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} \ - --fallback build \ - --artifact-source github-actions diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index e698443590e50..dfd3945596555 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false max-parallel: 13 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -23,23 +23,10 @@ jobs: run: | wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - name: Restore cache - id: cache - uses: actions/cache@v3 - with: - path: /opt/mambaforge - key: ${{ runner.os }}--bulk--${{ hashFiles('**/install-and-set-up-conda.sh') }} - - name: Set up bioconda-utils - if: steps.cache.outputs.cache-hit != 'true' run: bash install-and-set-up-conda.sh # This script can be used to reconfigure conda to use the right channel setup. - # This has to be done after the cache is restored, because - # the channel setup is not cached as it resides in the home directory. - # We could use a system-wide (and therefore cached) channel setup, - # but mamba does not support that at the time of implementation - # (it ignores settings made with --system). - name: Configure conda run: bash configure-conda.sh @@ -68,15 +55,15 @@ jobs: --docker --mulled-test --anaconda-upload --mulled-upload-target biocontainers docker rmi quay.io/dpryan79/mulled_container:latest - build-osx: - name: OSX Upload + build-osx-64: + name: OSX-64 Upload if: github.repository == 'bioconda/bioconda-recipes' runs-on: macos-13 strategy: fail-fast: false max-parallel: 4 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -87,23 +74,10 @@ jobs: run: | wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - name: Restore cache - id: cache - uses: actions/cache@v3 - with: - path: /opt/mambaforge - key: ${{ runner.os }}--bulk--${{ hashFiles('**/install-and-set-up-conda.sh') }} - - name: Set up bioconda-utils - if: steps.cache.outputs.cache-hit != 'true' run: bash install-and-set-up-conda.sh # This script can be used to reconfigure conda to use the right channel setup. - # This has to be done after the cache is restored, because - # the channel setup is not cached as it resides in the home directory. - # We could use a system-wide (and therefore cached) channel setup, - # but mamba does not support that at the time of implementation - # (it ignores settings made with --system). - name: Configure conda run: bash configure-conda.sh @@ -131,3 +105,66 @@ jobs: # --fallback build bioconda-utils build recipes config.yml \ --anaconda-upload --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} + + build_and_upload-osx-arm64: + name: OSX-ARM64 Upload + if: github.repository == 'bioconda/bioconda-recipes' + runs-on: macOS-14 # M1 + strategy: + fail-fast: false + max-parallel: 4 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # bail if there's no osx-arm64 recipes + - name: Check for Additional Platforms + id: additional_platforms + run: | + result=$(./scripts/check-for-additional-platforms.sh "${GITHUB_SHA}~1 ${GITHUB_SHA}" "build_and_upload" "${GITHUB_JOB}") + if [[ ${result} != "build" ]] + then + echo "No recipes using this platform, skipping rest of job." + echo "skip_build=true" >> $GITHUB_OUTPUT + fi + + - name: set path + run: | + echo "/opt/mambaforge/bin" >> $GITHUB_PATH + + - name: Fetch conda install script + if: steps.additional_platforms.outputs.skip_build != 'true' + run: | + wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh + + - name: Set up bioconda-utils + if: steps.additional_platforms.outputs.skip_build != 'true' + run: bash install-and-set-up-conda.sh + + - name: Configure conda + if: steps.additional_platforms.outputs.skip_build != 'true' + run: bash configure-conda.sh + + - name: Build and Upload + if: steps.additional_platforms.outputs.skip_build != 'true' + env: + QUAY_LOGIN: ${{ secrets.QUAY_LOGIN }} + QUAY_OAUTH_TOKEN: ${{ secrets.QUAY_OAUTH_TOKEN }} + ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + INVOLUCRO_AUTH: ${{ secrets.INVOLUCRO_AUTH }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Mimic circleci + OSTYPE: "darwin" + CI: "true" + run: | + set -e + eval "$(conda shell.bash hook)" + conda activate bioconda + source common.sh + + bioconda-utils handle-merged-pr recipes config.yml \ + --repo bioconda/bioconda-recipes \ + --git-range ${GITHUB_SHA}~1 ${GITHUB_SHA} \ + --fallback build \ + --artifact-source github-actions From 16fbbf3ca7974e1ab374af56dd452948087cb8e7 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 9 Jul 2024 10:02:52 -0400 Subject: [PATCH 44/53] test run_conda_forge_build_setup --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d864cefc4685..d4d2960e139d4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,6 +58,10 @@ jobs: # a basic unit of work in a run conda activate bioconda source common.sh + if [[ $(uname) = "Darwin" ]]; then + run_conda_forge_build_setup + fi + if command -V docker >/dev/null; then bioconda-utils build recipes config.yml \ --lint --docker --mulled-test \ From 86cb5c7e9073bbc167f617e1310af6d42ba89b65 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 9 Jul 2024 11:01:16 -0400 Subject: [PATCH 45/53] test run_conda_forge_build_setup on GitHub Actions --- .github/workflows/PR-arm.yml | 88 ++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 .github/workflows/PR-arm.yml diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml new file mode 100644 index 0000000000000..883c2e20536a0 --- /dev/null +++ b/.github/workflows/PR-arm.yml @@ -0,0 +1,88 @@ +name: PR (osx-arm64) +on: + pull_request: + types: [opened, synchronize, reopened] +concurrency: + group: build-${{ github.event.pull_request.number || github.head_ref }} + cancel-in-progress: true +jobs: + build_and_test-osx-arm64: + name: Build and Test OSX-ARM64 + runs-on: macOS-14 # M1 + strategy: + fail-fast: true + max-parallel: 13 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # bail if there's no osx-arm64 recipes + - name: Check for Additional Platforms + id: additional_platforms + run: | + result=$(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${GITHUB_JOB}") + if [[ ${result} != "build" ]] + then + echo "No recipes using this platform, skipping rest of job." + echo "skip_build=true" >> $GITHUB_OUTPUT + fi + + - name: set path + run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH + + - name: Fetch conda install script + if: steps.additional_platforms.outputs.skip_build != 'true' + run: | + wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh + + - name: Set up bioconda-utils + if: steps.additional_platforms.outputs.skip_build != 'true' + run: bash install-and-set-up-conda.sh + + - name: Configure conda + if: steps.additional_platforms.outputs.skip_build != 'true' + run: bash configure-conda.sh + + - name: Build and Test + if: steps.additional_platforms.outputs.skip_build != 'true' + env: + # Mimic circleci + OSTYPE: "darwin" + CI: "true" + run: | + set -e + eval "$(conda shell.bash hook)" + conda activate bioconda + source common.sh + + run_conda_forge_build_setup + + if [ -z "$GITHUB_BASE_REF" ] ; then + export GITHUB_BASE_REF="master" + fi + git fetch origin "$GITHUB_BASE_REF" + + bioconda-utils build recipes config.yml \ + --lint --git-range origin/"$GITHUB_BASE_REF" HEAD + + - name: Prepare artifacts + if: steps.additional_platforms.outputs.skip_build != 'true' + run: | + ( + rm -rf /tmp/artifacts + mkdir -p /tmp/artifacts/packages + cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0 + find -name .cache | xargs rm -rf || true + for n in index.html channeldata.json osx-arm64 noarch; do + cp -rv $n /tmp/artifacts/packages || true + done + ) || true + + - name: Archive packages + if: steps.additional_platforms.outputs.skip_build != 'true' + uses: actions/upload-artifact@v4 + with: + name: osx-arm64-packages + path: | + /tmp/artifacts \ No newline at end of file From 88afef6baa1f0b20fd92a77d58bd440abdff35e4 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 9 Jul 2024 11:32:06 -0400 Subject: [PATCH 46/53] debug sdk location --- .circleci/config.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d4d2960e139d4..8279bd51b08d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -59,6 +59,17 @@ jobs: # a basic unit of work in a run source common.sh if [[ $(uname) = "Darwin" ]]; then + if [[ "${OSX_SDK_DIR:-}" == "" ]]; then + if [[ "${CI:-}" == "" ]]; then + OSX_SDK_DIR=$(mktemp -d) + else + OSX_SDK_DIR="$(xcode-select -p)/Platforms/MacOSX.platform/Developer/SDKs" + USING_SYSTEM_SDK_DIR=1 + fi + fi + echo "OSX_SDK_DIR is ${OSX_SDK_DIR}" + + run_conda_forge_build_setup fi From 8ad3623577aa4565f90f3e0d3b203fc46be9083a Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 9 Jul 2024 11:37:26 -0400 Subject: [PATCH 47/53] try permissions --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8279bd51b08d7..84643e5e5b832 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,6 +68,7 @@ jobs: # a basic unit of work in a run fi fi echo "OSX_SDK_DIR is ${OSX_SDK_DIR}" + sudo chmod o+rwx ${OSX_SDK_DIR} run_conda_forge_build_setup From dcd97dc7b91a0e09a9cc26bb067837ce34679db2 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 9 Jul 2024 11:38:45 -0400 Subject: [PATCH 48/53] run_conda_forge_build_setup is working on GHA --- .github/workflows/Bulk.yml | 2 +- .github/workflows/PR-arm.yml | 88 ------------------------------------ 2 files changed, 1 insertion(+), 89 deletions(-) delete mode 100644 .github/workflows/PR-arm.yml diff --git a/.github/workflows/Bulk.yml b/.github/workflows/Bulk.yml index 3121021a10ab9..e89fb18296d63 100644 --- a/.github/workflows/Bulk.yml +++ b/.github/workflows/Bulk.yml @@ -169,7 +169,7 @@ jobs: conda activate bioconda source common.sh - # run_conda_forge_build_setup causes errors on arm + run_conda_forge_build_setup echo '============' conda info --all diff --git a/.github/workflows/PR-arm.yml b/.github/workflows/PR-arm.yml deleted file mode 100644 index 883c2e20536a0..0000000000000 --- a/.github/workflows/PR-arm.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: PR (osx-arm64) -on: - pull_request: - types: [opened, synchronize, reopened] -concurrency: - group: build-${{ github.event.pull_request.number || github.head_ref }} - cancel-in-progress: true -jobs: - build_and_test-osx-arm64: - name: Build and Test OSX-ARM64 - runs-on: macOS-14 # M1 - strategy: - fail-fast: true - max-parallel: 13 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # bail if there's no osx-arm64 recipes - - name: Check for Additional Platforms - id: additional_platforms - run: | - result=$(./scripts/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test" "${GITHUB_JOB}") - if [[ ${result} != "build" ]] - then - echo "No recipes using this platform, skipping rest of job." - echo "skip_build=true" >> $GITHUB_OUTPUT - fi - - - name: set path - run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH - - - name: Fetch conda install script - if: steps.additional_platforms.outputs.skip_build != 'true' - run: | - wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{install-and-set-up-conda,configure-conda,common}.sh - - - name: Set up bioconda-utils - if: steps.additional_platforms.outputs.skip_build != 'true' - run: bash install-and-set-up-conda.sh - - - name: Configure conda - if: steps.additional_platforms.outputs.skip_build != 'true' - run: bash configure-conda.sh - - - name: Build and Test - if: steps.additional_platforms.outputs.skip_build != 'true' - env: - # Mimic circleci - OSTYPE: "darwin" - CI: "true" - run: | - set -e - eval "$(conda shell.bash hook)" - conda activate bioconda - source common.sh - - run_conda_forge_build_setup - - if [ -z "$GITHUB_BASE_REF" ] ; then - export GITHUB_BASE_REF="master" - fi - git fetch origin "$GITHUB_BASE_REF" - - bioconda-utils build recipes config.yml \ - --lint --git-range origin/"$GITHUB_BASE_REF" HEAD - - - name: Prepare artifacts - if: steps.additional_platforms.outputs.skip_build != 'true' - run: | - ( - rm -rf /tmp/artifacts - mkdir -p /tmp/artifacts/packages - cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0 - find -name .cache | xargs rm -rf || true - for n in index.html channeldata.json osx-arm64 noarch; do - cp -rv $n /tmp/artifacts/packages || true - done - ) || true - - - name: Archive packages - if: steps.additional_platforms.outputs.skip_build != 'true' - uses: actions/upload-artifact@v4 - with: - name: osx-arm64-packages - path: | - /tmp/artifacts \ No newline at end of file From b2320e06efb7ff95f5c267fb6ad56861c4e8f8ef Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Tue, 9 Jul 2024 12:15:32 -0400 Subject: [PATCH 49/53] clean up --- .circleci/config.yml | 18 ++++++++++-------- .github/workflows/Bulk.yml | 1 + .github/workflows/PR.yml | 2 ++ .github/workflows/master.yml | 4 +++- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 84643e5e5b832..c2a683fc0b551 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,19 +58,12 @@ jobs: # a basic unit of work in a run conda activate bioconda source common.sh + # Sets up OSX SDK if [[ $(uname) = "Darwin" ]]; then if [[ "${OSX_SDK_DIR:-}" == "" ]]; then - if [[ "${CI:-}" == "" ]]; then - OSX_SDK_DIR=$(mktemp -d) - else OSX_SDK_DIR="$(xcode-select -p)/Platforms/MacOSX.platform/Developer/SDKs" - USING_SYSTEM_SDK_DIR=1 - fi fi - echo "OSX_SDK_DIR is ${OSX_SDK_DIR}" sudo chmod o+rwx ${OSX_SDK_DIR} - - run_conda_forge_build_setup fi @@ -157,6 +150,15 @@ jobs: # a basic unit of work in a run conda activate bioconda source common.sh + # Sets up OSX SDK + if [[ $(uname) = "Darwin" ]]; then + if [[ "${OSX_SDK_DIR:-}" == "" ]]; then + OSX_SDK_DIR="$(xcode-select -p)/Platforms/MacOSX.platform/Developer/SDKs" + fi + sudo chmod o+rwx ${OSX_SDK_DIR} + run_conda_forge_build_setup + fi + bioconda-utils handle-merged-pr recipes config.yml \ --repo bioconda/bioconda-recipes \ --git-range ${CIRCLE_SHA1}~1 ${CIRCLE_SHA1} \ diff --git a/.github/workflows/Bulk.yml b/.github/workflows/Bulk.yml index e89fb18296d63..1d85e277121fe 100644 --- a/.github/workflows/Bulk.yml +++ b/.github/workflows/Bulk.yml @@ -169,6 +169,7 @@ jobs: conda activate bioconda source common.sh + # Sets up OSX SDK run_conda_forge_build_setup echo '============' diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index c7d5d3fce07e6..4a420edd4c603 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -263,6 +263,8 @@ jobs: eval "$(conda shell.bash hook)" conda activate bioconda source common.sh + # Sets up OSX SDK + run_conda_forge_build_setup if [ -z "$GITHUB_BASE_REF" ] ; then export GITHUB_BASE_REF="master" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index dfd3945596555..13af917a60b7d 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -96,7 +96,7 @@ jobs: eval "$(conda shell.bash hook)" conda activate bioconda - # The SDK isn't actually cached, so reinstall it + # Sets up OSX SDK run_conda_forge_build_setup # bioconda-utils handle-merged-pr recipes config.yml \ @@ -162,6 +162,8 @@ jobs: eval "$(conda shell.bash hook)" conda activate bioconda source common.sh + # Sets up OSX SDK + run_conda_forge_build_setup bioconda-utils handle-merged-pr recipes config.yml \ --repo bioconda/bioconda-recipes \ From 0c559ada94e22927fbac82f19ae04ce1bf399708 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Wed, 10 Jul 2024 10:51:04 -0400 Subject: [PATCH 50/53] Try newer xcode version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c2a683fc0b551..0ff031589ff1b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ version: 2.1 executors: osx-arm64: macos: - xcode: 15.0.0 + xcode: 15.4.0 resource_class: macos.m1.medium.gen1 linux-aarch64: machine: From 06eb06723680900960906d237bee0f49e1c135d2 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Wed, 10 Jul 2024 10:51:41 -0400 Subject: [PATCH 51/53] try x flag to see how noisy it is --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ff031589ff1b..286f1479699fd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,7 +53,7 @@ jobs: # a basic unit of work in a run - run: name: Build and test command: | - set -e + set -xe eval "$(conda shell.bash hook)" conda activate bioconda source common.sh From 61866d8e3b2441e9721dd61507aa1920a2d5c559 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Wed, 10 Jul 2024 10:58:14 -0400 Subject: [PATCH 52/53] bump builds --- recipes/bamtools/meta.yaml | 2 +- recipes/bioawk/meta.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/bamtools/meta.yaml b/recipes/bamtools/meta.yaml index af163092eab03..0f5b309f2a25e 100644 --- a/recipes/bamtools/meta.yaml +++ b/recipes/bamtools/meta.yaml @@ -12,7 +12,7 @@ source: # - 0001-Const-qualify-all-functors-operator-member-functions.patch build: - number: 4 + number: 5 run_exports: - {{ pin_subpackage(name, max_pin='x.x') }} diff --git a/recipes/bioawk/meta.yaml b/recipes/bioawk/meta.yaml index ea6a9dee20bf7..70ec45f58e648 100644 --- a/recipes/bioawk/meta.yaml +++ b/recipes/bioawk/meta.yaml @@ -7,7 +7,7 @@ source: sha256: 5cbef3f39b085daba45510ff450afcf943cfdfdd483a546c8a509d3075ff51b5 build: - number: 11 + number: 12 run_exports: - {{ pin_subpackage("bioawk", max_pin="x.x.x") }} From e9156156a70b780978fc1403176e7546b3e57d0b Mon Sep 17 00:00:00 2001 From: "Alicia A. Evans" <108547992+aliciaaevans@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:53:43 -0400 Subject: [PATCH 53/53] Apply suggestions from code review This does add a little clutter from conda activate, but it could be useful for debugging some of the osx-specific stuff. Co-authored-by: Martin Grigorov --- .circleci/config.yml | 2 +- .github/workflows/Bulk.yml | 2 +- .github/workflows/PR.yml | 2 +- .github/workflows/master.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 286f1479699fd..725c78f5e7e8a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -145,7 +145,7 @@ jobs: # a basic unit of work in a run - run: name: Build and push command: | - set -e + set -xe eval "$(conda shell.bash hook)" conda activate bioconda source common.sh diff --git a/.github/workflows/Bulk.yml b/.github/workflows/Bulk.yml index 1d85e277121fe..38e3f8248cf8b 100644 --- a/.github/workflows/Bulk.yml +++ b/.github/workflows/Bulk.yml @@ -164,7 +164,7 @@ jobs: OSTYPE: "darwin" CI: "true" run: | - set -e + set -xe eval "$(conda shell.bash hook)" conda activate bioconda diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml index 4a420edd4c603..82f17b7c4a736 100644 --- a/.github/workflows/PR.yml +++ b/.github/workflows/PR.yml @@ -259,7 +259,7 @@ jobs: OSTYPE: "darwin" CI: "true" run: | - set -e + set -xe eval "$(conda shell.bash hook)" conda activate bioconda source common.sh diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 13af917a60b7d..7e237b314da22 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -158,7 +158,7 @@ jobs: OSTYPE: "darwin" CI: "true" run: | - set -e + set -xe eval "$(conda shell.bash hook)" conda activate bioconda source common.sh