Skip to content

CD - Syft

CD - Syft #220

Workflow file for this run

name: CD - Syft
on:
# schedule:
# - cron: "00 12 * * */7" # At 12:00 UTC on every seven days
workflow_dispatch:
inputs:
skip_tests:
description: "If true, skip tests"
required: false
default: "false"
release_platform:
description: "Release Platform"
required: true
default: "REAL_PYPI"
type: choice
options:
- REAL_PYPI
- TEST_PYPI
- REAL_AND_TEST_PYPI
jobs:
call-pr-tests-linting:
if: github.repository == 'OpenMined/PySyft' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
uses: OpenMined/PySyft/.github/workflows/pr-tests-linting.yml@${GITHUB_HEAD_REF}
secrets: inherit
call-pr-tests-syft:
if: github.repository == 'OpenMined/PySyft' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
uses: OpenMined/PySyft/.github/workflows/pr-tests-syft.yml@${GITHUB_HEAD_REF}
secrets: inherit
call-pr-tests-stack:
if: github.repository == 'OpenMined/PySyft' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
uses: OpenMined/PySyft/.github/workflows/pr-tests-stack.yml@${GITHUB_HEAD_REF}

Check failure on line 37 in .github/workflows/cd-syft.yml

View workflow run for this annotation

GitHub Actions / .github/workflows/cd-syft.yml

Invalid workflow file

error parsing called workflow ".github/workflows/cd-syft.yml" -> "OpenMined/PySyft/.github/workflows/pr-tests-stack.yml@${GITHUB_HEAD_REF}" : failed to fetch workflow: reference to workflow should be either a valid branch, tag, or commit
secrets: inherit
# build-and-push-docker-images:
# needs: [call-pr-tests-linting, call-pr-tests-syft, call-pr-tests-stack]
# if: always() && (needs.call-pr-tests-linting.result == 'success' && needs.call-pr-tests-syft.result == 'success' && needs.call-pr-tests-stack.result == 'success' || github.event.inputs.skip_tests == 'true')
# strategy:
# matrix:
# runner: [sh-arc-linux-x64, sh-arc-linux-arm64]
# runs-on: ${{ matrix.runner }}
# outputs:
# release_tag: ${{ steps.release_metadata.outputs.release_tag }}
# grid_version: ${{ steps.release_metadata.outputs.grid_version }}
# steps:
# - uses: actions/checkout@v4
# # actions/setup-python doesn't yet support ARM
# - name: Setup Python on x64
# if: ${{ !endsWith(matrix.runner, '-arm64') }}
# uses: actions/setup-python@v4
# with:
# python-version: "3.11"
# # Currently psutil package requires gcc to be installed on arm
# # for building psutil from source
# # as linux/aarch64 wheels not avaialble for psutil
# # We could remove once we have aarch64 wheels
# # https://github.com/giampaolo/psutil/issues/1972
# - name: Install Metadata packages for arm64
# if: ${{ endsWith(matrix.runner, '-arm64') }}
# run: |
# sudo apt update -y
# sudo apt install software-properties-common -y
# sudo apt install gcc -y
# - name: Setup Python on arm64
# if: ${{ endsWith(matrix.runner, '-arm64') }}
# uses: deadsnakes/[email protected]
# with:
# python-version: "3.11"
# - name: Check python version
# run: |
# python --version
# python3 --version
# which python
# - name: Install dependencies
# run: |
# python -m pip install --upgrade pip
# pip install --upgrade bump2version tox
# # - name: Bump the Version
# # id: bump-version
# # run: |
# # ls **/VERSION | xargs -I {} python {}
# # cat packages/grid/devspace.yaml | grep '0\.'
# # bump2version prenum --allow-dirty --no-commit
# # ls **/VERSION | xargs -I {} python {}
# # cat packages/grid/devspace.yaml | grep '0\.'
# - name: Generate Release Metadata
# id: release_metadata
# run: |
# if [[ $(python packages/grid/VERSION) == *"beta"* ]]; then
# echo "release_tag=beta" >> $GITHUB_OUTPUT
# else
# echo "release_tag=latest" >> $GITHUB_OUTPUT
# fi
# if [[ ${{matrix.runner}} == *"x64"* ]]; then
# echo "release_platform=linux/amd64" >> $GITHUB_OUTPUT
# echo "short_release_platform=amd64" >> $GITHUB_OUTPUT
# else
# echo "release_platform=linux/arm64" >> $GITHUB_OUTPUT
# echo "short_release_platform=arm64" >> $GITHUB_OUTPUT
# fi
# echo "grid_version=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
# echo "seaweedfs_version=$(grep 'SEAWEEDFS_VERSION' packages/grid/default.env | cut -d '=' -f2)" >> $GITHUB_OUTPUT
# # TODO: Optimize redundant bump protocol version checks
# - name: Check and Bump Protocol Version
# run: |
# if [[ $(python packages/grid/VERSION) != *"beta"* ]]; then
# export BUMP=True
# fi
# tox -e syft.protocol.check
# - name: Set up Docker Buildx
# id: buildx
# uses: docker/setup-buildx-action@v3
# - name: Login to Docker
# uses: docker/login-action@v3
# with:
# username: ${{ secrets.DOCKER_LOGIN }}
# password: ${{ secrets.DOCKER_PASSWORD }}
# - name: Build and push `grid-backend` image to DockerHub
# id: grid-backend-build
# uses: docker/build-push-action@v5
# with:
# context: ./packages
# file: ./packages/grid/backend/backend.dockerfile
# platforms: ${{ steps.release_metadata.outputs.release_platform }}
# target: backend
# outputs: type=image,name=openmined/grid-backend,push-by-digest=true,name-canonical=true,push=true
# cache-from: type=registry,ref=openmined/grid-backend:cache-${{ steps.release_metadata.outputs.short_release_platform }}
# cache-to: type=registry,ref=openmined/grid-backend:cache-${{ steps.release_metadata.outputs.short_release_platform }},mode=max
# # - name: Build and push `syft-base-cpu` image to DockerHub
# # id: syft-base-cpu-build
# # uses: docker/build-push-action@v5
# # with:
# # context: ./packages
# # file: ./packages/grid/backend/worker_cpu.dockerfile
# # platforms: ${{ steps.release_metadata.outputs.release_platform }}
# # target: backend
# # outputs: type=image,name=openmined/syft-base-cpu,push-by-digest=true,name-canonical=true,push=true
# # cache-from: type=registry,ref=openmined/syft-base-cpu:cache-${{ steps.release_metadata.outputs.short_release_platform }}
# # cache-to: type=registry,ref=openmined/syft-base-cpu:cache-${{ steps.release_metadata.outputs.short_release_platform }},mode=max
# - name: Export digest for grid-backend
# run: |
# mkdir -p /tmp/digests/grid-backend
# digest="${{ steps.grid-backend-build.outputs.digest }}"
# touch "/tmp/digests/grid-backend/${digest#sha256:}"
# - name: Build and push `grid-frontend` image to DockerHub
# id: grid-frontend-build
# uses: docker/build-push-action@v5
# with:
# context: ./packages/grid/frontend
# file: ./packages/grid/frontend/frontend.dockerfile
# platforms: ${{ steps.release_metadata.outputs.release_platform }}
# outputs: type=image,name=openmined/grid-frontend,push-by-digest=true,name-canonical=true,push=true
# target: grid-ui-development
# cache-from: type=registry,ref=openmined/grid-frontend:cache-${{ steps.release_metadata.outputs.short_release_platform }}
# cache-to: type=registry,ref=openmined/grid-frontend:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max
# - name: Export digest for grid-frontend
# run: |
# mkdir -p /tmp/digests/grid-frontend
# digest="${{ steps.grid-frontend-build.outputs.digest }}"
# touch "/tmp/digests/grid-frontend/${digest#sha256:}"
# - name: Build and push `grid-seaweedfs` image to DockerHub
# id: grid-seaweedfs-build
# uses: docker/build-push-action@v5
# with:
# context: ./packages/grid/seaweedfs
# file: ./packages/grid/seaweedfs/seaweedfs.dockerfile
# platforms: ${{ steps.release_metadata.outputs.release_platform }}
# build-args: |
# SEAWEEDFS_VERSION=${{ steps.release_metadata.outputs.seaweedfs_version }}
# outputs: type=image,name=openmined/grid-seaweedfs,push-by-digest=true,name-canonical=true,push=true
# cache-from: type=registry,ref=openmined/grid-seaweedfs:cache-${{ steps.release_metadata.outputs.short_release_platform }}
# cache-to: type=registry,ref=openmined/grid-seaweedfs:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max
# - name: Export digest for grid-seaweedfs
# run: |
# mkdir -p /tmp/digests/grid-seaweedfs
# digest="${{ steps.grid-seaweedfs-build.outputs.digest }}"
# touch "/tmp/digests/grid-seaweedfs/${digest#sha256:}"
# - name: Upload digest for grid-backend, grid-frontend and grid-seaweedfs
# uses: actions/upload-artifact@v3
# with:
# name: digests-${{ steps.release_metadata.outputs.grid_version }}
# path: /tmp/digests/*
# if-no-files-found: error
# retention-days: 1
# #Used to merge x64 and arm64 into one docker image
# merge-docker-images:
# needs: [build-and-push-docker-images]
# if: always() && (needs.build-and-push-docker-images.result == 'success')
# runs-on: sh-arc-linux-x64
# steps:
# - name: Download digests
# uses: actions/download-artifact@v3
# with:
# name: digests-${{ needs.build-and-push-docker-images.outputs.grid_version }}
# path: /tmp/digests
# - name: Set up Docker Buildx
# id: buildx
# uses: docker/setup-buildx-action@v3
# - name: Login to Docker
# uses: docker/login-action@v3
# with:
# username: ${{ secrets.DOCKER_LOGIN }}
# password: ${{ secrets.DOCKER_PASSWORD }}
# - name: Create manifest list and push for grid-backend
# working-directory: /tmp/digests/grid-backend
# run: |
# docker buildx imagetools create \
# -t openmined/grid-backend:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
# -t openmined/grid-backend:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
# $(printf 'openmined/grid-backend@sha256:%s ' *)
# - name: Create manifest list and push for grid-frontend
# working-directory: /tmp/digests/grid-frontend
# run: |
# docker buildx imagetools create \
# -t openmined/grid-frontend:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
# -t openmined/grid-frontend:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
# $(printf 'openmined/grid-frontend@sha256:%s ' *)
# - name: Create manifest list and push for grid-seaweedfs
# working-directory: /tmp/digests/grid-seaweedfs
# run: |
# docker buildx imagetools create \
# -t openmined/grid-seaweedfs:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
# -t openmined/grid-seaweedfs:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
# $(printf 'openmined/grid-seaweedfs@sha256:%s ' *)
# deploy-syft:
# needs: [merge-docker-images]
# if: always() && needs.merge-docker-images.result == 'success'
# runs-on: ubuntu-latest
# permissions:
# contents: write # For tag and release notes.
# steps:
# - name: Permission to home directory
# run: |
# sudo chown -R $USER:$USER $HOME
# - uses: actions/checkout@v4
# with:
# token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}
# # free 10GB of space
# - name: Remove unnecessary files
# run: |
# sudo rm -rf /usr/share/dotnet
# sudo rm -rf "$AGENT_TOOLSDIRECTORY"
# docker image prune --all --force
# docker builder prune --all --force
# docker system prune --all --force
# - name: Set up Python
# uses: actions/setup-python@v4
# with:
# python-version: "3.11"
# - name: Install dependencies
# run: |
# python -m pip install --upgrade pip
# pip install --upgrade tox setuptools wheel twine bump2version PyYAML
# # - name: Bump the Version
# # id: bump-version
# # run: |
# # ls **/VERSION | xargs -I {} python {}
# # cat packages/grid/devspace.yaml | grep '0\.'
# # bump2version prenum --allow-dirty --no-commit
# # tox -e lint || true
# # ls **/VERSION | xargs -I {} python {}
# # cat packages/grid/devspace.yaml | grep '0\.'
# # python packages/hagrid/scripts/update_manifest.py $(python packages/grid/VERSION)
# - name: Check Protocol Version
# run: |
# tox -e syft.protocol.check
# - name: Build Helm Chart
# shell: bash
# run: |
# # install k3d
# K3D_VERSION=v5.6.0
# wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
# mv k3d-linux-amd64 k3d
# chmod +x k3d
# export PATH=`pwd`:$PATH
# k3d version
# #Install Devspace
# DEVSPACE_VERSION=v6.3.3
# curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
# chmod +x devspace
# devspace version
# # Install helm
# curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
# helm version
# tox -e syft.build.helm
# tox -e syft.package.helm
# # Check if the version is a pre-release
# - name: Check if the version is a pre-release and modify version string
# id: release_checks
# run: |
# if [[ $(python packages/grid/VERSION) == *"beta"* ]]; then
# echo "is_pre_release=true" >> $GITHUB_OUTPUT
# echo "github_release_version=$(python packages/grid/VERSION | sed 's/-beta./b/')" >> $GITHUB_OUTPUT
# else
# echo "is_pre_release=false" >> $GITHUB_OUTPUT
# echo "github_release_version=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
# fi
# echo "future_stable_version=$(python packages/grid/VERSION | sed 's/-beta.*//')" >> $GITHUB_OUTPUT
# - name: Check and Bump Protocol Version
# run: |
# if [[ "${{ steps.release_checks.outputs.is_pre_release }}" == "false" ]]; then
# export BUMP=True
# fi
# tox -e syft.protocol.check
# - name: Update PyPI Readme
# run: |
# python scripts/convert_to_pypi_readme.py --input-file packages/syft/README.md --output-file packages/syft/PYPI.md --version ${{ steps.release_checks.outputs.future_stable_version }}
# # - name: Commit changes to Syft
# # uses: EndBug/add-and-commit@v9
# # with:
# # author_name: ${{ secrets.OM_BOT_NAME }}
# # author_email: ${{ secrets.OM_BOT_EMAIL }}
# # message: "[syft]bump version"
# # add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION','packages/syft/PYPI.md', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' ,'packages/grid/podman/podman-kube/podman-syft-kube-config.yaml', 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json', 'packages/grid/backend/worker_cpu.dockerfile']"
# - name: Scheduled Build and Publish
# if: github.event_name == 'schedule'
# run: |
# tox -e syft.publish
# twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
# - name: Manual Build and Publish
# if: github.event_name != 'schedule'
# run: |
# tox -e syft.publish
# if [[ "${{ github.event.inputs.release_platform }}" == "REAL_PYPI" ]]; then
# twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
# fi
# if [[ "${{ github.event.inputs.release_platform }}" == "TEST_PYPI" ]]; then
# twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/*
# fi
# if [[ "${{ github.event.inputs.release_platform }}" == "REAL_AND_TEST_PYPI" ]]; then
# twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
# twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/*
# fi
# # Checkout Infra repo (nested)
# - name: Checkout Infra Repo
# uses: actions/checkout@v4
# with:
# repository: ${{ secrets.INFRA_REPO }}
# ref: "main"
# token: ${{ secrets.INFRA_BOT_COMMIT_TOKEN }}
# path: infrastructure
# # This step will copy the generated K8s manifest files to the correct directory in Infra repo
# - name: Copy files to Infra Repo
# run: |
# if $is_pre_release; then
# rm -rf infrastructure/gitops/environments/pre-release/.
# cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/pre_release/
# else
# rm -rf infrastructure/gitops/environments/stable/.
# cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/stable/
# fi
# - name: Commit changes to Infra Repo
# uses: EndBug/add-and-commit@v9
# with:
# author_name: ${{ secrets.OM_BOT_NAME }}
# author_email: ${{ secrets.OM_BOT_EMAIL }}
# message: "Update K8s Manifests from Syft Repo"
# add: "."
# push: "origin main"
# cwd: "./infrastructure/"
# - name: Create SyftCLI Config assets
# run: |
# pip install pyyaml
# python scripts/create_syftcli_config.py
# - name: GitHub Release
# if: github.event.inputs.release_platform != 'TEST_PYPI'
# uses: softprops/action-gh-release@v1
# with:
# name: v${{ steps.release_checks.outputs.github_release_version }}
# generate_release_notes: true
# prerelease: ${{ steps.release_checks.outputs.is_pre_release }}
# files: |
# ./packages/syftcli/manifest.yml
# ./build/syftcli-config/*
# ./packages/hagrid/hagrid/manifest_template.yml
# tag_name: v${{ steps.release_checks.outputs.github_release_version }}
# # Checkout to gh-pages and update helm repo
# - name: Checkout to gh-pages
# uses: actions/checkout@v4
# with:
# ref: gh-pages
# token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}
# path: ghpages
# - name: Copy helm repo files from Syft Repo
# run: |
# rm -rf ghpages/helm/*
# cp -R packages/grid/helm/repo/. ghpages/helm/
# - name: Commit changes to gh-pages
# uses: EndBug/add-and-commit@v9
# with:
# author_name: ${{ secrets.OM_BOT_NAME }}
# author_email: ${{ secrets.OM_BOT_EMAIL }}
# message: "Update Helm package from Syft Repo"
# add: "helm/"
# push: "origin gh-pages"
# cwd: "./ghpages/"