Skip to content

Commit

Permalink
merge
Browse files Browse the repository at this point in the history
  • Loading branch information
bitsofsteve committed Mar 18, 2024
2 parents c51e144 + 2933921 commit 5ad3368
Show file tree
Hide file tree
Showing 378 changed files with 13,453 additions and 6,820 deletions.
2 changes: 1 addition & 1 deletion .bumpversion.cfg
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[bumpversion]
current_version = 0.8.5-beta.1
current_version = 0.8.5-beta.9
tag = False
tag_name = {new_version}
commit = True
Expand Down
5 changes: 3 additions & 2 deletions .github/workflows/cd-docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
cd-docs:
strategy:
matrix:
python-version: ["3.11"]
python-version: ["3.12"]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
Expand All @@ -27,7 +27,8 @@ jobs:

- name: Install tox
run: |
pip install -U tox
pip install --upgrade pip uv==0.1.18 tox tox-uv==1.5.1
uv --version
- name: Build the docs
run: |
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/cd-hagrid.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
python-version: "3.12"

- name: Install dependencies
if: ${{needs.hagrid-deploy.outputs.current_hash}} != ${{needs.hagrid-deploy.outputs.previous_hash}}
Expand Down
File renamed without changes.
13 changes: 12 additions & 1 deletion .github/workflows/cd-syft-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
python-version: "3.12"

- name: Set up Docker Buildx
id: buildx
Expand Down Expand Up @@ -121,6 +121,17 @@ jobs:
${{ secrets.ACR_SERVER }}/openmined/grid-seaweedfs:dev-${{ github.sha }}
${{ secrets.ACR_SERVER }}/openmined/grid-seaweedfs:${{ steps.grid.outputs.GRID_VERSION }}
- name: Build and push `grid-veilid` image to registry
uses: docker/build-push-action@v5
with:
context: ./packages/grid/veilid
file: ./packages/grid/veilid/veilid.dockerfile
push: true
tags: |
${{ secrets.ACR_SERVER }}/openmined/grid-veilid:dev
${{ secrets.ACR_SERVER }}/openmined/grid-veilid:dev-${{ github.sha }}
${{ secrets.ACR_SERVER }}/openmined/grid-veilid:${{ steps.grid.outputs.GRID_VERSION }}
- name: Build Helm Chart & Copy to infra
if: github.ref == 'refs/heads/dev' || github.event.inputs.deploy-helm == 'true'
shell: bash
Expand Down
50 changes: 40 additions & 10 deletions .github/workflows/cd-syft.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
python-version: "3.12"

# The steps ensure that the cron job is able to run only for
# for beta releases and not for stable releases
Expand Down Expand Up @@ -100,7 +100,7 @@ jobs:
if: ${{ !endsWith(matrix.runner, '-arm64') }}
uses: actions/setup-python@v5
with:
python-version: "3.11"
python-version: "3.12"

# Currently psutil package requires gcc to be installed on arm
# for building psutil from source
Expand All @@ -118,7 +118,12 @@ jobs:
if: ${{ endsWith(matrix.runner, '-arm64') }}
uses: deadsnakes/[email protected]
with:
python-version: "3.11"
python-version: "3.12"

- name: Install Git
run: |
sudo apt-get update
sudo apt-get install git -y
- name: Check python version
run: |
Expand All @@ -128,8 +133,8 @@ jobs:
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install --upgrade bump2version tox
pip install --upgrade pip uv==0.1.18 bump2version tox tox-uv==1.5.1
uv --version
- name: Get Release tag
id: get_release_tag
Expand Down Expand Up @@ -246,7 +251,24 @@ jobs:
digest="${{ steps.grid-seaweedfs-build.outputs.digest }}"
touch "/tmp/digests/grid-seaweedfs/${digest#sha256:}"
- name: Upload digest for grid-backend, grid-frontend and grid-seaweedfs
- name: Build and push `grid-veilid` image to DockerHub
id: grid-veilid-build
uses: docker/build-push-action@v5
with:
context: ./packages/grid/veilid
file: ./packages/grid/veilid/veilid.dockerfile
platforms: ${{ steps.release_metadata.outputs.release_platform }}
outputs: type=image,name=openmined/grid-veilid,push-by-digest=true,name-canonical=true,push=true
cache-from: type=registry,ref=openmined/grid-veilid:cache-${{ steps.release_metadata.outputs.short_release_platform }}
cache-to: type=registry,ref=openmined/grid-veilid:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max

- name: Export digest for grid-veilid
run: |
mkdir -p /tmp/digests/grid-veilid
digest="${{ steps.grid-veilid-build.outputs.digest }}"
touch "/tmp/digests/grid-veilid/${digest#sha256:}"
- name: Upload digest for grid-backend, grid-frontend and grid-seaweedfs, grid-veilid
uses: actions/upload-artifact@v4
with:
name: digests-${{ steps.release_metadata.outputs.grid_version }}-${{ steps.release_metadata.outputs.short_release_platform }}
Expand Down Expand Up @@ -305,6 +327,14 @@ jobs:
-t openmined/grid-seaweedfs:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
$(printf 'openmined/grid-seaweedfs@sha256:%s ' *)
- name: Create manifest list and push for grid-veilid
working-directory: /tmp/digests/grid-veilid
run: |
docker buildx imagetools create \
-t openmined/grid-veilid:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
-t openmined/grid-veilid:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
$(printf 'openmined/grid-veilid@sha256:%s ' *)
deploy-syft:
needs: [merge-docker-images]
if: always() && needs.merge-docker-images.result == 'success'
Expand Down Expand Up @@ -337,11 +367,11 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
python-version: "3.12"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install --upgrade tox setuptools wheel twine bump2version PyYAML
pip install --upgrade pip uv==0.1.18 tox tox-uv==1.5.1 setuptools wheel twine bump2version PyYAML
uv --version
- name: Bump the Version
if: needs.merge-docker-images.outputs.release_tag == 'beta'
Expand Down Expand Up @@ -486,7 +516,7 @@ jobs:
- name: GitHub Release
if: github.event.inputs.release_platform != 'TEST_PYPI'
uses: softprops/action-gh-release@v1
uses: softprops/action-gh-release@v2
with:
name: v${{ steps.release_checks.outputs.github_release_version }}
generate_release_notes: true
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/cd-syftcli.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
python-version: "3.12"

- name: Install dependencies
if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }}
Expand Down Expand Up @@ -117,7 +117,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
python-version: "3.12"

- name: Install build dependencies for syftcli
run: |
Expand Down Expand Up @@ -181,7 +181,7 @@ jobs:

- name: GitHub Release
if: steps.check-pypi.outputs.released == 'true'
uses: softprops/action-gh-release@v1
uses: softprops/action-gh-release@v2
with:
name: syftcli-v${{ needs.deploy-syft-cli.outputs.deployed_version }}
generate_release_notes: false
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/container-scan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
python-version: "3.12"

#Generate SBOM
- name: Generate SBOM
Expand Down
90 changes: 90 additions & 0 deletions .github/workflows/e2e-tests-notebook.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
name: E2E - Notebook Tests

on:
workflow_dispatch:
inputs:
syft_version:
description: "Syft version to test"
required: true
type: string
node_url:
description: "Node URL to use"
required: true
type: string
node_port:
description: "Node port"
required: true
type: number
exclude_notebooks:
description: "Notebooks to exclude ex: not 11-container-images-k8s.ipynb"
required: false
type: string

workflow_call:
inputs:
syft_version:
description: "Syft version to test"
required: true
type: string
node_url:
description: "Node URL to use"
required: true
type: string
node_port:
description: "Node port"
required: true
type: number
exclude_notebooks:
description: "Notebooks to exclude ex: not 11-container-images-k8s.ipynb"
required: false
type: string

jobs:
notebook-test-hagrid:
strategy:
max-parallel: 99
matrix:
os: [ubuntu-latest]
python-version: ["3.12"]

runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4

- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5

with:
python-version: ${{ matrix.python-version }}

- name: Upgrade pip
run: |
python -m pip install --upgrade --user pip
- name: Get pip cache dir
id: pip-cache
shell: bash
run: |
echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
- name: pip cache
uses: actions/cache@v4
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
restore-keys: |
${{ runner.os }}-pip-py${{ matrix.python-version }}-
- name: Install tox
run: |
pip install tox
- name: Run Notebook tests
env:
SYFT_VERSION: ${{ inputs.syft_version }}
NODE_URL: ${{ inputs.node_url }}
NODE_PORT: ${{ inputs.node_port }}
EXCLUDE_NOTEBOOKS: ${{ inputs.exclude_notebooks }}
run: |
tox -e e2e.test.notebook
2 changes: 1 addition & 1 deletion .github/workflows/manual-delete-buildjet-cache.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
strategy:
matrix:
os: [ubuntu-latest]
python-version: ["3.9", "3.10", "3.11"]
python-version: ["3.10", "3.11", "3.12"]

runs-on: ${{ matrix.os }}
steps:
Expand Down
35 changes: 18 additions & 17 deletions .github/workflows/pr-tests-enclave.yml
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
name: PR Tests - Enclave

on:
workflow_call:
# Temporarily disabled oblv tests
# workflow_call:

pull_request:
branches:
- dev
- main
- "0.8"
# pull_request:
# branches:
# - dev
# - main
# - "0.8"

workflow_dispatch:
inputs:
Expand All @@ -25,7 +26,7 @@ jobs:
max-parallel: 4
matrix:
os: [ubuntu-latest]
python-version: ["3.11"]
python-version: ["3.12"]

runs-on: ${{ matrix.os }}
steps:
Expand Down Expand Up @@ -58,31 +59,31 @@ jobs:
- name: Upgrade pip
if: steps.changes.outputs.syft == 'true'
run: |
python -m pip install --upgrade --user pip
pip install --upgrade pip uv==0.1.18
uv --version
- name: Get pip cache dir
id: pip-cache
if: steps.changes.outputs.syft == 'true'
shell: bash
run: |
echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT
- name: pip cache
uses: actions/cache@v4
if: steps.changes.outputs.syft == 'true'
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
restore-keys: |
${{ runner.os }}-pip-py${{ matrix.python-version }}-
${{ runner.os }}-uv-py${{ matrix.python-version }}-
- name: Install Dependencies
if: steps.changes.outputs.syft == 'true'
run: |
pip install --upgrade tox packaging wheel --default-timeout=60
pip install --upgrade tox tox-uv==1.5.1
# Temporarily disabled oblv tests
# - name: Run Enclave tests
# if: steps.changes.outputs.syft == 'true'
# run: |
# tox -e stack.test.integration.enclave.oblv
- name: Run Enclave tests
if: steps.changes.outputs.syft == 'true'
run: |
tox -e stack.test.integration.enclave.oblv
Loading

0 comments on commit 5ad3368

Please sign in to comment.