Merge pull request #7795 from OpenMined/dev

Update 0.8.1
OpenMined · Jun 21, 2023 · 511cb40 · 511cb40
2 parents e715bde + b91deab
commit 511cb40
Show file tree

Hide file tree

Showing 130 changed files with 7,302 additions and 1,926 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.8.1-beta.7
+current_version = 0.8.1-beta.12
 tag = False
 tag_name = {new_version}
 commit = True
@@ -40,4 +40,6 @@ first_value = 1
 
 [bumpversion:file:packages/grid/frontend/package.json]
 
+[bumpversion:file:packages/grid/helm/syft/Chart.yaml]
+
 [bumpversion:file:packages/hagrid/hagrid/manifest_template.yml]
diff --git a/.github/workflows/cd-hagrid.yml b/.github/workflows/cd-hagrid.yml
@@ -2,7 +2,7 @@ name: CD - HAGrid
 
 on:
   schedule:
-    - cron: "00 15 * * */2" # At 15:00 on every 2nd day-of-week.
+    - cron: "00 15 * * */1" # At 15:00 UTC on every day
   workflow_dispatch:
     inputs:
       none:

diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml
@@ -2,7 +2,7 @@ name: CD - Syft
 
 on:
   schedule:
-    - cron: "00 12 * * */2" # At 12:00 on every 2nd day-of-week.
+    - cron: "00 12 * * */1" # At 12:00 UTC on every day
   workflow_dispatch:
     inputs:
       none:
@@ -56,7 +56,7 @@ jobs:
           author_email: [email protected]
           message: "bump version"
           add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml']"
-          pull_strategy: NO-PULL
+          pull: NO-PULL
 
       - name: Build and publish
         env:

diff --git a/.github/workflows/nightlies.yml b/.github/workflows/nightlies.yml
@@ -37,6 +37,6 @@ jobs:
     uses: OpenMined/PySyft/.github/workflows/pr-tests-stack-public.yml@dev
     secrets: inherit
 
-  call-vm-tests:
-    if: github.repository == 'OpenMined/PySyft' # don't run on forks
-    uses: OpenMined/PySyft/.github/workflows/vm-tests.yml@dev
+  # call-vm-tests:
+  #   if: github.repository == 'OpenMined/PySyft' # don't run on forks
+  #   uses: OpenMined/PySyft/.github/workflows/vm-tests.yml@dev
diff --git a/.github/workflows/pr-tests-stack-public.yml b/.github/workflows/pr-tests-stack-public.yml
@@ -157,7 +157,6 @@ jobs:
         with:
           timeout_seconds: 1800
           max_attempts: 3
-          retry_on: timeout
           command: tox -e stack.test.integration
 
       - name: Reboot node

diff --git a/.github/workflows/pr-tests-stack.yml b/.github/workflows/pr-tests-stack.yml
@@ -219,3 +219,97 @@ jobs:
           mention_users_when: "failure,warnings"
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK_WEBHOOK_URL }}
+
+  pr-tests-stack-k8s:
+    strategy:
+      max-parallel: 99
+      matrix:
+        # os: [ubuntu-latest, macos-latest, windows-latest, windows]
+        os: [ubuntu-latest]
+        python-version: ["3.11"]
+        pytest-modules: ["frontend network"]
+      fail-fast: false
+
+    runs-on: ${{matrix.os}}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Check for file changes
+        uses: dorny/paths-filter@v2
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        if: steps.changes.outputs.stack == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Get pip cache dir
+        if: steps.changes.outputs.stack == 'true'
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+
+      - name: pip cache
+        uses: actions/cache@v3
+        if: steps.changes.outputs.stack == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ matrix.python-version }}
+
+      - name: Upgrade pip
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          python -m pip install --upgrade --user pip
+
+      - name: Install tox
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          pip install -U tox
+
+      - name: Install kubectl
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          # cleanup apt version
+          sudo apt remove kubectl || true
+          # install kubectl 1.27
+          curl -LO https://dl.k8s.io/release/v1.27.2/bin/linux/amd64/kubectl
+          chmod +x kubectl
+          sudo install kubectl /usr/local/bin;
+
+      - name: Install k9s
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          # install k9s
+          wget https://github.com/derailed/k9s/releases/download/v0.27.4/k9s_Linux_amd64.tar.gz
+          tar -xvf k9s_Linux_amd64.tar.gz
+          chmod +x k9s
+          sudo install k9s /usr/local/bin;
+
+      - name: Run integration tests
+        if: steps.changes.outputs.stack == 'true'
+        timeout-minutes: 60
+        env:
+          HAGRID_ART: false
+          PYTEST_MODULES: "${{ matrix.pytest-modules }}"
+        shell: bash
+        run: |
+          # install k3d
+          wget https://github.com/k3d-io/k3d/releases/download/v5.5.1/k3d-linux-amd64
+          mv k3d-linux-amd64 k3d
+          chmod +x k3d
+          export PATH=`pwd`:$PATH
+          k3d version
+          DEVSPACE_VERSION=v6.3.2
+          curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
+          chmod +x devspace
+          devspace version
+          tox -e syft.build.helm
+          # tox -e syft.test.helm
diff --git a/.github/workflows/pr-tests-syft.yml b/.github/workflows/pr-tests-syft.yml
@@ -160,12 +160,16 @@ jobs:
           python -c "import platform; import os; os.system('pip install jaxlib==0.4.10 -f https://whls.blob.core.windows.net/unstable/index.html') if platform.system().lower() == 'windows' else ''"
 
       - name: Run notebook tests
+        uses: nick-fields/retry@v2
         if: steps.changes.outputs.syft == 'true'
         env:
           ORCHESTRA_NODE_TYPE: "${{ matrix.node-type }}"
           TEST_NOTEBOOK_PATHS: "${{ matrix.notebook-paths }}"
-        run: |
-          tox -e syft.test.notebook
+        with:
+          timeout_seconds: 1800
+          max_attempts: 3
+          retry_on: timeout
+          command: tox -e syft.test.notebook
 
   pr-tests-syft-notebook-container:
     strategy:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -19,7 +19,7 @@ repos:
         exclude: ^(packages/grid/backend/wheels/.*|docs/img/header.png|docs/img/terminalizer.gif)
       - id: check-yaml
         always_run: true
-        exclude: ^(packages/grid/k8s/rendered/)
+        exclude: ^(packages/grid/k8s/rendered/|packages/grid/helm/)
       - id: check-merge-conflict
         always_run: true
         args: ["--assume-in-merge"]
@@ -176,6 +176,7 @@ repos:
     rev: "v3.0.0-alpha.9-for-vscode"
     hooks:
       - id: prettier
+        exclude: ^(packages/grid/helm)
 
   # - repo: meta
   #   hooks:

diff --git a/README.md b/README.md
@@ -42,6 +42,20 @@ sy.requires(">=0.8,<0.8.1")
 domain_client = sy.login(port=8080, email="[email protected]", password="changethis")
 ```
 
+## Deploy Kubernetes Helm Chart
+
+```
+$ kubectl create namespace syft
+$ helm install my-domain syft --namespace syft --version 0.8.1-beta.12 --repo https://openmined.github.io/PySyft/helm
+```
+
+# Azure or GKE Ingress
+
+```
+$ helm install ... --set ingress.ingressClass="azure/application-gateway"
+$ helm install ... --set ingress.ingressClass="gce"
+```
+
 ## Deploy to a Container Engine or Cloud
 
 1. Install our handy 🛵 cli tool which makes deploying a Domain or Gateway server a one-liner:  
@@ -55,7 +69,7 @@ domain_client = sy.login(port=8080, email="[email protected]", password="change
 
    `PyGrid` = our 🐳 `docker` / 🐧 `vm` `Domain` & `Gateway` Servers where `private data` lives
 
-4. During quickstart we will deploy `PyGrid` to localhost with 🐳 `docker`, however 🛵 HAGrid can deploy to `podman` or a 🐧 `ubuntu` VM on `azure` / `gcp` / `ANY_IP_ADDRESS` by using 🔨 `ansible`†
+4. During quickstart we will deploy `PyGrid` to localhost with 🐳 `docker`, however 🛵 HAGrid can deploy to a 🐧 `ubuntu` VM on `azure` / `gcp` / `ANY_IP_ADDRESS` by using 🔨 `ansible`†
 
 ## Docs and Support
 
@@ -71,7 +85,7 @@ domain_client = sy.login(port=8080, email="[email protected]", password="change
 - PySyft 0.8 Requires: 🐍 `python 3.9 - 3.11` - Run: `pip install -U syft`  
   \*`macOS` Apple Silicon users might need cmake: `brew install cmake`  
   ‡`Windows` users must run this first: `pip install jaxlib==0.4.10 -f https://whls.blob.core.windows.net/unstable/index.html`
-- PyGrid Requires: 🐳 `docker` or 🐧 `ubuntu` VM - Run: `hagrid launch ...`
+- PyGrid Requires: 🐳 `docker`, ☸️ `kubernetes` or 🐧 `ubuntu` VM - Run: `hagrid launch ...`
 
 # Versions
 

diff --git a/VERSION b/VERSION
@@ -1,5 +1,5 @@
 # Mono Repo Global Version
-__version__ = "0.8.1-beta.7"
+__version__ = "0.8.1-beta.12"
 # elsewhere we can call this file: `python VERSION` and simply take the stdout
 
 # stdlib

diff --git a/notebooks/api/0.8/00-load-data.ipynb b/notebooks/api/0.8/00-load-data.ipynb
@@ -337,15 +337,13 @@
    },
    "outputs": [],
    "source": [
-    "dataset.add_contributor(role=sy.roles.UPLOADER, \n",
-    "                                name=\"Andrew Trask\", \n",
-    "                                email=\"[email protected]\",\n",
-    "                                note=\"Andrew runs this domain and prepared the dataset metadata.\")\n",
+    "dataset.add_contributor(name=\"Andrew Trask\", \n",
+    "                        email=\"[email protected]\",\n",
+    "                        note=\"Andrew runs this domain and prepared the dataset metadata.\")\n",
     "\n",
-    "dataset.add_contributor(role=sy.roles.EDITOR, \n",
-    "                                name=\"Madhava Jay\", \n",
-    "                                email=\"[email protected]\",\n",
-    "                                note=\"Madhava tweaked the description to add the URL because Andrew forgot.\")"
+    "dataset.add_contributor(name=\"Madhava Jay\", \n",
+    "                        email=\"[email protected]\",\n",
+    "                        note=\"Madhava tweaked the description to add the URL because Andrew forgot.\")"
    ]
   },
   {
@@ -390,10 +388,9 @@
    },
    "outputs": [],
    "source": [
-    "ctf.add_contributor(role=sy.roles.UPLOADER, \n",
-    "                      name=\"Andrew Trask\", \n",
-    "                      email=\"[email protected]\",\n",
-    "                      note=\"Andrew runs this domain and prepared the asset.\")"
+    "ctf.add_contributor(name=\"Andrew Trask\", \n",
+    "                    email=\"[email protected]\",\n",
+    "                    note=\"Andrew runs this domain and prepared the asset.\")"
    ]
   },
   {
@@ -545,7 +542,7 @@
    },
    "outputs": [],
    "source": [
-    "assert mock_ca_data.equals(mock.syft_action_data)"
+    "assert mock_ca_data.equals(mock)"
    ]
   },
   {
@@ -570,6 +567,19 @@
     "assert ca_data.equals(real.syft_action_data)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Create account for guest user\n",
+    "### Signup is disabled by default\n",
+    "### An Admin/DO can enable it by `domain_client.settings.allow_guest_signup(enable=True)`\n",
+    "\n",
+    "domain_client.register(name=\"Jane Doe\", email=\"[email protected]\", password=\"abc123\", institution=\"Caltech\", website=\"https://www.caltech.edu/\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -581,6 +591,13 @@
     "if node.node_type.value == \"python\":\n",
     "    node.land()"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -599,7 +616,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.3"
+   "version": "3.9.16"
   },
   "toc": {
    "base_numbering": 1,