diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e05b5405d9c..cd48a82318d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.8.7-beta.10 +current_version = 0.8.7-beta.13 tag = False tag_name = {new_version} commit = True diff --git a/.github/workflows/cd-docs.yml b/.github/workflows/cd-docs.yml index ff042e74017..0e741baacc6 100644 --- a/.github/workflows/cd-docs.yml +++ b/.github/workflows/cd-docs.yml @@ -27,7 +27,7 @@ jobs: - name: Install tox run: | - pip install --upgrade pip uv==0.1.35 tox tox-uv==1.5.1 + pip install --upgrade pip==24.0 uv==0.2.13 tox tox-uv==1.9.0 uv --version - name: Build the docs diff --git a/.github/workflows/cd-post-release-tests.yml b/.github/workflows/cd-post-release-tests.yml index 370469ea0bb..adaafb47f9a 100644 --- a/.github/workflows/cd-post-release-tests.yml +++ b/.github/workflows/cd-post-release-tests.yml @@ -132,7 +132,7 @@ jobs: - name: Install tox run: | - pip install --upgrade pip uv==0.1.35 tox tox-uv==1.5.1 + pip install --upgrade pip==24.0 uv==0.2.13 tox tox-uv==1.9.0 - name: Run K8s tests env: @@ -193,7 +193,7 @@ jobs: - name: Install tox and uv run: | - pip install --upgrade pip uv==0.1.35 tox tox-uv==1.5.1 tox-current-env + pip install --upgrade pip==24.0 uv==0.2.13 tox tox-uv==1.9.0 tox-current-env - name: Run unit tests run: | diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index 0231f97e172..00dd4acb816 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -84,7 +84,7 @@ jobs: echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT - name: Build and push `syft` image to registry - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages file: ./packages/grid/syft-client/syft.Dockerfile @@ -95,7 +95,7 @@ jobs: ${{ secrets.ACR_SERVER }}/openmined/syft-client:${{ steps.grid.outputs.GRID_VERSION }} - name: Build and push `grid-backend` image to registry - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages file: ./packages/grid/backend/backend.dockerfile @@ -107,7 +107,7 @@ jobs: ${{ secrets.ACR_SERVER }}/openmined/grid-backend:${{ steps.grid.outputs.GRID_VERSION }} - name: Build and push `grid-frontend` image to registry - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/frontend file: ./packages/grid/frontend/frontend.dockerfile @@ -119,7 +119,7 @@ jobs: target: grid-ui-development - name: Build and push `grid-seaweedfs` image to registry - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/seaweedfs file: ./packages/grid/seaweedfs/seaweedfs.dockerfile @@ -130,7 +130,7 @@ jobs: ${{ secrets.ACR_SERVER }}/openmined/grid-seaweedfs:${{ steps.grid.outputs.GRID_VERSION }} - name: Build and push `grid-enclave-attestation` image to registry - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/enclave/attestation file: ./packages/grid/enclave/attestation/attestation.dockerfile @@ -164,7 +164,7 @@ jobs: helm version # install tox - python -m pip install --upgrade pip + python -m pip install --upgrade pip==24.0 pip install tox tox -e syft.build.helm diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index 486196ecbdb..fc0818a0814 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -112,7 +112,8 @@ jobs: run: | sudo apt update -y sudo apt install software-properties-common -y - sudo apt install gcc -y + sudo apt install gcc curl -y + sudo apt-get install python3-dev -y - name: Setup Python on arm64 if: ${{ endsWith(matrix.runner, '-arm64') }} @@ -133,7 +134,7 @@ jobs: - name: Install dependencies run: | - pip install --upgrade pip uv==0.1.35 bump2version tox tox-uv==1.5.1 + pip install --upgrade pip==24.0 uv==0.2.13 bump2version tox tox-uv==1.9.0 uv --version - name: Get Release tag @@ -185,7 +186,7 @@ jobs: - name: Build and push `grid-backend` image to DockerHub id: grid-backend-build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages file: ./packages/grid/backend/backend.dockerfile @@ -203,7 +204,7 @@ jobs: - name: Build and push `grid-frontend` image to DockerHub id: grid-frontend-build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/frontend file: ./packages/grid/frontend/frontend.dockerfile @@ -221,7 +222,7 @@ jobs: - name: Build and push `grid-seaweedfs` image to DockerHub id: grid-seaweedfs-build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/seaweedfs file: ./packages/grid/seaweedfs/seaweedfs.dockerfile @@ -241,7 +242,7 @@ jobs: - name: Build and push `grid-enclave-attestation` image to DockerHub if: ${{ endsWith(matrix.runner, '-x64') }} id: grid-enclave-attestation-build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/enclave/attestation file: ./packages/grid/enclave/attestation/attestation.dockerfile @@ -259,7 +260,7 @@ jobs: - name: Build and push `syft` image to registry id: syft-build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/ file: ./packages/grid/syft-client/syft.Dockerfile @@ -385,7 +386,7 @@ jobs: python-version: "3.12" - name: Install dependencies run: | - pip install --upgrade pip uv==0.1.35 tox tox-uv==1.5.1 setuptools wheel twine bump2version PyYAML + pip install --upgrade pip==24.0 uv==0.2.13 tox tox-uv==1.9.0 setuptools wheel twine bump2version PyYAML uv --version - name: Bump the Version diff --git a/.github/workflows/cd-syftcli.yml b/.github/workflows/cd-syftcli.yml index 65f2c37662e..7b29caea012 100644 --- a/.github/workflows/cd-syftcli.yml +++ b/.github/workflows/cd-syftcli.yml @@ -65,7 +65,7 @@ jobs: - name: Install dependencies if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }} run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip==24.0 pip install --upgrade tox setuptools wheel twine bump2version PyYAML - name: Bump the Version @@ -121,7 +121,7 @@ jobs: - name: Install build dependencies for syftcli run: | - pip install --upgrade pip + pip install --upgrade pip==24.0 - name: Install Tox run: | diff --git a/.github/workflows/e2e-tests-notebook.yml b/.github/workflows/e2e-tests-notebook.yml index 10c3eb84e2d..ebf843f10a6 100644 --- a/.github/workflows/e2e-tests-notebook.yml +++ b/.github/workflows/e2e-tests-notebook.yml @@ -63,7 +63,7 @@ jobs: - name: Install Deps run: | - pip install --upgrade pip uv==0.1.35 tox tox-uv==1.5.1 + pip install --upgrade pip==24.0 uv==0.2.13 tox tox-uv==1.9.0 - name: Get pip cache dir id: pip-cache diff --git a/.github/workflows/pr-tests-frontend.yml b/.github/workflows/pr-tests-frontend.yml index bf36991a385..99b6aa79f3c 100644 --- a/.github/workflows/pr-tests-frontend.yml +++ b/.github/workflows/pr-tests-frontend.yml @@ -46,7 +46,7 @@ jobs: - name: Upgrade pip if: steps.changes.outputs.frontend == 'true' run: | - pip install --upgrade pip uv==0.1.35 + pip install --upgrade pip==24.0 uv==0.2.13 uv --version - name: Get pip cache dir @@ -67,12 +67,12 @@ jobs: - name: Docker on MacOS if: steps.changes.outputs.frontend == 'true' && matrix.os == 'macos-latest' - uses: crazy-max/ghaction-setup-docker@v3.2.0 + uses: crazy-max/ghaction-setup-docker@v3.3.0 - name: Install Tox if: steps.changes.outputs.frontend == 'true' run: | - pip install --upgrade tox tox-uv==1.5.1 + pip install --upgrade tox tox-uv==1.9.0 - name: Remove existing containers if: steps.changes.outputs.frontend == 'true' diff --git a/.github/workflows/pr-tests-helm-lint.yml b/.github/workflows/pr-tests-helm-lint.yml index 1ef21e5a5f9..c55479f9f7f 100644 --- a/.github/workflows/pr-tests-helm-lint.yml +++ b/.github/workflows/pr-tests-helm-lint.yml @@ -33,7 +33,7 @@ jobs: brew install kube-linter FairwindsOps/tap/polaris # Install python deps - pip install --upgrade pip + pip install --upgrade pip==24.0 pip install tox kube-linter version diff --git a/.github/workflows/pr-tests-helm-upgrade.yml b/.github/workflows/pr-tests-helm-upgrade.yml index be8bbc21996..0419a125369 100644 --- a/.github/workflows/pr-tests-helm-upgrade.yml +++ b/.github/workflows/pr-tests-helm-upgrade.yml @@ -37,7 +37,7 @@ jobs: brew update # Install python deps - pip install --upgrade pip + pip install --upgrade pip==24.0 pip install tox # Install kubernetes diff --git a/.github/workflows/pr-tests-linting.yml b/.github/workflows/pr-tests-linting.yml index fdb90728c2a..c2b71998087 100644 --- a/.github/workflows/pr-tests-linting.yml +++ b/.github/workflows/pr-tests-linting.yml @@ -29,7 +29,7 @@ jobs: - name: Install pip packages run: | - pip install --upgrade pip uv==0.1.35 + pip install --upgrade pip==24.0 uv==0.2.13 uv --version - name: Get pip cache dir @@ -49,7 +49,7 @@ jobs: - name: Install Tox run: | - pip install --upgrade tox tox-uv==1.5.1 + pip install --upgrade tox tox-uv==1.9.0 - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/pr-tests-stack.yml b/.github/workflows/pr-tests-stack.yml index 34620e3fa80..b0aefd3f23c 100644 --- a/.github/workflows/pr-tests-stack.yml +++ b/.github/workflows/pr-tests-stack.yml @@ -52,7 +52,7 @@ jobs: - name: Upgrade pip if: steps.changes.outputs.stack == 'true' run: | - pip install --upgrade pip uv==0.1.35 + pip install --upgrade pip==24.0 uv==0.2.13 uv --version - name: Get pip cache dir @@ -74,7 +74,7 @@ jobs: - name: Install tox if: steps.changes.outputs.stack == 'true' run: | - pip install --upgrade tox tox-uv==1.5.1 + pip install --upgrade tox tox-uv==1.9.0 - name: Run syft backend base image building test if: steps.changes.outputs.stack == 'true' @@ -113,7 +113,7 @@ jobs: - name: Upgrade pip if: steps.changes.outputs.stack == 'true' run: | - pip install --upgrade pip uv==0.1.35 + pip install --upgrade pip==24.0 uv==0.2.13 uv --version - name: Get pip cache dir @@ -135,7 +135,7 @@ jobs: - name: Install tox if: steps.changes.outputs.stack == 'true' run: | - pip install --upgrade tox tox-uv==1.5.1 + pip install --upgrade tox tox-uv==1.9.0 - name: Run Syft Integration Tests if: steps.changes.outputs.stack == 'true' @@ -200,7 +200,7 @@ jobs: - name: Upgrade pip if: steps.changes.outputs.stack == 'true' run: | - pip install --upgrade pip uv==0.1.35 + pip install --upgrade pip==24.0 uv==0.2.13 uv --version - name: Get pip cache dir @@ -222,7 +222,7 @@ jobs: - name: Install tox if: steps.changes.outputs.stack == 'true' run: | - pip install --upgrade tox tox-uv==1.5.1 + pip install --upgrade tox tox-uv==1.9.0 - name: Install kubectl if: steps.changes.outputs.stack == 'true' @@ -353,7 +353,7 @@ jobs: - name: Upgrade pip if: steps.changes.outputs.stack == 'true' run: | - pip install --upgrade pip uv==0.1.35 + pip install --upgrade pip==24.0 uv==0.2.13 uv --version - name: Get pip cache dir @@ -375,7 +375,7 @@ jobs: - name: Install tox if: steps.changes.outputs.stack == 'true' run: | - pip install --upgrade tox tox-uv==1.5.1 + pip install --upgrade tox tox-uv==1.9.0 - name: Install kubectl if: steps.changes.outputs.stack == 'true' diff --git a/.github/workflows/pr-tests-syft.yml b/.github/workflows/pr-tests-syft.yml index 046dea143e0..e416700f149 100644 --- a/.github/workflows/pr-tests-syft.yml +++ b/.github/workflows/pr-tests-syft.yml @@ -65,7 +65,7 @@ jobs: - name: Upgrade pip if: steps.changes.outputs.syft == 'true' run: | - pip install --upgrade pip uv==0.1.35 + pip install --upgrade pip==24.0 uv==0.2.13 uv --version - name: Get pip cache dir @@ -86,14 +86,14 @@ jobs: # - name: Docker on MacOS # if: steps.changes.outputs.syft == 'true' && matrix.os == 'macos-latest' - # uses: crazy-max/ghaction-setup-docker@v3.2.0 + # uses: crazy-max/ghaction-setup-docker@v3.3.0 # with: # set-host: true - name: Install Dependencies if: steps.changes.outputs.syft == 'true' run: | - pip install --upgrade tox tox-uv==1.5.1 + pip install --upgrade tox tox-uv==1.9.0 - name: Run unit tests if: steps.changes.outputs.syft == 'true' @@ -160,7 +160,7 @@ jobs: - name: Upgrade pip if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks == 'true' run: | - pip install --upgrade pip uv==0.1.35 + pip install --upgrade pip==24.0 uv==0.2.13 uv --version - name: Get pip cache dir @@ -182,7 +182,7 @@ jobs: - name: Install Dependencies if: steps.changes.outputs.syft == 'true' || steps.changes.outputs.notebooks == 'true' run: | - pip install --upgrade tox tox-uv==1.5.1 + pip install --upgrade tox tox-uv==1.9.0 - name: Run notebook tests uses: nick-fields/retry@v3 @@ -242,7 +242,7 @@ jobs: - name: Upgrade pip if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true' run: | - pip install --upgrade pip uv==0.1.35 + pip install --upgrade pip==24.0 uv==0.2.13 uv --version - name: Get pip cache dir @@ -264,7 +264,7 @@ jobs: - name: Install Dependencies if: steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true' run: | - pip install --upgrade tox tox-uv==1.5.1 + pip install --upgrade tox tox-uv==1.9.0 - name: Docker Compose on Linux if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'ubuntu-latest' @@ -278,7 +278,7 @@ jobs: - name: Docker on MacOS if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'macos-latest' - uses: crazy-max/ghaction-setup-docker@v3.2.0 + uses: crazy-max/ghaction-setup-docker@v3.3.0 - name: Docker Compose on MacOS if: (steps.changes.outputs.stack == 'true' || steps.changes.outputs.notebooks == 'true') && matrix.os == 'macos-latest' @@ -343,7 +343,7 @@ jobs: - name: Upgrade pip if: steps.changes.outputs.syft == 'true' run: | - pip install --upgrade pip uv==0.1.35 + pip install --upgrade pip==24.0 uv==0.2.13 uv --version - name: Get pip cache dir @@ -365,7 +365,7 @@ jobs: - name: Install Dependencies if: steps.changes.outputs.syft == 'true' run: | - pip install --upgrade tox tox-uv==1.5.1 + pip install --upgrade tox tox-uv==1.9.0 - name: Scan for security issues if: steps.changes.outputs.syft == 'true' diff --git a/VERSION b/VERSION index 61a3b991302..df1dec5602a 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.7-beta.10" +__version__ = "0.8.7-beta.13" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/docs/requirements.txt b/docs/requirements.txt index a16817917de..b16716ced72 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -11,3 +11,4 @@ sphinx-autoapi==1.8.4 sphinx-code-include==1.1.1 sphinx-copybutton==0.4.0 sphinx-panels==0.6.0 +urllib3>=2.2.2 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/notebooks/api/0.8/01-submit-code.ipynb b/notebooks/api/0.8/01-submit-code.ipynb index 761d1a96e7a..3d360c88b18 100644 --- a/notebooks/api/0.8/01-submit-code.ipynb +++ b/notebooks/api/0.8/01-submit-code.ipynb @@ -306,7 +306,7 @@ " dp.enable_features(\"contrib\")\n", "\n", " aggregate = 0.0\n", - " base_lap = dp.m.make_base_laplace(\n", + " base_lap = dp.m.make_laplace(\n", " dp.atom_domain(T=float),\n", " dp.absolute_distance(T=float),\n", " scale=5.0,\n", diff --git a/notebooks/api/0.8/02-review-code-and-approve.ipynb b/notebooks/api/0.8/02-review-code-and-approve.ipynb index 4faedd441c0..9612144b952 100644 --- a/notebooks/api/0.8/02-review-code-and-approve.ipynb +++ b/notebooks/api/0.8/02-review-code-and-approve.ipynb @@ -290,6 +290,21 @@ "print(op.policy_code)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Policies provided by Syft are available before approving the code,\n", + "# Custom policies are only safe to use once the code is approved.\n", + "\n", + "assert func.output_policy is not None\n", + "assert func.input_policy is not None\n", + "\n", + "func.output_policy" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -346,23 +361,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Sharing results back to the Data Scientist" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "By calling this function we attach the result of the function to the original request\n", + "### Approving a request\n", "\n", - "`request.accept_by_depositing_result(real_result)`" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's say we accidentally submit incorrect results, we can correct it by using override it using the `force=True` flag" + "By calling `request.approve()`, the data scientist can execute their function on the real data, and obtain the result" ] }, { @@ -374,31 +375,7 @@ "outputs": [], "source": [ "# Uploaded wrong result - we shared mock_result instead of the real_result\n", - "result = request.accept_by_depositing_result(mock_result)\n", - "result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "assert isinstance(result, sy.SyftSuccess)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# upload correct result\n", - "result = request.accept_by_depositing_result(real_result, force=True)\n", + "result = request.approve()\n", "result" ] }, @@ -472,7 +449,7 @@ "metadata": {}, "outputs": [], "source": [ - "result = request.accept_by_depositing_result(real_result)\n", + "result = request.approve()\n", "result" ] }, @@ -524,7 +501,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.2" + "version": "3.10.13" }, "toc": { "base_numbering": 1, diff --git a/notebooks/api/0.8/04-pytorch-example.ipynb b/notebooks/api/0.8/04-pytorch-example.ipynb index 05a18badf4e..5cbc89ecaea 100644 --- a/notebooks/api/0.8/04-pytorch-example.ipynb +++ b/notebooks/api/0.8/04-pytorch-example.ipynb @@ -128,7 +128,7 @@ }, "outputs": [], "source": [ - "train_domain_obj = domain_client.api.services.action.set(train)\n", + "train_domain_obj = train.send(domain_client)\n", "type(train_domain_obj)" ] }, @@ -139,7 +139,7 @@ "metadata": {}, "outputs": [], "source": [ - "assert torch.round(train_domain_obj.syft_action_data.sum()) == 1557" + "train_domain_obj" ] }, { @@ -148,6 +148,16 @@ "id": "11", "metadata": {}, "outputs": [], + "source": [ + "assert torch.round(train_domain_obj.syft_action_data.sum()) == 1557" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], "source": [ "class MLP(nn.Module):\n", " def __init__(self, out_dims):\n", @@ -171,7 +181,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -181,7 +191,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "14", "metadata": { "tags": [] }, @@ -193,7 +203,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "15", "metadata": { "tags": [] }, @@ -205,7 +215,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "16", "metadata": { "tags": [] }, @@ -217,19 +227,19 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "17", "metadata": { "tags": [] }, "outputs": [], "source": [ - "weight_domain_obj = domain_client.api.services.action.set(w)" + "weight_domain_obj = w.send(domain_client)" ] }, { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "18", "metadata": { "tags": [] }, @@ -276,7 +286,7 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "19", "metadata": { "tags": [] }, @@ -289,7 +299,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "20", "metadata": {}, "outputs": [], "source": [ @@ -299,7 +309,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "21", "metadata": { "tags": [] }, @@ -312,7 +322,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "22", "metadata": { "tags": [] }, @@ -324,7 +334,7 @@ { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "23", "metadata": { "tags": [] }, @@ -337,7 +347,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "24", "metadata": { "tags": [] }, @@ -349,7 +359,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "25", "metadata": {}, "outputs": [], "source": [ @@ -359,7 +369,7 @@ { "cell_type": "code", "execution_count": null, - "id": "25", + "id": "26", "metadata": { "tags": [] }, @@ -371,7 +381,7 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "27", "metadata": { "tags": [] }, @@ -384,7 +394,7 @@ { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "28", "metadata": {}, "outputs": [], "source": [] @@ -406,7 +416,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.2" + "version": "3.1.-1" }, "toc": { "base_numbering": 1, diff --git a/notebooks/api/0.8/05-custom-policy.ipynb b/notebooks/api/0.8/05-custom-policy.ipynb index fac27bfcbe8..8c7b18c9328 100644 --- a/notebooks/api/0.8/05-custom-policy.ipynb +++ b/notebooks/api/0.8/05-custom-policy.ipynb @@ -231,7 +231,7 @@ "metadata": {}, "outputs": [], "source": [ - "domain_client.api.services.action.set(x_pointer)" + "x_pointer = x_pointer.send(domain_client)" ] }, { @@ -297,14 +297,6 @@ " if kwarg_value.is_err():\n", " return Err(kwarg_value.err())\n", " code_inputs[var_name] = kwarg_value.ok()\n", - "\n", - " elif context.node.node_type == NodeType.ENCLAVE:\n", - " dict_object = action_service.get(context=root_context, uid=code_item_id)\n", - " if dict_object.is_err():\n", - " return Err(dict_object.err())\n", - " for value in dict_object.ok().syft_action_data.values():\n", - " code_inputs.update(value)\n", - "\n", " else:\n", " raise Exception(\n", " f\"Invalid Node Type for Code Submission:{context.node.node_type}\"\n", @@ -328,11 +320,6 @@ " verify_key=context.node.signing_key.verify_key,\n", " )\n", " allowed_inputs = allowed_inputs.get(node_identity, {})\n", - " elif context.node.node_type == NodeType.ENCLAVE:\n", - " base_dict = {}\n", - " for key in allowed_inputs.values():\n", - " base_dict.update(key)\n", - " allowed_inputs = base_dict\n", " else:\n", " raise Exception(\n", " f\"Invalid Node Type for Code Submission:{context.node.node_type}\"\n", @@ -403,11 +390,6 @@ " verify_key=context.node.signing_key.verify_key,\n", " )\n", " allowed_inputs = allowed_inputs.get(node_identity, {})\n", - " elif context.node.node_type == NodeType.ENCLAVE:\n", - " base_dict = {}\n", - " for key in allowed_inputs.values():\n", - " base_dict.update(key)\n", - " allowed_inputs = base_dict\n", " else:\n", " raise Exception(\n", " f\"Invalid Node Type for Code Submission:{context.node.node_type}\"\n", @@ -508,13 +490,12 @@ "cell_type": "code", "execution_count": null, "id": "23", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ - "result = func.unsafe_function(x=x_pointer)\n", - "result" + "# Custom policies need to be approved before they can be viewed and used\n", + "assert func.input_policy is None\n", + "assert func.output_policy is None" ] }, { @@ -526,11 +507,8 @@ }, "outputs": [], "source": [ - "# syft absolute\n", - "from syft.service.response import SyftError\n", - "\n", - "final_result = request.accept_by_depositing_result(result)\n", - "assert isinstance(final_result, SyftError)" + "result = func.unsafe_function(x=x_pointer)\n", + "result" ] }, { @@ -547,6 +525,17 @@ "cell_type": "code", "execution_count": null, "id": "26", + "metadata": {}, + "outputs": [], + "source": [ + "assert func.input_policy is not None\n", + "assert func.output_policy is not None" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", "metadata": { "tags": [] }, @@ -559,7 +548,7 @@ { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "28", "metadata": {}, "outputs": [], "source": [ @@ -570,7 +559,7 @@ { "cell_type": "code", "execution_count": null, - "id": "28", + "id": "29", "metadata": { "tags": [] }, @@ -582,7 +571,7 @@ { "cell_type": "code", "execution_count": null, - "id": "29", + "id": "30", "metadata": { "tags": [] }, @@ -594,7 +583,7 @@ { "cell_type": "code", "execution_count": null, - "id": "30", + "id": "31", "metadata": { "tags": [] }, @@ -610,7 +599,7 @@ { "cell_type": "code", "execution_count": null, - "id": "31", + "id": "32", "metadata": { "tags": [] }, @@ -637,7 +626,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0rc1" + "version": "3.10.13" }, "toc": { "base_numbering": 1, diff --git a/notebooks/api/0.8/06-multiple-code-requests.ipynb b/notebooks/api/0.8/06-multiple-code-requests.ipynb index 4be948cc00b..a52b5c6b38f 100644 --- a/notebooks/api/0.8/06-multiple-code-requests.ipynb +++ b/notebooks/api/0.8/06-multiple-code-requests.ipynb @@ -203,7 +203,7 @@ "\n", " # compute sum\n", " res = data.sum()\n", - " base_lap = dp.m.make_base_laplace(\n", + " base_lap = dp.m.make_laplace(\n", " dp.atom_domain(T=float),\n", " dp.absolute_distance(T=float),\n", " scale=10.0,\n", @@ -304,7 +304,7 @@ "\n", " # compute mean\n", " mean = data.mean()\n", - " base_lap = dp.m.make_base_laplace(\n", + " base_lap = dp.m.make_laplace(\n", " dp.atom_domain(T=float),\n", " dp.absolute_distance(T=float),\n", " scale=10.0,\n", diff --git a/notebooks/api/0.8/10-container-images.ipynb b/notebooks/api/0.8/10-container-images.ipynb index b0bacf0295f..e080146b63e 100644 --- a/notebooks/api/0.8/10-container-images.ipynb +++ b/notebooks/api/0.8/10-container-images.ipynb @@ -831,7 +831,7 @@ "data = np.array([1, 2, 3])\n", "data_action_obj = sy.ActionObject.from_obj(data)\n", "\n", - "data_pointer = domain_client.api.services.action.set(data_action_obj)\n", + "data_pointer = data_action_obj.send(domain_client)\n", "data_pointer" ] }, @@ -1483,7 +1483,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.2" + "version": "3.11.8" } }, "nbformat": 4, diff --git a/notebooks/api/0.8/11-container-images-k8s.ipynb b/notebooks/api/0.8/11-container-images-k8s.ipynb index c685e6f8d7d..59bf6b5da9e 100644 --- a/notebooks/api/0.8/11-container-images-k8s.ipynb +++ b/notebooks/api/0.8/11-container-images-k8s.ipynb @@ -246,12 +246,14 @@ "metadata": {}, "outputs": [], "source": [ + "# syft absolute\n", + "from syft.util.util import get_latest_tag\n", + "\n", "registry = os.getenv(\"SYFT_BASE_IMAGE_REGISTRY\", \"docker.io\")\n", "repo = \"openmined/grid-backend\"\n", "\n", "if \"k3d\" in registry:\n", - " res = requests.get(url=f\"http://{registry}/v2/{repo}/tags/list\")\n", - " tag = res.json()[\"tags\"][0]\n", + " tag = get_latest_tag(registry, repo)\n", "else:\n", " tag = sy.__version__" ] @@ -795,7 +797,7 @@ "data = np.array([1, 2, 3])\n", "data_action_obj = sy.ActionObject.from_obj(data)\n", "\n", - "data_pointer = domain_client.api.services.action.set(data_action_obj)\n", + "data_pointer = data_action_obj.send(domain_client)\n", "data_pointer" ] }, @@ -1500,7 +1502,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.11.8" } }, "nbformat": 4, diff --git a/notebooks/api/0.8/12-custom-api-endpoint.ipynb b/notebooks/api/0.8/12-custom-api-endpoint.ipynb index f84ca9c5c3f..20c269ea286 100644 --- a/notebooks/api/0.8/12-custom-api-endpoint.ipynb +++ b/notebooks/api/0.8/12-custom-api-endpoint.ipynb @@ -574,7 +574,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0rc1" + "version": "3.11.8" } }, "nbformat": 4, diff --git a/notebooks/real_jobs_ux.ipynb b/notebooks/real_jobs_ux.ipynb deleted file mode 100644 index 24b8685e542..00000000000 --- a/notebooks/real_jobs_ux.ipynb +++ /dev/null @@ -1,5276 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "import syft as sy" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Staging Protocol Changes...\n", - "Creating default worker image with tag='local-dev'\n", - "Building default worker image with tag=local-dev\n", - "Setting up worker poolname=default-pool workers=4 image_uid=533f068bce124b228c6f21262613832f in_memory=True\n", - "Created default worker pool.\n", - "Data Migrated to latest version !!!\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "22/04/24 17:10:40 FUNCTION LOG (9a1cd5d70dde491ba6900ad2d9823d91): Job Iter 0\n", - "22/04/24 17:10:40 FUNCTION LOG (9a1cd5d70dde491ba6900ad2d9823d91): Job Iter 1\n", - "22/04/24 17:10:42 FUNCTION LOG (fd88bc3e255842a49ce329069eda2f1b): Subjob Iter 0\n", - "22/04/24 17:10:43 FUNCTION LOG (ccb8f5413883402db1f3d2c8965681c9): Subjob Iter 0\n", - "22/04/24 17:10:44 FUNCTION LOG (fd88bc3e255842a49ce329069eda2f1b): Subjob Iter 1\n", - "22/04/24 17:10:44 FUNCTION LOG (ccb8f5413883402db1f3d2c8965681c9): Subjob Iter 1\n", - "22/04/24 17:10:45 FUNCTION LOG (fd88bc3e255842a49ce329069eda2f1b): Subjob Iter 2\n", - "22/04/24 17:10:46 FUNCTION LOG (ccb8f5413883402db1f3d2c8965681c9): Subjob Iter 2\n", - "22/04/24 17:10:46 FUNCTION LOG (fd88bc3e255842a49ce329069eda2f1b): Subjob Iter 3\n", - "22/04/24 17:10:47 FUNCTION LOG (ccb8f5413883402db1f3d2c8965681c9): Subjob Iter 3\n", - "22/04/24 17:10:47 FUNCTION LOG (fd88bc3e255842a49ce329069eda2f1b): Subjob Iter 4\n", - "22/04/24 17:10:48 FUNCTION LOG (ccb8f5413883402db1f3d2c8965681c9): Subjob Iter 4\n", - "22/04/24 17:10:48 FUNCTION LOG (fd88bc3e255842a49ce329069eda2f1b): Subjob Iter 5\n", - "22/04/24 17:10:49 FUNCTION LOG (ccb8f5413883402db1f3d2c8965681c9): Subjob Iter 5\n", - "22/04/24 17:10:49 FUNCTION LOG (fd88bc3e255842a49ce329069eda2f1b): Subjob Iter 6\n", - "22/04/24 17:10:50 FUNCTION LOG (ccb8f5413883402db1f3d2c8965681c9): Subjob Iter 6\n", - "22/04/24 17:10:50 FUNCTION LOG (fd88bc3e255842a49ce329069eda2f1b): Subjob Iter 7\n", - "22/04/24 17:10:51 FUNCTION LOG (ccb8f5413883402db1f3d2c8965681c9): Subjob Iter 7\n", - "22/04/24 17:10:51 FUNCTION LOG (fd88bc3e255842a49ce329069eda2f1b): Subjob Iter 8\n", - "22/04/24 17:10:52 FUNCTION LOG (ccb8f5413883402db1f3d2c8965681c9): Subjob Iter 8\n", - "22/04/24 17:10:52 FUNCTION LOG (fd88bc3e255842a49ce329069eda2f1b): Subjob Iter 9\n", - "22/04/24 17:10:53 FUNCTION LOG (ccb8f5413883402db1f3d2c8965681c9): Subjob Iter 9\n" - ] - } - ], - "source": [ - "node = sy.orchestra.launch(\n", - " name=\"test\",\n", - " dev_mode=True,\n", - " reset=True,\n", - " local_db=True,\n", - " n_consumers=4,\n", - " create_producer=True,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logged into as \n" - ] - }, - { - "data": { - "text/html": [ - "
SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`.

" - ], - "text/plain": [ - "SyftWarning: You are using a default password. Please change the password using `[your_client].me.set_password([new_password])`." - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "domain_client = node.login(email=\"info@openmined.org\", password=\"changethis\")\n", - "# domain_client = sy.login(email=\"info@openmined.org\", password=\"changethis\", port=8080)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SyftSuccess: Syft function 'subjob' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" - ], - "text/plain": [ - "SyftSuccess: Syft function 'subjob' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "@sy.syft_function()\n", - "def subjob(domain):\n", - " # stdlib\n", - " import time\n", - "\n", - " n_iters = 10\n", - " domain.init_progress(n_iters=n_iters)\n", - " for i in range(n_iters):\n", - " time.sleep(1)\n", - " print(f\"Subjob Iter {i}\")\n", - " domain.increment_progress()\n", - " return \"Done\"" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SyftSuccess: User Code Submitted

" - ], - "text/plain": [ - "SyftSuccess: User Code Submitted" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "domain_client.code.submit(subjob)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
SyftSuccess: Syft function 'job' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`.

" - ], - "text/plain": [ - "SyftSuccess: Syft function 'job' successfully created. To add a code request, please create a project using `project = syft.Project(...)`, then use command `project.create_code_request`." - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "@sy.syft_function_single_use()\n", - "def job(domain):\n", - " # stdlib\n", - " import time\n", - "\n", - " n_iters = 2\n", - " domain.init_progress(n_iters=n_iters)\n", - " for i in range(n_iters):\n", - " _ = domain.launch_job(subjob)\n", - " time.sleep(0.1)\n", - " print(f\"Job Iter {i}\")\n", - " domain.increment_progress()\n", - " return \"Done\"" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - "
\n", - "

Request

\n", - "

Id: 9a38e069840b4c42a45b24d6dcf0dd89

\n", - "

Request time: 2024-04-22 13:10:38

\n", - " \n", - " \n", - "

Status: RequestStatus.PENDING

\n", - "

Requested on: Test of type Domain

\n", - "

Requested by: Jane Doe (info@openmined.org)

\n", - "

Changes: Request to change job (Pool Id: default-pool) to permission RequestStatus.APPROVED. Nested Requests not resolved.

\n", - "
\n", - "\n", - " " - ], - "text/markdown": [ - "```python\n", - "class Request:\n", - " id: str = 9a38e069840b4c42a45b24d6dcf0dd89\n", - " request_time: str = 2024-04-22 13:10:38\n", - " updated_at: str = None\n", - " status: str = RequestStatus.PENDING\n", - " changes: str = ['Request to change job (Pool Id: default-pool) to permission RequestStatus.APPROVED. Nested Requests not resolved']\n", - " requesting_user_verify_key: str = 67d3b5eaf0c0bf6b5a602d359daecc86a7a74053490ec37ae08e71360587c870\n", - "\n", - "```" - ], - "text/plain": [ - "syft.service.request.request.Request" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "domain_client.code.request_code_execution(job)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Approving request for domain test\n" - ] - }, - { - "data": { - "text/html": [ - "
SyftSuccess: Request 9a38e069840b4c42a45b24d6dcf0dd89 changes applied

" - ], - "text/plain": [ - "SyftSuccess: Request 9a38e069840b4c42a45b24d6dcf0dd89 changes applied" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "domain_client.requests[-1].approve(approve_nested=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "res = domain_client.code.job(blocking=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - "\n", - "\n", - " \n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "
\n", - "
test/jobs/
\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - " \n", - " JOB\n", - "
\n", - " job\n", - "
\n", - " \n", - " \n", - "
\n", - " #9a1cd5d70dde491ba6900ad2d9823d91\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " UserCode:\n", - " job\n", - "
\n", - "
\n", - " Status:\n", - " Created\n", - "
\n", - "
\n", - " \n", - " Started At:\n", - " 2024-04-22 17:10:39 by Jane Doe info@openmined.org\n", - "
\n", - "
\n", - " \n", - " Updated At:\n", - " --\n", - "
\n", - " \n", - "
\n", - " Subjobs:\n", - " 0\n", - "
\n", - "
\n", - "
\n", - " \n", - " \n", - "
\n", - "
\n", - " \n", - " \n", - "
\n", - "
\n", - "
\n", - "
\n", - " syft.service.action.action_data_empty.ObjectNotReady\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " " - ], - "text/markdown": [ - "```python\n", - "class Job:\n", - " id: UID = 9a1cd5d70dde491ba6900ad2d9823d91\n", - " status: JobStatus.CREATED\n", - " has_parent: False\n", - " result: syft.service.action.action_data_empty.ObjectNotReady\n", - " logs:\n", - "\n", - "0 \n", - " \n", - "```" - ], - "text/plain": [ - "syft.service.job.job_stash.Job" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "res" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "x = res.wait()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "```python\n", - "Pointer\n", - "'Done'```\n" - ], - "text/plain": [ - "Pointer:\n", - "'Done'" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "x" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "

Job List

\n", - "
\n", - "\n", - "
\n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - "\n", - "

0

\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "domain_client.jobs" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "

Job List

\n", - "
\n", - "\n", - "
\n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - "\n", - "

0

\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "domain_client.jobs" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "subjob = domain_client.jobs[1]" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - "\n", - "\n", - " \n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "
\n", - "
test/jobs/
\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - " \n", - " JOB\n", - "
\n", - " job\n", - "
\n", - " \n", - " \n", - "
\n", - " #9a1cd5d70dde491ba6900ad2d9823d91\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " UserCode:\n", - " job\n", - "
\n", - "
\n", - " Status:\n", - " Completed\n", - "
\n", - "
\n", - " \n", - " Started At:\n", - " 2024-04-22 17:10:39 by Jane Doe info@openmined.org\n", - "
\n", - "
\n", - " \n", - " Updated At:\n", - " --\n", - "
\n", - " \n", - "
\n", - " Subjobs:\n", - " 2\n", - "
\n", - "
\n", - "
\n", - " \n", - " \n", - "
\n", - "
\n", - " \n", - " \n", - "
\n", - "
\n", - "
\n", - "
\n", - " Done\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "
\n", - " \n", - " #\n", - " \n", - " \n", - " \n", - " Message\n", - " \n", - "
\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " " - ], - "text/markdown": [ - "```python\n", - "class Job:\n", - " id: UID = 9a1cd5d70dde491ba6900ad2d9823d91\n", - " status: JobStatus.COMPLETED\n", - " has_parent: False\n", - " result: Done\n", - " logs:\n", - "\n", - "0 Job Iter 0\n", - "1 Job Iter 1\n", - "JOB COMPLETED\n", - " \n", - "```" - ], - "text/plain": [ - "syft.service.job.job_stash.Job" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "res" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Job Iter 0\n", - "Job Iter 1\n", - "\n", - "\n" - ] - } - ], - "source": [ - "res.logs()" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "j = domain_client.jobs[1]" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - "\n", - "\n", - " \n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "
\n", - "
test/jobs/job
\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - " \n", - " SUBJOB\n", - "
\n", - " subjob\n", - "
\n", - " \n", - " \n", - "
\n", - " #fd88bc3e255842a49ce329069eda2f1b\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " UserCode:\n", - " subjob\n", - "
\n", - "
\n", - " Status:\n", - " Processing\n", - "
\n", - "
\n", - " \n", - " Started At:\n", - " 2024-04-22 17:10:41 by Jane Doe info@openmined.org\n", - "
\n", - "
\n", - " \n", - " Updated At:\n", - " 2024-04-22 1\n", - "
\n", - " \n", - "
\n", - " \n", - " Worker Pool:\n", - " default-pool-2 on worker \n", - " \n", - "
\n", - " #default-pool\n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - " Subjobs:\n", - " 0\n", - "
\n", - "
\n", - "
\n", - " \n", - " \n", - "
\n", - "
\n", - " \n", - " \n", - "
\n", - "
\n", - "
\n", - "
\n", - " syft.service.action.action_data_empty.ObjectNotReady\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - "
\n", - " \n", - " #\n", - " \n", - " \n", - " \n", - " Message\n", - " \n", - "
\n", - "
\n", - " 0\n", - "
\n", - "
\n", - "
\n", - " Job Iter 0\n", - "
\n", - "
\n", - "
\n", - " 1\n", - "
\n", - "
\n", - "
\n", - " Job Iter 1\n", - "
\n", - "
\n", - "
\n", - " 2\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " " - ], - "text/markdown": [ - "```python\n", - "class Job:\n", - " id: UID = fd88bc3e255842a49ce329069eda2f1b\n", - " status: JobStatus.PROCESSING\n", - " has_parent: True\n", - " result: syft.service.action.action_data_empty.ObjectNotReady\n", - " logs:\n", - "\n", - "0 Subjob Iter 0\n", - "1 Subjob Iter 1\n", - "2 Subjob Iter 2\n", - " \n", - "```" - ], - "text/plain": [ - "syft.service.job.job_stash.Job" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "j" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "# subjob" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "

Job List

\n", - "
\n", - "\n", - "
\n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - "\n", - "

0

\n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n", - " \n", - "
\n", - "
\n" - ], - "text/plain": [ - "[syft.service.job.job_stash.Job,\n", - " syft.service.job.job_stash.Job,\n", - " syft.service.job.job_stash.Job,\n", - " syft.service.job.job_stash.Job,\n", - " syft.service.job.job_stash.Job,\n", - " syft.service.job.job_stash.Job,\n", - " syft.service.job.job_stash.Job,\n", - " syft.service.job.job_stash.Job,\n", - " syft.service.job.job_stash.Job,\n", - " syft.service.job.job_stash.Job]" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "res.subjobs" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": true - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/scenarios/bigquery/.gitignore b/notebooks/scenarios/bigquery/.gitignore new file mode 100644 index 00000000000..63b990492d4 --- /dev/null +++ b/notebooks/scenarios/bigquery/.gitignore @@ -0,0 +1 @@ +service_account.json \ No newline at end of file diff --git a/notebooks/scenarios/bigquery/01-setup-on-gcp.ipynb b/notebooks/scenarios/bigquery/01-setup-on-gcp.ipynb new file mode 100644 index 00000000000..ee727e8c9cb --- /dev/null +++ b/notebooks/scenarios/bigquery/01-setup-on-gcp.ipynb @@ -0,0 +1,48 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "453d340a-4dac-4407-9003-3e99e8de4aca", + "metadata": {}, + "outputs": [], + "source": [ + "# -- creating a project\n", + "# -- gke\n", + "# -- artifact registry\n", + "# -- gcs bucket\n", + "# -- bigquery instance\n", + "# -- service account" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "78e4a96e-9981-4021-9f08-6a29342755c9", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/bigquery/02-deploying-to-gke.ipynb b/notebooks/scenarios/bigquery/02-deploying-to-gke.ipynb new file mode 100644 index 00000000000..fcdfe9597f7 --- /dev/null +++ b/notebooks/scenarios/bigquery/02-deploying-to-gke.ipynb @@ -0,0 +1,45 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "6e61a7e1-ff80-436e-a7ed-25e88d148e64", + "metadata": {}, + "outputs": [], + "source": [ + "# -- helm install\n", + "# -- configuring seaweed blob storage\n", + "# -- ingress" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7246355d-a13a-429c-88ca-4afe4b6e7a4d", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/bigquery/03-custom-worker-pool.ipynb b/notebooks/scenarios/bigquery/03-custom-worker-pool.ipynb new file mode 100644 index 00000000000..0ff89e8753f --- /dev/null +++ b/notebooks/scenarios/bigquery/03-custom-worker-pool.ipynb @@ -0,0 +1,41 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "786ccd6c-d55a-40e9-8d5f-27deba80332d", + "metadata": {}, + "outputs": [], + "source": [ + "# -- configuring artifact registry\n", + "# -- building a custom image with bigquery and pushing\n", + "# -- adding worker pool\n", + "# -- optional: deploying an image which is pre-built\n", + "# -- optional: adding service account via labels\n", + "# -- setting default worker pool to custom pool\n", + "# -- scaling down `default-worker-pool`" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/bigquery/04-adding-users.ipynb b/notebooks/scenarios/bigquery/04-adding-users.ipynb new file mode 100644 index 00000000000..2721865882a --- /dev/null +++ b/notebooks/scenarios/bigquery/04-adding-users.ipynb @@ -0,0 +1,37 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "3ba045e3-68be-4a0e-8e99-7721c66d1999", + "metadata": {}, + "outputs": [], + "source": [ + "# -- how to enable user registration\n", + "# -- how to create users\n", + "# -- how to reset user passwords" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/bigquery/05-custom-api-endpoints.ipynb b/notebooks/scenarios/bigquery/05-custom-api-endpoints.ipynb new file mode 100644 index 00000000000..47ff98c6d80 --- /dev/null +++ b/notebooks/scenarios/bigquery/05-custom-api-endpoints.ipynb @@ -0,0 +1,45 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "248f62a8-94b6-481d-a32b-03d5c2b67ae0", + "metadata": {}, + "outputs": [], + "source": [ + "# -- adding custom endpoints that use custom pools\n", + "# -- mock vs private\n", + "# -- context.node\n", + "# -- settings and state\n", + "# -- service account via settings or k8s labels\n", + "# -- how to clear state\n", + "# -- how to implement a basic rate limiter using context.user\n", + "# -- creating a big query endpoint with a try catch\n", + "# -- how to delete / replace api endpoints\n", + "# -- testing endpoints and their mock / private versions\n", + "# -- executing a large query and checking blob storage" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/bigquery/06-working-with-remote-apis-and-worker-pools.ipynb b/notebooks/scenarios/bigquery/06-working-with-remote-apis-and-worker-pools.ipynb new file mode 100644 index 00000000000..744923e5b68 --- /dev/null +++ b/notebooks/scenarios/bigquery/06-working-with-remote-apis-and-worker-pools.ipynb @@ -0,0 +1,42 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "ddc9b9f5-a236-4ab2-a99d-0a2bd4a28f5f", + "metadata": {}, + "outputs": [], + "source": [ + "# -- browsing api endpoints\n", + "# -- listing worker pools\n", + "# -- mock vs private execution\n", + "# -- calling an endpoint directly\n", + "# -- using an endpoint in a syft function with worker-pool\n", + "# -- doing some small analysis on the results\n", + "# -- testing and submitting for approval\n", + "# -- users calling" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/bigquery/07-reviewing-user-code.ipynb b/notebooks/scenarios/bigquery/07-reviewing-user-code.ipynb new file mode 100644 index 00000000000..3791163a12c --- /dev/null +++ b/notebooks/scenarios/bigquery/07-reviewing-user-code.ipynb @@ -0,0 +1,46 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "1f2da72c-33a1-4edc-8ab5-dc9b83ce2af8", + "metadata": {}, + "outputs": [], + "source": [ + "# -- requests queue\n", + "# -- reviewing code\n", + "# -- carefully testing code\n", + "# -- approve / deny code" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "446fefdc-50a0-4d44-9ce4-d870432857c9", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/bigquery/08-running-code.ipynb b/notebooks/scenarios/bigquery/08-running-code.ipynb new file mode 100644 index 00000000000..b64a240fe90 --- /dev/null +++ b/notebooks/scenarios/bigquery/08-running-code.ipynb @@ -0,0 +1,40 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "d7ad91a8-7b45-4d3f-8733-568d07a2b3b5", + "metadata": {}, + "outputs": [], + "source": [ + "# -- executing approved code\n", + "# -- working with jobs\n", + "# -- viewing logs\n", + "# -- refreshing\n", + "# -- getting final result\n", + "# -- success" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/bigquery/09-debugging.ipynb b/notebooks/scenarios/bigquery/09-debugging.ipynb new file mode 100644 index 00000000000..b1e5b0237e4 --- /dev/null +++ b/notebooks/scenarios/bigquery/09-debugging.ipynb @@ -0,0 +1,45 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "af997f01-452e-4f18-affa-d5e3f7feb87a", + "metadata": {}, + "outputs": [], + "source": [ + "# -- common issues and where to look\n", + "# -- jobs, blob storage, queues, service accounts, seaweedfs, gcs\n", + "# -- custom worker images, registry, pull permissions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "92ee79f2-07b5-4fb8-a2f4-05f006b2fabb", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/packages/grid/backend/grid/logger/__init__.py b/notebooks/scenarios/bigquery/README.md similarity index 100% rename from packages/grid/backend/grid/logger/__init__.py rename to notebooks/scenarios/bigquery/README.md diff --git a/notebooks/scenarios/enclave/01-primary-domain-setup.ipynb b/notebooks/scenarios/enclave/01-primary-domain-setup.ipynb new file mode 100644 index 00000000000..77086333a0a --- /dev/null +++ b/notebooks/scenarios/enclave/01-primary-domain-setup.ipynb @@ -0,0 +1,38 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "f9f7fba1-43f8-48bc-a45d-46530574d010", + "metadata": {}, + "outputs": [], + "source": [ + "# -- upload model tensor\n", + "# -- create user account\n", + "# -- phase 2 add model hosting\n", + "# -- phase 3 run on kubernetes" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/enclave/02-manual-enclave-setup.ipynb b/notebooks/scenarios/enclave/02-manual-enclave-setup.ipynb new file mode 100644 index 00000000000..0f1b68dddf2 --- /dev/null +++ b/notebooks/scenarios/enclave/02-manual-enclave-setup.ipynb @@ -0,0 +1,38 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "fac336b0-c1a6-46a0-8133-3a2b0704a2b3", + "metadata": {}, + "outputs": [], + "source": [ + "# -- create enclave node\n", + "# -- attach to primary domain\n", + "# -- phase 2 launch python enclave dynamically instead\n", + "# -- phase 3 run on cloud enclave with k3d (dynamically after)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/enclave/03-secondary-domain-setup.ipynb b/notebooks/scenarios/enclave/03-secondary-domain-setup.ipynb new file mode 100644 index 00000000000..bc8251c65a4 --- /dev/null +++ b/notebooks/scenarios/enclave/03-secondary-domain-setup.ipynb @@ -0,0 +1,37 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0cb79e18-a7f7-4096-b20f-31aef7b049c3", + "metadata": {}, + "outputs": [], + "source": [ + "# -- upload inference tensor\n", + "# -- phase 2 inference eval dataset\n", + "# -- create user account" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/enclave/04-data-scientist-join.ipynb b/notebooks/scenarios/enclave/04-data-scientist-join.ipynb new file mode 100644 index 00000000000..d49d09ccbe0 --- /dev/null +++ b/notebooks/scenarios/enclave/04-data-scientist-join.ipynb @@ -0,0 +1,51 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "52c96d72-c333-4b5b-8631-caaf3c48e4d0", + "metadata": {}, + "outputs": [], + "source": [ + "# -- connect to domains\n", + "# -- associate domains?\n", + "# -- list enclaves\n", + "# -- find datasets\n", + "# -- execution policies\n", + "# -- phase 2 - add a hf model and custom worker image to execution policy\n", + "# -- phase 3 eager data scientist inference inputs in InputPolicy\n", + "# -- create usercode sum(a, b)\n", + "# -- submit project" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ebf6dc1-6b71-4c6b-826b-c35018a041e7", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/enclave/05-domains-review.ipynb b/notebooks/scenarios/enclave/05-domains-review.ipynb new file mode 100644 index 00000000000..0220db2d7d0 --- /dev/null +++ b/notebooks/scenarios/enclave/05-domains-review.ipynb @@ -0,0 +1,41 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "6fe704db-90b5-4511-8b29-0056eb82e967", + "metadata": {}, + "outputs": [], + "source": [ + "# -- review project\n", + "# -- inspect code\n", + "# -- step through execution policy\n", + "# -- query enclave attestation\n", + "# -- approve execution\n", + "# -- phase 2 - once approved everywhere, setup custom image on enclave\n", + "# -- phase 3 - once approved deploy with terraform etc" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/enclave/06-manual-execution.ipynb b/notebooks/scenarios/enclave/06-manual-execution.ipynb new file mode 100644 index 00000000000..6d5f77a8f01 --- /dev/null +++ b/notebooks/scenarios/enclave/06-manual-execution.ipynb @@ -0,0 +1,46 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "6a7cf74a-a267-4e4d-a167-aa02364ca860", + "metadata": {}, + "outputs": [], + "source": [ + "# -- get project\n", + "# -- check project status\n", + "# -- run code\n", + "# -- get result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "611f94c5-a6fd-4cb6-a581-d878bc11bcdc", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/enclave/07-audit-project-logs.ipynb b/notebooks/scenarios/enclave/07-audit-project-logs.ipynb new file mode 100644 index 00000000000..3af993e7572 --- /dev/null +++ b/notebooks/scenarios/enclave/07-audit-project-logs.ipynb @@ -0,0 +1,36 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "c36fced0-3a9d-439f-b237-64a71a1ee3ac", + "metadata": {}, + "outputs": [], + "source": [ + "# -- domain owners view logs from enclave on domain\n", + "# -- step through execution policy at each step who did what" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/enclave/08-enclave-shutdown.ipynb b/notebooks/scenarios/enclave/08-enclave-shutdown.ipynb new file mode 100644 index 00000000000..2f0e245e8fd --- /dev/null +++ b/notebooks/scenarios/enclave/08-enclave-shutdown.ipynb @@ -0,0 +1,35 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "a3cfb45c-9dbd-485d-a71a-e024c9889715", + "metadata": {}, + "outputs": [], + "source": [ + "# -- primary terminates enclave" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/enclave/README.md b/notebooks/scenarios/enclave/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/notebooks/scenarios/getting-started/01-installing-syft.ipynb b/notebooks/scenarios/getting-started/01-installing-syft.ipynb new file mode 100644 index 00000000000..1926fbff596 --- /dev/null +++ b/notebooks/scenarios/getting-started/01-installing-syft.ipynb @@ -0,0 +1,47 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "1340105f-d81f-453a-8ef7-6d51453dd3ad", + "metadata": {}, + "outputs": [], + "source": [ + "# -- pip install syft\n", + "# -- conda? (pycapnp lib bug??)\n", + "# -- accessing betas\n", + "# -- checking the version you have installed\n", + "# -- using the same version as the server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ddb5738-99a2-4d93-9f14-cfa3ac06d3c9", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/getting-started/02-running-python-server.ipynb b/notebooks/scenarios/getting-started/02-running-python-server.ipynb new file mode 100644 index 00000000000..54eca1a1431 --- /dev/null +++ b/notebooks/scenarios/getting-started/02-running-python-server.ipynb @@ -0,0 +1,47 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "5ac9669d-4c1f-4b29-b6fb-abec508649e7", + "metadata": {}, + "outputs": [], + "source": [ + "# -- configuration options for demo / dev mode etc\n", + "# -- how to see the web server is running on port x\n", + "# -- how to make a server accessible to other users (brief explanation of networking)\n", + "# -- Optional: for testing purposes you can use bore: https://github.com/ekzhang/bore for free\n", + "# -- Note: production mode is recommended to use kubernetes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c204ff56-a6c6-4031-9ce2-f6b7a875af20", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/getting-started/03-configuring-domain.ipynb b/notebooks/scenarios/getting-started/03-configuring-domain.ipynb new file mode 100644 index 00000000000..b152a87d872 --- /dev/null +++ b/notebooks/scenarios/getting-started/03-configuring-domain.ipynb @@ -0,0 +1,37 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "9278cdde-2ddc-493f-ab1a-7a224f2c2338", + "metadata": {}, + "outputs": [], + "source": [ + "# -- all the configuration and node settings options\n", + "# -- optional: adding email settings (smtp with sendgrid free account)\n", + "# -- changing passwords (importance of root account)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/getting-started/04-uploading-data.ipynb b/notebooks/scenarios/getting-started/04-uploading-data.ipynb new file mode 100644 index 00000000000..1ba607547bb --- /dev/null +++ b/notebooks/scenarios/getting-started/04-uploading-data.ipynb @@ -0,0 +1,48 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "c4db03f1-f663-42e3-b033-09aa817acbbb", + "metadata": {}, + "outputs": [], + "source": [ + "# -- what kinds of data are supported\n", + "# -- how to structure your data\n", + "# -- mock data and how to create some\n", + "# -- how much data you can store (Note: k8s requires blob storage configuration)\n", + "# -- adding metadata and uploading\n", + "# -- how to change the data later" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "150862d6-0395-4fbb-ad19-9bdae91e33fa", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/getting-started/05-adding-users.ipynb b/notebooks/scenarios/getting-started/05-adding-users.ipynb new file mode 100644 index 00000000000..881fb6a5a01 --- /dev/null +++ b/notebooks/scenarios/getting-started/05-adding-users.ipynb @@ -0,0 +1,45 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "e5d3de5f-afee-4743-89c8-8298b12edfe7", + "metadata": {}, + "outputs": [], + "source": [ + "# -- how to enable / disable user registration\n", + "# -- how to create users\n", + "# -- how to reset user passwords" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ac0c932-601f-4424-ad80-f02da291b2a3", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/getting-started/06-working-with-remote-data.ipynb b/notebooks/scenarios/getting-started/06-working-with-remote-data.ipynb new file mode 100644 index 00000000000..6b343e978c1 --- /dev/null +++ b/notebooks/scenarios/getting-started/06-working-with-remote-data.ipynb @@ -0,0 +1,53 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "bc1a03b7-05d5-41bb-bc9e-0361769d5c7e", + "metadata": {}, + "outputs": [], + "source": [ + "# -- browsing datasets\n", + "# -- getting a pointer\n", + "# -- mock vs private\n", + "# -- Pointer UIDs\n", + "# -- choosing an input policy\n", + "# -- choosing an output policy\n", + "# -- using the syft function decorator\n", + "# -- testing code locally\n", + "# -- submitting code for approval\n", + "# -- code is denied\n", + "# -- changing code and re-uploading a new version" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37adcb64-bf59-4a3f-9eab-31814fd1c675", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/getting-started/07-reviewing-user-code.ipynb b/notebooks/scenarios/getting-started/07-reviewing-user-code.ipynb new file mode 100644 index 00000000000..a35e4695f42 --- /dev/null +++ b/notebooks/scenarios/getting-started/07-reviewing-user-code.ipynb @@ -0,0 +1,48 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "f9535594-f1a2-48a6-87b1-a608f1fa7520", + "metadata": {}, + "outputs": [], + "source": [ + "# -- notifications of code requests\n", + "# -- requests queue\n", + "# -- reviewing code\n", + "# -- carefully testing code\n", + "# -- approve / deny code\n", + "# -- substituting a result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa2e6a39-1c10-4eec-bbce-452afde0f10b", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/getting-started/08-running-code.ipynb b/notebooks/scenarios/getting-started/08-running-code.ipynb new file mode 100644 index 00000000000..383f19f4f82 --- /dev/null +++ b/notebooks/scenarios/getting-started/08-running-code.ipynb @@ -0,0 +1,48 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "6f674a77-fcb7-4e99-87a1-daf12094d3aa", + "metadata": {}, + "outputs": [], + "source": [ + "# -- executing approved code\n", + "# -- working with jobs\n", + "# -- refreshing\n", + "# -- viewing logs\n", + "# -- getting final result\n", + "# -- success" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "363c56db-4c7c-458d-9a63-108eda0fcef7", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/getting-started/README.md b/notebooks/scenarios/getting-started/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/notebooks/scenarios/reverse-tunnel/01-why-reverse-tunnel.ipynb b/notebooks/scenarios/reverse-tunnel/01-why-reverse-tunnel.ipynb new file mode 100644 index 00000000000..7161744c040 --- /dev/null +++ b/notebooks/scenarios/reverse-tunnel/01-why-reverse-tunnel.ipynb @@ -0,0 +1,37 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "9a9dbce4-7d8d-4bad-a758-25d4696f0bb8", + "metadata": {}, + "outputs": [], + "source": [ + "# -- include cleaned up diagram?\n", + "# -- NAT Firewall problem\n", + "# -- Solution: rathole" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/reverse-tunnel/02-creating-gateway.ipynb b/notebooks/scenarios/reverse-tunnel/02-creating-gateway.ipynb new file mode 100644 index 00000000000..44003fc1edc --- /dev/null +++ b/notebooks/scenarios/reverse-tunnel/02-creating-gateway.ipynb @@ -0,0 +1,36 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "66b4e68c-73a9-4064-802a-4fd96887b3f6", + "metadata": {}, + "outputs": [], + "source": [ + "# -- helm install\n", + "# -- deploy gateway k8s to azure" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/reverse-tunnel/03-network-configuration.ipynb b/notebooks/scenarios/reverse-tunnel/03-network-configuration.ipynb new file mode 100644 index 00000000000..477148a7c33 --- /dev/null +++ b/notebooks/scenarios/reverse-tunnel/03-network-configuration.ipynb @@ -0,0 +1,37 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "6296126b-2575-44b4-9f09-2d73f444ca96", + "metadata": {}, + "outputs": [], + "source": [ + "# -- ingress\n", + "# -- open port for rathole server\n", + "# -- nodeport vs websockets" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/reverse-tunnel/04-setup-domain-with-tunnel.ipynb b/notebooks/scenarios/reverse-tunnel/04-setup-domain-with-tunnel.ipynb new file mode 100644 index 00000000000..decca53c96d --- /dev/null +++ b/notebooks/scenarios/reverse-tunnel/04-setup-domain-with-tunnel.ipynb @@ -0,0 +1,36 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "a2746b76-cc79-4853-8ef5-98c63b516eab", + "metadata": {}, + "outputs": [], + "source": [ + "# -- deploy local docker k3d\n", + "# -- enable rathole reverse tunnel" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/reverse-tunnel/05-connect-to-gateway-over-tunnel.ipynb b/notebooks/scenarios/reverse-tunnel/05-connect-to-gateway-over-tunnel.ipynb new file mode 100644 index 00000000000..c6391b82185 --- /dev/null +++ b/notebooks/scenarios/reverse-tunnel/05-connect-to-gateway-over-tunnel.ipynb @@ -0,0 +1,36 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "ad6b54cd-2898-4b67-af54-ddc7a13173e2", + "metadata": {}, + "outputs": [], + "source": [ + "# -- run connection request\n", + "# -- approve request" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/reverse-tunnel/06-proxy-clients.ipynb b/notebooks/scenarios/reverse-tunnel/06-proxy-clients.ipynb new file mode 100644 index 00000000000..536dce15404 --- /dev/null +++ b/notebooks/scenarios/reverse-tunnel/06-proxy-clients.ipynb @@ -0,0 +1,36 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "1c3868bb-a140-451a-ac51-a4fd17bf2ab8", + "metadata": {}, + "outputs": [], + "source": [ + "# -- how to list domains on gateway\n", + "# -- getting a proxy client" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/reverse-tunnel/07-blob-storage-streaming.ipynb b/notebooks/scenarios/reverse-tunnel/07-blob-storage-streaming.ipynb new file mode 100644 index 00000000000..b2d388246e9 --- /dev/null +++ b/notebooks/scenarios/reverse-tunnel/07-blob-storage-streaming.ipynb @@ -0,0 +1,35 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "2927dc60-0a36-4a47-8dac-98c8aca71cfe", + "metadata": {}, + "outputs": [], + "source": [ + "# -- checking upload and download to blob storage work" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/reverse-tunnel/08-debugging.ipynb b/notebooks/scenarios/reverse-tunnel/08-debugging.ipynb new file mode 100644 index 00000000000..24d912d0938 --- /dev/null +++ b/notebooks/scenarios/reverse-tunnel/08-debugging.ipynb @@ -0,0 +1,40 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "c3692642-b407-4471-9467-bcc9ca9bb7a9", + "metadata": {}, + "outputs": [], + "source": [ + "# -- include cleaned up diagram?\n", + "# -- tunnel config file and config maps\n", + "# -- determining ip addresses and testing ports are available\n", + "# -- running curl from inside the containers\n", + "# -- the internal host / header on the gateway\n", + "# -- checking logs" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/scenarios/reverse-tunnel/README.md b/notebooks/scenarios/reverse-tunnel/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/notebooks/tutorials/data-owner/03-messages-and-requests.ipynb b/notebooks/tutorials/data-owner/03-messages-and-requests.ipynb index 8e7a1618425..c05d6f7d556 100644 --- a/notebooks/tutorials/data-owner/03-messages-and-requests.ipynb +++ b/notebooks/tutorials/data-owner/03-messages-and-requests.ipynb @@ -360,67 +360,6 @@ "cell_type": "markdown", "id": "32", "metadata": {}, - "source": [ - "### Substituting" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "33", - "metadata": {}, - "outputs": [], - "source": [ - "mean_request = admin_client.requests[-2]\n", - "mean_request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "34", - "metadata": {}, - "outputs": [], - "source": [ - "admin_asset = admin_client.datasets[0].assets[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "35", - "metadata": {}, - "outputs": [], - "source": [ - "result = mean_request.code.unsafe_function(data=admin_asset)\n", - "result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "36", - "metadata": {}, - "outputs": [], - "source": [ - "mean_request.accept_by_depositing_result(result)\n", - "mean_request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "37", - "metadata": {}, - "outputs": [], - "source": [ - "admin_client.projects[0].requests" - ] - }, - { - "cell_type": "markdown", - "id": "38", - "metadata": {}, "source": [ "### Rejecting" ] @@ -428,7 +367,7 @@ { "cell_type": "code", "execution_count": null, - "id": "39", + "id": "33", "metadata": {}, "outputs": [], "source": [ @@ -439,7 +378,7 @@ { "cell_type": "code", "execution_count": null, - "id": "40", + "id": "34", "metadata": {}, "outputs": [], "source": [ @@ -449,7 +388,7 @@ { "cell_type": "code", "execution_count": null, - "id": "41", + "id": "35", "metadata": {}, "outputs": [], "source": [ @@ -459,20 +398,12 @@ { "cell_type": "code", "execution_count": null, - "id": "42", + "id": "36", "metadata": {}, "outputs": [], "source": [ "admin_client.projects[0]" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "43", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -491,7 +422,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.8" + "version": "3.10.13" }, "toc": { "base_numbering": 1, diff --git a/notebooks/tutorials/enclaves/Enclave-single-notebook-DO-DS.ipynb b/notebooks/tutorials/enclaves/Enclave-single-notebook-DO-DS.ipynb deleted file mode 100644 index 62d7fc48c99..00000000000 --- a/notebooks/tutorials/enclaves/Enclave-single-notebook-DO-DS.ipynb +++ /dev/null @@ -1,735 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "0", - "metadata": {}, - "outputs": [], - "source": [ - "# third party\n", - "from recordlinkage.datasets import load_febrl4\n", - "\n", - "# syft absolute\n", - "import syft as sy" - ] - }, - { - "cell_type": "markdown", - "id": "1", - "metadata": {}, - "source": [ - "# Create Nodes and connect to gateway" - ] - }, - { - "cell_type": "markdown", - "id": "2", - "metadata": {}, - "source": [ - "create enclave node" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "# Local Python Node\n", - "enclave_node = sy.orchestra.launch(\n", - " name=\"Enclave\",\n", - " node_type=sy.NodeType.ENCLAVE,\n", - " local_db=True,\n", - " dev_mode=True,\n", - " reset=True,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "from syft.abstract_node import NodeType\n", - "\n", - "assert enclave_node.python_node.node_type == NodeType.ENCLAVE" - ] - }, - { - "cell_type": "markdown", - "id": "5", - "metadata": {}, - "source": [ - "Create canada node & italy node" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "ca_node = sy.orchestra.launch(name=\"Canada\", local_db=True, reset=True, dev_mode=True)\n", - "it_node = sy.orchestra.launch(name=\"Italy\", local_db=True, reset=True, dev_mode=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [ - "assert ca_node.python_node.node_type == NodeType.DOMAIN\n", - "assert it_node.python_node.node_type == NodeType.DOMAIN" - ] - }, - { - "cell_type": "markdown", - "id": "8", - "metadata": {}, - "source": [ - "Create gateway Node" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9", - "metadata": {}, - "outputs": [], - "source": [ - "gateway_node = sy.orchestra.launch(\n", - " name=\"gateway\",\n", - " node_type=sy.NodeType.GATEWAY,\n", - " local_db=True,\n", - " reset=True,\n", - " dev_mode=True,\n", - " association_request_auto_approval=True,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "10", - "metadata": {}, - "source": [ - "Connect nodes to gateway" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11", - "metadata": {}, - "outputs": [], - "source": [ - "enclave_guest_client = enclave_node.client\n", - "ca_guest_client = ca_node.client\n", - "it_guest_client = it_node.client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "12", - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "from syft.client.domain_client import DomainClient\n", - "from syft.client.enclave_client import EnclaveClient\n", - "from syft.client.gateway_client import GatewayClient\n", - "\n", - "assert isinstance(enclave_guest_client, EnclaveClient)\n", - "assert isinstance(ca_guest_client, DomainClient)\n", - "assert isinstance(it_guest_client, DomainClient)\n", - "assert isinstance(gateway_node.client, GatewayClient)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "13", - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "# Connect enclave to gateway\n", - "from syft.service.response import SyftSuccess\n", - "\n", - "res = enclave_guest_client.connect_to_gateway(handle=gateway_node)\n", - "assert isinstance(res, SyftSuccess)\n", - "res" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "14", - "metadata": {}, - "outputs": [], - "source": [ - "# Connect Canada to gateway\n", - "res = ca_guest_client.connect_to_gateway(handle=gateway_node)\n", - "assert isinstance(res, SyftSuccess)\n", - "res" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "15", - "metadata": {}, - "outputs": [], - "source": [ - "# Connect Italy to gateway\n", - "res = it_guest_client.connect_to_gateway(handle=gateway_node)\n", - "assert isinstance(res, SyftSuccess)\n", - "res" - ] - }, - { - "cell_type": "markdown", - "id": "16", - "metadata": {}, - "source": [ - "# DOs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "17", - "metadata": {}, - "outputs": [], - "source": [ - "do_ca_client = ca_node.login(email=\"info@openmined.org\", password=\"changethis\")\n", - "do_it_client = it_node.login(email=\"info@openmined.org\", password=\"changethis\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "18", - "metadata": {}, - "outputs": [], - "source": [ - "# syft absolute\n", - "from syft.client.domain_client import DomainClient\n", - "\n", - "assert isinstance(do_ca_client, DomainClient)\n", - "assert isinstance(do_it_client, DomainClient)" - ] - }, - { - "cell_type": "markdown", - "id": "19", - "metadata": {}, - "source": [ - "## Upload dataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "20", - "metadata": {}, - "outputs": [], - "source": [ - "# Using public datasets from Freely Extensible Biomedical Record Linkage (Febrl)\n", - "canada_census_data, italy_census_data = load_febrl4()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "21", - "metadata": {}, - "outputs": [], - "source": [ - "for dataset, client, country in zip(\n", - " [canada_census_data, italy_census_data],\n", - " [do_ca_client, do_it_client],\n", - " [\"Canada\", \"Italy\"],\n", - "):\n", - " private_data, mock_data = dataset[:2500], dataset[2500:]\n", - " dataset = sy.Dataset(\n", - " name=f\"{country} - FEBrl Census Data\",\n", - " description=\"abc\",\n", - " asset_list=[\n", - " sy.Asset(\n", - " name=\"census_data\",\n", - " mock=mock_data,\n", - " data=private_data,\n", - " shape=private_data.shape,\n", - " mock_is_real=True,\n", - " )\n", - " ],\n", - " )\n", - " client.upload_dataset(dataset)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "22", - "metadata": {}, - "outputs": [], - "source": [ - "assert len(do_ca_client.datasets.get_all()) == 1\n", - "assert len(do_it_client.datasets.get_all()) == 1" - ] - }, - { - "cell_type": "markdown", - "id": "23", - "metadata": {}, - "source": [ - "## create accounts for DS" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24", - "metadata": {}, - "outputs": [], - "source": [ - "for client in [do_ca_client, do_it_client]:\n", - " res = client.register(\n", - " name=\"Sheldon\",\n", - " email=\"sheldon@caltech.edu\",\n", - " password=\"changethis\",\n", - " password_verify=\"changethis\",\n", - " )\n", - " assert isinstance(res, SyftSuccess)" - ] - }, - { - "cell_type": "markdown", - "id": "25", - "metadata": {}, - "source": [ - "# DS" - ] - }, - { - "cell_type": "markdown", - "id": "26", - "metadata": {}, - "source": [ - "## Login into gateway as guest" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "27", - "metadata": {}, - "outputs": [], - "source": [ - "ds_gateway_client = gateway_node.client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "28", - "metadata": {}, - "outputs": [], - "source": [ - "# Explore the domains and enclaves connected to the gateway\n", - "ds_gateway_client.domains" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "29", - "metadata": {}, - "outputs": [], - "source": [ - "# Log into canada as proxy_client\n", - "ds_ca_proxy_client = ds_gateway_client.domains[0]\n", - "ds_ca_proxy_client = ds_ca_proxy_client.login(\n", - " email=\"sheldon@caltech.edu\", password=\"changethis\"\n", - ")\n", - "assert ds_ca_proxy_client.name == \"Canada\"\n", - "assert ds_ca_proxy_client.connection.proxy_target_uid == do_ca_client.id\n", - "assert isinstance(ds_ca_proxy_client, DomainClient)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "30", - "metadata": {}, - "outputs": [], - "source": [ - "# Log into italy as proxy_client\n", - "ds_it_proxy_client = ds_gateway_client.domains[1]\n", - "ds_it_proxy_client = ds_it_proxy_client.login(\n", - " email=\"sheldon@caltech.edu\", password=\"changethis\"\n", - ")\n", - "assert ds_it_proxy_client.name == \"Italy\"\n", - "assert ds_it_proxy_client.connection.proxy_target_uid == do_it_client.id\n", - "assert isinstance(ds_it_proxy_client, DomainClient)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "31", - "metadata": {}, - "outputs": [], - "source": [ - "# Create an account and log into enclave as proxy client\n", - "ds_enclave_proxy_client = ds_gateway_client.enclaves[0]\n", - "ds_enclave_proxy_client = ds_enclave_proxy_client.login(\n", - " email=\"sheldon@caltech.edu\", password=\"changethis\", name=\"Sheldon\", register=True\n", - ")\n", - "assert ds_enclave_proxy_client.name == \"Enclave\"\n", - "assert ds_enclave_proxy_client.connection.proxy_target_uid == enclave_guest_client.id\n", - "assert isinstance(ds_enclave_proxy_client, EnclaveClient)" - ] - }, - { - "cell_type": "markdown", - "id": "32", - "metadata": {}, - "source": [ - "## Find datasets" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "33", - "metadata": {}, - "outputs": [], - "source": [ - "canada_census_data = ds_ca_proxy_client.datasets[-1].assets[0]\n", - "italy_census_data = ds_it_proxy_client.datasets[-1].assets[0]" - ] - }, - { - "cell_type": "markdown", - "id": "34", - "metadata": {}, - "source": [ - "## Create Request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "35", - "metadata": {}, - "outputs": [], - "source": [ - "@sy.syft_function_single_use(\n", - " canada_census_data=canada_census_data,\n", - " italy_census_data=italy_census_data,\n", - " share_results_with_owners=True,\n", - ")\n", - "def compute_census_matches(canada_census_data, italy_census_data):\n", - " # third party\n", - " import recordlinkage\n", - "\n", - " # Index step\n", - " indexer = recordlinkage.Index()\n", - " indexer.block(\"given_name\")\n", - "\n", - " candidate_links = indexer.index(canada_census_data, italy_census_data)\n", - "\n", - " # Comparison step\n", - " compare_cl = recordlinkage.Compare()\n", - "\n", - " compare_cl.exact(\"given_name\", \"given_name\", label=\"given_name\")\n", - " compare_cl.string(\n", - " \"surname\", \"surname\", method=\"jarowinkler\", threshold=0.85, label=\"surname\"\n", - " )\n", - " compare_cl.exact(\"date_of_birth\", \"date_of_birth\", label=\"date_of_birth\")\n", - " compare_cl.exact(\"suburb\", \"suburb\", label=\"suburb\")\n", - " compare_cl.exact(\"state\", \"state\", label=\"state\")\n", - " compare_cl.string(\"address_1\", \"address_1\", threshold=0.85, label=\"address_1\")\n", - "\n", - " features = compare_cl.compute(\n", - " candidate_links, canada_census_data, italy_census_data\n", - " )\n", - "\n", - " # Classification step\n", - " matches = features[features.sum(axis=1) > 3]\n", - "\n", - " return len(matches)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "36", - "metadata": {}, - "outputs": [], - "source": [ - "# Check result of mock data execution\n", - "mock_result = compute_census_matches(\n", - " canada_census_data=canada_census_data.mock,\n", - " italy_census_data=italy_census_data.mock,\n", - ")\n", - "mock_result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "37", - "metadata": {}, - "outputs": [], - "source": [ - "req = ds_enclave_proxy_client.request_code_execution(compute_census_matches)\n", - "req" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "38", - "metadata": {}, - "outputs": [], - "source": [ - "assert isinstance(req, sy.service.request.request.Request)" - ] - }, - { - "cell_type": "markdown", - "id": "39", - "metadata": {}, - "source": [ - "# DOs" - ] - }, - { - "cell_type": "markdown", - "id": "40", - "metadata": {}, - "source": [ - "## Approve" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "41", - "metadata": {}, - "outputs": [], - "source": [ - "for client in [do_ca_client, do_it_client]:\n", - " res = client.requests[-1].approve()\n", - " assert isinstance(res, SyftSuccess)" - ] - }, - { - "cell_type": "markdown", - "id": "42", - "metadata": {}, - "source": [ - "# DS" - ] - }, - { - "cell_type": "markdown", - "id": "43", - "metadata": {}, - "source": [ - "## Get result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "44", - "metadata": {}, - "outputs": [], - "source": [ - "status = ds_enclave_proxy_client.code.get_all()[-1].status\n", - "status" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "45", - "metadata": {}, - "outputs": [], - "source": [ - "for st, _ in status.status_dict.values():\n", - " assert st == sy.service.request.request.UserCodeStatus.APPROVED" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "46", - "metadata": {}, - "outputs": [], - "source": [ - "ds_enclave_proxy_client.code[-1].output_policy" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "47", - "metadata": {}, - "outputs": [], - "source": [ - "result_pointer = ds_enclave_proxy_client.code.compute_census_matches(\n", - " canada_census_data=canada_census_data, italy_census_data=italy_census_data\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "48", - "metadata": {}, - "outputs": [], - "source": [ - "result_pointer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "49", - "metadata": {}, - "outputs": [], - "source": [ - "result_pointer.syft_action_data == 858" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "50", - "metadata": {}, - "outputs": [], - "source": [ - "real_result = result_pointer.get()\n", - "real_result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "51", - "metadata": {}, - "outputs": [], - "source": [ - "assert real_result == 813" - ] - }, - { - "cell_type": "markdown", - "id": "52", - "metadata": {}, - "source": [ - "# DO" - ] - }, - { - "cell_type": "markdown", - "id": "53", - "metadata": {}, - "source": [ - "## Can also get the result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "54", - "metadata": {}, - "outputs": [], - "source": [ - "request = do_ca_client.requests[0]\n", - "request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "55", - "metadata": {}, - "outputs": [], - "source": [ - "result_ptr = request.get_results()\n", - "result_ptr" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "56", - "metadata": {}, - "outputs": [], - "source": [ - "assert result_ptr.syft_action_data == 813" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "57", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.16" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": { - "height": "calc(100% - 180px)", - "left": "10px", - "top": "150px", - "width": "358.398px" - }, - "toc_section_display": true, - "toc_window_display": true - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/tutorials/enclaves/Enclave-single-notebook-high-low-network.ipynb b/notebooks/tutorials/enclaves/Enclave-single-notebook-high-low-network.ipynb deleted file mode 100644 index 771f0ad4389..00000000000 --- a/notebooks/tutorials/enclaves/Enclave-single-notebook-high-low-network.ipynb +++ /dev/null @@ -1,1087 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "0", - "metadata": {}, - "outputs": [], - "source": [ - "# third party\n", - "from recordlinkage.datasets import load_febrl4\n", - "\n", - "# syft absolute\n", - "import syft as sy" - ] - }, - { - "cell_type": "markdown", - "id": "1", - "metadata": {}, - "source": [ - "# Create Nodes" - ] - }, - { - "cell_type": "markdown", - "id": "2", - "metadata": {}, - "source": [ - "## Staging Low side" - ] - }, - { - "cell_type": "markdown", - "id": "3", - "metadata": {}, - "source": [ - "create enclave node" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "embassador_node_low = sy.orchestra.launch(\n", - " name=\"ambassador node\",\n", - " node_side_type=\"low\",\n", - " local_db=True,\n", - " reset=True,\n", - " # enable_warnings=True,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "5", - "metadata": {}, - "source": [ - "Create canada node & italy node" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "ca_node_low = sy.orchestra.launch(\n", - " name=\"canada-1\",\n", - " node_side_type=\"low\",\n", - " local_db=True,\n", - " reset=True,\n", - " # enable_warnings=True,\n", - ")\n", - "it_node_low = sy.orchestra.launch(\n", - " name=\"italy-1\",\n", - " node_side_type=\"low\",\n", - " local_db=True,\n", - " reset=True,\n", - " # enable_warnings=True,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [ - "gateway_node_low = sy.orchestra.launch(\n", - " name=\"gateway-1\",\n", - " node_type=\"gateway\",\n", - " node_side_type=\"low\",\n", - " local_db=True,\n", - " reset=True,\n", - " dev_mode=True,\n", - " association_request_auto_approval=True,\n", - " # enable_warnings=True\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "8", - "metadata": {}, - "source": [ - "## High side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9", - "metadata": {}, - "outputs": [], - "source": [ - "enclave_node_high = sy.orchestra.launch(\n", - " name=\"enclave node\",\n", - " node_type=\"enclave\",\n", - " reset=True,\n", - " # enable_warnings=True,\n", - ")\n", - "ca_node_high = sy.orchestra.launch(\n", - " name=\"canada-2\",\n", - " local_db=True,\n", - " reset=True,\n", - " # enable_warnings=True,\n", - ")\n", - "it_node_high = sy.orchestra.launch(\n", - " name=\"italy-2\",\n", - " local_db=True,\n", - " reset=True,\n", - " # enable_warnings=True,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "10", - "metadata": {}, - "outputs": [], - "source": [ - "gateway_node_high = sy.orchestra.launch(\n", - " name=\"gateway-2\",\n", - " node_type=\"gateway\",\n", - " local_db=True,\n", - " reset=True,\n", - " dev_mode=True,\n", - " association_request_auto_approval=True,\n", - " # enable_warnings=True,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "11", - "metadata": {}, - "source": [ - "# DOs" - ] - }, - { - "cell_type": "markdown", - "id": "12", - "metadata": {}, - "source": [ - "## Login" - ] - }, - { - "cell_type": "markdown", - "id": "13", - "metadata": {}, - "source": [ - "### Staging Low side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "14", - "metadata": {}, - "outputs": [], - "source": [ - "do_ca_client_low = ca_node_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", - "do_it_client_low = it_node_low.login(email=\"info@openmined.org\", password=\"changethis\")\n", - "embassador_client_low = embassador_node_low.login(\n", - " email=\"info@openmined.org\", password=\"changethis\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "15", - "metadata": {}, - "source": [ - "### Production High side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "16", - "metadata": {}, - "outputs": [], - "source": [ - "do_ca_client_high = ca_node_high.login(\n", - " email=\"info@openmined.org\", password=\"changethis\"\n", - ")\n", - "do_it_client_high = it_node_high.login(\n", - " email=\"info@openmined.org\", password=\"changethis\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "17", - "metadata": {}, - "source": [ - "## Connect to network" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "18", - "metadata": {}, - "outputs": [], - "source": [ - "# TODO: add security layer here" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "19", - "metadata": {}, - "outputs": [], - "source": [ - "enclave_client_high = enclave_node_high.client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "20", - "metadata": {}, - "outputs": [], - "source": [ - "# gateway_root_client.register(name=\"\", email=\"info@openmined.org\", password=\"changethis\")\n", - "# gateway_root_client.register(name=\"\", email=\"info@openmined.org\", password=\"changethis\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "21", - "metadata": {}, - "outputs": [], - "source": [ - "res = do_ca_client_low.connect_to_gateway(\n", - " handle=gateway_node_low\n", - ") # add credentials here\n", - "res" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "22", - "metadata": {}, - "outputs": [], - "source": [ - "res = do_it_client_low.connect_to_gateway(\n", - " handle=gateway_node_low\n", - ") # add credentials here\n", - "res" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "23", - "metadata": {}, - "outputs": [], - "source": [ - "res = do_ca_client_high.connect_to_gateway(handle=gateway_node_high)\n", - "res = do_it_client_high.connect_to_gateway(handle=gateway_node_high)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24", - "metadata": {}, - "outputs": [], - "source": [ - "## Also for ambassador" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "25", - "metadata": {}, - "outputs": [], - "source": [ - "# TODO: who is going to be responsible for connecting the enclave to the gateway\n", - "res = enclave_client_high.connect_to_gateway(handle=gateway_node_high)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "26", - "metadata": {}, - "outputs": [], - "source": [ - "res = embassador_client_low.connect_to_gateway(\n", - " handle=gateway_node_low\n", - ") # add credentials here" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "27", - "metadata": {}, - "outputs": [], - "source": [ - "## Upload dataset" - ] - }, - { - "cell_type": "markdown", - "id": "28", - "metadata": {}, - "source": [ - "### Staging Low side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "29", - "metadata": {}, - "outputs": [], - "source": [ - "# Using public datasets from Freely Extensible Biomedical Record Linkage (Febrl)\n", - "canada_census_data_low, italy_census_data_low = load_febrl4()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "30", - "metadata": {}, - "outputs": [], - "source": [ - "for dataset, client, country in zip(\n", - " [canada_census_data_low, italy_census_data_low],\n", - " [do_ca_client_low, do_it_client_low],\n", - " [\"Canada\", \"Italy\"],\n", - "):\n", - " private_data, mock_data = dataset[:2500], dataset[2500:]\n", - " dataset = sy.Dataset(\n", - " name=f\"{country} - FEBrl Census Data\",\n", - " description=\"abc\",\n", - " asset_list=[\n", - " sy.Asset(\n", - " name=\"census_data\",\n", - " mock=mock_data,\n", - " data=private_data,\n", - " shape=private_data.shape,\n", - " mock_is_real=True,\n", - " )\n", - " ],\n", - " )\n", - " client.upload_dataset(dataset)" - ] - }, - { - "cell_type": "markdown", - "id": "31", - "metadata": {}, - "source": [ - "### Production High side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "32", - "metadata": {}, - "outputs": [], - "source": [ - "# Using public datasets from Freely Extensible Biomedical Record Linkage (Febrl)\n", - "canada_census_data_high, italy_census_data_high = load_febrl4()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "33", - "metadata": {}, - "outputs": [], - "source": [ - "for dataset, client, country in zip(\n", - " [canada_census_data_high, italy_census_data_high],\n", - " [do_ca_client_high, do_it_client_high],\n", - " [\"Canada\", \"Italy\"],\n", - "):\n", - " private_data, mock_data = dataset[:2500], dataset[2500:]\n", - " dataset = sy.Dataset(\n", - " name=f\"{country} - FEBrl Census Data\",\n", - " description=\"abc\",\n", - " asset_list=[\n", - " sy.Asset(\n", - " name=\"census_data\",\n", - " mock=mock_data,\n", - " data=private_data,\n", - " shape=private_data.shape,\n", - " mock_is_real=True,\n", - " )\n", - " ],\n", - " )\n", - " client.upload_dataset(dataset)" - ] - }, - { - "cell_type": "markdown", - "id": "34", - "metadata": {}, - "source": [ - "## create accounts for DS" - ] - }, - { - "cell_type": "markdown", - "id": "35", - "metadata": {}, - "source": [ - "### Staging Low side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "36", - "metadata": {}, - "outputs": [], - "source": [ - "for client in [do_ca_client_low, do_it_client_low]:\n", - " client.register(\n", - " name=\"Sheldon\",\n", - " email=\"sheldon@caltech.edu\",\n", - " password=\"changethis\",\n", - " password_verify=\"changethis\",\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "37", - "metadata": {}, - "outputs": [], - "source": [ - "embassador_client_low.register(\n", - " name=\"Sheldon\",\n", - " email=\"sheldon@caltech.edu\",\n", - " password=\"changethis\",\n", - " password_verify=\"changethis\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "38", - "metadata": {}, - "source": [ - "## Create account for embassador" - ] - }, - { - "cell_type": "markdown", - "id": "39", - "metadata": {}, - "source": [ - "### Production High Side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "40", - "metadata": {}, - "outputs": [], - "source": [ - "for client in [do_ca_client_high, do_it_client_high]:\n", - " client.register(\n", - " name=\"Sheldon\",\n", - " email=\"sheldon@caltech.edu\",\n", - " password=\"changethis\",\n", - " password_verify=\"changethis\",\n", - " )" - ] - }, - { - "cell_type": "markdown", - "id": "41", - "metadata": {}, - "source": [ - "# DS Low Side" - ] - }, - { - "cell_type": "markdown", - "id": "42", - "metadata": {}, - "source": [ - "## DS Get proxy clients" - ] - }, - { - "cell_type": "markdown", - "id": "43", - "metadata": {}, - "source": [ - "### Staging Low side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "44", - "metadata": {}, - "outputs": [], - "source": [ - "ds_gateway_client_low = gateway_node_low.client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "45", - "metadata": {}, - "outputs": [], - "source": [ - "assert len(ds_gateway_client_low.domains) == 3\n", - "ds_gateway_client_low.domains" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "46", - "metadata": {}, - "outputs": [], - "source": [ - "ds_ca_proxy_client_low = ds_gateway_client_low.domains[1].login(\n", - " email=\"sheldon@caltech.edu\", password=\"changethis\"\n", - ")\n", - "ds_it_proxy_client_low = ds_gateway_client_low.domains[2].login(\n", - " email=\"sheldon@caltech.edu\", password=\"changethis\"\n", - ")\n", - "ds_amb_proxy_client_low = ds_gateway_client_low.domains[0].login(\n", - " email=\"sheldon@caltech.edu\", password=\"changethis\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "47", - "metadata": {}, - "source": [ - "## Find datasets" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "48", - "metadata": {}, - "outputs": [], - "source": [ - "canada_census_data = ds_ca_proxy_client_low.datasets[-1].assets[0]\n", - "italy_census_data = ds_it_proxy_client_low.datasets[-1].assets[0]" - ] - }, - { - "cell_type": "markdown", - "id": "49", - "metadata": {}, - "source": [ - "## Create Request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "50", - "metadata": {}, - "outputs": [], - "source": [ - "@sy.syft_function_single_use(\n", - " canada_census_data=canada_census_data, italy_census_data=italy_census_data\n", - ")\n", - "def compute_census_matches(canada_census_data, italy_census_data):\n", - " # third party\n", - " import recordlinkage\n", - "\n", - " # Index step\n", - " indexer = recordlinkage.Index()\n", - " indexer.block(\"given_name\")\n", - "\n", - " candidate_links = indexer.index(canada_census_data, italy_census_data)\n", - "\n", - " # Comparison step\n", - " compare_cl = recordlinkage.Compare()\n", - "\n", - " compare_cl.exact(\"given_name\", \"given_name\", label=\"given_name\")\n", - " compare_cl.string(\n", - " \"surname\", \"surname\", method=\"jarowinkler\", threshold=0.85, label=\"surname\"\n", - " )\n", - " compare_cl.exact(\"date_of_birth\", \"date_of_birth\", label=\"date_of_birth\")\n", - " compare_cl.exact(\"suburb\", \"suburb\", label=\"suburb\")\n", - " compare_cl.exact(\"state\", \"state\", label=\"state\")\n", - " compare_cl.string(\"address_1\", \"address_1\", threshold=0.85, label=\"address_1\")\n", - "\n", - " features = compare_cl.compute(\n", - " candidate_links, canada_census_data, italy_census_data\n", - " )\n", - "\n", - " # Classification step\n", - " matches = features[features.sum(axis=1) > 3]\n", - "\n", - " return len(matches)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "51", - "metadata": {}, - "outputs": [], - "source": [ - "# Checking result of mock data execution\n", - "mock_result = compute_census_matches(\n", - " canada_census_data=canada_census_data.mock,\n", - " italy_census_data=italy_census_data.mock,\n", - ")\n", - "mock_result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "52", - "metadata": {}, - "outputs": [], - "source": [ - "ds_amb_proxy_client_low.code.request_code_execution(compute_census_matches)" - ] - }, - { - "cell_type": "markdown", - "id": "53", - "metadata": {}, - "source": [ - "# Ambassador flow" - ] - }, - { - "cell_type": "markdown", - "id": "54", - "metadata": {}, - "source": [ - "## Check Code Staging Low Side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "55", - "metadata": {}, - "outputs": [], - "source": [ - "embassador_client_low.requests[0].code" - ] - }, - { - "cell_type": "markdown", - "id": "56", - "metadata": {}, - "source": [ - "## Login to Production High Side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "57", - "metadata": {}, - "outputs": [], - "source": [ - "amb_gateway_client_high = gateway_node_high.client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "58", - "metadata": {}, - "outputs": [], - "source": [ - "assert len(amb_gateway_client_high.domains) == 2\n", - "amb_gateway_client_high.domains" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "59", - "metadata": {}, - "outputs": [], - "source": [ - "amb_ca_proxy_client_high = amb_gateway_client_high.domains[1].login(\n", - " email=\"sheldon@caltech.edu\", password=\"changethis\"\n", - ")\n", - "amb_it_proxy_client_high = amb_gateway_client_high.domains[0].login(\n", - " email=\"sheldon@caltech.edu\", password=\"changethis\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "60", - "metadata": {}, - "outputs": [], - "source": [ - "assert len(amb_gateway_client_high.enclaves) == 1" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "61", - "metadata": {}, - "outputs": [], - "source": [ - "amb_ca_proxy_client_high" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "62", - "metadata": {}, - "outputs": [], - "source": [ - "amb_enclave_proxy_client_high = amb_gateway_client_high.enclaves[0].login(\n", - " name=\"Sheldon\", email=\"sheldon@caltech.edu\", password=\"changethis\", register=True\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "63", - "metadata": {}, - "outputs": [], - "source": [ - "# # this also creates a guest client\n", - "# embassador_client_high = enclave_node_high.login(email=\"info@openmined.org\", password=\"changethis\",\n", - "# name=\"Signor Ambassador\", register=True)" - ] - }, - { - "cell_type": "markdown", - "id": "64", - "metadata": {}, - "source": [ - "## Find Datasets Production High side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "65", - "metadata": {}, - "outputs": [], - "source": [ - "canada_census_data_high = amb_ca_proxy_client_high.datasets[-1].assets[0]\n", - "italy_census_data_high = amb_it_proxy_client_high.datasets[-1].assets[0]" - ] - }, - { - "cell_type": "markdown", - "id": "66", - "metadata": {}, - "source": [ - "Copy code from the request" - ] - }, - { - "cell_type": "markdown", - "id": "67", - "metadata": {}, - "source": [ - "## Submit code Production High side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "68", - "metadata": {}, - "outputs": [], - "source": [ - "@sy.syft_function_single_use(\n", - " canada_census_data=canada_census_data_high, italy_census_data=italy_census_data_high\n", - ")\n", - "def compute_census_matches_high(canada_census_data, italy_census_data):\n", - " # third party\n", - " import recordlinkage\n", - "\n", - " # Index step\n", - " indexer = recordlinkage.Index()\n", - " indexer.block(\"given_name\")\n", - "\n", - " candidate_links = indexer.index(canada_census_data, italy_census_data)\n", - "\n", - " # Comparison step\n", - " compare_cl = recordlinkage.Compare()\n", - "\n", - " compare_cl.exact(\"given_name\", \"given_name\", label=\"given_name\")\n", - " compare_cl.string(\n", - " \"surname\", \"surname\", method=\"jarowinkler\", threshold=0.85, label=\"surname\"\n", - " )\n", - " compare_cl.exact(\"date_of_birth\", \"date_of_birth\", label=\"date_of_birth\")\n", - " compare_cl.exact(\"suburb\", \"suburb\", label=\"suburb\")\n", - " compare_cl.exact(\"state\", \"state\", label=\"state\")\n", - " compare_cl.string(\"address_1\", \"address_1\", threshold=0.85, label=\"address_1\")\n", - "\n", - " features = compare_cl.compute(\n", - " candidate_links, canada_census_data, italy_census_data\n", - " )\n", - "\n", - " # Classification step\n", - " matches = features[features.sum(axis=1) > 3]\n", - "\n", - " return len(matches)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "69", - "metadata": {}, - "outputs": [], - "source": [ - "# Checking result of mock data execution\n", - "mock_result = compute_census_matches_high(\n", - " canada_census_data=canada_census_data_high.mock,\n", - " italy_census_data=italy_census_data_high.mock,\n", - ")\n", - "mock_result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "70", - "metadata": {}, - "outputs": [], - "source": [ - "# note that this is not embassador_client_high.**code**.request_code_execution\n", - "amb_enclave_proxy_client_high.request_code_execution(compute_census_matches_high)" - ] - }, - { - "cell_type": "markdown", - "id": "71", - "metadata": {}, - "source": [ - "## DOs Approve Production High Side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "72", - "metadata": {}, - "outputs": [], - "source": [ - "do_ca_client_high.requests[0].approve()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "73", - "metadata": {}, - "outputs": [], - "source": [ - "do_it_client_high.requests[0].approve()" - ] - }, - { - "cell_type": "markdown", - "id": "74", - "metadata": {}, - "source": [ - "## Embassdor gets result from Production High Side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "75", - "metadata": {}, - "outputs": [], - "source": [ - "amb_enclave_proxy_client_high.code[-1].status" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "76", - "metadata": {}, - "outputs": [], - "source": [ - "result_pointer = amb_enclave_proxy_client_high.code.compute_census_matches_high(\n", - " canada_census_data=canada_census_data_high,\n", - " italy_census_data=italy_census_data_high,\n", - ")\n", - "\n", - "result_pointer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "77", - "metadata": {}, - "outputs": [], - "source": [ - "real_result = result_pointer.get()\n", - "real_result" - ] - }, - { - "cell_type": "markdown", - "id": "78", - "metadata": {}, - "source": [ - "## Ambassador Deposits Result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "79", - "metadata": {}, - "outputs": [], - "source": [ - "embassador_client_low.requests[0].accept_by_depositing_result(real_result)" - ] - }, - { - "cell_type": "markdown", - "id": "80", - "metadata": {}, - "source": [ - "# DS" - ] - }, - { - "cell_type": "markdown", - "id": "81", - "metadata": {}, - "source": [ - "## Get result from Staging Low Side" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "82", - "metadata": {}, - "outputs": [], - "source": [ - "ds_amb_proxy_client_low.code[-1].status" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "83", - "metadata": {}, - "outputs": [], - "source": [ - "result_pointer = ds_amb_proxy_client_low.code.compute_census_matches(\n", - " canada_census_data=canada_census_data,\n", - " italy_census_data=italy_census_data,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "84", - "metadata": {}, - "outputs": [], - "source": [ - "result_pointer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "85", - "metadata": {}, - "outputs": [], - "source": [ - "real_result = result_pointer.get()\n", - "real_result" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "86", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": { - "height": "calc(100% - 180px)", - "left": "10px", - "top": "150px", - "width": "358.391px" - }, - "toc_section_display": true, - "toc_window_display": true - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/tutorials/hello-syft/01-hello-syft.ipynb b/notebooks/tutorials/hello-syft/01-hello-syft.ipynb index 12f01679e3c..858b1042e69 100644 --- a/notebooks/tutorials/hello-syft/01-hello-syft.ipynb +++ b/notebooks/tutorials/hello-syft/01-hello-syft.ipynb @@ -432,7 +432,7 @@ "id": "39", "metadata": {}, "source": [ - "### Share the real result with the Data Scientist" + "### Approving the request" ] }, { @@ -442,9 +442,9 @@ "metadata": {}, "outputs": [], "source": [ - "result = request.accept_by_depositing_result(real_result)\n", - "print(result)\n", - "assert isinstance(result, sy.SyftSuccess)" + "result = request.approve()\n", + "assert isinstance(result, sy.SyftSuccess)\n", + "result" ] }, { @@ -454,7 +454,7 @@ "source": [ "## Data Scientist - Part 2\n", "\n", - "### Fetch Real Result" + "### Computing the Real Result" ] }, { @@ -580,14 +580,6 @@ "source": [ "node.land()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "54", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -606,7 +598,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.2" + "version": "3.10.13" }, "toc": { "base_numbering": 1, diff --git a/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb b/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb index 036c21f9ed6..eb0d3df04a9 100644 --- a/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb +++ b/notebooks/tutorials/model-auditing/colab/01-user-log.ipynb @@ -572,7 +572,7 @@ "id": "53", "metadata": {}, "source": [ - "Once the model owner feels confident that this code is not malicious, we can run the function on the real data." + "Once the model owner feels confident that this code is not malicious, we can run the function on the real data to inspect the result." ] }, { @@ -601,7 +601,7 @@ "id": "56", "metadata": {}, "source": [ - "This gives us a result which we can attach to the request" + "If everything looks good, we can approve the request" ] }, { @@ -611,7 +611,7 @@ "metadata": {}, "outputs": [], "source": [ - "request.accept_by_depositing_result(real_result)" + "request.approve()" ] }, { @@ -619,7 +619,7 @@ "id": "58", "metadata": {}, "source": [ - "## Auditor Receives Final Results" + "## Auditor computes Final Results" ] }, { @@ -661,14 +661,6 @@ " \n", "" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "63", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -687,7 +679,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.10.13" }, "toc": { "base_numbering": 1, diff --git a/notebooks/tutorials/model-training/02-data-owner-review-approve-code.ipynb b/notebooks/tutorials/model-training/02-data-owner-review-approve-code.ipynb index 841c39c021b..a919a9e2a8e 100644 --- a/notebooks/tutorials/model-training/02-data-owner-review-approve-code.ipynb +++ b/notebooks/tutorials/model-training/02-data-owner-review-approve-code.ipynb @@ -199,7 +199,7 @@ "id": "18", "metadata": {}, "source": [ - "## 2. DO runs the submitted code on private data, then deposits the results to the domain so the DS can retrieve them" + "## 2. DO runs the submitted code on private data, then approves the request so the DS can execute the function" ] }, { @@ -257,7 +257,7 @@ "metadata": {}, "outputs": [], "source": [ - "res = request.accept_by_depositing_result((train_accs, params))" + "res = request.approve()" ] }, { @@ -278,14 +278,6 @@ "source": [ "### 📓 Now switch to the [second DS's notebook](./03-data-scientist-download-results.ipynb)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "26", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -304,7 +296,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.2" + "version": "3.10.13" }, "toc": { "base_numbering": 1, diff --git a/notebooks/tutorials/pandas-cookbook/01-reading-from-a-csv.ipynb b/notebooks/tutorials/pandas-cookbook/01-reading-from-a-csv.ipynb index d5cdc94cc9d..5026c62450f 100644 --- a/notebooks/tutorials/pandas-cookbook/01-reading-from-a-csv.ipynb +++ b/notebooks/tutorials/pandas-cookbook/01-reading-from-a-csv.ipynb @@ -612,7 +612,7 @@ "id": "50", "metadata": {}, "source": [ - "# Data owner: execute function" + "# Data owner: approve request" ] }, { @@ -688,38 +688,33 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "id": "57", - "metadata": { - "tags": [] - }, - "outputs": [], + "metadata": {}, "source": [ - "request = project_notification.link.events[0].request" + "### Review and approve request" ] }, { "cell_type": "code", "execution_count": null, "id": "58", - "metadata": {}, + "metadata": { + "tags": [] + }, "outputs": [], "source": [ - "func = request.code" + "request = project_notification.link.events[0].request" ] }, { "cell_type": "code", "execution_count": null, "id": "59", - "metadata": { - "tags": [] - }, + "metadata": {}, "outputs": [], "source": [ - "# func = request.code\n", - "#" + "func = request.code" ] }, { @@ -791,7 +786,7 @@ }, "outputs": [], "source": [ - "result = request.accept_by_depositing_result(real_result)\n", + "result = request.approve()\n", "assert isinstance(result, sy.SyftSuccess)" ] }, @@ -800,7 +795,7 @@ "id": "66", "metadata": {}, "source": [ - "# Data scientist: fetch result" + "# Data scientist: compute result" ] }, { @@ -851,26 +846,6 @@ "real_result = result_ptr.get()\n", "real_result.plot()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "71", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "node.land()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "72", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -889,7 +864,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.10.13" }, "toc": { "base_numbering": 1, diff --git a/notebooks/tutorials/pandas-cookbook/02-selecting-data-finding-common-complain.ipynb b/notebooks/tutorials/pandas-cookbook/02-selecting-data-finding-common-complain.ipynb index 09e1e25b8dc..97482b84975 100644 --- a/notebooks/tutorials/pandas-cookbook/02-selecting-data-finding-common-complain.ipynb +++ b/notebooks/tutorials/pandas-cookbook/02-selecting-data-finding-common-complain.ipynb @@ -818,7 +818,7 @@ "id": "66", "metadata": {}, "source": [ - "# Data owner: execute function" + "# Data owner: approve request" ] }, { @@ -961,7 +961,7 @@ }, "outputs": [], "source": [ - "result = request.accept_by_depositing_result(real_result)\n", + "result = request.approve()\n", "assert isinstance(result, sy.SyftSuccess)" ] }, @@ -970,7 +970,7 @@ "id": "79", "metadata": {}, "source": [ - "# Data scientist: fetch result" + "# Data scientist: compute result" ] }, { @@ -1040,7 +1040,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.10.13" }, "toc": { "base_numbering": 1, diff --git a/notebooks/tutorials/pandas-cookbook/03-which-borough-has-the-most-noise-complaints.ipynb b/notebooks/tutorials/pandas-cookbook/03-which-borough-has-the-most-noise-complaints.ipynb index 51443872eb7..152da5c6c45 100644 --- a/notebooks/tutorials/pandas-cookbook/03-which-borough-has-the-most-noise-complaints.ipynb +++ b/notebooks/tutorials/pandas-cookbook/03-which-borough-has-the-most-noise-complaints.ipynb @@ -21,6 +21,9 @@ "execution_count": null, "id": "2", "metadata": { + "jupyter": { + "source_hidden": true + }, "tags": [] }, "outputs": [], @@ -932,7 +935,7 @@ "id": "75", "metadata": {}, "source": [ - "# Data owner: execute function" + "# Data owner: approve request" ] }, { @@ -1075,7 +1078,7 @@ }, "outputs": [], "source": [ - "result = request.accept_by_depositing_result(real_result)\n", + "result = request.approve()\n", "assert isinstance(result, sy.SyftSuccess)" ] }, @@ -1084,7 +1087,7 @@ "id": "88", "metadata": {}, "source": [ - "# Data scientist: fetch result" + "# Data scientist: compute result" ] }, { @@ -1154,7 +1157,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.10.13" }, "toc": { "base_numbering": 1, diff --git a/notebooks/tutorials/pandas-cookbook/04-weekday-bike-most-groupby-aggregate.ipynb b/notebooks/tutorials/pandas-cookbook/04-weekday-bike-most-groupby-aggregate.ipynb index 29878fd826c..2ba4b0cfe7c 100644 --- a/notebooks/tutorials/pandas-cookbook/04-weekday-bike-most-groupby-aggregate.ipynb +++ b/notebooks/tutorials/pandas-cookbook/04-weekday-bike-most-groupby-aggregate.ipynb @@ -692,7 +692,7 @@ "id": "57", "metadata": {}, "source": [ - "# Data owner: execute syft_function" + "# Data owner: approve request" ] }, { @@ -847,7 +847,7 @@ }, "outputs": [], "source": [ - "result = request.accept_by_depositing_result(real_result)\n", + "result = request.approve()\n", "assert isinstance(result, sy.SyftSuccess)" ] }, @@ -856,7 +856,7 @@ "id": "71", "metadata": {}, "source": [ - "# Data scientist: fetch result" + "# Data scientist: compute result" ] }, { @@ -926,7 +926,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.10.13" }, "toc": { "base_numbering": 1, diff --git a/notebooks/tutorials/pandas-cookbook/05-combining-dataframes-scraping-weather-data.ipynb b/notebooks/tutorials/pandas-cookbook/05-combining-dataframes-scraping-weather-data.ipynb index 9afc01da2ec..56718085058 100644 --- a/notebooks/tutorials/pandas-cookbook/05-combining-dataframes-scraping-weather-data.ipynb +++ b/notebooks/tutorials/pandas-cookbook/05-combining-dataframes-scraping-weather-data.ipynb @@ -879,7 +879,7 @@ "id": "72", "metadata": {}, "source": [ - "# Data owner: execute syft function" + "# Data owner: approve request" ] }, { @@ -1049,7 +1049,7 @@ }, "outputs": [], "source": [ - "result = request.accept_by_depositing_result(real_result)\n", + "result = request.approve()\n", "assert isinstance(result, sy.SyftSuccess)" ] }, @@ -1058,7 +1058,7 @@ "id": "87", "metadata": {}, "source": [ - "# Data scientist: fetch result" + "# Data scientist: compute result" ] }, { @@ -1195,7 +1195,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.10.13" }, "toc": { "base_numbering": 1, diff --git a/notebooks/tutorials/pandas-cookbook/06-string-operations-which-month-was-the-snowiest.ipynb b/notebooks/tutorials/pandas-cookbook/06-string-operations-which-month-was-the-snowiest.ipynb index 3544f6b82f4..cbdb061df8d 100644 --- a/notebooks/tutorials/pandas-cookbook/06-string-operations-which-month-was-the-snowiest.ipynb +++ b/notebooks/tutorials/pandas-cookbook/06-string-operations-which-month-was-the-snowiest.ipynb @@ -781,7 +781,7 @@ "id": "63", "metadata": {}, "source": [ - "# Data owner: execute syft_function" + "# Data owner: approve request" ] }, { @@ -937,7 +937,7 @@ }, "outputs": [], "source": [ - "result = request.accept_by_depositing_result(real_result)\n", + "result = request.approve()\n", "assert isinstance(result, sy.SyftSuccess)" ] }, @@ -946,7 +946,7 @@ "id": "77", "metadata": {}, "source": [ - "# Data scientist: fetch result" + "# Data scientist: compute result" ] }, { @@ -1092,7 +1092,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.8" + "version": "3.10.13" }, "toc": { "base_numbering": 1, diff --git a/notebooks/tutorials/pandas-cookbook/07-cleaning-up-messy-data.ipynb b/notebooks/tutorials/pandas-cookbook/07-cleaning-up-messy-data.ipynb index f64f8728793..801509179d4 100644 --- a/notebooks/tutorials/pandas-cookbook/07-cleaning-up-messy-data.ipynb +++ b/notebooks/tutorials/pandas-cookbook/07-cleaning-up-messy-data.ipynb @@ -836,7 +836,7 @@ "id": "64", "metadata": {}, "source": [ - "# Data owner: execute syft_function" + "# Data owner: approve request" ] }, { @@ -1004,7 +1004,7 @@ }, "outputs": [], "source": [ - "result = request.accept_by_depositing_result(real_result)\n", + "result = request.approve()\n", "assert isinstance(result, sy.SyftSuccess)" ] }, @@ -1025,7 +1025,7 @@ "id": "80", "metadata": {}, "source": [ - "# Data scientist: fetch result" + "# Data scientist: compute result" ] }, { @@ -1106,7 +1106,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.10.13" }, "toc": { "base_numbering": 1, diff --git a/notebooks/tutorials/pandas-cookbook/08-how-to-deal-with-timestamps.ipynb b/notebooks/tutorials/pandas-cookbook/08-how-to-deal-with-timestamps.ipynb index 6d1c11f3153..58b0132bd25 100644 --- a/notebooks/tutorials/pandas-cookbook/08-how-to-deal-with-timestamps.ipynb +++ b/notebooks/tutorials/pandas-cookbook/08-how-to-deal-with-timestamps.ipynb @@ -787,7 +787,7 @@ "id": "62", "metadata": {}, "source": [ - "# Data owner: execute syft_function" + "# Data owner: approve request" ] }, { @@ -943,7 +943,7 @@ }, "outputs": [], "source": [ - "result = request.accept_by_depositing_result(real_result)\n", + "result = request.approve()\n", "assert isinstance(result, sy.SyftSuccess)" ] }, @@ -953,7 +953,7 @@ "id": "76", "metadata": {}, "source": [ - "# Data Owner: fetch result" + "# Data Owner: compute result" ] }, { @@ -1034,7 +1034,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5" + "version": "3.10.13" }, "toc": { "base_numbering": 1, diff --git a/packages/grid/VERSION b/packages/grid/VERSION index 61a3b991302..df1dec5602a 100644 --- a/packages/grid/VERSION +++ b/packages/grid/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.7-beta.10" +__version__ = "0.8.7-beta.13" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/packages/grid/backend/backend.dockerfile b/packages/grid/backend/backend.dockerfile index 08ea2c9a72a..606569c49f4 100644 --- a/packages/grid/backend/backend.dockerfile +++ b/packages/grid/backend/backend.dockerfile @@ -1,6 +1,11 @@ ARG PYTHON_VERSION="3.12" -ARG UV_VERSION="0.1.41-r0" -ARG TORCH_VERSION="2.3.0" +ARG UV_VERSION="0.2.13-r0" +ARG TORCH_VERSION="2.2.2" + +# wolfi-os pkg definition links +# https://github.com/wolfi-dev/os/blob/main/python-3.12.yaml +# https://github.com/wolfi-dev/os/blob/main/py3-pip.yaml +# https://github.com/wolfi-dev/os/blob/main/uv.yaml # ==================== [BUILD STEP] Python Dev Base ==================== # @@ -12,7 +17,9 @@ ARG TORCH_VERSION # Setup Python DEV RUN apk update && apk upgrade && \ - apk add build-base gcc python-$PYTHON_VERSION-dev-default uv=$UV_VERSION + apk add build-base gcc python-$PYTHON_VERSION-dev uv=$UV_VERSION && \ + # preemptive fix for wolfi-os breaking python entrypoint + (test -f /usr/bin/python || ln -s /usr/bin/python3.12 /usr/bin/python) WORKDIR /root/app @@ -44,7 +51,9 @@ ARG PYTHON_VERSION ARG UV_VERSION RUN apk update && apk upgrade && \ - apk add --no-cache git bash python-$PYTHON_VERSION-default py$PYTHON_VERSION-pip uv=$UV_VERSION + apk add --no-cache git bash python-$PYTHON_VERSION py$PYTHON_VERSION-pip uv=$UV_VERSION && \ + # preemptive fix for wolfi-os breaking python entrypoint + (test -f /usr/bin/python || ln -s /usr/bin/python3.12 /usr/bin/python) WORKDIR /root/app/ diff --git a/packages/grid/backend/grid/images/worker_cpu.dockerfile b/packages/grid/backend/grid/images/worker_cpu.dockerfile index f1b6207ce90..94297fee0a9 100644 --- a/packages/grid/backend/grid/images/worker_cpu.dockerfile +++ b/packages/grid/backend/grid/images/worker_cpu.dockerfile @@ -5,13 +5,10 @@ # NOTE: This dockerfile will be built inside a grid-backend container in PROD # Hence COPY will not work the same way in DEV vs. PROD -# FIXME: Due to dependency on grid-backend base, python can only be changed from 3.11 to 3.11-dev -# Later we'd want to uninstall old python, and then install a new python runtime... -# ... but pre-built syft deps may break! - -ARG SYFT_VERSION_TAG="0.8.7-beta.10" +ARG SYFT_VERSION_TAG="0.8.7-beta.13" FROM openmined/grid-backend:${SYFT_VERSION_TAG} +# should match base image python version ARG PYTHON_VERSION="3.12" ARG SYSTEM_PACKAGES="" ARG PIP_PACKAGES="pip --dry-run" @@ -22,7 +19,9 @@ ENV SYFT_WORKER="true" \ SYFT_VERSION_TAG=${SYFT_VERSION_TAG} \ UV_HTTP_TIMEOUT=600 -RUN apk update && apk upgrade && \ +# dont run `apk upgrade` here, as it runs upgrades on the base image +# which may break syft or carry over breaking changes by wolfi-os +RUN apk update && \ apk add --no-cache ${SYSTEM_PACKAGES} && \ # if uv is present then run uv pip install else simple pip install if [ -x "$(command -v uv)" ]; then uv pip install --no-cache ${PIP_PACKAGES}; else pip install --user ${PIP_PACKAGES}; fi && \ diff --git a/packages/grid/backend/grid/logger/config.py b/packages/grid/backend/grid/logger/config.py deleted file mode 100644 index 000a9c9c713..00000000000 --- a/packages/grid/backend/grid/logger/config.py +++ /dev/null @@ -1,59 +0,0 @@ -"""This file defines the configuration for `loguru` which is used as the python logging client. -For more information refer to `loguru` documentation: https://loguru.readthedocs.io/en/stable/overview.html -""" - -# stdlib -from datetime import time -from datetime import timedelta -from enum import Enum -from functools import lru_cache - -# third party -from pydantic_settings import BaseSettings - - -# LOGURU_LEVEL type for version>3.8 -class LogLevel(Enum): - """Types of logging levels.""" - - TRACE = "TRACE" - DEBUG = "DEBUG" - INFO = "INFO" - SUCCESS = "SUCCESS" - WARNING = "WARNING" - ERROR = "ERROR" - CRITICAL = "CRITICAL" - - -class LogConfig(BaseSettings): - """Configuration for the logging client.""" - - # Logging format - LOGURU_FORMAT: str = ( - "{time:YYYY-MM-DD HH:mm:ss} | " - "{level: <8} | " - "{name}:{function}:{line}: " - "{message}" - ) - - LOGURU_LEVEL: str = LogLevel.INFO.value - LOGURU_SINK: str | None = "/var/log/pygrid/grid.log" - LOGURU_COMPRESSION: str | None = None - LOGURU_ROTATION: str | int | time | timedelta | None = None - LOGURU_RETENTION: str | int | timedelta | None = None - LOGURU_COLORIZE: bool | None = True - LOGURU_SERIALIZE: bool | None = False - LOGURU_BACKTRACE: bool | None = True - LOGURU_DIAGNOSE: bool | None = False - LOGURU_ENQUEUE: bool | None = True - LOGURU_AUTOINIT: bool | None = False - - -@lru_cache -def get_log_config() -> LogConfig: - """Returns the configuration for the logging client. - - Returns: - LogConfig: configuration for the logging client. - """ - return LogConfig() diff --git a/packages/grid/backend/grid/logger/handler.py b/packages/grid/backend/grid/logger/handler.py deleted file mode 100644 index 7f198bbcece..00000000000 --- a/packages/grid/backend/grid/logger/handler.py +++ /dev/null @@ -1,108 +0,0 @@ -# future -from __future__ import annotations - -# stdlib -from functools import lru_cache -import logging -from pprint import pformat -import sys - -# third party -import loguru -from loguru import logger - -# relative -from .config import get_log_config - - -class LogHandler: - def __init__(self) -> None: - self.config = get_log_config() - - def format_record(self, record: loguru.Record) -> str: - """ - Custom loguru log message format for handling JSON (in record['extra']) - """ - format_string: str = self.config.LOGURU_FORMAT - - if record["extra"] is not None: - for key in record["extra"].keys(): - record["extra"][key] = pformat( - record["extra"][key], indent=2, compact=False, width=88 - ) - format_string += "\n{extra[" + key + "]}" - - format_string += "{exception}\n" - - return format_string - - def init_logger(self) -> None: - """ - Redirects all registered std logging handlers to a loguru sink. - Call init_logger() on fastapi startup. - """ - intercept_handler = InterceptHandler() - - # Generalizes log level for all root loggers, including third party - logging.root.setLevel(self.config.LOGURU_LEVEL) - logging.root.handlers = [intercept_handler] - - for log in logging.root.manager.loggerDict.keys(): - log_instance = logging.getLogger(log) - log_instance.handlers = [] - log_instance.propagate = True - - logger.configure( - handlers=[ - { - "sink": sys.stdout, - "level": self.config.LOGURU_LEVEL, - "serialize": self.config.LOGURU_SERIALIZE, - "format": self.format_record, - } - ], - ) - - try: - if ( - self.config.LOGURU_SINK is not ("sys.stdout" or "sys.stderr") - and self.config.LOGURU_SINK is not None - ): - logger.add( - self.config.LOGURU_SINK, - retention=self.config.LOGURU_RETENTION, - rotation=self.config.LOGURU_ROTATION, - compression=self.config.LOGURU_COMPRESSION, - ) - logger.debug(f"Logging to {self.config.LOGURU_SINK}") - - except Exception as err: - logger.debug( - f"Failed creating a new sink. Check your log config. error: {err}" - ) - - -class InterceptHandler(logging.Handler): - """ - Check https://loguru.readthedocs.io/en/stable/overview.html#entirely-compatible-with-standard-logging - """ - - def emit(self, record: logging.LogRecord) -> None: - try: - level = logger.level(record.levelname).name - except ValueError: - level = record.levelno - - frame, depth = logging.currentframe(), 2 - while frame.f_code.co_filename == logging.__file__: - frame = frame.f_back # type: ignore - depth += 1 - - logger.opt(depth=depth, exception=record.exc_info).log( - level, record.getMessage() - ) - - -@lru_cache -def get_log_handler() -> LogHandler: - return LogHandler() diff --git a/packages/grid/backend/grid/logging.yaml b/packages/grid/backend/grid/logging.yaml new file mode 100644 index 00000000000..b41eb783038 --- /dev/null +++ b/packages/grid/backend/grid/logging.yaml @@ -0,0 +1,46 @@ +version: 1 +disable_existing_loggers: True +formatters: + default: + format: "%(asctime)s - %(levelname)s - %(name)s - %(message)s" + datefmt: "%Y-%m-%d %H:%M:%S" + uvicorn.default: + "()": uvicorn.logging.DefaultFormatter + format: "%(asctime)s - %(levelname)s - %(name)s - %(message)s" + uvicorn.access: + "()": "uvicorn.logging.AccessFormatter" + format: "%(asctime)s - %(levelname)s - %(name)s - %(message)s" + datefmt: "%Y-%m-%d %H:%M:%S" +handlers: + default: + formatter: default + class: logging.StreamHandler + stream: ext://sys.stdout + uvicorn.default: + formatter: uvicorn.default + class: logging.StreamHandler + stream: ext://sys.stdout + uvicorn.access: + formatter: uvicorn.access + class: logging.StreamHandler + stream: ext://sys.stdout +loggers: + uvicorn.error: + level: INFO + handlers: + - uvicorn.default + propagate: no + uvicorn.access: + level: INFO + handlers: + - uvicorn.access + propagate: no + syft: + level: INFO + handlers: + - default + propagate: no +root: + level: INFO + handlers: + - default diff --git a/packages/grid/backend/grid/main.py b/packages/grid/backend/grid/main.py index 9ca43dadee8..459448c5f01 100644 --- a/packages/grid/backend/grid/main.py +++ b/packages/grid/backend/grid/main.py @@ -1,7 +1,6 @@ -# stdlib - # stdlib from contextlib import asynccontextmanager +import logging from typing import Any # third party @@ -16,7 +15,15 @@ from grid.api.router import api_router from grid.core.config import settings from grid.core.node import worker -from grid.logger.handler import get_log_handler + + +class EndpointFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + return record.getMessage().find("/api/v2/?probe=livenessProbe") == -1 + + +logger = logging.getLogger("uvicorn.error") +logging.getLogger("uvicorn.access").addFilter(EndpointFilter()) @asynccontextmanager @@ -25,7 +32,7 @@ async def lifespan(app: FastAPI) -> Any: yield finally: worker.stop() - print("Worker Stop !!!") + logger.info("Worker Stop") app = FastAPI( @@ -34,7 +41,6 @@ async def lifespan(app: FastAPI) -> Any: lifespan=lifespan, ) -app.add_event_handler("startup", get_log_handler().init_logger) # Set all CORS enabled origins if settings.BACKEND_CORS_ORIGINS: @@ -47,13 +53,13 @@ async def lifespan(app: FastAPI) -> Any: ) app.include_router(api_router, prefix=settings.API_V2_STR) -print("Included routes, app should now be reachable") +logger.info("Included routes, app should now be reachable") if settings.DEV_MODE: - print("Staging protocol changes...") + logger.info("Staging protocol changes...") status = stage_protocol_changes() - print(status) + logger.info(f"Staging protocol result: {status}") # needed for Google Kubernetes Engine LoadBalancer Healthcheck diff --git a/packages/grid/backend/grid/start.sh b/packages/grid/backend/grid/start.sh index bcb36c5e5a9..4b3d5de4cf2 100755 --- a/packages/grid/backend/grid/start.sh +++ b/packages/grid/backend/grid/start.sh @@ -33,4 +33,4 @@ export NODE_TYPE=$NODE_TYPE echo "NODE_UID=$NODE_UID" echo "NODE_TYPE=$NODE_TYPE" -exec $DEBUG_CMD uvicorn $RELOAD --host $HOST --port $PORT --log-level $LOG_LEVEL "$APP_MODULE" +exec $DEBUG_CMD uvicorn $RELOAD --host $HOST --port $PORT --log-config=$APPDIR/grid/logging.yaml --log-level $LOG_LEVEL "$APP_MODULE" diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 5b01f23aad9..6e12cf245cc 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -27,7 +27,7 @@ vars: DOCKER_IMAGE_SEAWEEDFS: openmined/grid-seaweedfs DOCKER_IMAGE_ENCLAVE_ATTESTATION: openmined/grid-enclave-attestation CONTAINER_REGISTRY: "docker.io" - VERSION: "0.8.7-beta.10" + VERSION: "0.8.7-beta.13" PLATFORM: $(uname -m | grep -q 'arm64' && echo "arm64" || echo "amd64") # This is a list of `images` that DevSpace can build for this project diff --git a/packages/grid/frontend/package.json b/packages/grid/frontend/package.json index f5115c53976..15fddc2a6d2 100644 --- a/packages/grid/frontend/package.json +++ b/packages/grid/frontend/package.json @@ -1,6 +1,6 @@ { "name": "pygrid-ui", - "version": "0.8.7-beta.10", + "version": "0.8.7-beta.13", "private": true, "scripts": { "dev": "pnpm i && vite dev --host --port 80", diff --git a/packages/grid/helm/repo/index.yaml b/packages/grid/helm/repo/index.yaml index 4c88b674ff0..56fdc4275cb 100644 --- a/packages/grid/helm/repo/index.yaml +++ b/packages/grid/helm/repo/index.yaml @@ -1,9 +1,48 @@ apiVersion: v1 entries: syft: + - apiVersion: v2 + appVersion: 0.8.7-beta.13 + created: "2024-06-25T14:52:54.915182775Z" + description: Perform numpy-like analysis on data that remains in someone elses + server + digest: 1dbe3ecdfec57bf25020cbcff783fab908f0eb0640ad684470b2fd1da1928005 + home: https://github.com/OpenMined/PySyft/ + icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png + name: syft + type: application + urls: + - https://openmined.github.io/PySyft/helm/syft-0.8.7-beta.13.tgz + version: 0.8.7-beta.13 + - apiVersion: v2 + appVersion: 0.8.7-beta.12 + created: "2024-06-25T14:52:54.914499475Z" + description: Perform numpy-like analysis on data that remains in someone elses + server + digest: e92b2f3a522dabb3a79ff762a7042ae16d2bf3a53eebbb2885a69b9f834d109c + home: https://github.com/OpenMined/PySyft/ + icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png + name: syft + type: application + urls: + - https://openmined.github.io/PySyft/helm/syft-0.8.7-beta.12.tgz + version: 0.8.7-beta.12 + - apiVersion: v2 + appVersion: 0.8.7-beta.11 + created: "2024-06-25T14:52:54.913815474Z" + description: Perform numpy-like analysis on data that remains in someone elses + server + digest: 099f6cbd44b699ee2410a4be012ed1a8a65bcacb06a43057b2779d7fe34fc0ad + home: https://github.com/OpenMined/PySyft/ + icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png + name: syft + type: application + urls: + - https://openmined.github.io/PySyft/helm/syft-0.8.7-beta.11.tgz + version: 0.8.7-beta.11 - apiVersion: v2 appVersion: 0.8.7-beta.10 - created: "2024-06-03T13:45:21.377002407Z" + created: "2024-06-25T14:52:54.913093962Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 00773cb241522e281c1915339fc362e047650e08958a736e93d6539f44cb5e25 @@ -16,7 +55,7 @@ entries: version: 0.8.7-beta.10 - apiVersion: v2 appVersion: 0.8.7-beta.9 - created: "2024-06-03T13:45:21.382840443Z" + created: "2024-06-25T14:52:54.921044649Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: a3f8e85d9ddef7a644b959fcc2fcb0fc08f7b6abae1045e893d0d62fa4ae132e @@ -29,7 +68,7 @@ entries: version: 0.8.7-beta.9 - apiVersion: v2 appVersion: 0.8.7-beta.8 - created: "2024-06-03T13:45:21.382193467Z" + created: "2024-06-25T14:52:54.920407065Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: a422ac88d8fd1fb80d5004d5eb6e95fa9efc7f6a87da12e5ac04829da7f04c4d @@ -42,7 +81,7 @@ entries: version: 0.8.7-beta.8 - apiVersion: v2 appVersion: 0.8.7-beta.7 - created: "2024-06-03T13:45:21.381537725Z" + created: "2024-06-25T14:52:54.919767347Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 0dc313a1092e6256a7c8aad002c8ec380b3add2c289d680db1e238a336399b7a @@ -55,7 +94,7 @@ entries: version: 0.8.7-beta.7 - apiVersion: v2 appVersion: 0.8.7-beta.6 - created: "2024-06-03T13:45:21.380874049Z" + created: "2024-06-25T14:52:54.91914935Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 052a2ec1102d2a4c9915f95647abd4a6012f56fa05a106f4952ee9b55bf7bae8 @@ -68,7 +107,7 @@ entries: version: 0.8.7-beta.6 - apiVersion: v2 appVersion: 0.8.7-beta.5 - created: "2024-06-03T13:45:21.380230309Z" + created: "2024-06-25T14:52:54.918496807Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 1728af756907c3fcbe87c2fd2de014a2d963c22a4c2eb6af6596b525a9b9a18a @@ -81,7 +120,7 @@ entries: version: 0.8.7-beta.5 - apiVersion: v2 appVersion: 0.8.7-beta.4 - created: "2024-06-03T13:45:21.379600085Z" + created: "2024-06-25T14:52:54.917833966Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 387a57a3904a05ed61e92ee48605ef6fd5044ff7e822e0924e0d4c485e2c88d2 @@ -94,7 +133,7 @@ entries: version: 0.8.7-beta.4 - apiVersion: v2 appVersion: 0.8.7-beta.3 - created: "2024-06-03T13:45:21.378911612Z" + created: "2024-06-25T14:52:54.916857511Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 3668002b7a4118516b2ecd61d6275f60d83fc12841587ab8f62e1c1200731c67 @@ -107,7 +146,7 @@ entries: version: 0.8.7-beta.3 - apiVersion: v2 appVersion: 0.8.7-beta.2 - created: "2024-06-03T13:45:21.377596593Z" + created: "2024-06-25T14:52:54.915789101Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: e62217ffcadee2b8896ab0543f9ccc42f2df898fd979438ac9376d780b802af7 @@ -120,7 +159,7 @@ entries: version: 0.8.7-beta.2 - apiVersion: v2 appVersion: 0.8.7-beta.1 - created: "2024-06-03T13:45:21.37632278Z" + created: "2024-06-25T14:52:54.912442092Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 553981fe1d5c980e6903b3ff2f1b9b97431f6dd8aee91e3976bcc5594285235e @@ -133,7 +172,7 @@ entries: version: 0.8.7-beta.1 - apiVersion: v2 appVersion: 0.8.6 - created: "2024-06-03T13:45:21.375759532Z" + created: "2024-06-25T14:52:54.911941434Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: ddbbe6fea1702e57404875eb3019a3b1a341017bdbb5fbc6ce418507e5c15756 @@ -146,7 +185,7 @@ entries: version: 0.8.6 - apiVersion: v2 appVersion: 0.8.6-beta.1 - created: "2024-06-03T13:45:21.375198047Z" + created: "2024-06-25T14:52:54.911419657Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: cc2c81ef6796ac853dce256e6bf8a6af966c21803e6534ea21920af681c62e61 @@ -159,7 +198,7 @@ entries: version: 0.8.6-beta.1 - apiVersion: v2 appVersion: 0.8.5 - created: "2024-06-03T13:45:21.37461952Z" + created: "2024-06-25T14:52:54.910887971Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: db5d90d44006209fd5ecdebd88f5fd56c70f7c76898343719a0ff8da46da948a @@ -172,7 +211,7 @@ entries: version: 0.8.5 - apiVersion: v2 appVersion: 0.8.5-post.2 - created: "2024-06-03T13:45:21.3738741Z" + created: "2024-06-25T14:52:54.910132145Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: ea3f7269b55f773fa165d7008c054b7cf3ec4c62eb40a96f08cd3a9b77fd2165 @@ -185,7 +224,7 @@ entries: version: 0.8.5-post.2 - apiVersion: v2 appVersion: 0.8.5-post.1 - created: "2024-06-03T13:45:21.373337562Z" + created: "2024-06-25T14:52:54.909328308Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 9deb844d3dc2d8480c60f8c631dcc7794adfb39cec3aa3b1ce22ea26fdf87d02 @@ -198,7 +237,7 @@ entries: version: 0.8.5-post.1 - apiVersion: v2 appVersion: 0.8.5-beta.10 - created: "2024-06-03T13:45:21.365760974Z" + created: "2024-06-25T14:52:54.901637641Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 9cfe01e8f57eca462261a24a805b41509be2de9a0fee76e331d124ed98c4bc49 @@ -211,7 +250,7 @@ entries: version: 0.8.5-beta.10 - apiVersion: v2 appVersion: 0.8.5-beta.9 - created: "2024-06-03T13:45:21.372587593Z" + created: "2024-06-25T14:52:54.908047519Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 057f1733f2bc966e15618f62629315c8207773ef6211c79c4feb557dae15c32b @@ -224,7 +263,7 @@ entries: version: 0.8.5-beta.9 - apiVersion: v2 appVersion: 0.8.5-beta.8 - created: "2024-06-03T13:45:21.37183012Z" + created: "2024-06-25T14:52:54.907301222Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 921cbce836c3032ef62b48cc82b5b4fcbe44fb81d473cf4d69a4bf0f806eb298 @@ -237,7 +276,7 @@ entries: version: 0.8.5-beta.8 - apiVersion: v2 appVersion: 0.8.5-beta.7 - created: "2024-06-03T13:45:21.371012243Z" + created: "2024-06-25T14:52:54.906546207Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 75482e955b2b9853a80bd653afb1d56535f78f3bfb7726798522307eb3effbbd @@ -250,7 +289,7 @@ entries: version: 0.8.5-beta.7 - apiVersion: v2 appVersion: 0.8.5-beta.6 - created: "2024-06-03T13:45:21.369543119Z" + created: "2024-06-25T14:52:54.905804338Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 6a2dfaf65ca855e1b3d7b966d4ff291e6fcbe761e2fc2a78033211ccd3a75de0 @@ -263,7 +302,7 @@ entries: version: 0.8.5-beta.6 - apiVersion: v2 appVersion: 0.8.5-beta.5 - created: "2024-06-03T13:45:21.368795474Z" + created: "2024-06-25T14:52:54.905005392Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: fead03823bef04d66901d563aa755c68ab277f72b126aaa6f0dce76a6f3bdb6d @@ -276,7 +315,7 @@ entries: version: 0.8.5-beta.5 - apiVersion: v2 appVersion: 0.8.5-beta.4 - created: "2024-06-03T13:45:21.368047108Z" + created: "2024-06-25T14:52:54.904263382Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 93e4539d5726a7fd0d6a3e93d1c17c6a358a923ddc01d102eab22f37377502ab @@ -289,7 +328,7 @@ entries: version: 0.8.5-beta.4 - apiVersion: v2 appVersion: 0.8.5-beta.3 - created: "2024-06-03T13:45:21.367286349Z" + created: "2024-06-25T14:52:54.90351485Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: f91e9390edf3441469048f5da646099af98f8b6d199409d0e2c1e6da3a51f054 @@ -302,7 +341,7 @@ entries: version: 0.8.5-beta.3 - apiVersion: v2 appVersion: 0.8.5-beta.2 - created: "2024-06-03T13:45:21.36650959Z" + created: "2024-06-25T14:52:54.902672242Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 59159c3aa4888038edc3c0135c83402363d7a0639fe62966a1e9d4928a364fa8 @@ -315,7 +354,7 @@ entries: version: 0.8.5-beta.2 - apiVersion: v2 appVersion: 0.8.5-beta.1 - created: "2024-06-03T13:45:21.364910646Z" + created: "2024-06-25T14:52:54.900351402Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 65aeb74c52ed8ba5474af500b4c1188a570ee4cb1f2a2da356b3488d28356ed9 @@ -327,7 +366,7 @@ entries: version: 0.8.5-beta.1 - apiVersion: v2 appVersion: 0.8.4 - created: "2024-06-03T13:45:21.364529059Z" + created: "2024-06-25T14:52:54.899982841Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 08afea8e3a9eef225b7e611f0bc1216c140053ef8e51439b02337faeac621fd0 @@ -339,7 +378,7 @@ entries: version: 0.8.4 - apiVersion: v2 appVersion: 0.8.4-beta.31 - created: "2024-06-03T13:45:21.361126791Z" + created: "2024-06-25T14:52:54.897562879Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: fabf3e2f37e53fa623f5d3d99b00feae06e278e5cd63bce419089946312ab1fc @@ -351,7 +390,7 @@ entries: version: 0.8.4-beta.31 - apiVersion: v2 appVersion: 0.8.4-beta.30 - created: "2024-06-03T13:45:21.360708906Z" + created: "2024-06-25T14:52:54.897124859Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 6e8f792709f73ec14eab48a268bdf50a4505b340bd142cddd7c7bfffd94009ad @@ -363,7 +402,7 @@ entries: version: 0.8.4-beta.30 - apiVersion: v2 appVersion: 0.8.4-beta.29 - created: "2024-06-03T13:45:21.359963866Z" + created: "2024-06-25T14:52:54.896384402Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 4c985d6a9b3456769c4013f9e85e7374c0f963d2d27627e61f914f5537de1971 @@ -375,7 +414,7 @@ entries: version: 0.8.4-beta.29 - apiVersion: v2 appVersion: 0.8.4-beta.28 - created: "2024-06-03T13:45:21.359560718Z" + created: "2024-06-25T14:52:54.895981798Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: bd2aa3c92c768c47c502e31a326f341addcb34e64d22cdcbf5cc3f19689d859c @@ -387,7 +426,7 @@ entries: version: 0.8.4-beta.28 - apiVersion: v2 appVersion: 0.8.4-beta.27 - created: "2024-06-03T13:45:21.35915197Z" + created: "2024-06-25T14:52:54.895572461Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: e8ad0869993af39d7adda8cb868dc0b24cfb63b4bb9820dc579939c1007a60ba @@ -399,7 +438,7 @@ entries: version: 0.8.4-beta.27 - apiVersion: v2 appVersion: 0.8.4-beta.26 - created: "2024-06-03T13:45:21.358714849Z" + created: "2024-06-25T14:52:54.895153567Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 30dccf630aa25a86a03c67572fe5411687d8ce6d58def448ea10efdba2b85e3a @@ -411,7 +450,7 @@ entries: version: 0.8.4-beta.26 - apiVersion: v2 appVersion: 0.8.4-beta.25 - created: "2024-06-03T13:45:21.358309608Z" + created: "2024-06-25T14:52:54.894459409Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: b6e2043bcf5a0335967d770c7939f5a7832955359a7d871c90b265660ff26e5f @@ -423,7 +462,7 @@ entries: version: 0.8.4-beta.25 - apiVersion: v2 appVersion: 0.8.4-beta.24 - created: "2024-06-03T13:45:21.357903745Z" + created: "2024-06-25T14:52:54.893613495Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: b19efa95394d50bb8d76da6ec306de5d3bb9ea55371fafea95a1282a697fa33e @@ -435,7 +474,7 @@ entries: version: 0.8.4-beta.24 - apiVersion: v2 appVersion: 0.8.4-beta.23 - created: "2024-06-03T13:45:21.357499837Z" + created: "2024-06-25T14:52:54.893200211Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 5c5d05c15bff548574896118ce92335ae10c5b78f5307fe9b2618e5a5aa71a5c @@ -447,7 +486,7 @@ entries: version: 0.8.4-beta.23 - apiVersion: v2 appVersion: 0.8.4-beta.22 - created: "2024-06-03T13:45:21.357090377Z" + created: "2024-06-25T14:52:54.892776578Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 0160dbce938198132ca9cd2a5cb362816344687291f5b6d7cf6de8f2855e9414 @@ -459,7 +498,7 @@ entries: version: 0.8.4-beta.22 - apiVersion: v2 appVersion: 0.8.4-beta.21 - created: "2024-06-03T13:45:21.356672332Z" + created: "2024-06-25T14:52:54.892368944Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 7dce153d2fcae7513e9c132e139b2721fd975ea3cc43a370e34dbeb2a1b7f683 @@ -471,7 +510,7 @@ entries: version: 0.8.4-beta.21 - apiVersion: v2 appVersion: 0.8.4-beta.20 - created: "2024-06-03T13:45:21.356230252Z" + created: "2024-06-25T14:52:54.891969626Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: c51189a187bbf24135382e25cb00964e0330dfcd3b2f0c884581a6686f05dd28 @@ -483,7 +522,7 @@ entries: version: 0.8.4-beta.20 - apiVersion: v2 appVersion: 0.8.4-beta.19 - created: "2024-06-03T13:45:21.354339461Z" + created: "2024-06-25T14:52:54.891018374Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 8219575dedb42fa2ddbf2768a4e9afbfacbc2dff7e953d77c7b10a41b78dc687 @@ -495,7 +534,7 @@ entries: version: 0.8.4-beta.19 - apiVersion: v2 appVersion: 0.8.4-beta.18 - created: "2024-06-03T13:45:21.353946513Z" + created: "2024-06-25T14:52:54.890618786Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 6418cde559cf12f1f7fea5a2b123bba950e50eeb3be002441827d2ab7f9e4ef7 @@ -505,21 +544,9 @@ entries: urls: - https://openmined.github.io/PySyft/helm/syft-0.8.4-beta.18.tgz version: 0.8.4-beta.18 - - apiVersion: v2 - appVersion: 0.8.4-beta.17 - created: "2024-06-03T13:45:21.35354575Z" - description: Perform numpy-like analysis on data that remains in someone elses - server - digest: 71b39c5a4c64037eadbb154f7029282ba90d9a0d703f8d4c7dfc1ba2f5d81498 - icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png - name: syft - type: application - urls: - - https://openmined.github.io/PySyft/helm/syft-0.8.4-beta.17.tgz - version: 0.8.4-beta.17 - apiVersion: v2 appVersion: 0.8.4-beta.16 - created: "2024-06-03T13:45:21.353142151Z" + created: "2024-06-25T14:52:54.890227603Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 9c9840a7c9476dbb08e0ac83926330718fe50c89879752dd8f92712b036109c0 @@ -531,7 +558,7 @@ entries: version: 0.8.4-beta.16 - apiVersion: v2 appVersion: 0.8.4-beta.15 - created: "2024-06-03T13:45:21.352732211Z" + created: "2024-06-25T14:52:54.889830279Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 0955fd22da028315e30c68132cbfa4bdc82bae622039bcfce0de339707bb82eb @@ -543,7 +570,7 @@ entries: version: 0.8.4-beta.15 - apiVersion: v2 appVersion: 0.8.4-beta.14 - created: "2024-06-03T13:45:21.352334795Z" + created: "2024-06-25T14:52:54.88942517Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 56208571956abe20ed7a5cc1867cab2667ed792c63e53d0e8bb70a9b438b7bf6 @@ -555,7 +582,7 @@ entries: version: 0.8.4-beta.14 - apiVersion: v2 appVersion: 0.8.4-beta.13 - created: "2024-06-03T13:45:21.351989927Z" + created: "2024-06-25T14:52:54.889036452Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: d7222c72412b6ee5833fbb07d2549be179cdfc7ccd89e0ad947d112fce799b83 @@ -567,7 +594,7 @@ entries: version: 0.8.4-beta.13 - apiVersion: v2 appVersion: 0.8.4-beta.12 - created: "2024-06-03T13:45:21.351643917Z" + created: "2024-06-25T14:52:54.888678191Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: af08c723756e397962b2d5190dedfd50797b771c5caf58b93a6f65d8fa24785c @@ -579,7 +606,7 @@ entries: version: 0.8.4-beta.12 - apiVersion: v2 appVersion: 0.8.4-beta.11 - created: "2024-06-03T13:45:21.35129452Z" + created: "2024-06-25T14:52:54.888112171Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: a0235835ba57d185a83dd8a26281fa37b2077c3a37fe3a1c50585005695927e3 @@ -591,7 +618,7 @@ entries: version: 0.8.4-beta.11 - apiVersion: v2 appVersion: 0.8.4-beta.10 - created: "2024-06-03T13:45:21.350902194Z" + created: "2024-06-25T14:52:54.88720381Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 910ddfeba0c5e66651500dd11404afff092adc0f768ed68e0d93b04b83aa4388 @@ -603,7 +630,7 @@ entries: version: 0.8.4-beta.10 - apiVersion: v2 appVersion: 0.8.4-beta.9 - created: "2024-06-03T13:45:21.364116334Z" + created: "2024-06-25T14:52:54.899577432Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: c25ca8a9f072d6a5d02232448deaef5668aca05f24dfffbba3ebe30a4f75bb26 @@ -615,7 +642,7 @@ entries: version: 0.8.4-beta.9 - apiVersion: v2 appVersion: 0.8.4-beta.8 - created: "2024-06-03T13:45:21.36376841Z" + created: "2024-06-25T14:52:54.89925044Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 7249a39d4137e457b369384ba0a365c271c780d93a8327ce25083df763c39999 @@ -627,7 +654,7 @@ entries: version: 0.8.4-beta.8 - apiVersion: v2 appVersion: 0.8.4-beta.7 - created: "2024-06-03T13:45:21.363407161Z" + created: "2024-06-25T14:52:54.898923167Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: ee750c7c8d6ea05bd447375e624fdd7f66dd87680ab81f7b7e73df7379a9024a @@ -639,7 +666,7 @@ entries: version: 0.8.4-beta.7 - apiVersion: v2 appVersion: 0.8.4-beta.6 - created: "2024-06-03T13:45:21.362698854Z" + created: "2024-06-25T14:52:54.898597537Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 0e046be9f73df7444a995608c59af16fab9030b139b2acb4d6db6185b8eb5337 @@ -651,7 +678,7 @@ entries: version: 0.8.4-beta.6 - apiVersion: v2 appVersion: 0.8.4-beta.5 - created: "2024-06-03T13:45:21.361807359Z" + created: "2024-06-25T14:52:54.898264563Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: b56e9a23d46810eccdb4cf5272cc05126da3f6db314e541959c3efb5f260620b @@ -663,7 +690,7 @@ entries: version: 0.8.4-beta.5 - apiVersion: v2 appVersion: 0.8.4-beta.4 - created: "2024-06-03T13:45:21.361471147Z" + created: "2024-06-25T14:52:54.897896133Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 1d5808ecaf55391f3b27ae6236400066508acbd242e33db24a1ab4bffa77409e @@ -675,7 +702,7 @@ entries: version: 0.8.4-beta.4 - apiVersion: v2 appVersion: 0.8.4-beta.3 - created: "2024-06-03T13:45:21.360307792Z" + created: "2024-06-25T14:52:54.896727544Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: b64efa8529d82be56c6ab60487ed24420a5614d96d2509c1f93c1003eda71a54 @@ -687,7 +714,7 @@ entries: version: 0.8.4-beta.3 - apiVersion: v2 appVersion: 0.8.4-beta.2 - created: "2024-06-03T13:45:21.355072969Z" + created: "2024-06-25T14:52:54.891563686Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -703,7 +730,7 @@ entries: version: 0.8.4-beta.2 - apiVersion: v2 appVersion: 0.8.4-beta.1 - created: "2024-06-03T13:45:21.350539272Z" + created: "2024-06-25T14:52:54.886840739Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -719,7 +746,7 @@ entries: version: 0.8.4-beta.1 - apiVersion: v2 appVersion: 0.8.3 - created: "2024-06-03T13:45:21.349972637Z" + created: "2024-06-25T14:52:54.886301389Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -735,7 +762,7 @@ entries: version: 0.8.3 - apiVersion: v2 appVersion: 0.8.3-beta.6 - created: "2024-06-03T13:45:21.349141275Z" + created: "2024-06-25T14:52:54.88564517Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -751,7 +778,7 @@ entries: version: 0.8.3-beta.6 - apiVersion: v2 appVersion: 0.8.3-beta.5 - created: "2024-06-03T13:45:21.347981356Z" + created: "2024-06-25T14:52:54.885028145Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -767,7 +794,7 @@ entries: version: 0.8.3-beta.5 - apiVersion: v2 appVersion: 0.8.3-beta.4 - created: "2024-06-03T13:45:21.347397349Z" + created: "2024-06-25T14:52:54.884468206Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -781,25 +808,9 @@ entries: urls: - https://openmined.github.io/PySyft/helm/syft-0.8.3-beta.4.tgz version: 0.8.3-beta.4 - - apiVersion: v2 - appVersion: 0.8.3-beta.3 - created: "2024-06-03T13:45:21.346709046Z" - dependencies: - - name: component-chart - repository: https://charts.devspace.sh - version: 0.9.1 - description: Perform numpy-like analysis on data that remains in someone elses - server - digest: 9162bc14e40021b56111c3c9dbeba2596ce1ff469263b0a1e0a4679174c14ef7 - icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png - name: syft - type: application - urls: - - https://openmined.github.io/PySyft/helm/syft-0.8.3-beta.3.tgz - version: 0.8.3-beta.3 - apiVersion: v2 appVersion: 0.8.3-beta.2 - created: "2024-06-03T13:45:21.346158131Z" + created: "2024-06-25T14:52:54.883815253Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -815,7 +826,7 @@ entries: version: 0.8.3-beta.2 - apiVersion: v2 appVersion: 0.8.3-beta.1 - created: "2024-06-03T13:45:21.345552283Z" + created: "2024-06-25T14:52:54.883268449Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -831,7 +842,7 @@ entries: version: 0.8.3-beta.1 - apiVersion: v2 appVersion: 0.8.2 - created: "2024-06-03T13:45:21.344859932Z" + created: "2024-06-25T14:52:54.882705244Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -847,7 +858,7 @@ entries: version: 0.8.2 - apiVersion: v2 appVersion: 0.8.2-beta.60 - created: "2024-06-03T13:45:21.344215902Z" + created: "2024-06-25T14:52:54.881823299Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -863,7 +874,7 @@ entries: version: 0.8.2-beta.60 - apiVersion: v2 appVersion: 0.8.2-beta.59 - created: "2024-06-03T13:45:21.343556895Z" + created: "2024-06-25T14:52:54.880463332Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -879,7 +890,7 @@ entries: version: 0.8.2-beta.59 - apiVersion: v2 appVersion: 0.8.2-beta.58 - created: "2024-06-03T13:45:21.342837122Z" + created: "2024-06-25T14:52:54.879820297Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -895,7 +906,7 @@ entries: version: 0.8.2-beta.58 - apiVersion: v2 appVersion: 0.8.2-beta.57 - created: "2024-06-03T13:45:21.341469051Z" + created: "2024-06-25T14:52:54.87918108Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -911,7 +922,7 @@ entries: version: 0.8.2-beta.57 - apiVersion: v2 appVersion: 0.8.2-beta.56 - created: "2024-06-03T13:45:21.340829309Z" + created: "2024-06-25T14:52:54.878467744Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -925,25 +936,9 @@ entries: urls: - https://openmined.github.io/PySyft/helm/syft-0.8.2-beta.56.tgz version: 0.8.2-beta.56 - - apiVersion: v2 - appVersion: 0.8.2-beta.53 - created: "2024-06-03T13:45:21.340191661Z" - dependencies: - - name: component-chart - repository: https://charts.devspace.sh - version: 0.9.1 - description: Perform numpy-like analysis on data that remains in someone elses - server - digest: ab91512010a81ca45ff65030c366c6d16873913199a5346013cd24ee6348df84 - icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png - name: syft - type: application - urls: - - https://openmined.github.io/PySyft/helm/syft-0.8.2-beta.53.tgz - version: 0.8.2-beta.53 - apiVersion: v2 appVersion: 0.8.2-beta.52 - created: "2024-06-03T13:45:21.339543493Z" + created: "2024-06-25T14:52:54.877795425Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -959,7 +954,7 @@ entries: version: 0.8.2-beta.52 - apiVersion: v2 appVersion: 0.8.2-beta.51 - created: "2024-06-03T13:45:21.338845252Z" + created: "2024-06-25T14:52:54.877128356Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -975,7 +970,7 @@ entries: version: 0.8.2-beta.51 - apiVersion: v2 appVersion: 0.8.2-beta.50 - created: "2024-06-03T13:45:21.338148503Z" + created: "2024-06-25T14:52:54.876499117Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -991,7 +986,7 @@ entries: version: 0.8.2-beta.50 - apiVersion: v2 appVersion: 0.8.2-beta.49 - created: "2024-06-03T13:45:21.337497129Z" + created: "2024-06-25T14:52:54.87585986Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -1007,7 +1002,7 @@ entries: version: 0.8.2-beta.49 - apiVersion: v2 appVersion: 0.8.2-beta.48 - created: "2024-06-03T13:45:21.336820537Z" + created: "2024-06-25T14:52:54.874640847Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -1023,7 +1018,7 @@ entries: version: 0.8.2-beta.48 - apiVersion: v2 appVersion: 0.8.2-beta.47 - created: "2024-06-03T13:45:21.335899539Z" + created: "2024-06-25T14:52:54.873983756Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -1039,7 +1034,7 @@ entries: version: 0.8.2-beta.47 - apiVersion: v2 appVersion: 0.8.2-beta.46 - created: "2024-06-03T13:45:21.334762583Z" + created: "2024-06-25T14:52:54.873429929Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -1055,7 +1050,7 @@ entries: version: 0.8.2-beta.46 - apiVersion: v2 appVersion: 0.8.2-beta.45 - created: "2024-06-03T13:45:21.334005641Z" + created: "2024-06-25T14:52:54.87285366Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -1071,7 +1066,7 @@ entries: version: 0.8.2-beta.45 - apiVersion: v2 appVersion: 0.8.2-beta.44 - created: "2024-06-03T13:45:21.333450688Z" + created: "2024-06-25T14:52:54.872314971Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -1087,7 +1082,7 @@ entries: version: 0.8.2-beta.44 - apiVersion: v2 appVersion: 0.8.2-beta.43 - created: "2024-06-03T13:45:21.332882671Z" + created: "2024-06-25T14:52:54.871773206Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -1103,7 +1098,7 @@ entries: version: 0.8.2-beta.43 - apiVersion: v2 appVersion: 0.8.2-beta.41 - created: "2024-06-03T13:45:21.332237018Z" + created: "2024-06-25T14:52:54.871123439Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -1119,7 +1114,7 @@ entries: version: 0.8.2-beta.41 - apiVersion: v2 appVersion: 0.8.2-beta.40 - created: "2024-06-03T13:45:21.331568342Z" + created: "2024-06-25T14:52:54.870471407Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -1135,7 +1130,7 @@ entries: version: 0.8.2-beta.40 - apiVersion: v2 appVersion: 0.8.2-beta.39 - created: "2024-06-03T13:45:21.330974116Z" + created: "2024-06-25T14:52:54.869913914Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -1151,7 +1146,7 @@ entries: version: 0.8.2-beta.39 - apiVersion: v2 appVersion: 0.8.2-beta.38 - created: "2024-06-03T13:45:21.330415346Z" + created: "2024-06-25T14:52:54.869352352Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -1167,7 +1162,7 @@ entries: version: 0.8.2-beta.38 - apiVersion: v2 appVersion: 0.8.2-beta.37 - created: "2024-06-03T13:45:21.329830537Z" + created: "2024-06-25T14:52:54.868708877Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -1183,7 +1178,7 @@ entries: version: 0.8.2-beta.37 - apiVersion: v2 appVersion: 0.8.1 - created: "2024-06-03T13:45:21.329215361Z" + created: "2024-06-25T14:52:54.86743294Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -1197,4 +1192,4 @@ entries: urls: - https://openmined.github.io/PySyft/helm/syft-0.8.1.tgz version: 0.8.1 -generated: "2024-06-03T13:45:21.328440806Z" +generated: "2024-06-25T14:52:54.866566006Z" diff --git a/packages/grid/helm/repo/syft-0.8.2-beta.53.tgz b/packages/grid/helm/repo/syft-0.8.2-beta.53.tgz deleted file mode 100644 index a4baf184d7a..00000000000 Binary files a/packages/grid/helm/repo/syft-0.8.2-beta.53.tgz and /dev/null differ diff --git a/packages/grid/helm/repo/syft-0.8.3-beta.3.tgz b/packages/grid/helm/repo/syft-0.8.3-beta.3.tgz deleted file mode 100644 index b7a2322c490..00000000000 Binary files a/packages/grid/helm/repo/syft-0.8.3-beta.3.tgz and /dev/null differ diff --git a/packages/grid/helm/repo/syft-0.8.4-beta.17.tgz b/packages/grid/helm/repo/syft-0.8.4-beta.17.tgz deleted file mode 100644 index 9d9a5fcfc78..00000000000 Binary files a/packages/grid/helm/repo/syft-0.8.4-beta.17.tgz and /dev/null differ diff --git a/packages/grid/helm/repo/syft-0.8.7-beta.11.tgz b/packages/grid/helm/repo/syft-0.8.7-beta.11.tgz new file mode 100644 index 00000000000..32078a0eae3 Binary files /dev/null and b/packages/grid/helm/repo/syft-0.8.7-beta.11.tgz differ diff --git a/packages/grid/helm/repo/syft-0.8.7-beta.12.tgz b/packages/grid/helm/repo/syft-0.8.7-beta.12.tgz new file mode 100644 index 00000000000..9bb5f0888ab Binary files /dev/null and b/packages/grid/helm/repo/syft-0.8.7-beta.12.tgz differ diff --git a/packages/grid/helm/repo/syft-0.8.7-beta.13.tgz b/packages/grid/helm/repo/syft-0.8.7-beta.13.tgz new file mode 100644 index 00000000000..212dd783e39 Binary files /dev/null and b/packages/grid/helm/repo/syft-0.8.7-beta.13.tgz differ diff --git a/packages/grid/helm/syft/Chart.yaml b/packages/grid/helm/syft/Chart.yaml index 4b925cc05c1..6965335a744 100644 --- a/packages/grid/helm/syft/Chart.yaml +++ b/packages/grid/helm/syft/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: syft description: Perform numpy-like analysis on data that remains in someone elses server type: application -version: "0.8.7-beta.10" -appVersion: "0.8.7-beta.10" +version: "0.8.7-beta.13" +appVersion: "0.8.7-beta.13" home: https://github.com/OpenMined/PySyft/ icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png diff --git a/packages/grid/helm/syft/templates/NOTES.txt b/packages/grid/helm/syft/templates/NOTES.txt index b412c4ff833..85dc72ecda4 100644 --- a/packages/grid/helm/syft/templates/NOTES.txt +++ b/packages/grid/helm/syft/templates/NOTES.txt @@ -200,6 +200,11 @@ "version": 2, "hash": "925f1b8ccd4b9d542700a111f9c4bdd28bfa55978d805ddb2fb3c108cc940d19", "action": "add" + }, + "3": { + "version": 3, + "hash": "1b5fd28919cb496f8073a64a57736d477ace1ed969962b1b049cecf766f2661c", + "action": "add" } }, "NodePeer": { @@ -244,12 +249,108 @@ "action": "remove" } }, + "CreateCustomWorkerPoolChange": { + "3": { + "version": 3, + "hash": "e982f2ebcdc6fe23a65a014109e33ba7c487bb7ca5623723cf5ec7642f86828c", + "action": "add" + } + }, "NodePeerUpdate": { "1": { "version": 1, "hash": "9e7cd39f6a9f90e8c595452865525e0989df1688236acfd1a665ed047ba47de9", "action": "add" } + }, + "JobInfo": { + "2": { + "version": 2, + "hash": "89dbd4a810586b49498be1f5299b565a19871487e14a120433b0a4cf607b6dee", + "action": "remove" + } + }, + "HTTPConnection": { + "3": { + "version": 3, + "hash": "54b452bb4ab76691ac1e704b62e7bcec740850fea00805145259b37973ecd0f4", + "action": "add" + } + }, + "UserCode": { + "4": { + "version": 4, + "hash": "0a7181cd5f76800b6566175ffa7276d0cf38c4ddc5110114430147dfc8bfdb2a", + "action": "remove" + }, + "5": { + "version": 5, + "hash": "128705a5fdf308055ef857b25c80966c928938a05ec03459dae9b36bd6122aa2", + "action": "add" + } + }, + "SyncedUserCodeStatusChange": { + "3": { + "version": 3, + "hash": "9b8ab2d513d84006bdd1329cd0bb636e7e62100a6227d8b772a5bf7c0c45b72f", + "action": "add" + } + }, + "PolicyRule": { + "1": { + "version": 1, + "hash": "31a982b94654ce27ad27a6622c6fa26dfe3f759a7824ac21d104390f10a5aa82", + "action": "add" + } + }, + "CreatePolicyRule": { + "1": { + "version": 1, + "hash": "9b82e36c63e10c5b7b76b3b8ec1da1d2dfdce39f2cce98603a418ec221621874", + "action": "add" + } + }, + "CreatePolicyRuleConstant": { + "1": { + "version": 1, + "hash": "9e821ddd383b6472f95dad2b56ebaefad225ff49c96b89b4ce35dc99c422ba76", + "action": "add" + } + }, + "Matches": { + "1": { + "version": 1, + "hash": "d1e875a6332a481458e83db364dfdf92bd34a87093d9762dfe8e136e5088bc4e", + "action": "add" + } + }, + "PreFill": { + "1": { + "version": 1, + "hash": "22c38b8ad68409493810362e6c48822d3e2919760f26eba2d1de3f2ad394e314", + "action": "add" + } + }, + "UserOwned": { + "1": { + "version": 1, + "hash": "b5cbb44d742fa51b9adf2a48bb56d9ff5ca82a25f8568a2505961bd906d9d084", + "action": "add" + } + }, + "MixedInputPolicy": { + "1": { + "version": 1, + "hash": "0e84e4c91e378717e1a4703574b07e3b1e6a3e5707401b4e0cc8d30088a506b9", + "action": "add" + } + }, + "Request": { + "3": { + "version": 3, + "hash": "ba9ebb04cc3e8b3ae3302fd42a67e47261a0a330bae5f189d8f4819cf2804711", + "action": "add" + } } } diff --git a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml index 106d2fee893..be0a35d6245 100644 --- a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml @@ -109,7 +109,14 @@ spec: - name: SMTP_USERNAME value: {{ .Values.node.smtp.username | quote }} - name: SMTP_PASSWORD + {{- if .Values.node.smtp.existingSecret }} + valueFrom: + secretKeyRef: + name: {{ .Values.node.smtp.existingSecret }} + key: smtpPassword + {{ else }} value: {{ .Values.node.smtp.password | quote }} + {{ end }} - name: EMAIL_SENDER value: {{ .Values.node.smtp.from | quote}} # SeaweedFS diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 2644eac26e4..72907e2aa81 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -1,7 +1,7 @@ global: # Affects only backend, frontend, and seaweedfs containers registry: docker.io - version: 0.8.7-beta.10 + version: 0.8.7-beta.13 # Force default secret values for development. DO NOT SET THIS TO FALSE IN PRODUCTION randomizedSecrets: true @@ -175,6 +175,8 @@ node: # SMTP Settings smtp: + # Existing secret for SMTP with key 'smtpPassword' + existingSecret: null host: smtp.sendgrid.net port: 587 from: noreply@openmined.org @@ -195,7 +197,7 @@ node: resourcesPreset: xlarge resources: null - # Seaweed secret name. Override this if you want to use a self-managed secret. + # Backend secret name. Override this if you want to use a self-managed secret. # Secret must contain the following keys: # - defaultRootPassword secretKeyName: backend-secret diff --git a/packages/grid/syft-client/syft.Dockerfile b/packages/grid/syft-client/syft.Dockerfile index 8f94e38b81b..abfed99480a 100644 --- a/packages/grid/syft-client/syft.Dockerfile +++ b/packages/grid/syft-client/syft.Dockerfile @@ -8,13 +8,16 @@ ARG PYTHON_VERSION ENV PATH="/root/.local/bin:$PATH" +# Setup Python DEV RUN apk update && apk upgrade && \ - apk add --no-cache build-base gcc python-$PYTHON_VERSION-dev-default py$PYTHON_VERSION-pip + apk add build-base gcc python-$PYTHON_VERSION-dev py$PYTHON_VERSION-pip && \ + # preemptive fix for wolfi-os breaking python entrypoint + (test -f /usr/bin/python || ln -s /usr/bin/python3.12 /usr/bin/python) COPY ./syft /tmp/syft RUN --mount=type=cache,target=/root/.cache,sharing=locked \ - pip install --user jupyterlab==4.1.6 /tmp/syft + pip install --user jupyterlab==4.2.2 /tmp/syft # ==================== [Final] Setup Syft Client ==================== # @@ -25,7 +28,7 @@ ARG PYTHON_VERSION ENV PATH="/root/.local/bin:$PATH" RUN apk update && apk upgrade && \ - apk add --no-cache git python-$PYTHON_VERSION-dev-default py$PYTHON_VERSION-pip + apk add --no-cache git python-$PYTHON_VERSION-dev py$PYTHON_VERSION-pip COPY --from=syft_deps /root/.local /root/.local diff --git a/packages/syft/PYPI.md b/packages/syft/PYPI.md index 0a2b08b1495..e2dbd7e2880 100644 --- a/packages/syft/PYPI.md +++ b/packages/syft/PYPI.md @@ -70,7 +70,10 @@ domain_client = sy.login( ## Deploy Kubernetes Helm Chart -#### 0. Deploy Kubernetes with 8+ Cores and 16GB RAM +#### 0. Deploy Kubernetes + +Required resources: 1 CPU and 4GB RAM. However, you will need some special instructions to deploy, please consult [these instructions](https://github.com/OpenMined/PySyft/blob/dev/notebooks/tutorials/deployments/03-deploy-k8s-k3d.ipynb) or look at the resource constraint testing [here](https://github.com/OpenMined/PySyft/pull/8828#issue-2300774645). +Recommended resources: 8+ Cores and 16GB RAM If you're using Docker Desktop to deploy your Kubernetes, you may need to go into Settings > Resources and increase CPUs and Memory. diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 59bfee973ea..6c8ed7d741b 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = syft -version = attr: "0.8.7-beta.10" +version = attr: "0.8.7-beta.13" description = Perform numpy-like analysis on data that remains in someone elses server author = OpenMined author_email = info@openmined.org @@ -30,7 +30,6 @@ syft = bcrypt==4.1.2 boto3==1.34.56 forbiddenfruit==0.1.4 - loguru==0.7.2 packaging>=23.0 pyarrow==15.0.0 pycapnp==2.0.0 @@ -49,7 +48,7 @@ syft = uvicorn[standard]==0.30.0 markdown==3.5.2 fastapi==0.111.0 - psutil==5.9.8 + psutil==6.0.0 itables==1.7.1 argon2-cffi==23.1.0 matplotlib>=3.7.1,<3.9.1 @@ -65,6 +64,7 @@ syft = rich==13.7.1 jinja2==3.1.4 tenacity==8.3.0 + nh3==0.2.17 install_requires = %(syft)s @@ -86,7 +86,7 @@ data_science = evaluate==0.4.2 recordlinkage==0.16 # backend.dockerfile installs torch separately, so update the version over there as well! - torch==2.3.0 + torch==2.2.2 dev = %(test_plugins)s diff --git a/packages/syft/src/syft/VERSION b/packages/syft/src/syft/VERSION index 61a3b991302..df1dec5602a 100644 --- a/packages/syft/src/syft/VERSION +++ b/packages/syft/src/syft/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.7-beta.10" +__version__ = "0.8.7-beta.13" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/packages/syft/src/syft/__init__.py b/packages/syft/src/syft/__init__.py index d7183898935..23ac10fd52f 100644 --- a/packages/syft/src/syft/__init__.py +++ b/packages/syft/src/syft/__init__.py @@ -1,87 +1,90 @@ -__version__ = "0.8.7-beta.10" +__version__ = "0.8.7-beta.13" # stdlib from collections.abc import Callable import pathlib from pathlib import Path import sys -from types import MethodType from typing import Any # relative -from .abstract_node import NodeSideType # noqa: F401 -from .abstract_node import NodeType # noqa: F401 -from .client.client import connect # noqa: F401 -from .client.client import login # noqa: F401 -from .client.client import login_as_guest # noqa: F401 -from .client.client import register # noqa: F401 -from .client.domain_client import DomainClient # noqa: F401 -from .client.gateway_client import GatewayClient # noqa: F401 -from .client.registry import DomainRegistry # noqa: F401 -from .client.registry import EnclaveRegistry # noqa: F401 -from .client.registry import NetworkRegistry # noqa: F401 -from .client.search import Search # noqa: F401 -from .client.search import SearchResults # noqa: F401 -from .client.user_settings import UserSettings # noqa: F401 -from .client.user_settings import settings # noqa: F401 -from .custom_worker.config import DockerWorkerConfig # noqa: F401 -from .custom_worker.config import PrebuiltWorkerConfig # noqa: F401 -from .node.credentials import SyftSigningKey # noqa: F401 -from .node.domain import Domain # noqa: F401 -from .node.enclave import Enclave # noqa: F401 -from .node.gateway import Gateway # noqa: F401 -from .node.server import serve_node # noqa: F401 -from .node.server import serve_node as bind_worker # noqa: F401 -from .node.worker import Worker # noqa: F401 -from .orchestra import Orchestra as orchestra # noqa: F401 -from .protocol.data_protocol import bump_protocol_version # noqa: F401 -from .protocol.data_protocol import check_or_stage_protocol # noqa: F401 -from .protocol.data_protocol import get_data_protocol # noqa: F401 -from .protocol.data_protocol import stage_protocol_changes # noqa: F401 -from .serde import NOTHING # noqa: F401 -from .serde.deserialize import _deserialize as deserialize # noqa: F401 -from .serde.serializable import serializable # noqa: F401 -from .serde.serialize import _serialize as serialize # noqa: F401 -from .service.action.action_data_empty import ActionDataEmpty # noqa: F401 -from .service.action.action_object import ActionObject # noqa: F401 -from .service.action.plan import Plan # noqa: F401 -from .service.action.plan import planify # noqa: F401 -from .service.api.api import api_endpoint # noqa: F401 -from .service.api.api import api_endpoint_method # noqa: F401 -from .service.api.api import create_new_api_endpoint as TwinAPIEndpoint # noqa: F401 -from .service.code.user_code import UserCodeStatus # noqa: F401; noqa: F401 -from .service.code.user_code import syft_function # noqa: F401; noqa: F401 -from .service.code.user_code import syft_function_single_use # noqa: F401; noqa: F401 -from .service.data_subject import DataSubjectCreate as DataSubject # noqa: F401 -from .service.dataset.dataset import Contributor # noqa: F401 -from .service.dataset.dataset import CreateAsset as Asset # noqa: F401 -from .service.dataset.dataset import CreateDataset as Dataset # noqa: F401 -from .service.notification.notifications import NotificationStatus # noqa: F401 -from .service.policy.policy import CustomInputPolicy # noqa: F401 -from .service.policy.policy import CustomOutputPolicy # noqa: F401 -from .service.policy.policy import ExactMatch # noqa: F401 -from .service.policy.policy import SingleExecutionExactOutput # noqa: F401 -from .service.policy.policy import UserInputPolicy # noqa: F401 -from .service.policy.policy import UserOutputPolicy # noqa: F401 -from .service.project.project import ProjectSubmit as Project # noqa: F401 -from .service.request.request import SubmitRequest as Request # noqa: F401 -from .service.response import SyftError # noqa: F401 -from .service.response import SyftNotReady # noqa: F401 -from .service.response import SyftSuccess # noqa: F401 -from .service.user.roles import Roles as roles # noqa: F401 -from .service.user.user_service import UserService # noqa: F401 +from .abstract_node import NodeSideType +from .abstract_node import NodeType +from .client.client import connect +from .client.client import login +from .client.client import login_as_guest +from .client.client import register +from .client.domain_client import DomainClient +from .client.gateway_client import GatewayClient +from .client.registry import DomainRegistry +from .client.registry import EnclaveRegistry +from .client.registry import NetworkRegistry +from .client.search import Search +from .client.search import SearchResults +from .client.syncing import compare_clients +from .client.syncing import compare_states +from .client.syncing import sync +from .client.user_settings import UserSettings +from .client.user_settings import settings +from .custom_worker.config import DockerWorkerConfig +from .custom_worker.config import PrebuiltWorkerConfig +from .node.credentials import SyftSigningKey +from .node.domain import Domain +from .node.enclave import Enclave +from .node.gateway import Gateway +from .node.server import serve_node +from .node.server import serve_node as bind_worker +from .node.worker import Worker +from .orchestra import Orchestra as orchestra +from .protocol.data_protocol import bump_protocol_version +from .protocol.data_protocol import check_or_stage_protocol +from .protocol.data_protocol import get_data_protocol +from .protocol.data_protocol import stage_protocol_changes +from .serde import NOTHING +from .serde.deserialize import _deserialize as deserialize +from .serde.serializable import serializable +from .serde.serialize import _serialize as serialize +from .service.action.action_data_empty import ActionDataEmpty +from .service.action.action_object import ActionObject +from .service.action.plan import Plan +from .service.action.plan import planify +from .service.api.api import api_endpoint +from .service.api.api import api_endpoint_method +from .service.api.api import create_new_api_endpoint as TwinAPIEndpoint +from .service.code.user_code import UserCodeStatus +from .service.code.user_code import syft_function +from .service.code.user_code import syft_function_single_use +from .service.data_subject import DataSubjectCreate as DataSubject +from .service.dataset.dataset import Contributor +from .service.dataset.dataset import CreateAsset as Asset +from .service.dataset.dataset import CreateDataset as Dataset +from .service.notification.notifications import NotificationStatus +from .service.policy.policy import CreatePolicyRuleConstant as Constant +from .service.policy.policy import CustomInputPolicy +from .service.policy.policy import CustomOutputPolicy +from .service.policy.policy import ExactMatch +from .service.policy.policy import MixedInputPolicy +from .service.policy.policy import SingleExecutionExactOutput +from .service.policy.policy import UserInputPolicy +from .service.policy.policy import UserOutputPolicy +from .service.project.project import ProjectSubmit as Project +from .service.request.request import SubmitRequest as Request +from .service.response import SyftError +from .service.response import SyftNotReady +from .service.response import SyftSuccess +from .service.user.roles import Roles as roles +from .service.user.user_service import UserService from .stable_version import LATEST_STABLE_SYFT -from .types.syft_object import SyftObject -from .types.twin_object import TwinObject # noqa: F401 -from .types.uid import UID # noqa: F401 -from .util import filterwarnings # noqa: F401 -from .util import logger # noqa: F401 -from .util import options # noqa: F401 -from .util.autoreload import disable_autoreload # noqa: F401 -from .util.autoreload import enable_autoreload # noqa: F401 -from .util.telemetry import instrument # noqa: F401 -from .util.util import autocache # noqa: F401 -from .util.util import get_root_data_path # noqa: F401 +from .types.twin_object import TwinObject +from .types.uid import UID +from .util import filterwarnings +from .util import options +from .util.autoreload import disable_autoreload +from .util.autoreload import enable_autoreload +from .util.patch_ipython import patch_ipython +from .util.telemetry import instrument +from .util.util import autocache +from .util.util import get_root_data_path from .util.version_compare import make_requires requires = make_requires(LATEST_STABLE_SYFT, __version__) @@ -92,103 +95,8 @@ sys.path.append(str(Path(__file__))) -logger.start() - -try: - # third party - from IPython import get_ipython - - get_ipython() # noqa: F821 - # TODO: add back later or auto detect - # display( - # Markdown( - # "\nWarning: syft is imported in light mode by default. \ - # \nTo switch to dark mode, please run `sy.options.color_theme = 'dark'`" - # ) - # ) -except: # noqa: E722 - pass # nosec - - -def _patch_ipython_autocompletion() -> None: - try: - # third party - from IPython.core.guarded_eval import EVALUATION_POLICIES - except ImportError: - return - - ipython = get_ipython() - if ipython is None: - return - - try: - # this allows property getters to be used in nested autocomplete - ipython.Completer.evaluation = "limited" - ipython.Completer.use_jedi = False - policy = EVALUATION_POLICIES["limited"] - - policy.allowed_getattr_external.update( - [ - ("syft.client.api", "APIModule"), - ("syft.client.api", "SyftAPI"), - ] - ) - original_can_get_attr = policy.can_get_attr - - def patched_can_get_attr(value: Any, attr: str) -> bool: - attr_name = "__syft_allow_autocomplete__" - # first check if exist to prevent side effects - if hasattr(value, attr_name) and attr in getattr(value, attr_name, []): - if attr in dir(value): - return True - else: - return False - else: - return original_can_get_attr(value, attr) - - policy.can_get_attr = patched_can_get_attr - except Exception: - print("Failed to patch ipython autocompletion for syft property getters") - - try: - # this constraints the completions for autocomplete. - # if __syft_dir__ is defined we only autocomplete those properties - # stdlib - import re - - original_attr_matches = ipython.Completer.attr_matches - - def patched_attr_matches(self, text: str) -> list[str]: # type: ignore - res = original_attr_matches(text) - m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) - if not m2: - return res - expr, _ = m2.group(1, 2) - obj = self._evaluate_expr(expr) - if isinstance(obj, SyftObject) and hasattr(obj, "__syft_dir__"): - # here we filter all autocomplete results to only contain those - # defined in __syft_dir__, however the original autocomplete prefixes - # have the full path, while __syft_dir__ only defines the attr - attrs = set(obj.__syft_dir__()) - new_res = [] - for r in res: - splitted = r.split(".") - if len(splitted) > 1: - attr_name = splitted[-1] - if attr_name in attrs: - new_res.append(r) - return new_res - else: - return res - - ipython.Completer.attr_matches = MethodType( - patched_attr_matches, ipython.Completer - ) - except Exception: - print("Failed to patch syft autocompletion for __syft_dir__") - - -_patch_ipython_autocompletion() + +patch_ipython() def module_property(func: Any) -> Callable: diff --git a/packages/syft/src/syft/assets/css/style.css b/packages/syft/src/syft/assets/css/style.css index 528046d6668..beece1fa2c0 100644 --- a/packages/syft/src/syft/assets/css/style.css +++ b/packages/syft/src/syft/assets/css/style.css @@ -5,6 +5,7 @@ body.vscode-dark { --tertiary-color: #cfcdd6; --button-color: #111111; --colors-black: #ffffff; + --surface-color: #fff; } body { @@ -13,6 +14,7 @@ body { --tertiary-color: #000000de; --button-color: #d1d5db; --colors-black: #17161d; + --surface-color: #464158; } .header-1 { @@ -564,3 +566,52 @@ body { .syft-widget li a:hover { background-color: #c2def0; } + +.syft-user_code, +.syft-project, +.syft-project-create, +.syft-settings, +.syft-dataset, +.syft-asset, +.syft-contributor, +.syft-request, +.syft-syncstate, +.job-info { + color: var(--surface-color); +} + +.syft-dataset h3, +.syft-dataset p, +.syft-asset h3, +.syft-asset p, +.syft-syncstate h3, +.syft-syncstate p { + font-family: "Open Sans"; +} + +.diff-container { + border: 0.5px solid #b4b0bf; +} + +.syft-container { + padding: 5px; + font-family: "Open Sans"; +} + +.syft-alert-info { + color: #1f567a; + background-color: #c2def0; + border-radius: 4px; + padding: 5px; + padding: 13px 10px; +} + +.syft-code-block { + background-color: #f7f7f7; + border: 1px solid #cfcfcf; + padding: 0px 2px; +} + +.syft-space { + margin-top: 1em; +} diff --git a/packages/syft/src/syft/assets/svg/copy.svg b/packages/syft/src/syft/assets/svg/copy.svg index aadd5116ebb..9e43a5b27f2 100644 --- a/packages/syft/src/syft/assets/svg/copy.svg +++ b/packages/syft/src/syft/assets/svg/copy.svg @@ -1,5 +1,3 @@ - - \ No newline at end of file + + diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index c4f87e5d53a..9c8b244b129 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -62,7 +62,7 @@ from ..types.uid import UID from ..util.autoreload import autoreload_enabled from ..util.markdown import as_markdown_python_code -from ..util.table import list_dict_repr_html +from ..util.notebook_ui.components.tabulator_template import build_tabulator_table from ..util.telemetry import instrument from ..util.util import prompt_warning_message from .connection import NodeConnection @@ -226,6 +226,9 @@ def sign(self, credentials: SyftSigningKey) -> SignedSyftAPICall: signature=signed_message.signature, ) + def __repr__(self) -> str: + return f"SyftAPICall(path={self.path}, args={self.args}, kwargs={self.kwargs}, blocking={self.blocking})" + @instrument @serializable() @@ -730,9 +733,9 @@ def recursively_get_submodules( APISubModulesView(submodule=submodule_name, endpoints=child_paths) ) - return list_dict_repr_html(views) - # return NotImplementedError + return build_tabulator_table(views) + # should never happen? results = self.get_all() return results._repr_html_() @@ -753,7 +756,7 @@ def debox_signed_syftapicall_response( def downgrade_signature(signature: Signature, object_versions: dict) -> Signature: migrated_parameters = [] - for _, parameter in signature.parameters.items(): + for parameter in signature.parameters.values(): annotation = unwrap_and_migrate_annotation( parameter.annotation, object_versions ) @@ -1062,8 +1065,11 @@ def make_call(self, api_call: SyftAPICall, cache_result: bool = True) -> Result: if isinstance(result, CachedSyftObject): if result.error_msg is not None: if cache_result: + msg = "Loading results from cache." + if result.error_msg: + msg = f"{result.error_msg}. {msg}" prompt_warning_message( - message=f"{result.error_msg}. Loading results from cache." + message=msg, ) else: result = SyftError(message=result.error_msg) @@ -1114,7 +1120,7 @@ def build_endpoint_tree( endpoints: dict[str, LibEndpoint], communication_protocol: PROTOCOL_TYPE ) -> APIModule: api_module = APIModule(path="", refresh_callback=self.refresh_api_callback) - for _, v in endpoints.items(): + for v in endpoints.values(): signature = v.signature if not v.has_self: signature = signature_remove_self(signature) @@ -1183,7 +1189,9 @@ def __repr__(self) -> str: if hasattr(module_or_func, "_modules"): for func_name in module_or_func._modules: func = getattr(module_or_func, func_name) - sig = func.__ipython_inspector_signature_override__ + sig = getattr( + func, "__ipython_inspector_signature_override__", "" + ) _repr_str += f"{module_path_str}.{func_name}{sig}\n\n" return _repr_str @@ -1261,7 +1269,6 @@ def monkey_patch_getdef(self: Any, obj: Any, oname: str = "") -> str | None: Inspector._getdef_bak = Inspector._getdef Inspector._getdef = types.MethodType(monkey_patch_getdef, Inspector) except Exception: - # print("Failed to monkeypatch IPython Signature Override") pass # nosec diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 498c22e7536..eb5b1d1cc44 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -4,16 +4,18 @@ # stdlib import base64 from collections.abc import Callable -from copy import deepcopy from enum import Enum from getpass import getpass import json +import logging from typing import Any from typing import TYPE_CHECKING from typing import cast # third party from argon2 import PasswordHasher +from cachetools import TTLCache +from cachetools import cached from pydantic import field_validator import requests from requests import Response @@ -48,8 +50,8 @@ from ..service.user.user_service import UserService from ..types.grid_url import GridURL from ..types.syft_object import SYFT_OBJECT_VERSION_2 +from ..types.syft_object import SYFT_OBJECT_VERSION_3 from ..types.uid import UID -from ..util.logger import debug from ..util.telemetry import instrument from ..util.util import prompt_warning_message from ..util.util import thread_ident @@ -63,6 +65,8 @@ from .connection import NodeConnection from .protocol import SyftProtocol +logger = logging.getLogger(__name__) + if TYPE_CHECKING: # relative from ..service.network.node_peer import NodePeer @@ -78,7 +82,7 @@ def upgrade_tls(url: GridURL, response: Response) -> GridURL: if response.url.startswith("https://") and url.protocol == "http": # we got redirected to https https_url = GridURL.from_url(response.url).with_path("") - debug(f"GridURL Upgraded to HTTPS. {https_url}") + logger.debug(f"GridURL Upgraded to HTTPS. {https_url}") return https_url except Exception as e: print(f"Failed to upgrade to HTTPS. {e}") @@ -130,7 +134,7 @@ class Routes(Enum): @serializable(attrs=["proxy_target_uid", "url"]) -class HTTPConnection(NodeConnection): +class HTTPConnectionV2(NodeConnection): __canonical_name__ = "HTTPConnection" __version__ = SYFT_OBJECT_VERSION_2 @@ -139,6 +143,18 @@ class HTTPConnection(NodeConnection): routes: type[Routes] = Routes session_cache: Session | None = None + +@serializable(attrs=["proxy_target_uid", "url"]) +class HTTPConnection(NodeConnection): + __canonical_name__ = "HTTPConnection" + __version__ = SYFT_OBJECT_VERSION_3 + + url: GridURL + proxy_target_uid: UID | None = None + routes: type[Routes] = Routes + session_cache: Session | None = None + headers: dict[str, str] | None = None + @field_validator("url", mode="before") @classmethod def make_url(cls, v: Any) -> Any: @@ -148,6 +164,9 @@ def make_url(cls, v: Any) -> Any: else v ) + def set_headers(self, headers: dict[str, str]) -> None: + self.headers = headers + def with_proxy(self, proxy_target_uid: UID) -> Self: return HTTPConnection(url=self.url, proxy_target_uid=proxy_target_uid) @@ -183,9 +202,35 @@ def session(self) -> Session: return self.session_cache def _make_get(self, path: str, params: dict | None = None) -> bytes: + if params is None: + return self._make_get_no_params(path) url = self.url.with_path(path) response = self.session.get( - str(url), verify=verify_tls(), proxies={}, params=params + str(url), + headers=self.headers, + verify=verify_tls(), + proxies={}, + params=params, + ) + if response.status_code != 200: + raise requests.ConnectionError( + f"Failed to fetch {url}. Response returned with code {response.status_code}" + ) + + # upgrade to tls if available + self.url = upgrade_tls(self.url, response) + + return response.content + + @cached(cache=TTLCache(maxsize=128, ttl=300)) + def _make_get_no_params(self, path: str) -> bytes: + print(path) + url = self.url.with_path(path) + response = self.session.get( + str(url), + headers=self.headers, + verify=verify_tls(), + proxies={}, ) if response.status_code != 200: raise requests.ConnectionError( @@ -205,7 +250,12 @@ def _make_post( ) -> bytes: url = self.url.with_path(path) response = self.session.post( - str(url), verify=verify_tls(), json=json, proxies={}, data=data + str(url), + headers=self.headers, + verify=verify_tls(), + json=json, + proxies={}, + data=data, ) if response.status_code != 200: raise requests.ConnectionError( @@ -220,7 +270,7 @@ def _make_post( def stream_data(self, credentials: SyftSigningKey) -> Response: url = self.url.with_path(self.routes.STREAM.value) response = self.session.get( - str(url), verify=verify_tls(), proxies={}, stream=True + str(url), verify=verify_tls(), proxies={}, stream=True, headers=self.headers ) return response @@ -310,6 +360,7 @@ def make_call(self, signed_call: SignedSyftAPICall) -> Any | SyftError: response = requests.post( # nosec url=str(self.api_url), data=msg_bytes, + headers=self.headers, ) if response.status_code != 200: @@ -531,6 +582,15 @@ def post_init(self) -> None: self.metadata.supported_protocols ) + def set_headers(self, headers: dict[str, str]) -> None | SyftError: + if isinstance(self.connection, HTTPConnection): + self.connection.set_headers(headers) + return None + return SyftError( # type: ignore + message="Incompatible connection type." + + f"Expected HTTPConnection, got {type(self.connection)}" + ) + def _get_communication_protocol( self, protocols_supported_by_server: list ) -> int | str: @@ -569,60 +629,6 @@ def create_project( project = project_create.send() return project - # TODO: type of request should be REQUEST, but it will give circular import error - def sync_code_from_request(self, request: Any) -> SyftSuccess | SyftError: - # relative - from ..service.code.user_code import UserCode - from ..service.code.user_code import UserCodeStatusCollection - from ..store.linked_obj import LinkedObject - - code: UserCode | SyftError = request.code - if isinstance(code, SyftError): - return code - - code = deepcopy(code) - code.node_uid = self.id - code.user_verify_key = self.verify_key - - def get_nested_codes(code: UserCode) -> list[UserCode]: - result: list[UserCode] = [] - if code.nested_codes is None: - return result - - for _, (linked_code_obj, _) in code.nested_codes.items(): - nested_code = linked_code_obj.resolve - nested_code = deepcopy(nested_code) - nested_code.node_uid = code.node_uid - nested_code.user_verify_key = code.user_verify_key - result.append(nested_code) - result += get_nested_codes(nested_code) - - return result - - def get_code_statusses(codes: list[UserCode]) -> list[UserCodeStatusCollection]: - statusses = [] - for code in codes: - status = deepcopy(code.status) - statusses.append(status) - code.status_link = LinkedObject.from_obj(status, node_uid=code.node_uid) - return statusses - - nested_codes = get_nested_codes(code) - statusses = get_code_statusses(nested_codes + [code]) - - for c in nested_codes + [code]: - res = self.code.submit(c) - if isinstance(res, SyftError): - return res - - for status in statusses: - res = self.api.services.code_status.create(status) - if isinstance(res, SyftError): - return res - - self._fetch_api(self.credentials) - return SyftSuccess(message="User Code Submitted") - @property def authed(self) -> bool: return bool(self.credentials) @@ -777,6 +783,22 @@ def login_as_guest(self) -> Self: return _guest_client + def login_as(self, email: str) -> Self: + user_private_key = self.api.services.user.key_for_email(email=email) + if not isinstance(user_private_key, UserPrivateKey): + return user_private_key + if self.metadata is not None: + print( + f"Logged into <{self.name}: {self.metadata.node_side_type.capitalize()}-side " + f"{self.metadata.node_type.capitalize()}> as {email}" + ) + + return self.__class__( + connection=self.connection, + credentials=user_private_key.signing_key, + metadata=self.metadata, + ) + def login( self, email: str | None = None, diff --git a/packages/syft/src/syft/client/domain_client.py b/packages/syft/src/syft/client/domain_client.py index 1c768710040..cc39ac05e49 100644 --- a/packages/syft/src/syft/client/domain_client.py +++ b/packages/syft/src/syft/client/domain_client.py @@ -2,14 +2,15 @@ from __future__ import annotations # stdlib +import logging from pathlib import Path import re from string import Template +import traceback from typing import TYPE_CHECKING from typing import cast # third party -from loguru import logger import markdown from result import Result from tqdm import tqdm @@ -41,6 +42,8 @@ from .connection import NodeConnection from .protocol import SyftProtocol +logger = logging.getLogger(__name__) + if TYPE_CHECKING: # relative from ..orchestra import NodeHandle @@ -128,18 +131,23 @@ def upload_dataset(self, dataset: CreateDataset) -> SyftSuccess | SyftError: ) as pbar: for asset in dataset.asset_list: try: + contains_empty = asset.contains_empty() twin = TwinObject( - private_obj=asset.data, - mock_obj=asset.mock, + private_obj=ActionObject.from_obj(asset.data), + mock_obj=ActionObject.from_obj(asset.mock), syft_node_location=self.id, syft_client_verify_key=self.verify_key, ) - twin._save_to_blob_storage() + res = twin._save_to_blob_storage(allow_empty=contains_empty) + if isinstance(res, SyftError): + return res except Exception as e: tqdm.write(f"Failed to create twin for {asset.name}. {e}") return SyftError(message=f"Failed to create twin. {e}") - response = self.api.services.action.set(twin) + response = self.api.services.action.set( + twin, ignore_detached_objs=contains_empty + ) if isinstance(response, SyftError): tqdm.write(f"Failed to upload asset: {asset.name}") return response @@ -158,27 +166,6 @@ def upload_dataset(self, dataset: CreateDataset) -> SyftSuccess | SyftError: return valid return self.api.services.dataset.add(dataset=dataset) - # def get_permissions_for_other_node( - # self, - # items: list[Union[ActionObject, SyftObject]], - # ) -> dict: - # if len(items) > 0: - # if not len({i.syft_node_location for i in items}) == 1 or ( - # not len({i.syft_client_verify_key for i in items}) == 1 - # ): - # raise ValueError("permissions from different nodes") - # item = items[0] - # api = APIRegistry.api_for( - # item.syft_node_location, item.syft_client_verify_key - # ) - # if api is None: - # raise ValueError( - # f"Can't access the api. Please log in to {item.syft_node_location}" - # ) - # return api.services.sync.get_permissions(items) - # else: - # return {} - def refresh(self) -> None: if self.credentials: self._fetch_node_metadata(self.credentials) @@ -194,7 +181,6 @@ def get_sync_state(self) -> SyncState | SyftError: for uid, obj in state.objects.items(): if isinstance(obj, ActionObject): obj = obj.refresh_object(resolve_nested=False) - obj.reload_cache() state.objects[uid] = obj return state @@ -206,8 +192,10 @@ def apply_state(self, resolved_state: ResolvedSyncState) -> SyftSuccess | SyftEr action_objects = [x for x in items if isinstance(x, ActionObject)] for action_object in action_objects: + action_object.reload_cache() # NOTE permissions are added separately server side - action_object._send(self, add_storage_permission=False) + action_object._send(self.id, self.verify_key, add_storage_permission=False) + action_object._clear_cache() ignored_batches = resolved_state.ignored_batches @@ -287,8 +275,9 @@ def upload_files( return ActionObject.from_obj(result).send(self) except Exception as err: - logger.debug("upload_files: Error creating action_object: {}", err) - return SyftError(message=f"Failed to upload files: {err}") + return SyftError( + message=f"Failed to upload files: {err}.\n{traceback.format_exc()}" + ) def connect_to_gateway( self, diff --git a/packages/syft/src/syft/client/enclave_client.py b/packages/syft/src/syft/client/enclave_client.py index 32eebdf3189..6252817f630 100644 --- a/packages/syft/src/syft/client/enclave_client.py +++ b/packages/syft/src/syft/client/enclave_client.py @@ -2,12 +2,10 @@ from __future__ import annotations # stdlib -from typing import Any from typing import TYPE_CHECKING # relative from ..abstract_node import NodeSideType -from ..client.api import APIRegistry from ..serde.serializable import serializable from ..service.metadata.node_metadata import NodeMetadataJSON from ..service.network.routes import NodeRouteType @@ -15,7 +13,6 @@ from ..service.response import SyftSuccess from ..types.syft_object import SYFT_OBJECT_VERSION_3 from ..types.syft_object import SyftObject -from ..types.uid import UID from ..util.assets import load_png_base64 from ..util.notebook_ui.styles import FONT_CSS from .api import APIModule @@ -27,7 +24,6 @@ if TYPE_CHECKING: # relative from ..orchestra import NodeHandle - from ..service.code.user_code import SubmitUserCode @serializable() @@ -109,54 +105,6 @@ def connect_to_gateway( def get_enclave_metadata(self) -> EnclaveMetadata: return EnclaveMetadata(route=self.connection.route) - def request_code_execution(self, code: SubmitUserCode) -> Any | SyftError: - # relative - from ..service.code.user_code_service import SubmitUserCode - - if not isinstance(code, SubmitUserCode): - raise Exception( - f"The input code should be of type: {SubmitUserCode} got:{type(code)}" - ) - if code.input_policy_init_kwargs is None: - raise ValueError(f"code {code}'s input_policy_init_kwargs is None") - - enclave_metadata = self.get_enclave_metadata() - - code_id = UID() - code.id = code_id - code.enclave_metadata = enclave_metadata - - apis = [] - for k, v in code.input_policy_init_kwargs.items(): - # We would need the verify key of the data scientist to be able to index the correct client - # Since we do not want the data scientist to pass in the clients to the enclave client - # from a UX perspecitve. - # we will use the recent node id to find the correct client - # assuming that it is the correct client - # Warning: This could lead to inconsistent results, when we have multiple clients - # in the same node pointing to the same node. - # One way, by which we could solve this in the long term, - # by forcing the user to pass only assets to the sy.ExactMatch, - # by which we could extract the verify key of the data scientist - # as each object comes with a verify key and node_uid - # the asset object would contain the verify key of the data scientist. - api = APIRegistry.get_by_recent_node_uid(k.node_id) - if api is None: - raise ValueError(f"could not find client for input {v}") - else: - apis += [api] - - for api in apis: - res = api.services.code.request_code_execution(code=code) - if isinstance(res, SyftError): - return res - - # we are using the real method here, see the .code property getter - _ = self.code - res = self._request_code_execution(code=code) - - return res - def _repr_html_(self) -> str: commands = """
  • <your_client> diff --git a/packages/syft/src/syft/client/registry.py b/packages/syft/src/syft/client/registry.py index ee57b642f53..4f239e2265d 100644 --- a/packages/syft/src/syft/client/registry.py +++ b/packages/syft/src/syft/client/registry.py @@ -4,6 +4,7 @@ # stdlib from concurrent import futures import json +import logging import os from typing import Any @@ -18,10 +19,9 @@ from ..service.response import SyftException from ..types.grid_url import GridURL from ..util.constants import DEFAULT_TIMEOUT -from ..util.logger import error -from ..util.logger import warning from .client import SyftClient as Client +logger = logging.getLogger(__name__) NETWORK_REGISTRY_URL = ( "https://raw.githubusercontent.com/OpenMined/NetworkRegistry/main/gateways.json" ) @@ -43,7 +43,7 @@ def __init__(self) -> None: network_json=network_json, version="2.0.0" ) except Exception as e: - warning( + logger.warning( f"Failed to get Network Registry, go checkout: {NETWORK_REGISTRY_REPO}. Exception: {e}" ) @@ -64,7 +64,7 @@ def load_network_registry_json() -> dict: return network_json except Exception as e: - warning( + logger.warning( f"Failed to get Network Registry from {NETWORK_REGISTRY_REPO}. Exception: {e}" ) return {} @@ -169,7 +169,6 @@ def create_client(network: dict[str, Any]) -> Client: client = connect(url=str(grid_url)) return client.guest() except Exception as e: - error(f"Failed to login with: {network}. {e}") raise SyftException(f"Failed to login with: {network}. {e}") def __getitem__(self, key: str | int) -> Client: @@ -194,7 +193,7 @@ def __init__(self) -> None: ) self._get_all_domains() except Exception as e: - warning( + logger.warning( f"Failed to get Network Registry, go checkout: {NETWORK_REGISTRY_REPO}. {e}" ) @@ -263,7 +262,7 @@ def online_domains(self) -> list[tuple[NodePeer, NodeMetadataJSON | None]]: try: network_client = NetworkRegistry.create_client(network) except Exception as e: - print(f"Error in creating network client with exception {e}") + logger.error(f"Error in creating network client {e}") continue domains: list[NodePeer] = network_client.domains.retrieve_nodes() @@ -334,7 +333,6 @@ def create_client(self, peer: NodePeer) -> Client: try: return peer.guest_client except Exception as e: - error(f"Failed to login to: {peer}. {e}") raise SyftException(f"Failed to login to: {peer}. {e}") def __getitem__(self, key: str | int) -> Client: @@ -364,7 +362,7 @@ def __init__(self) -> None: enclaves_json = response.json() self.all_enclaves = enclaves_json["2.0.0"]["enclaves"] except Exception as e: - warning( + logger.warning( f"Failed to get Enclave Registry, go checkout: {ENCLAVE_REGISTRY_REPO}. {e}" ) @@ -405,10 +403,7 @@ def check_enclave(enclave: dict) -> dict[Any, Any] | None: executor.map(lambda enclave: check_enclave(enclave), enclaves) ) - online_enclaves = [] - for each in _online_enclaves: - if each is not None: - online_enclaves.append(each) + online_enclaves = [each for each in _online_enclaves if each is not None] return online_enclaves def _repr_html_(self) -> str: @@ -436,7 +431,6 @@ def create_client(enclave: dict[str, Any]) -> Client: client = connect(url=str(grid_url)) return client.guest() except Exception as e: - error(f"Failed to login with: {enclave}. {e}") raise SyftException(f"Failed to login with: {enclave}. {e}") def __getitem__(self, key: str | int) -> Client: diff --git a/packages/syft/src/syft/client/syncing.py b/packages/syft/src/syft/client/syncing.py index 156866b26ff..0489ae0507c 100644 --- a/packages/syft/src/syft/client/syncing.py +++ b/packages/syft/src/syft/client/syncing.py @@ -1,5 +1,9 @@ # stdlib +# stdlib +from collections.abc import Collection +import logging + # relative from ..abstract_node import NodeSideType from ..node.credentials import SyftVerifyKey @@ -13,10 +17,39 @@ from ..service.sync.sync_state import SyncState from ..types.uid import UID from ..util.decorators import deprecated +from ..util.util import prompt_warning_message from .domain_client import DomainClient from .sync_decision import SyncDecision from .sync_decision import SyncDirection +logger = logging.getLogger(__name__) + + +def sync( + from_client: DomainClient, + to_client: DomainClient, + include_ignored: bool = False, + include_same: bool = False, + filter_by_email: str | None = None, + include_types: Collection[str | type] | None = None, + exclude_types: Collection[str | type] | None = None, + hide_usercode: bool = True, +) -> PaginatedResolveWidget | SyftError | SyftSuccess: + diff = compare_clients( + from_client=from_client, + to_client=to_client, + include_ignored=include_ignored, + include_same=include_same, + filter_by_email=filter_by_email, + include_types=include_types, + exclude_types=exclude_types, + hide_usercode=hide_usercode, + ) + if isinstance(diff, SyftError): + return diff + + return diff.resolve() + def compare_states( from_state: SyncState, @@ -24,7 +57,9 @@ def compare_states( include_ignored: bool = False, include_same: bool = False, filter_by_email: str | None = None, - filter_by_type: str | type | None = None, + include_types: Collection[str | type] | None = None, + exclude_types: Collection[str | type] | None = None, + hide_usercode: bool = True, ) -> NodeDiff | SyftError: # NodeDiff if ( @@ -45,6 +80,15 @@ def compare_states( return SyftError( "Invalid node side types: can only compare a high and low node" ) + + if hide_usercode: + prompt_warning_message( + "UserCodes are hidden by default, and are part of the Requests." + " If you want to include them as separate objects, set `hide_usercode=False`" + ) + exclude_types = exclude_types or [] + exclude_types.append("usercode") + return NodeDiff.from_sync_state( low_state=low_state, high_state=high_state, @@ -52,7 +96,8 @@ def compare_states( include_ignored=include_ignored, include_same=include_same, filter_by_email=filter_by_email, - filter_by_type=filter_by_type, + include_types=include_types, + exclude_types=exclude_types, ) @@ -62,7 +107,9 @@ def compare_clients( include_ignored: bool = False, include_same: bool = False, filter_by_email: str | None = None, - filter_by_type: type | None = None, + include_types: Collection[str | type] | None = None, + exclude_types: Collection[str | type] | None = None, + hide_usercode: bool = True, ) -> NodeDiff | SyftError: from_state = from_client.get_sync_state() if isinstance(from_state, SyftError): @@ -78,7 +125,9 @@ def compare_clients( include_ignored=include_ignored, include_same=include_same, filter_by_email=filter_by_email, - filter_by_type=filter_by_type, + include_types=include_types, + exclude_types=exclude_types, + hide_usercode=hide_usercode, ) @@ -134,7 +183,7 @@ def handle_sync_batch( obj_diff_batch.decision = decision sync_instructions = [] - for diff in obj_diff_batch.get_dependents(include_roots=True): + for diff in obj_diff_batch.get_dependencies(include_roots=True): # figure out the right verify key to share to # in case of a job with user code, share to user code owner # without user code, share to job owner @@ -154,7 +203,7 @@ def handle_sync_batch( ) sync_instructions.append(instruction) - print(f"Decision: Syncing {len(sync_instructions)} objects") + logger.debug(f"Decision: Syncing {len(sync_instructions)} objects") # Apply empty state to source side to signal that we are done syncing res_src = src_client.apply_state(src_resolved_state) @@ -186,7 +235,7 @@ def handle_ignore_batch( for other_batch in other_ignore_batches: other_batch.decision = SyncDecision.IGNORE - print(f"Ignoring other batch with root {other_batch.root_type.__name__}") + logger.debug(f"Ignoring other batch with root {other_batch.root_type.__name__}") src_client = obj_diff_batch.source_client tgt_client = obj_diff_batch.target_client @@ -220,7 +269,7 @@ def handle_unignore_batch( other_batches = [b for b in all_batches if b is not obj_diff_batch] other_unignore_batches = get_other_unignore_batches(obj_diff_batch, other_batches) for other_batch in other_unignore_batches: - print(f"Ignoring other batch with root {other_batch.root_type.__name__}") + logger.debug(f"Ignoring other batch with root {other_batch.root_type.__name__}") other_batch.decision = None src_resolved_state.add_unignored(other_batch.root_id) tgt_resolved_state.add_unignored(other_batch.root_id) diff --git a/packages/syft/src/syft/custom_worker/k8s.py b/packages/syft/src/syft/custom_worker/k8s.py index cb4b5765e62..7f76cb94337 100644 --- a/packages/syft/src/syft/custom_worker/k8s.py +++ b/packages/syft/src/syft/custom_worker/k8s.py @@ -120,14 +120,11 @@ def resolve_pod(client: kr8s.Api, pod: str | Pod) -> Pod | None: @staticmethod def get_logs(pods: list[Pod]) -> str: - """Combine and return logs for all the pods as string""" - logs = [] - for pod in pods: - logs.append(f"----------Logs for pod={pod.metadata.name}----------") - for log in pod.logs(): - logs.append(log) - - return "\n".join(logs) + """Combine and return logs for all the pods as a single string.""" + return "\n".join( + f"----------Logs for pod={pod.metadata.name}----------\n{''.join(pod.logs())}" + for pod in pods + ) @staticmethod def get_pod_status(pod: Pod) -> PodStatus | None: @@ -150,11 +147,11 @@ def get_pod_env(pod: Pod) -> list[dict] | None: @staticmethod def get_container_exit_code(pods: list[Pod]) -> list[int]: """Return the exit codes of all the containers in the given pods.""" - exit_codes = [] - for pod in pods: - for container_status in pod.status.containerStatuses: - exit_codes.append(container_status.state.terminated.exitCode) - return exit_codes + return [ + container_status.state.terminated.exitCode + for pod in pods + for container_status in pod.status.containerStatuses + ] @staticmethod def get_container_exit_message(pods: list[Pod]) -> str | None: diff --git a/packages/syft/src/syft/custom_worker/runner_k8s.py b/packages/syft/src/syft/custom_worker/runner_k8s.py index 3e739ef4fdb..ddb9765042c 100644 --- a/packages/syft/src/syft/custom_worker/runner_k8s.py +++ b/packages/syft/src/syft/custom_worker/runner_k8s.py @@ -230,6 +230,23 @@ def _create_stateful_set( "image": tag, "env": env_vars, "volumeMounts": volume_mounts, + "livenessProbe": { + "httpGet": { + "path": "/api/v2/metadata?probe=livenessProbe", + "port": 80, + }, + "periodSeconds": 15, + "timeoutSeconds": 5, + "failureThreshold": 3, + }, + "startupProbe": { + "httpGet": { + "path": "/api/v2/metadata?probe=startupProbe", + "port": 80, + }, + "failureThreshold": 30, + "periodSeconds": 10, + }, } ], "volumes": volumes, diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 1e2c00c6f24..ea9ab3f8a47 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -4,10 +4,12 @@ # stdlib from collections import OrderedDict from collections.abc import Callable +from datetime import MINYEAR from datetime import datetime from functools import partial import hashlib import json +import logging import os from pathlib import Path import shutil @@ -17,9 +19,9 @@ from time import sleep import traceback from typing import Any +from typing import cast # third party -from loguru import logger from nacl.signing import SigningKey from result import Err from result import Result @@ -64,6 +66,7 @@ from ..service.job.job_service import JobService from ..service.job.job_stash import Job from ..service.job.job_stash import JobStash +from ..service.job.job_stash import JobStatus from ..service.job.job_stash import JobType from ..service.log.log_service import LogService from ..service.metadata.metadata_service import MetadataService @@ -101,6 +104,7 @@ from ..service.sync.sync_service import SyncService from ..service.user.user import User from ..service.user.user import UserCreate +from ..service.user.user import UserView from ..service.user.user_roles import ServiceRole from ..service.user.user_service import UserService from ..service.user.user_stash import UserStash @@ -123,6 +127,7 @@ from ..store.mongo_document_store import MongoStoreConfig from ..store.sqlite_document_store import SQLiteStoreClientConfig from ..store.sqlite_document_store import SQLiteStoreConfig +from ..types.datetime import DATETIME_FORMAT from ..types.syft_metaclass import Empty from ..types.syft_object import PartialSyftObject from ..types.syft_object import SYFT_OBJECT_VERSION_2 @@ -140,6 +145,8 @@ from .credentials import SyftVerifyKey from .worker_settings import WorkerSettings +logger = logging.getLogger(__name__) + # if user code needs to be serded and its not available we can call this to refresh # the code for a specific node UID and thread CODE_RELOADER: dict[int, Callable] = {} @@ -464,7 +471,7 @@ def get_default_store(self, use_sqlite: bool, store_type: str) -> StoreConfig: path = self.get_temp_dir("db") file_name: str = f"{self.id}.sqlite" if self.dev_mode: - print(f"{store_type}'s SQLite DB path: {path/file_name}") + logger.debug(f"{store_type}'s SQLite DB path: {path/file_name}") return SQLiteStoreConfig( client_config=SQLiteStoreClientConfig( filename=file_name, @@ -535,7 +542,7 @@ def create_queue_config( queue_config_ = queue_config elif queue_port is not None or n_consumers > 0 or create_producer: if not create_producer and queue_port is None: - print("No queue port defined to bind consumers.") + logger.warn("No queue port defined to bind consumers.") queue_config_ = ZMQQueueConfig( client_config=ZMQClientConfig( create_producer=create_producer, @@ -590,7 +597,7 @@ def init_queue_manager(self, queue_config: QueueConfig) -> None: else: # Create consumer for given worker pool syft_worker_uid = get_syft_worker_uid() - print( + logger.info( f"Running as consumer with uid={syft_worker_uid} service={service_name}" ) @@ -619,7 +626,7 @@ def add_consumer_for_service( consumer.run() def remove_consumer_with_id(self, syft_worker_id: UID) -> None: - for _, consumers in self.queue_manager.consumers.items(): + for consumers in self.queue_manager.consumers.values(): # Grab the list of consumers for the given queue consumer_to_pop = None for consumer_idx, consumer in enumerate(consumers): @@ -750,9 +757,8 @@ def find_and_migrate_data(self) -> None: ) if object_pending_migration: - print( - "Object in Document Store that needs migration: ", - object_pending_migration, + logger.debug( + f"Object in Document Store that needs migration: {object_pending_migration}" ) # Migrate data for objects in document store @@ -762,7 +768,7 @@ def find_and_migrate_data(self) -> None: if object_partition is None: continue - print(f"Migrating data for: {canonical_name} table.") + logger.debug(f"Migrating data for: {canonical_name} table.") migration_status = object_partition.migrate_data( to_klass=object_type, context=context ) @@ -779,9 +785,8 @@ def find_and_migrate_data(self) -> None: ) if action_object_pending_migration: - print( - "Object in Action Store that needs migration: ", - action_object_pending_migration, + logger.info( + f"Object in Action Store that needs migration: {action_object_pending_migration}", ) # Migrate data for objects in action store @@ -795,7 +800,7 @@ def find_and_migrate_data(self) -> None: raise Exception( f"Failed to migrate data for {canonical_name}. Error: {migration_status.err()}" ) - print("Data Migrated to latest version !!!") + logger.info("Data Migrated to latest version !!!") @property def guest_client(self) -> SyftClient: @@ -817,7 +822,7 @@ def get_guest_client(self, verbose: bool = True) -> SyftClient: ) if self.node_type: message += f"side {self.node_type.value.capitalize()} > as GUEST" - print(message) + logger.debug(message) client_type = connection.get_client_type() if isinstance(client_type, SyftError): @@ -833,9 +838,7 @@ def get_guest_client(self, verbose: bool = True) -> SyftClient: def __repr__(self) -> str: service_string = "" if not self.is_subprocess: - services = [] - for service in self.services: - services.append(service.__name__) + services = [service.__name__ for service in self.services] service_string = ", ".join(sorted(services)) service_string = f"\n\nServices:\n{service_string}" return f"{type(self).__name__}: {self.name} - {self.id} - {self.node_type}{service_string}" @@ -1267,6 +1270,7 @@ def handle_api_call_with_unsigned_result( _private_api_path = user_config_registry.private_path_for(api_call.path) method = self.get_service_method(_private_api_path) try: + logger.info(f"API Call: {api_call}") result = method(context, *api_call.args, **api_call.kwargs) except PySyftException as e: return e.handle() @@ -1448,8 +1452,10 @@ def add_queueitem_to_queue( ) # 🟡 TODO 36: Needs distributed lock + job_res = self.job_stash.set(credentials, job) + if job_res.is_err(): + return SyftError(message=f"{job_res.err()}") self.queue_stash.set_placeholder(credentials, queue_item) - self.job_stash.set(credentials, job) log_service = self.get_service("logservice") @@ -1458,21 +1464,47 @@ def add_queueitem_to_queue( return result return job + def _sort_jobs(self, jobs: list[Job]) -> list[Job]: + job_datetimes = {} + for job in jobs: + try: + d = datetime.strptime(job.creation_time, DATETIME_FORMAT) + except Exception: + d = datetime(MINYEAR, 1, 1) + job_datetimes[job.id] = d + + jobs.sort( + key=lambda job: (job.status != JobStatus.COMPLETED, job_datetimes[job.id]), + reverse=True, + ) + + return jobs + def _get_existing_user_code_jobs( self, context: AuthedServiceContext, user_code_id: UID ) -> list[Job] | SyftError: job_service = self.get_service("jobservice") - return job_service.get_by_user_code_id( + jobs = job_service.get_by_user_code_id( context=context, user_code_id=user_code_id ) + if isinstance(jobs, SyftError): + return jobs + + return self._sort_jobs(jobs) + def _is_usercode_call_on_owned_kwargs( - self, context: AuthedServiceContext, api_call: SyftAPICall + self, + context: AuthedServiceContext, + api_call: SyftAPICall, + user_code_id: UID, ) -> bool: if api_call.path != "code.call": return False user_code_service = self.get_service("usercodeservice") - return user_code_service.is_execution_on_owned_args(api_call.kwargs, context) + return user_code_service.is_execution_on_owned_args( + context, user_code_id, api_call.kwargs + ) def add_api_call_to_queue( self, api_call: SyftAPICall, parent_job_id: UID | None = None @@ -1495,18 +1527,25 @@ def add_api_call_to_queue( action = None if is_user_code: action = Action.from_api_call(unsigned_call) + user_code_id = action.user_code_id + + user = self.get_service(UserService).get_current_user(context) + if isinstance(user, SyftError): + return user + user = cast(UserView, user) + is_execution_on_owned_kwargs_allowed = ( + user.mock_execution_permission or context.role == ServiceRole.ADMIN + ) is_usercode_call_on_owned_kwargs = self._is_usercode_call_on_owned_kwargs( - context, unsigned_call + context, unsigned_call, user_code_id ) # Low side does not execute jobs, unless this is a mock execution if ( not is_usercode_call_on_owned_kwargs and self.node_side_type == NodeSideType.LOW_SIDE ): - existing_jobs = self._get_existing_user_code_jobs( - context, action.user_code_id - ) + existing_jobs = self._get_existing_user_code_jobs(context, user_code_id) if isinstance(existing_jobs, SyftError): return existing_jobs elif len(existing_jobs) > 0: @@ -1523,6 +1562,14 @@ def add_api_call_to_queue( message="Please wait for the admin to allow the execution of this code" ) + elif ( + is_usercode_call_on_owned_kwargs + and not is_execution_on_owned_kwargs_allowed + ): + return SyftError( + message="You do not have the permissions for mock execution, please contact the admin" + ) + return self.add_action_to_queue( action, api_call.credentials, parent_job_id=parent_job_id ) @@ -1606,7 +1653,9 @@ def create_initial_settings(self, admin_email: str) -> NodeSettings | None: try: settings_stash = SettingsStash(store=self.document_store) if self.signing_key is None: - print("create_initial_settings failed as there is no signing key") + logger.debug( + "create_initial_settings failed as there is no signing key" + ) return None settings_exists = settings_stash.get_all(self.signing_key.verify_key).ok() if settings_exists: @@ -1641,7 +1690,7 @@ def create_initial_settings(self, admin_email: str) -> NodeSettings | None: return result.ok() return None except Exception as e: - print(f"create_initial_settings failed with error {e}") + logger.error("create_initial_settings failed", exc_info=e) return None @@ -1681,7 +1730,7 @@ def create_admin_new( else: raise Exception(f"Could not create user: {result}") except Exception as e: - print("Unable to create new admin", e) + logger.error("Unable to create new admin", exc_info=e) return None @@ -1741,11 +1790,12 @@ def create_default_worker_pool(node: Node) -> SyftError | None: if isinstance(default_worker_pool, SyftError): logger.error( - f"Failed to get default worker pool {default_pool_name}. Error: {default_worker_pool.message}" + f"Failed to get default worker pool {default_pool_name}. " + f"Error: {default_worker_pool.message}" ) return default_worker_pool - print(f"Creating default worker image with tag='{default_worker_tag}'") + logger.info(f"Creating default worker image with tag='{default_worker_tag}'") # Get/Create a default worker SyftWorkerImage default_image = create_default_image( credentials=credentials, @@ -1754,11 +1804,11 @@ def create_default_worker_pool(node: Node) -> SyftError | None: in_kubernetes=in_kubernetes(), ) if isinstance(default_image, SyftError): - print("Failed to create default worker image: ", default_image.message) + logger.error(f"Failed to create default worker image: {default_image.message}") return default_image if not default_image.is_built: - print(f"Building default worker image with tag={default_worker_tag}") + logger.info(f"Building default worker image with tag={default_worker_tag}") image_build_method = node.get_service_method(SyftWorkerImageService.build) # Build the Image for given tag result = image_build_method( @@ -1769,11 +1819,11 @@ def create_default_worker_pool(node: Node) -> SyftError | None: ) if isinstance(result, SyftError): - print("Failed to build default worker image: ", result.message) + logger.error(f"Failed to build default worker image: {result.message}") return None # Create worker pool if it doesn't exists - print( + logger.info( "Setting up worker pool" f"name={default_pool_name} " f"workers={worker_count} " @@ -1804,17 +1854,17 @@ def create_default_worker_pool(node: Node) -> SyftError | None: ) if isinstance(result, SyftError): - print(f"Default worker pool error. {result.message}") + logger.info(f"Default worker pool error. {result.message}") return None for n in range(worker_to_add_): container_status = result[n] if container_status.error: - print( + logger.error( f"Failed to create container: Worker: {container_status.worker}," f"Error: {container_status.error}" ) return None - print("Created default worker pool.") + logger.info("Created default worker pool.") return None diff --git a/packages/syft/src/syft/node/routes.py b/packages/syft/src/syft/node/routes.py index 5b25774ff18..8be45245190 100644 --- a/packages/syft/src/syft/node/routes.py +++ b/packages/syft/src/syft/node/routes.py @@ -1,6 +1,7 @@ # stdlib import base64 import binascii +import logging from typing import Annotated # third party @@ -12,7 +13,6 @@ from fastapi import Response from fastapi.responses import JSONResponse from fastapi.responses import StreamingResponse -from loguru import logger from pydantic import ValidationError import requests @@ -34,6 +34,8 @@ from .credentials import UserLoginCredentials from .worker import Worker +logger = logging.getLogger(__name__) + def make_routes(worker: Worker) -> APIRouter: if TRACE_MODE: @@ -42,8 +44,8 @@ def make_routes(worker: Worker) -> APIRouter: # third party from opentelemetry import trace from opentelemetry.propagate import extract - except Exception: - print("Failed to import opentelemetry") + except Exception as e: + logger.error("Failed to import opentelemetry", exc_info=e) router = APIRouter() @@ -171,7 +173,7 @@ def handle_login(email: str, password: str, node: AbstractNode) -> Response: result = method(context=context) if isinstance(result, SyftError): - logger.bind(payload={"email": email}).error(result.message) + logger.error(f"Login Error: {result.message}. user={email}") response = result else: user_private_key = result @@ -196,7 +198,9 @@ def handle_register(data: bytes, node: AbstractNode) -> Response: result = method(new_user=user_create) if isinstance(result, SyftError): - logger.bind(payload={"user": user_create}).error(result.message) + logger.error( + f"Register Error: {result.message}. user={user_create.model_dump()}" + ) response = SyftError(message=f"{result.message}") else: response = result diff --git a/packages/syft/src/syft/node/server.py b/packages/syft/src/syft/node/server.py index f5f05bf35ac..43b8359a1f9 100644 --- a/packages/syft/src/syft/node/server.py +++ b/packages/syft/src/syft/node/server.py @@ -2,7 +2,6 @@ import asyncio from collections.abc import Callable from enum import Enum -import logging import multiprocessing import os import platform @@ -144,14 +143,7 @@ async def _run_uvicorn( except Exception: # nosec print(f"Failed to kill python process on port: {port}") - log_level = "critical" - if dev_mode: - log_level = "info" - logging.getLogger("uvicorn").setLevel(logging.CRITICAL) - logging.getLogger("uvicorn.access").setLevel(logging.CRITICAL) - config = uvicorn.Config( - app, host=host, port=port, log_level=log_level, reload=dev_mode - ) + config = uvicorn.Config(app, host=host, port=port, reload=dev_mode) server = uvicorn.Server(config) await server.serve() @@ -285,14 +277,14 @@ def find_python_processes_on_port(port: int) -> list[int]: python_pids = [] for pid in pids: - try: - if system == "Windows": - command = ( - f"wmic process where (ProcessId='{pid}') get ProcessId,CommandLine" - ) - else: - command = f"ps -p {pid} -o pid,command" + if system == "Windows": + command = ( + f"wmic process where (ProcessId='{pid}') get ProcessId,CommandLine" + ) + else: + command = f"ps -p {pid} -o pid,command" + try: process = subprocess.Popen( # nosec command, shell=True, @@ -301,13 +293,13 @@ def find_python_processes_on_port(port: int) -> list[int]: text=True, ) output, _ = process.communicate() - lines = output.strip().split("\n") - - if len(lines) > 1 and "python" in lines[1].lower(): - python_pids.append(pid) - except Exception as e: print(f"Error checking process {pid}: {e}") + continue + + lines = output.strip().split("\n") + if len(lines) > 1 and "python" in lines[1].lower(): + python_pids.append(pid) return python_pids diff --git a/packages/syft/src/syft/protocol/data_protocol.py b/packages/syft/src/syft/protocol/data_protocol.py index 79f0d680658..170c103d7a8 100644 --- a/packages/syft/src/syft/protocol/data_protocol.py +++ b/packages/syft/src/syft/protocol/data_protocol.py @@ -3,6 +3,7 @@ from collections.abc import Iterable from collections.abc import MutableMapping from collections.abc import MutableSequence +from functools import cache import hashlib import json from operator import itemgetter @@ -507,7 +508,7 @@ def calculate_supported_protocols(self) -> dict: # we assume its supported until we prove otherwise protocol_supported[v] = True # iterate through each object - for canonical_name, _ in version_data["object_versions"].items(): + for canonical_name in version_data["object_versions"].keys(): if canonical_name not in self.state: protocol_supported[v] = False break @@ -529,12 +530,20 @@ def reset_dev_protocol(self) -> None: def get_data_protocol(raise_exception: bool = False) -> DataProtocol: - return DataProtocol( + return _get_data_protocol( filename=data_protocol_file_name(), raise_exception=raise_exception, ) +@cache +def _get_data_protocol(filename: str, raise_exception: bool = False) -> DataProtocol: + return DataProtocol( + filename=filename, + raise_exception=raise_exception, + ) + + def stage_protocol_changes() -> Result[SyftSuccess, SyftError]: data_protocol = get_data_protocol(raise_exception=True) return data_protocol.stage_protocol_changes() diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index 375aa1af66b..5054b847ee4 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -272,6 +272,107 @@ "hash": "89dbd4a810586b49498be1f5299b565a19871487e14a120433b0a4cf607b6dee", "action": "remove" } + }, + "HTTPConnection": { + "3": { + "version": 3, + "hash": "54b452bb4ab76691ac1e704b62e7bcec740850fea00805145259b37973ecd0f4", + "action": "add" + } + }, + "UserCode": { + "4": { + "version": 4, + "hash": "0a7181cd5f76800b6566175ffa7276d0cf38c4ddc5110114430147dfc8bfdb2a", + "action": "remove" + }, + "5": { + "version": 5, + "hash": "128705a5fdf308055ef857b25c80966c928938a05ec03459dae9b36bd6122aa2", + "action": "add" + }, + "6": { + "version": 6, + "hash": "c48ec3160bb34adf937e6306523c7ebc52861ff84a576a30a28cd45c224ded0f", + "action": "add" + } + }, + "SyncedUserCodeStatusChange": { + "3": { + "version": 3, + "hash": "9b8ab2d513d84006bdd1329cd0bb636e7e62100a6227d8b772a5bf7c0c45b72f", + "action": "add" + } + }, + "PolicyRule": { + "1": { + "version": 1, + "hash": "31a982b94654ce27ad27a6622c6fa26dfe3f759a7824ac21d104390f10a5aa82", + "action": "add" + } + }, + "CreatePolicyRule": { + "1": { + "version": 1, + "hash": "9b82e36c63e10c5b7b76b3b8ec1da1d2dfdce39f2cce98603a418ec221621874", + "action": "add" + } + }, + "CreatePolicyRuleConstant": { + "1": { + "version": 1, + "hash": "9e821ddd383b6472f95dad2b56ebaefad225ff49c96b89b4ce35dc99c422ba76", + "action": "add" + } + }, + "Matches": { + "1": { + "version": 1, + "hash": "d1e875a6332a481458e83db364dfdf92bd34a87093d9762dfe8e136e5088bc4e", + "action": "add" + } + }, + "PreFill": { + "1": { + "version": 1, + "hash": "22c38b8ad68409493810362e6c48822d3e2919760f26eba2d1de3f2ad394e314", + "action": "add" + } + }, + "UserOwned": { + "1": { + "version": 1, + "hash": "b5cbb44d742fa51b9adf2a48bb56d9ff5ca82a25f8568a2505961bd906d9d084", + "action": "add" + } + }, + "MixedInputPolicy": { + "1": { + "version": 1, + "hash": "0e84e4c91e378717e1a4703574b07e3b1e6a3e5707401b4e0cc8d30088a506b9", + "action": "add" + } + }, + "Request": { + "3": { + "version": 3, + "hash": "ba9ebb04cc3e8b3ae3302fd42a67e47261a0a330bae5f189d8f4819cf2804711", + "action": "add" + } + }, + "SubmitUserCode": { + "5": { + "version": 5, + "hash": "3135727b8f0ca7689d47c04e45a2bd6a7693f17c043f76fd2243135196c27232", + "action": "add" + } + }, + "CodeHistory": { + "3": { + "version": 3, + "hash": "1b9bd1d3d096abab5617c2ff597b4c80751f686d16482a2cff4efd8741b84d53", + "action": "add" + } } } } diff --git a/packages/syft/src/syft/serde/__init__.py b/packages/syft/src/syft/serde/__init__.py index 666be78ca11..00122b4769f 100644 --- a/packages/syft/src/syft/serde/__init__.py +++ b/packages/syft/src/syft/serde/__init__.py @@ -1,4 +1,4 @@ # relative -from .array import NOTHING # noqa: F401 F811 -from .recursive import NOTHING # noqa: F401 F811 -from .third_party import NOTHING # noqa: F401 F811 +from .array import NOTHING # noqa: F811 +from .recursive import NOTHING # noqa: F811 +from .third_party import NOTHING # noqa: F811 diff --git a/packages/syft/src/syft/serde/recursive.py b/packages/syft/src/syft/serde/recursive.py index 02957e5f23d..f009ea34299 100644 --- a/packages/syft/src/syft/serde/recursive.py +++ b/packages/syft/src/syft/serde/recursive.py @@ -311,7 +311,7 @@ def rs_proto2object(proto: _DynamicStructBuilder) -> Any: # relative from ..node.node import CODE_RELOADER - for _, load_user_code in CODE_RELOADER.items(): + for load_user_code in CODE_RELOADER.values(): load_user_code() try: class_type = getattr(sys.modules[".".join(module_parts)], klass) diff --git a/packages/syft/src/syft/serde/recursive_primitives.py b/packages/syft/src/syft/serde/recursive_primitives.py index cb90932247a..042234cd843 100644 --- a/packages/syft/src/syft/serde/recursive_primitives.py +++ b/packages/syft/src/syft/serde/recursive_primitives.py @@ -68,13 +68,14 @@ def deserialize_iterable(iterable_type: type, blob: bytes) -> Collection: from .deserialize import _deserialize MAX_TRAVERSAL_LIMIT = 2**64 - 1 - values = [] with iterable_schema.from_bytes( blob, traversal_limit_in_words=MAX_TRAVERSAL_LIMIT ) as msg: - for element in msg.values: - values.append(_deserialize(combine_bytes(element), from_bytes=True)) + values = [ + _deserialize(combine_bytes(element), from_bytes=True) + for element in msg.values + ] return iterable_type(values) diff --git a/packages/syft/src/syft/serde/third_party.py b/packages/syft/src/syft/serde/third_party.py index 4e94219951e..cf025aedb9c 100644 --- a/packages/syft/src/syft/serde/third_party.py +++ b/packages/syft/src/syft/serde/third_party.py @@ -183,20 +183,8 @@ def serialize_bytes_io(io: BytesIO) -> bytes: import torch from torch._C import _TensorMeta - def serialize_torch_tensor_meta(t: _TensorMeta) -> bytes: - buffer = BytesIO() - torch.save(t, buffer) - return buffer.getvalue() - - def deserialize_torch_tensor_meta(buf: bytes) -> _TensorMeta: - buffer = BytesIO(buf) - return torch.load(buffer) - - recursive_serde_register( - _TensorMeta, - serialize=serialize_torch_tensor_meta, - deserialize=deserialize_torch_tensor_meta, - ) + recursive_serde_register_type(_TensorMeta) + recursive_serde_register_type(torch.Tensor) def torch_serialize(tensor: torch.Tensor) -> bytes: return numpy_serialize(tensor.numpy()) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index dffa3d3d9de..b8e22138d26 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -7,6 +7,7 @@ from enum import Enum import inspect from io import BytesIO +import logging from pathlib import Path import sys import threading @@ -46,7 +47,6 @@ from ...types.syncable_object import SyncableSyftObject from ...types.uid import LineageID from ...types.uid import UID -from ...util.logger import debug from ...util.util import prompt_warning_message from ..context import AuthedServiceContext from ..response import SyftException @@ -59,6 +59,8 @@ from .action_types import action_type_for_type from .action_types import action_types +logger = logging.getLogger(__name__) + if TYPE_CHECKING: # relative from ..sync.diff_state import AttrDiff @@ -247,6 +249,7 @@ class ActionObjectPointer: "__repr_str__", # pydantic "__repr_args__", # pydantic "__post_init__", # syft + "_get_api", # syft "__validate_private_attrs__", # syft "id", # syft "to_mongo", # syft 🟡 TODO 23: Add composeable / inheritable object passthrough attrs @@ -318,6 +321,8 @@ class ActionObjectPointer: "_data_repr", "syft_eq", # syft "__table_coll_widths__", + "_clear_cache", + "_set_reprs", ] dont_wrap_output_attrs = [ "__repr__", @@ -341,6 +346,8 @@ class ActionObjectPointer: "get_sync_dependencies", # syft "syft_eq", # syft "__table_coll_widths__", + "_clear_cache", + "_set_reprs", ] dont_make_side_effects = [ "__repr_attrs__", @@ -362,6 +369,8 @@ class ActionObjectPointer: "get_sync_dependencies", "syft_eq", # syft "__table_coll_widths__", + "_clear_cache", + "_set_reprs", ] action_data_empty_must_run = [ "__repr__", @@ -436,9 +445,10 @@ def make_action_side_effect( action_type=context.action_type, ) context.action = action - except Exception: - print(f"make_action_side_effect failed with {traceback.format_exc()}") - return Err(f"make_action_side_effect failed with {traceback.format_exc()}") + except Exception as e: + msg = "make_action_side_effect failed" + logger.error(msg, exc_info=e) + return Err(f"{msg} with {traceback.format_exc()}") return Ok((context, args, kwargs)) @@ -514,7 +524,7 @@ def convert_to_pointers( arg.syft_node_uid = node_uid r = arg._save_to_blob_storage() if isinstance(r, SyftError): - print(r.message) + logger.error(r.message) arg = api.services.action.set(arg) arg_list.append(arg) @@ -532,7 +542,7 @@ def convert_to_pointers( arg.syft_node_uid = node_uid r = arg._save_to_blob_storage() if isinstance(r, SyftError): - print(r.message) + logger.error(r.message) arg = api.services.action.set(arg) kwarg_dict[k] = arg @@ -659,9 +669,19 @@ def debox_args_and_kwargs(args: Any, kwargs: Any) -> tuple[Any, Any]: "_data_repr", "syft_eq", "__table_coll_widths__", + "_clear_cache", + "_set_reprs", ] +def truncate_str(string: str, length: int = 100) -> str: + stringlen = len(string) + if stringlen > length: + n_hidden = stringlen - length + string = f"{string[:length]}... ({n_hidden} characters hidden)" + return string + + @serializable(without=["syft_pre_hooks__", "syft_post_hooks__"]) class ActionObject(SyncableSyftObject): """Action object for remote execution.""" @@ -735,13 +755,10 @@ def _set_obj_location_(self, node_uid: UID, credentials: SyftVerifyKey) -> None: @property def syft_action_data(self) -> Any: - if ( - self.syft_blob_storage_entry_id - and self.syft_created_at - and not TraceResultRegistry.current_thread_is_tracing() - ): - self.reload_cache() - + if self.syft_blob_storage_entry_id and self.syft_created_at: + res = self.reload_cache() + if isinstance(res, SyftError): + print(res) return self.syft_action_data_cache def reload_cache(self) -> SyftError | None: @@ -758,9 +775,8 @@ def reload_cache(self) -> SyftError | None: uid=self.syft_blob_storage_entry_id ) if isinstance(blob_retrieval_object, SyftError): - print( - "Could not fetch actionobject data\n", - blob_retrieval_object, + logger.error( + f"Could not fetch actionobject data: {blob_retrieval_object}" ) return blob_retrieval_object # relative @@ -781,8 +797,7 @@ def reload_cache(self) -> SyftError | None: self.syft_action_data_type = type(self.syft_action_data) return None else: - print("cannot reload cache") - return None + return SyftError("Could not reload cache, could not get read method") return None @@ -826,40 +841,47 @@ def _save_to_blob_storage_(self, data: Any) -> SyftError | None: blob_deposit_object.blob_storage_entry_id ) else: - print("cannot save to blob storage") + logger.warn("cannot save to blob storage. allocate_method=None") self.syft_action_data_type = type(data) - - if inspect.isclass(data): - self.syft_action_data_repr_ = repr_cls(data) - else: - self.syft_action_data_repr_ = ( - data._repr_markdown_() - if hasattr(data, "_repr_markdown_") - else data.__repr__() - ) - self.syft_action_data_str_ = str(data) + self._set_reprs(data) self.syft_has_bool_attr = hasattr(data, "__bool__") else: - debug("skipping writing action object to store, passed data was empty.") + logger.debug( + "skipping writing action object to store, passed data was empty." + ) self.syft_action_data_cache = data return None - def _save_to_blob_storage(self) -> SyftError | None: + def _set_reprs(self, data: any) -> None: + if inspect.isclass(data): + self.syft_action_data_repr_ = truncate_str(repr_cls(data)) + else: + self.syft_action_data_repr_ = truncate_str( + data._repr_markdown_() + if hasattr(data, "_repr_markdown_") + else data.__repr__() + ) + self.syft_action_data_str_ = truncate_str(str(data)) + + def _save_to_blob_storage(self, allow_empty: bool = False) -> SyftError | None: data = self.syft_action_data if isinstance(data, SyftError): return data - if isinstance(data, ActionDataEmpty): + if isinstance(data, ActionDataEmpty) and not allow_empty: return SyftError(message=f"cannot store empty object {self.id}") result = self._save_to_blob_storage_(data) if isinstance(result, SyftError): return result if not TraceResultRegistry.current_thread_is_tracing(): - self.syft_action_data_cache = self.as_empty_data() + self._clear_cache() return None + def _clear_cache(self) -> None: + self.syft_action_data_cache = self.as_empty_data() + @property def is_pointer(self) -> bool: return self.syft_node_uid is not None @@ -879,14 +901,14 @@ def __check_action_data(cls, values: dict) -> dict: values["syft_action_data_type"] = type(v) if not isinstance(v, ActionDataEmpty): if inspect.isclass(v): - values["syft_action_data_repr_"] = repr_cls(v) + values["syft_action_data_repr_"] = truncate_str(repr_cls(v)) else: - values["syft_action_data_repr_"] = ( + values["syft_action_data_repr_"] = truncate_str( v._repr_markdown_() if v is not None and hasattr(v, "_repr_markdown_") else v.__repr__() ) - values["syft_action_data_str_"] = str(v) + values["syft_action_data_str_"] = truncate_str(str(v)) values["syft_has_bool_attr"] = hasattr(v, "__bool__") return values @@ -1090,14 +1112,9 @@ def syft_make_action( if kwargs is None: kwargs = {} - arg_ids = [] - kwarg_ids = {} - - for obj in args: - arg_ids.append(self._syft_prepare_obj_uid(obj)) + arg_ids = [self._syft_prepare_obj_uid(obj) for obj in args] - for k, obj in kwargs.items(): - kwarg_ids[k] = self._syft_prepare_obj_uid(obj) + kwarg_ids = {k: self._syft_prepare_obj_uid(obj) for k, obj in kwargs.items()} action = Action( path=path, @@ -1191,13 +1208,28 @@ def wrapper( return wrapper def send(self, client: SyftClient) -> Any: - return self._send(client, add_storage_permission=True) + return self._send( + node_uid=client.id, + verify_key=client.verify_key, + add_storage_permission=True, + ) - def _send(self, client: SyftClient, add_storage_permission: bool = True) -> Self: - """Send the object to a Syft Client""" - self._set_obj_location_(client.id, client.verify_key) - self._save_to_blob_storage() - res = client.api.services.action.set( + def _send( + self, + node_uid: UID, + verify_key: SyftVerifyKey, + add_storage_permission: bool = True, + ) -> Self | SyftError: + self._set_obj_location_(node_uid, verify_key) + + blob_storage_res = self._save_to_blob_storage() + if isinstance(blob_storage_res, SyftError): + return blob_storage_res + + api = self._get_api() + if isinstance(api, SyftError): + return api + res = api.services.action.set( self, add_storage_permission=add_storage_permission ) if isinstance(res, ActionObject): @@ -1212,7 +1244,7 @@ def get_from(self, client: SyftClient) -> Any: else: return res.syft_action_data - def refresh_object(self, resolve_nested: bool = True) -> ActionObject: + def refresh_object(self, resolve_nested: bool = True) -> ActionObject | SyftError: # relative from ...client.api import APIRegistry @@ -1247,9 +1279,10 @@ def get(self, block: bool = False) -> Any: self.wait() res = self.refresh_object() - if not isinstance(res, ActionObject): return SyftError(message=f"{res}") # type: ignore + elif issubclass(res.syft_action_data_type, Err): + return SyftError(message=f"{res.syft_action_data.err()}") else: if not self.has_storage_permission(): prompt_warning_message( @@ -1387,7 +1420,7 @@ def remove_trace_hook(cls) -> bool: def as_empty_data(self) -> ActionDataEmpty: return ActionDataEmpty(syft_internal_type=self.syft_internal_type) - def wait(self, timeout: int | None = None) -> ActionObject: + def wait(self, timeout: int | None = None) -> ActionObject | SyftError: # relative from ...client.api import APIRegistry @@ -1401,12 +1434,18 @@ def wait(self, timeout: int | None = None) -> ActionObject: obj_id = self.id counter = 0 - while api and not api.services.action.is_resolved(obj_id): - time.sleep(1) - if timeout is not None: - counter += 1 - if counter > timeout: - return SyftError(message="Reached Timeout!") + while api: + obj_resolved: bool | str = api.services.action.is_resolved(obj_id) + if isinstance(obj_resolved, str): + return SyftError(message=obj_resolved) + if obj_resolved: + break + if not obj_resolved: + time.sleep(1) + if timeout is not None: + counter += 1 + if counter > timeout: + return SyftError(message="Reached Timeout!") return self @@ -1547,7 +1586,7 @@ def _syft_run_pre_hooks__( if result.is_ok(): context, result_args, result_kwargs = result.ok() else: - debug(f"Pre-hook failed with {result.err()}") + logger.debug(f"Pre-hook failed with {result.err()}") if name not in self._syft_dont_wrap_attrs(): if HOOK_ALWAYS in self.syft_pre_hooks__: for hook in self.syft_pre_hooks__[HOOK_ALWAYS]: @@ -1556,7 +1595,7 @@ def _syft_run_pre_hooks__( context, result_args, result_kwargs = result.ok() else: msg = result.err().replace("\\n", "\n") - debug(f"Pre-hook failed with {msg}") + logger.debug(f"Pre-hook failed with {msg}") if self.is_pointer: if name not in self._syft_dont_wrap_attrs(): @@ -1567,7 +1606,7 @@ def _syft_run_pre_hooks__( context, result_args, result_kwargs = result.ok() else: msg = result.err().replace("\\n", "\n") - debug(f"Pre-hook failed with {msg}") + logger.debug(f"Pre-hook failed with {msg}") return context, result_args, result_kwargs @@ -1582,7 +1621,7 @@ def _syft_run_post_hooks__( if result.is_ok(): new_result = result.ok() else: - debug(f"Post hook failed with {result.err()}") + logger.debug(f"Post hook failed with {result.err()}") if name not in self._syft_dont_wrap_attrs(): if HOOK_ALWAYS in self.syft_post_hooks__: @@ -1591,7 +1630,7 @@ def _syft_run_post_hooks__( if result.is_ok(): new_result = result.ok() else: - debug(f"Post hook failed with {result.err()}") + logger.debug(f"Post hook failed with {result.err()}") if self.is_pointer: if name not in self._syft_dont_wrap_attrs(): @@ -1601,7 +1640,7 @@ def _syft_run_post_hooks__( if result.is_ok(): new_result = result.ok() else: - debug(f"Post hook failed with {result.err()}") + logger.debug(f"Post hook failed with {result.err()}") return new_result @@ -1693,7 +1732,7 @@ def _syft_wrap_attribute_for_bool_on_nonbools(self, name: str) -> Any: "[_wrap_attribute_for_bool_on_nonbools] self.syft_action_data already implements the bool operator" ) - debug("[__getattribute__] Handling bool on nonbools") + logger.debug("[__getattribute__] Handling bool on nonbools") context = PreHookContext( obj=self, op_name=name, @@ -1726,7 +1765,7 @@ def _syft_wrap_attribute_for_properties(self, name: str) -> Any: raise RuntimeError( "[_wrap_attribute_for_properties] Use this only on properties" ) - debug(f"[__getattribute__] Handling property {name} ") + logger.debug(f"[__getattribute__] Handling property {name}") context = PreHookContext( obj=self, @@ -1750,7 +1789,7 @@ def _syft_wrap_attribute_for_methods(self, name: str) -> Any: def fake_func(*args: Any, **kwargs: Any) -> Any: return ActionDataEmpty(syft_internal_type=self.syft_internal_type) - debug(f"[__getattribute__] Handling method {name} ") + logger.debug(f"[__getattribute__] Handling method {name}") if ( issubclass(self.syft_action_data_type, ActionDataEmpty) and name not in action_data_empty_must_run @@ -1787,20 +1826,20 @@ def _base_wrapper(*args: Any, **kwargs: Any) -> Any: return post_result if inspect.ismethod(original_func) or inspect.ismethoddescriptor(original_func): - debug("Running method: ", name) + logger.debug(f"Running method: {name}") def wrapper(_self: Any, *args: Any, **kwargs: Any) -> Any: return _base_wrapper(*args, **kwargs) wrapper = types.MethodType(wrapper, type(self)) else: - debug("Running non-method: ", name) + logger.debug(f"Running non-method: {name}") wrapper = _base_wrapper try: wrapper.__doc__ = original_func.__doc__ - debug( + logger.debug( "Found original signature for ", name, inspect.signature(original_func), @@ -1809,7 +1848,7 @@ def wrapper(_self: Any, *args: Any, **kwargs: Any) -> Any: original_func ) except Exception: - debug("name", name, "has no signature") + logger.debug(f"name={name} has no signature") # third party return wrapper @@ -1903,7 +1942,7 @@ def is_link(self) -> bool: def __setattr__(self, name: str, value: Any) -> Any: defined_on_self = name in self.__dict__ or name in self.__private_attributes__ - debug(">> ", name, ", defined_on_self = ", defined_on_self) + logger.debug(f">> {name} defined_on_self={defined_on_self}") # use the custom defined version if defined_on_self: @@ -1945,7 +1984,7 @@ def _repr_markdown_(self, wrap_as_python: bool = True, indent: int = 0) -> str: else self.syft_action_data_cache.__repr__() ) - return f"```python\n{res}\n{data_repr_}\n```\n" + return f"\n**{res}**\n\n{data_repr_}\n" def _data_repr(self) -> str | None: if isinstance(self.syft_action_data_cache, ActionDataEmpty): @@ -2152,13 +2191,13 @@ def __int__(self) -> float: def debug_original_func(name: str, func: Callable) -> None: - debug(f"{name} func is:") - debug("inspect.isdatadescriptor", inspect.isdatadescriptor(func)) - debug("inspect.isgetsetdescriptor", inspect.isgetsetdescriptor(func)) - debug("inspect.isfunction", inspect.isfunction(func)) - debug("inspect.isbuiltin", inspect.isbuiltin(func)) - debug("inspect.ismethod", inspect.ismethod(func)) - debug("inspect.ismethoddescriptor", inspect.ismethoddescriptor(func)) + logger.debug(f"{name} func is:") + logger.debug(f"inspect.isdatadescriptor = {inspect.isdatadescriptor(func)}") + logger.debug(f"inspect.isgetsetdescriptor = {inspect.isgetsetdescriptor(func)}") + logger.debug(f"inspect.isfunction = {inspect.isfunction(func)}") + logger.debug(f"inspect.isbuiltin = {inspect.isbuiltin(func)}") + logger.debug(f"inspect.ismethod = {inspect.ismethod(func)}") + logger.debug(f"inspect.ismethoddescriptor = {inspect.ismethoddescriptor(func)}") def is_action_data_empty(obj: Any) -> bool: @@ -2172,7 +2211,7 @@ def has_action_data_empty(args: Any, kwargs: Any) -> bool: if is_action_data_empty(a): return True - for _, a in kwargs.items(): + for a in kwargs.values(): if is_action_data_empty(a): return True return False diff --git a/packages/syft/src/syft/service/action/action_service.py b/packages/syft/src/syft/service/action/action_service.py index 00f1414b247..10b864732bb 100644 --- a/packages/syft/src/syft/service/action/action_service.py +++ b/packages/syft/src/syft/service/action/action_service.py @@ -31,6 +31,7 @@ from ..user.user_roles import ADMIN_ROLE_LEVEL from ..user.user_roles import GUEST_ROLE_LEVEL from ..user.user_roles import ServiceRole +from .action_endpoint import CustomEndpointActionObject from .action_object import Action from .action_object import ActionObject from .action_object import ActionObjectPointer @@ -82,13 +83,39 @@ def set( context: AuthedServiceContext, action_object: ActionObject | TwinObject, add_storage_permission: bool = True, - ) -> Result[ActionObject, str]: - return self._set( + ignore_detached_objs: bool = False, + ) -> ActionObject | SyftError: + res = self._set( context, action_object, has_result_read_permission=True, add_storage_permission=add_storage_permission, + ignore_detached_objs=ignore_detached_objs, ) + if res.is_err(): + return SyftError(message=res.value) + else: + return res.ok() + + def is_detached_obj( + self, + action_object: ActionObject | TwinObject, + ignore_detached_obj: bool = False, + ) -> bool: + if ( + isinstance(action_object, TwinObject) + and ( + action_object.mock_obj.syft_blob_storage_entry_id is None + or action_object.private_obj.syft_blob_storage_entry_id is None + ) + and not ignore_detached_obj + ): + return True + if isinstance(action_object, ActionObject) and ( + action_object.syft_blob_storage_entry_id is None and not ignore_detached_obj + ): + return True + return False def _set( self, @@ -96,15 +123,26 @@ def _set( action_object: ActionObject | TwinObject, has_result_read_permission: bool = False, add_storage_permission: bool = True, + ignore_detached_objs: bool = False, + skip_clear_cache: bool = False, ) -> Result[ActionObject, str]: + if self.is_detached_obj(action_object, ignore_detached_objs): + return Err( + "you uploaded an ActionObject that is not yet in the blob storage" + ) """Save an object to the action store""" # 🟡 TODO 9: Create some kind of type checking / protocol for SyftSerializable if isinstance(action_object, ActionObject): action_object.syft_created_at = DateTime.now() + if not skip_clear_cache: + action_object._clear_cache() else: action_object.private_obj.syft_created_at = DateTime.now() # type: ignore[unreachable] action_object.mock_obj.syft_created_at = DateTime.now() + if not skip_clear_cache: + action_object.private_obj._clear_cache() + action_object.mock_obj._clear_cache() # If either context or argument is True, has_result_read_permission is True has_result_read_permission = ( @@ -121,6 +159,13 @@ def _set( ) if result.is_ok(): if isinstance(action_object, TwinObject): + # give read permission to the mock + blob_id = action_object.mock_obj.syft_blob_storage_entry_id + permission = ActionObjectPermission(blob_id, ActionPermission.ALL_READ) + blob_storage_service: AbstractService = context.node.get_service( + BlobStorageService + ) + blob_storage_service.stash.add_permission(permission) if has_result_read_permission: action_object = action_object.private else: @@ -139,8 +184,6 @@ def is_resolved( uid: UID, ) -> Result[Ok[bool], Err[str]]: """Get an object from the action store""" - # relative - result = self._get(context, uid) if result.is_ok(): obj = result.ok() @@ -148,7 +191,6 @@ def is_resolved( result = self.resolve_links( context, obj.syft_action_data.action_object_id.id ) - # Checking in case any error occurred if result.is_err(): return result @@ -161,7 +203,6 @@ def is_resolved( # If it's not an action data link or non resolved (empty). It's resolved return Ok(True) - # If it's not in the store or permission error, return the error return result @@ -337,6 +378,17 @@ def _user_code_execute( if isinstance(result, SyftError): return Err(result.message) filtered_kwargs = result.ok() + + if hasattr(input_policy, "transform_kwargs"): + filtered_kwargs_res = input_policy.transform_kwargs( # type: ignore + context, + filtered_kwargs, + ) + if filtered_kwargs_res.is_err(): + return filtered_kwargs_res + else: + filtered_kwargs = filtered_kwargs_res.ok() + # update input policy to track any input state has_twin_inputs = False @@ -352,8 +404,9 @@ def _user_code_execute( try: if not has_twin_inputs: # no twins + # allow python types from inputpolicy filtered_kwargs = filter_twin_kwargs( - real_kwargs, twin_mode=TwinMode.NONE + real_kwargs, twin_mode=TwinMode.NONE, allow_python_types=True ) exec_result = execute_byte_code(code_item, filtered_kwargs, context) if output_policy: @@ -362,7 +415,7 @@ def _user_code_execute( exec_result.result, update_policy=not override_execution_permission, ) - code_item.output_policy = output_policy + code_item.output_policy = output_policy # type: ignore user_code_service.update_code_state(context, code_item) if isinstance(exec_result.result, ActionObject): result_action_object = ActionObject.link( @@ -373,7 +426,7 @@ def _user_code_execute( else: # twins private_kwargs = filter_twin_kwargs( - real_kwargs, twin_mode=TwinMode.PRIVATE + real_kwargs, twin_mode=TwinMode.PRIVATE, allow_python_types=True ) private_exec_result = execute_byte_code( code_item, private_kwargs, context @@ -384,13 +437,15 @@ def _user_code_execute( private_exec_result.result, update_policy=not override_execution_permission, ) - code_item.output_policy = output_policy + code_item.output_policy = output_policy # type: ignore user_code_service.update_code_state(context, code_item) result_action_object_private = wrap_result( result_id, private_exec_result.result ) - mock_kwargs = filter_twin_kwargs(real_kwargs, twin_mode=TwinMode.MOCK) + mock_kwargs = filter_twin_kwargs( + real_kwargs, twin_mode=TwinMode.MOCK, allow_python_types=True + ) # relative from .action_data_empty import ActionDataEmpty @@ -414,8 +469,6 @@ def _user_code_execute( mock_obj=result_action_object_mock, ) except Exception as e: - # import traceback - # return Err(f"_user_code_execute failed. {e} {traceback.format_exc()}") return Err(f"_user_code_execute failed. {e}") return Ok(result_action_object) @@ -664,7 +717,6 @@ def execute( if action.action_type == ActionType.CREATEOBJECT: result_action_object = Ok(action.create_object) - # print(action.create_object, "already in blob storage") elif action.action_type == ActionType.SYFTFUNCTION: usercode_service = context.node.get_service("usercodeservice") kwarg_ids = {} @@ -979,7 +1031,9 @@ def filter_twin_args(args: list[Any], twin_mode: TwinMode) -> Any: return filtered -def filter_twin_kwargs(kwargs: dict, twin_mode: TwinMode) -> Any: +def filter_twin_kwargs( + kwargs: dict, twin_mode: TwinMode, allow_python_types: bool = False +) -> Any: filtered = {} for k, v in kwargs.items(): if isinstance(v, TwinObject): @@ -992,7 +1046,18 @@ def filter_twin_kwargs(kwargs: dict, twin_mode: TwinMode) -> Any: f"Filter can only use {TwinMode.PRIVATE} or {TwinMode.MOCK}" ) else: - filtered[k] = v.syft_action_data + if isinstance(v, ActionObject): + filtered[k] = v.syft_action_data + elif ( + isinstance(v, str | int | float | dict | CustomEndpointActionObject) + and allow_python_types + ): + filtered[k] = v + else: + # third party + raise ValueError( + f"unexepected value {v} passed to filtered twin kwargs" + ) return filtered diff --git a/packages/syft/src/syft/service/action/action_types.py b/packages/syft/src/syft/service/action/action_types.py index 9721a48ec8e..c7bd730d557 100644 --- a/packages/syft/src/syft/service/action/action_types.py +++ b/packages/syft/src/syft/service/action/action_types.py @@ -1,10 +1,12 @@ # stdlib +import logging from typing import Any # relative -from ...util.logger import debug from .action_data_empty import ActionDataEmpty +logger = logging.getLogger(__name__) + action_types: dict = {} @@ -21,7 +23,9 @@ def action_type_for_type(obj_or_type: Any) -> type: obj_or_type = type(obj_or_type) if obj_or_type not in action_types: - debug(f"WARNING: No Type for {obj_or_type}, returning {action_types[Any]}") + logger.debug( + f"WARNING: No Type for {obj_or_type}, returning {action_types[Any]}" + ) return action_types.get(obj_or_type, action_types[Any]) @@ -36,7 +40,7 @@ def action_type_for_object(obj: Any) -> type: _type = type(obj) if _type not in action_types: - debug(f"WARNING: No Type for {_type}, returning {action_types[Any]}") + logger.debug(f"WARNING: No Type for {_type}, returning {action_types[Any]}") return action_types[Any] return action_types[_type] diff --git a/packages/syft/src/syft/service/api/api.py b/packages/syft/src/syft/service/api/api.py index 6977f29e8c7..29846cb1134 100644 --- a/packages/syft/src/syft/service/api/api.py +++ b/packages/syft/src/syft/service/api/api.py @@ -4,6 +4,7 @@ import inspect from inspect import Signature import keyword +import linecache import re import textwrap from typing import Any @@ -19,6 +20,7 @@ # relative from ...abstract_node import AbstractNode +from ...client.client import SyftClient from ...serde.serializable import serializable from ...serde.signature import signature_remove_context from ...types.syft_object import PartialSyftObject @@ -35,6 +37,7 @@ from ..context import AuthedServiceContext from ..response import SyftError from ..user.user import UserView +from ..user.user_service import UserService NOT_ACCESSIBLE_STRING = "N / A" @@ -54,6 +57,8 @@ class TwinAPIAuthedContext(AuthedServiceContext): settings: dict[str, Any] | None = None code: HelperFunctionSet | None = None state: dict[Any, Any] | None = None + admin_client: SyftClient | None = None + user_client: SyftClient | None = None @serializable() @@ -73,6 +78,15 @@ def get_signature(func: Callable) -> Signature: return sig +def register_fn_in_linecache(fname: str, src: str) -> None: + """adds a function to linecache, such that inspect.getsource works for functions nested in this function. + This only works if the same function is compiled under the same filename""" + lines = [ + line + "\n" for line in src.splitlines() + ] # use same splitting method same as linecache 112 (py3.12) + linecache.cache[fname] = (137, None, lines, fname) + + @serializable() class TwinAPIEndpointView(SyftObject): # version @@ -191,7 +205,10 @@ def update_state(self, state: dict[Any, Any]) -> None: self.state = state def build_internal_context( - self, context: AuthedServiceContext + self, + context: AuthedServiceContext, + admin_client: SyftClient | None = None, + user_client: SyftClient | None = None, ) -> TwinAPIAuthedContext: helper_function_dict: dict[str, Callable] = {} self.helper_functions = self.helper_functions or {} @@ -220,6 +237,8 @@ def build_internal_context( code=helper_function_set, state=self.state or {}, user=user, + admin_client=admin_client, + user_client=user_client, ) def __call__(self, *args: Any, **kwargs: Any) -> Any: @@ -243,7 +262,7 @@ def call_locally( # load it exec(raw_byte_code) # nosec - internal_context = self.build_internal_context(context) + internal_context = self.build_internal_context(context=context) # execute it evil_string = f"{self.func_name}(*args, **kwargs,context=internal_context)" @@ -365,6 +384,10 @@ class TwinAPIEndpoint(SyncableSyftObject): # version __canonical_name__: str = "TwinAPIEndpoint" __version__ = SYFT_OBJECT_VERSION_1 + __exclude_sync_diff_attrs__ = ["private_function"] + __private_sync_attr_mocks__ = { + "private_function": None, + } def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) @@ -378,10 +401,6 @@ def __init__(self, **kwargs: Any) -> None: worker_pool: str | None = None endpoint_timeout: int = 60 - __private_sync_attr_mocks__ = { - "private_function": None, - } - __attr_searchable__ = ["path"] __attr_unique__ = ["path"] __repr_attrs__ = [ @@ -465,6 +484,25 @@ def exec_private_function( return SyftError(message="You're not allowed to run this code.") + def get_user_client_from_node(self, context: AuthedServiceContext) -> SyftClient: + # get a user client + guest_client = context.node.get_guest_client() + user_client = guest_client + signing_key_for_verify_key = context.node.get_service_method( + UserService.signing_key_for_verify_key + ) + private_key = signing_key_for_verify_key( + context=context, verify_key=context.credentials + ) + signing_key = private_key.signing_key + user_client.credentials = signing_key + return user_client + + def get_admin_client_from_node(self, context: AuthedServiceContext) -> SyftClient: + admin_client = context.node.get_guest_client() + admin_client.credentials = context.node.signing_key + return admin_client + def exec_code( self, code: PrivateAPIEndpoint | PublicAPIEndpoint, @@ -476,12 +514,18 @@ def exec_code( inner_function = ast.parse(code.api_code).body[0] inner_function.decorator_list = [] # compile the function - raw_byte_code = compile(ast.unparse(inner_function), "", "exec") + src = ast.unparse(inner_function) + raw_byte_code = compile(src, code.func_name, "exec") + register_fn_in_linecache(code.func_name, src) + user_client = self.get_user_client_from_node(context) + admin_client = self.get_admin_client_from_node(context) # load it exec(raw_byte_code) # nosec - internal_context = code.build_internal_context(context) + internal_context = code.build_internal_context( + context=context, admin_client=admin_client, user_client=user_client + ) # execute it evil_string = f"{code.func_name}(*args, **kwargs,context=internal_context)" @@ -507,7 +551,8 @@ def exec_code( return result except Exception as e: # If it's admin, return the error message. - if context.role.value == 128: + # TODO: cleanup typeerrors + if context.role.value == 128 or isinstance(e, TypeError): return SyftError( message=f"An error was raised during the execution of the API endpoint call: \n {str(e)}" ) @@ -610,6 +655,7 @@ def endpoint_to_private_endpoint() -> list[Callable]: "api_code", "func_name", "settings", + "view_access", "helper_functions", "state", "signature", @@ -703,6 +749,8 @@ def create_new_api_endpoint( description: MarkdownDescription | None = None, worker_pool: str | None = None, endpoint_timeout: int = 60, + hide_mock_definition: bool = False, + hide_private_definition: bool = True, ) -> CreateTwinAPIEndpoint | SyftError: try: # Parse the string to extract the function name @@ -712,7 +760,8 @@ def create_new_api_endpoint( if private_function.signature != mock_function.signature: return SyftError(message="Signatures don't match") endpoint_signature = mock_function.signature - private_function.view_access = False + private_function.view_access = not hide_private_definition + mock_function.view_access = not hide_mock_definition return CreateTwinAPIEndpoint( path=path, diff --git a/packages/syft/src/syft/service/api/api_service.py b/packages/syft/src/syft/service/api/api_service.py index 55d2df05bf0..a302a7805d2 100644 --- a/packages/syft/src/syft/service/api/api_service.py +++ b/packages/syft/src/syft/service/api/api_service.py @@ -65,19 +65,22 @@ def set( new_endpoint = endpoint if new_endpoint is None: - return SyftError(message="Invalid endpoint type.") + return SyftError(message="Invalid endpoint type.") # type: ignore except ValueError as e: return SyftError(message=str(e)) - endpoint_exists = self.stash.path_exists(context.credentials, new_endpoint.path) + if isinstance(endpoint, CreateTwinAPIEndpoint): + endpoint_exists = self.stash.path_exists( + context.credentials, new_endpoint.path + ) - if endpoint_exists.is_err(): - return SyftError(message=endpoint_exists.err()) + if endpoint_exists.is_err(): + return SyftError(message=endpoint_exists.err()) - if endpoint_exists.is_ok() and endpoint_exists.ok(): - return SyftError( - message="An API endpoint already exists at the given path." - ) + if endpoint_exists.is_ok() and endpoint_exists.ok(): + return SyftError( + message="An API endpoint already exists at the given path." + ) result = self.stash.upsert(context.credentials, endpoint=new_endpoint) if result.is_err(): @@ -91,7 +94,11 @@ def set( syft_client_verify_key=context.credentials, ) action_service = context.node.get_service("actionservice") - res = action_service.set(context=context, action_object=action_obj) + res = action_service.set_result_to_store( + context=context, + result_action_object=action_obj, + has_result_read_permission=True, + ) if res.is_err(): return SyftError(message=res.err()) @@ -222,6 +229,88 @@ def view( return api_endpoint.to(TwinAPIEndpointView, context=context) + @service_method( + path="api.get", + name="get", + roles=ADMIN_ROLE_LEVEL, + ) + def get( + self, context: AuthedServiceContext, api_path: str + ) -> TwinAPIEndpoint | SyftError: + """Retrieves an specific API endpoint.""" + result = self.stash.get_by_path(context.node.verify_key, api_path) + if result.is_err(): + return SyftError(message=result.err()) + api_endpoint = result.ok() + + return api_endpoint + + @service_method( + path="api.set_state", + name="set_state", + roles=ADMIN_ROLE_LEVEL, + ) + def set_state( + self, + context: AuthedServiceContext, + api_path: str, + state: dict, + private: bool = False, + mock: bool = False, + both: bool = False, + ) -> TwinAPIEndpoint | SyftError: + """Sets the state of a specific API endpoint.""" + if both: + private = True + mock = True + result = self.stash.get_by_path(context.node.verify_key, api_path) + if result.is_err(): + return SyftError(message=result.err()) + api_endpoint = result.ok() + + if private and api_endpoint.private_function: + api_endpoint.private_function.state = state + if mock and api_endpoint.mock_function: + api_endpoint.mock_function.state = state + + result = self.stash.upsert(context.credentials, endpoint=api_endpoint) + if result.is_err(): + return SyftError(message=result.err()) + return SyftSuccess(message=f"APIEndpoint {api_path} state updated.") + + @service_method( + path="api.set_settings", + name="set_settings", + roles=ADMIN_ROLE_LEVEL, + ) + def set_settings( + self, + context: AuthedServiceContext, + api_path: str, + settings: dict, + private: bool = False, + mock: bool = False, + both: bool = False, + ) -> TwinAPIEndpoint | SyftError: + """Sets the settings of a specific API endpoint.""" + if both: + private = True + mock = True + result = self.stash.get_by_path(context.node.verify_key, api_path) + if result.is_err(): + return SyftError(message=result.err()) + api_endpoint = result.ok() + + if private and api_endpoint.private_function: + api_endpoint.private_function.settings = settings + if mock and api_endpoint.mock_function: + api_endpoint.mock_function.settings = settings + + result = self.stash.upsert(context.credentials, endpoint=api_endpoint) + if result.is_err(): + return SyftError(message=result.err()) + return SyftSuccess(message=f"APIEndpoint {api_path} settings updated.") + @service_method( path="api.api_endpoints", name="api_endpoints", @@ -238,11 +327,10 @@ def api_endpoints( return SyftError(message=result.err()) all_api_endpoints = result.ok() - api_endpoint_view = [] - for api_endpoint in all_api_endpoints: - api_endpoint_view.append( - api_endpoint.to(TwinAPIEndpointView, context=context) - ) + api_endpoint_view = [ + api_endpoint.to(TwinAPIEndpointView, context=context) + for api_endpoint in all_api_endpoints + ] return api_endpoint_view @@ -360,7 +448,7 @@ def get_public_context( if isinstance(custom_endpoint, SyftError): return custom_endpoint - return custom_endpoint.mock_function.build_internal_context(context).to( + return custom_endpoint.mock_function.build_internal_context(context=context).to( TwinAPIContextView ) @@ -384,9 +472,9 @@ def get_private_context( PrivateAPIEndpoint, custom_endpoint.private_function ) - return custom_endpoint.private_function.build_internal_context(context).to( - TwinAPIContextView - ) + return custom_endpoint.private_function.build_internal_context( + context=context + ).to(TwinAPIContextView) @service_method(path="api.get_all", name="get_all", roles=ADMIN_ROLE_LEVEL) def get_all( diff --git a/packages/syft/src/syft/service/code/user_code.py b/packages/syft/src/syft/service/code/user_code.py index 2cbeaf31967..b8df87fc619 100644 --- a/packages/syft/src/syft/service/code/user_code.py +++ b/packages/syft/src/syft/service/code/user_code.py @@ -12,7 +12,9 @@ import inspect from io import StringIO import itertools +import keyword import random +import re import sys from textwrap import dedent from threading import Thread @@ -26,36 +28,50 @@ # third party from IPython.display import display +from pydantic import ValidationError from pydantic import field_validator from result import Err +from result import Ok +from result import Result from typing_extensions import Self # relative +from ...abstract_node import NodeSideType from ...abstract_node import NodeType from ...client.api import APIRegistry from ...client.api import NodeIdentity +from ...client.api import generate_remote_function from ...client.enclave_client import EnclaveMetadata from ...node.credentials import SyftVerifyKey from ...serde.deserialize import _deserialize from ...serde.serializable import serializable from ...serde.serialize import _serialize +from ...serde.signature import signature_remove_context +from ...serde.signature import signature_remove_self from ...store.document_store import PartitionKey from ...store.linked_obj import LinkedObject from ...types.datetime import DateTime +from ...types.syft_migration import migrate +from ...types.syft_object import PartialSyftObject from ...types.syft_object import SYFT_OBJECT_VERSION_1 from ...types.syft_object import SYFT_OBJECT_VERSION_2 from ...types.syft_object import SYFT_OBJECT_VERSION_4 +from ...types.syft_object import SYFT_OBJECT_VERSION_5 +from ...types.syft_object import SYFT_OBJECT_VERSION_6 from ...types.syft_object import SyftObject from ...types.syncable_object import SyncableSyftObject from ...types.transforms import TransformContext from ...types.transforms import add_node_uid_for_key +from ...types.transforms import drop from ...types.transforms import generate_id +from ...types.transforms import make_set_default from ...types.transforms import transform from ...types.uid import UID from ...util import options from ...util.colors import SURFACE from ...util.markdown import CodeMarkdown from ...util.markdown import as_markdown_code +from ...util.util import prompt_warning_message from ..action.action_endpoint import CustomEndpointActionObject from ..action.action_object import Action from ..action.action_object import ActionObject @@ -64,6 +80,7 @@ from ..job.job_stash import Job from ..output.output_service import ExecutionOutput from ..output.output_service import OutputService +from ..policy.policy import Constant from ..policy.policy import CustomInputPolicy from ..policy.policy import CustomOutputPolicy from ..policy.policy import EmpyInputPolicy @@ -83,7 +100,9 @@ from ..response import SyftNotReady from ..response import SyftSuccess from ..response import SyftWarning +from ..service import ServiceConfigRegistry from ..user.user import UserView +from ..user.user_roles import ServiceRole from .code_parse import GlobalsVisitor from .code_parse import LaunchJobVisitor from .unparse import unparse @@ -117,7 +136,6 @@ class UserCodeStatusCollection(SyncableSyftObject): __version__ = SYFT_OBJECT_VERSION_1 __repr_attrs__ = ["approved", "status_dict"] - status_dict: dict[NodeIdentity, tuple[UserCodeStatus, str]] = {} user_code_link: LinkedObject @@ -253,8 +271,7 @@ def get_sync_dependencies(self, context: AuthedServiceContext) -> list[UID]: return [self.user_code_link.object_uid] -@serializable() -class UserCode(SyncableSyftObject): +class UserCodeV4(SyncableSyftObject): # version __canonical_name__ = "UserCode" __version__ = SYFT_OBJECT_VERSION_4 @@ -279,9 +296,80 @@ class UserCode(SyncableSyftObject): input_kwargs: list[str] enclave_metadata: EnclaveMetadata | None = None submit_time: DateTime | None = None - uses_domain: bool = False # tracks if the code calls domain.something, variable is set during parsing + # tracks if the code calls domain.something, variable is set during parsing + uses_domain: bool = False + nested_codes: dict[str, tuple[LinkedObject, dict]] | None = {} + worker_pool_name: str | None = None + + +@serializable() +class UserCodeV5(SyncableSyftObject): + # version + __canonical_name__ = "UserCode" + __version__ = SYFT_OBJECT_VERSION_5 + + id: UID + node_uid: UID | None = None + user_verify_key: SyftVerifyKey + raw_code: str + input_policy_type: type[InputPolicy] | UserPolicy + input_policy_init_kwargs: dict[Any, Any] | None = None + input_policy_state: bytes = b"" + output_policy_type: type[OutputPolicy] | UserPolicy + output_policy_init_kwargs: dict[Any, Any] | None = None + output_policy_state: bytes = b"" + parsed_code: str + service_func_name: str + unique_func_name: str + user_unique_func_name: str + code_hash: str + signature: inspect.Signature + status_link: LinkedObject | None = None + input_kwargs: list[str] + enclave_metadata: EnclaveMetadata | None = None + submit_time: DateTime | None = None + # tracks if the code calls domain.something, variable is set during parsing + uses_domain: bool = False + nested_codes: dict[str, tuple[LinkedObject, dict]] | None = {} worker_pool_name: str | None = None + origin_node_side_type: NodeSideType + l0_deny_reason: str | None = None + + +@serializable() +class UserCode(SyncableSyftObject): + # version + __canonical_name__ = "UserCode" + __version__ = SYFT_OBJECT_VERSION_6 + + id: UID + node_uid: UID | None = None + user_verify_key: SyftVerifyKey + raw_code: str + input_policy_type: type[InputPolicy] | UserPolicy + input_policy_init_kwargs: dict[Any, Any] | None = None + input_policy_state: bytes = b"" + output_policy_type: type[OutputPolicy] | UserPolicy + output_policy_init_kwargs: dict[Any, Any] | None = None + output_policy_state: bytes = b"" + parsed_code: str + service_func_name: str + unique_func_name: str + user_unique_func_name: str + code_hash: str + signature: inspect.Signature + status_link: LinkedObject | None = None + input_kwargs: list[str] + submit_time: DateTime | None = None + # tracks if the code calls domain.something, variable is set during parsing + uses_domain: bool = False + + nested_codes: dict[str, tuple[LinkedObject, dict]] | None = {} + worker_pool_name: str | None = None + origin_node_side_type: NodeSideType + l0_deny_reason: str | None = None + _has_output_read_permissions_cache: bool | None = None __table_coll_widths__ = [ "min-content", @@ -305,10 +393,13 @@ class UserCode(SyncableSyftObject): "input_owners", "code_status", "worker_pool_name", + "l0_deny_reason", + "raw_code", ] __exclude_sync_diff_attrs__: ClassVar[list[str]] = [ "node_uid", + "code_status", "input_policy_type", "input_policy_init_kwargs", "input_policy_state", @@ -317,6 +408,14 @@ class UserCode(SyncableSyftObject): "output_policy_state", ] + @field_validator("service_func_name", mode="after") + @classmethod + def service_func_name_is_valid(cls, value: str) -> str: + res = is_valid_usercode_name(value) + if res.is_err(): + raise ValueError(res.err_value) + return value + def __setattr__(self, key: str, value: Any) -> None: # Get the attribute from the class, it might be a descriptor or None attr = getattr(type(self), key, None) @@ -350,6 +449,14 @@ def _coll_repr_(self) -> dict[str, Any]: "Submit time": str(self.submit_time), } + @property + def is_l0_deployment(self) -> bool: + return self.origin_node_side_type == NodeSideType.LOW_SIDE + + @property + def is_l2_deployment(self) -> bool: + return self.origin_node_side_type == NodeSideType.HIGH_SIDE + @property def user(self) -> UserView | SyftError: api = APIRegistry.api_for( @@ -362,24 +469,91 @@ def user(self) -> UserView | SyftError: ) return api.services.user.get_by_verify_key(self.user_verify_key) + def _compute_status_l0( + self, context: AuthedServiceContext | None = None + ) -> UserCodeStatusCollection | SyftError: + if context is None: + # Clientside + api = self._get_api() + if isinstance(api, SyftError): + return api + node_identity = NodeIdentity.from_api(api) + + if self._has_output_read_permissions_cache is None: + is_approved = api.output.has_output_read_permissions( + self.id, self.user_verify_key + ) + self._has_output_read_permissions_cache = is_approved + else: + is_approved = self._has_output_read_permissions_cache + else: + # Serverside + node_identity = NodeIdentity.from_node(context.node) + output_service = context.node.get_service("outputservice") + is_approved = output_service.has_output_read_permissions( + context, self.id, self.user_verify_key + ) + + if isinstance(is_approved, SyftError): + return is_approved + is_denied = self.l0_deny_reason is not None + + if is_denied: + if is_approved: + prompt_warning_message( + "This request already has results published to the data scientist. " + "They will still be able to access those results." + ) + message = self.l0_deny_reason + status = (UserCodeStatus.DENIED, message) + elif is_approved: + status = (UserCodeStatus.APPROVED, "") + else: + status = (UserCodeStatus.PENDING, "") + status_dict = {node_identity: status} + + return UserCodeStatusCollection( + status_dict=status_dict, + user_code_link=LinkedObject.from_obj(self), + ) + @property def status(self) -> UserCodeStatusCollection | SyftError: # Clientside only + + if self.is_l0_deployment: + if self.status_link is not None: + return SyftError( + message="Encountered a low side UserCode object with a status_link." + ) + return self._compute_status_l0() + + if self.status_link is None: + return SyftError( + message="This UserCode does not have a status. Please contact the Admin." + ) res = self.status_link.resolve return res def get_status( self, context: AuthedServiceContext ) -> UserCodeStatusCollection | SyftError: + if self.is_l0_deployment: + if self.status_link is not None: + return SyftError( + message="Encountered a low side UserCode object with a status_link." + ) + return self._compute_status_l0(context) + if self.status_link is None: + return SyftError( + message="This UserCode does not have a status. Please contact the Admin." + ) + status = self.status_link.resolve_with_context(context) if status.is_err(): return SyftError(message=status.err()) return status.ok() - @property - def is_enclave_code(self) -> bool: - return self.enclave_metadata is not None - @property def input_owners(self) -> list[str] | None: if self.input_policy_init_kwargs is not None: @@ -422,15 +596,15 @@ def code_status(self) -> list: @property def input_policy(self) -> InputPolicy | None: - if not self.status.approved: - return None - return self._get_input_policy() + if self.status.approved or self.input_policy_type.has_safe_serde: + return self._get_input_policy() + return None def get_input_policy(self, context: AuthedServiceContext) -> InputPolicy | None: status = self.get_status(context) - if not status.approved: - return None - return self._get_input_policy() + if status.approved or self.input_policy_type.has_safe_serde: + return self._get_input_policy() + return None def _get_input_policy(self) -> InputPolicy | None: if len(self.input_policy_state) == 0: @@ -442,7 +616,7 @@ def _get_input_policy(self) -> InputPolicy | None: ): # TODO: Tech Debt here node_view_workaround = False - for k, _ in self.input_policy_init_kwargs.items(): + for k in self.input_policy_init_kwargs.keys(): if isinstance(k, NodeIdentity): node_view_workaround = True @@ -486,13 +660,18 @@ def input_policy(self, value: Any) -> None: # type: ignore raise Exception(f"You can't set {type(value)} as input_policy_state") def get_output_policy(self, context: AuthedServiceContext) -> OutputPolicy | None: - if not self.get_status(context).approved: - return None - return self._get_output_policy() + status = self.get_status(context) + if status.approved or self.output_policy_type.has_safe_serde: + return self._get_output_policy() + return None + + @property + def output_policy(self) -> OutputPolicy | None: # type: ignore + if self.status.approved or self.output_policy_type.has_safe_serde: + return self._get_output_policy() + return None def _get_output_policy(self) -> OutputPolicy | None: - # if not self.status.approved: - # return None if len(self.output_policy_state) == 0: output_policy = None if isinstance(self.output_policy_type, type) and issubclass( @@ -529,10 +708,16 @@ def _get_output_policy(self) -> OutputPolicy | None: return None @property - def output_policy(self) -> OutputPolicy | None: # type: ignore - if not self.status.approved: - return None - return self._get_output_policy() + def output_policy_id(self) -> UID | None: + if self.output_policy_init_kwargs is not None: + return self.output_policy_init_kwargs.get("id", None) + return None + + @property + def input_policy_id(self) -> UID | None: + if self.input_policy_init_kwargs is not None: + return self.input_policy_init_kwargs.get("id", None) + return None @output_policy.setter # type: ignore def output_policy(self, value: Any) -> None: # type: ignore @@ -555,27 +740,22 @@ def output_history(self) -> list[ExecutionOutput] | SyftError: def get_output_history( self, context: AuthedServiceContext ) -> list[ExecutionOutput] | SyftError: - if not self.get_status(context).approved: - return SyftError( - message="Execution denied, Please wait for the code to be approved" - ) - output_service = cast(OutputService, context.node.get_service("outputservice")) return output_service.get_by_user_code_id(context, self.id) - def store_as_history( + def store_execution_output( self, context: AuthedServiceContext, outputs: Any, job_id: UID | None = None, input_ids: dict[str, UID] | None = None, ) -> ExecutionOutput | SyftError: + is_admin = context.role == ServiceRole.ADMIN output_policy = self.get_output_policy(context) - if output_policy is None: + if output_policy is None and not is_admin: return SyftError( message="You must wait for the output policy to be approved" ) - output_ids = filter_only_uids(outputs) output_service = context.node.get_service("outputservice") @@ -586,7 +766,7 @@ def store_as_history( output_ids=output_ids, executing_user_verify_key=self.user_verify_key, job_id=job_id, - output_policy_id=output_policy.id, + output_policy_id=self.output_policy_id, input_ids=input_ids, ) if isinstance(execution_result, SyftError): @@ -598,17 +778,6 @@ def store_as_history( def byte_code(self) -> PyCodeObject | None: return compile_byte_code(self.parsed_code) - def get_results(self) -> Any: - # relative - from ...client.api import APIRegistry - - api = APIRegistry.api_for(self.node_uid, self.syft_client_verify_key) - if api is None: - return SyftError( - message=f"Can't access the api. You must login to {self.node_uid}" - ) - return api.services.code.get_results(self) - @property def assets(self) -> list[Asset]: # relative @@ -647,7 +816,8 @@ def get_sync_dependencies( ] dependencies.extend(nested_code_ids) - dependencies.append(self.status_link.object_uid) + if self.status_link is not None: + dependencies.append(self.status_link.object_uid) return dependencies @@ -707,11 +877,21 @@ def _inner_repr(self, level: int = 0) -> str: f"outputs are *shared* with the owners of {owners_string} once computed" ) + constants_str = "" + args = [ + x + for _dict in self.input_policy_init_kwargs.values() # type: ignore + for x in _dict.values() + ] + constants = [x for x in args if isinstance(x, Constant)] + constants_str = "\n\t".join([f"{x.kw}: {x.val}" for x in constants]) + md = f"""class UserCode id: UID = {self.id} service_func_name: str = {self.service_func_name} shareholders: list = {self.input_owners} status: list = {self.code_status} + {constants_str} {shared_with_line} code: @@ -727,7 +907,7 @@ def _inner_repr(self, level: int = 0) -> str: [f"{' '*level}{substring}" for substring in md.split("\n")[:-1]] ) if self.nested_codes is not None: - for _, (obj, _) in self.nested_codes.items(): + for obj, _ in self.nested_codes.values(): code = obj.resolve md += "\n" md += code._inner_repr(level=level + 1) @@ -751,9 +931,36 @@ def show_code_cell(self) -> None: ip = get_ipython() ip.set_next_input(warning_message + self.raw_code) + def __call__(self, *args: Any, **kwargs: Any) -> Any: + api = self._get_api() + if isinstance(api, SyftError): + return api + + signature = self.signature + signature = signature_remove_self(signature) + signature = signature_remove_context(signature) + remote_user_function = generate_remote_function( + api=api, + node_uid=self.node_uid, + signature=self.signature, + path="code.call", + make_call=api.make_call, + pre_kwargs={"uid": self.id}, + warning=None, + communication_protocol=api.communication_protocol, + ) + return remote_user_function(*args, **kwargs) + + +class UserCodeUpdate(PartialSyftObject): + __canonical_name__ = "UserCodeUpdate" + __version__ = SYFT_OBJECT_VERSION_1 + + l0_deny_reason: str | None + @serializable(without=["local_function"]) -class SubmitUserCode(SyftObject): +class SubmitUserCodeV4(SyftObject): # version __canonical_name__ = "SubmitUserCode" __version__ = SYFT_OBJECT_VERSION_4 @@ -771,8 +978,35 @@ class SubmitUserCode(SyftObject): enclave_metadata: EnclaveMetadata | None = None worker_pool_name: str | None = None + +@serializable(without=["local_function"]) +class SubmitUserCode(SyftObject): + # version + __canonical_name__ = "SubmitUserCode" + __version__ = SYFT_OBJECT_VERSION_5 + + id: UID | None = None # type: ignore[assignment] + code: str + func_name: str + signature: inspect.Signature + input_policy_type: SubmitUserPolicy | UID | type[InputPolicy] + input_policy_init_kwargs: dict[Any, Any] | None = {} + output_policy_type: SubmitUserPolicy | UID | type[OutputPolicy] + output_policy_init_kwargs: dict[Any, Any] | None = {} + local_function: Callable | None = None + input_kwargs: list[str] + worker_pool_name: str | None = None + __repr_attrs__ = ["func_name", "code"] + @field_validator("func_name", mode="after") + @classmethod + def func_name_is_valid(cls, value: str) -> str: + res = is_valid_usercode_name(value) + if res.is_err(): + raise ValueError(res.err_value) + return value + @field_validator("output_policy_init_kwargs", mode="after") @classmethod def add_output_policy_ids(cls, values: Any) -> Any: @@ -860,7 +1094,10 @@ def _ephemeral_node_call( n_consumers=n_consumers, deploy_to="python", ) - ep_client = ep_node.login(email="info@openmined.org", password="changethis") # nosec + ep_client = ep_node.login( + email="info@openmined.org", + password="changethis", + ) # nosec self.input_policy_init_kwargs = cast(dict, self.input_policy_init_kwargs) for node_id, obj_dict in self.input_policy_init_kwargs.items(): # api = APIRegistry.api_for( @@ -876,7 +1113,7 @@ def _ephemeral_node_call( # And need only ActionObjects # Also, this works only on the assumption that all inputs # are ActionObjects, which might change in the future - for _, id in obj_dict.items(): + for id in obj_dict.values(): mock_obj = api.services.action.get_mock(id) if isinstance(mock_obj, SyftError): data_obj = api.services.action.get(id) @@ -895,7 +1132,7 @@ def _ephemeral_node_call( syft_node_location=node_id.node_id, syft_client_verify_key=node_id.verify_key, ) - res = ep_client.api.services.action.set(new_obj) + res = new_obj.send(ep_client) if isinstance(res, SyftError): return res @@ -932,6 +1169,29 @@ def input_owner_verify_keys(self) -> list[str] | None: return None +def get_code_hash(code: str, user_verify_key: SyftVerifyKey) -> str: + full_str = f"{code}{user_verify_key}" + return hashlib.sha256(full_str.encode()).hexdigest() + + +def is_valid_usercode_name(func_name: str) -> Result[Any, str]: + if len(func_name) == 0: + return Err("Function name cannot be empty") + if func_name == "_": + return Err("Cannot use anonymous function as syft function") + if not str.isidentifier(func_name): + return Err("Function name must be a valid Python identifier") + if keyword.iskeyword(func_name): + return Err("Function name is a reserved python keyword") + + service_method_path = f"code.{func_name}" + if ServiceConfigRegistry.path_exists(service_method_path): + return Err( + f"Could not create syft function with name {func_name}: a service with the same name already exists" + ) + return Ok(None) + + class ArgumentType(Enum): REAL = 1 MOCK = 2 @@ -965,11 +1225,19 @@ def syft_function_single_use( ) +def replace_func_name(src: str, new_func_name: str) -> str: + pattern = r"\bdef\s+(\w+)\s*\(" + replacement = f"def {new_func_name}(" + new_src = re.sub(pattern, replacement, src, count=1) + return new_src + + def syft_function( input_policy: InputPolicy | UID | None = None, output_policy: OutputPolicy | UID | None = None, share_results_with_owners: bool = False, worker_pool_name: str | None = None, + name: str | None = None, ) -> Callable: if input_policy is None: input_policy = EmpyInputPolicy() @@ -977,7 +1245,7 @@ def syft_function( init_input_kwargs = None if isinstance(input_policy, CustomInputPolicy): input_policy_type = SubmitUserPolicy.from_obj(input_policy) - init_input_kwargs = partition_by_node(input_policy.init_kwargs) + init_input_kwargs = partition_by_node(input_policy.init_kwargs) # type: ignore else: input_policy_type = type(input_policy) init_input_kwargs = getattr(input_policy, "init_kwargs", {}) @@ -991,18 +1259,35 @@ def syft_function( output_policy_type = type(output_policy) def decorator(f: Any) -> SubmitUserCode: - res = SubmitUserCode( - code=dedent(inspect.getsource(f)), - func_name=f.__name__, - signature=inspect.signature(f), - input_policy_type=input_policy_type, - input_policy_init_kwargs=init_input_kwargs, - output_policy_type=output_policy_type, - output_policy_init_kwargs=getattr(output_policy, "init_kwargs", {}), - local_function=f, - input_kwargs=f.__code__.co_varnames[: f.__code__.co_argcount], - worker_pool_name=worker_pool_name, - ) + try: + code = dedent(inspect.getsource(f)) + if name is not None: + fname = name + code = replace_func_name(code, fname) + else: + fname = f.__name__ + + res = SubmitUserCode( + code=code, + func_name=fname, + signature=inspect.signature(f), + input_policy_type=input_policy_type, + input_policy_init_kwargs=init_input_kwargs, + output_policy_type=output_policy_type, + output_policy_init_kwargs=getattr(output_policy, "init_kwargs", {}), + local_function=f, + input_kwargs=f.__code__.co_varnames[: f.__code__.co_argcount], + worker_pool_name=worker_pool_name, + ) + + except ValidationError as e: + errors = e.errors() + msg = "Failed to create syft function, encountered validation errors:\n" + for error in errors: + msg += f"\t{error['msg']}\n" + err = SyftError(message=msg) + display(err) + return err if share_results_with_owners and res.output_policy_init_kwargs is not None: res.output_policy_init_kwargs["output_readers"] = ( @@ -1160,10 +1445,12 @@ def compile_code(context: TransformContext) -> TransformContext: def hash_code(context: TransformContext) -> TransformContext: if context.output is None: return context + if not isinstance(context.obj, SubmitUserCode): + return context code = context.output["code"] context.output["raw_code"] = code - code_hash = hashlib.sha256(code.encode("utf8")).hexdigest() + code_hash = get_code_hash(code, context.credentials) context.output["code_hash"] = code_hash return context @@ -1219,6 +1506,10 @@ def create_code_status(context: TransformContext) -> TransformContext: if context.output is None: return context + # Low side requests have a computed status + if context.node.node_side_type == NodeSideType.LOW_SIDE: + return context + input_keys = list(context.output["input_policy_init_kwargs"].keys()) code_link = LinkedObject.from_uid( context.output["id"], @@ -1280,6 +1571,14 @@ def set_default_pool_if_empty(context: TransformContext) -> TransformContext: return context +def set_origin_node_side_type(context: TransformContext) -> TransformContext: + if context.node and context.output: + context.output["origin_node_side_type"] = ( + context.node.node_side_type or NodeSideType.HIGH_SIDE + ) + return context + + @transform(SubmitUserCode, UserCode) def submit_user_code_to_user_code() -> list[Callable]: return [ @@ -1295,6 +1594,7 @@ def submit_user_code_to_user_code() -> list[Callable]: add_node_uid_for_key("node_uid"), add_submit_time, set_default_pool_if_empty, + set_origin_node_side_type, ] @@ -1374,7 +1674,13 @@ def launch_job(func: UserCode, **kwargs: Any) -> Job | None: kw2id = {} for k, v in kwargs.items(): value = ActionObject.from_obj(v) - ptr = action_service._set(context, value) + ptr = action_service.set_result_to_store( + value, context, has_result_read_permissions=False + ) + if ptr.is_err(): + raise ValueError( + f"failed to create argument {k} for launch job using value {v}" + ) ptr = ptr.ok() kw2id[k] = ptr.id try: @@ -1611,3 +1917,19 @@ def load_approved_policy_code( load_policy_code(user_code.output_policy_type) except Exception as e: raise Exception(f"Failed to load code: {user_code}: {e}") + + +@migrate(UserCodeV4, UserCode) +def migrate_usercode_v4_to_v5() -> list[Callable]: + return [ + make_set_default("origin_node_side_type", NodeSideType.HIGH_SIDE), + make_set_default("l0_deny_reason", None), + ] + + +@migrate(UserCode, UserCodeV4) +def migrate_usercode_v5_to_v4() -> list[Callable]: + return [ + drop("origin_node_side_type"), + drop("l0_deny_reason"), + ] diff --git a/packages/syft/src/syft/service/code/user_code_service.py b/packages/syft/src/syft/service/code/user_code_service.py index 78a7c1c3170..41aba58575b 100644 --- a/packages/syft/src/syft/service/code/user_code_service.py +++ b/packages/syft/src/syft/service/code/user_code_service.py @@ -9,12 +9,11 @@ from result import Result # relative -from ...abstract_node import NodeType -from ...client.enclave_client import EnclaveClient from ...serde.serializable import serializable from ...store.document_store import DocumentStore from ...store.linked_obj import LinkedObject from ...types.cache_object import CachedSyftObject +from ...types.syft_metaclass import Empty from ...types.twin_object import TwinObject from ...types.uid import UID from ...util.telemetry import instrument @@ -22,11 +21,11 @@ from ..action.action_permissions import ActionObjectPermission from ..action.action_permissions import ActionPermission from ..context import AuthedServiceContext -from ..network.routes import route_to_connection from ..output.output_service import ExecutionOutput from ..policy.policy import OutputPolicy from ..request.request import Request from ..request.request import SubmitRequest +from ..request.request import SyncedUserCodeStatusChange from ..request.request import UserCodeStatusChange from ..request.request_service import RequestService from ..response import SyftError @@ -43,6 +42,8 @@ from .user_code import SubmitUserCode from .user_code import UserCode from .user_code import UserCodeStatus +from .user_code import UserCodeUpdate +from .user_code import get_code_hash from .user_code import load_approved_policy_code from .user_code_stash import UserCodeStash @@ -59,23 +60,95 @@ def __init__(self, store: DocumentStore) -> None: @service_method(path="code.submit", name="submit", roles=GUEST_ROLE_LEVEL) def submit( - self, context: AuthedServiceContext, code: UserCode | SubmitUserCode + self, context: AuthedServiceContext, code: SubmitUserCode ) -> UserCode | SyftError: """Add User Code""" - result = self._submit(context=context, code=code) + result = self._submit(context, code, exists_ok=False) if result.is_err(): return SyftError(message=str(result.err())) return SyftSuccess(message="User Code Submitted", require_api_update=True) def _submit( - self, context: AuthedServiceContext, code: UserCode | SubmitUserCode + self, + context: AuthedServiceContext, + submit_code: SubmitUserCode, + exists_ok: bool = False, ) -> Result[UserCode, str]: - if not isinstance(code, UserCode): - code = code.to(UserCode, context=context) # type: ignore[unreachable] + """ + Submit a UserCode. + + If exists_ok is True, the function will return the existing code if it exists. + + Args: + context (AuthedServiceContext): context + submit_code (SubmitUserCode): UserCode to submit + exists_ok (bool, optional): If True, return the existing code if it exists. + If false, existing codes returns Err. Defaults to False. + + Returns: + Result[UserCode, str]: New UserCode or error + """ + existing_code_or_err = self.stash.get_by_code_hash( + context.credentials, + code_hash=get_code_hash(submit_code.code, context.credentials), + ) + + if existing_code_or_err.is_err(): + return existing_code_or_err + existing_code = existing_code_or_err.ok() + if existing_code is not None: + if not exists_ok: + return Err("The code to be submitted already exists") + return Ok(existing_code) + + code = submit_code.to(UserCode, context=context) + + result = self._post_user_code_transform_ops(context, code) + if result.is_err(): + # if the validation fails, we should remove the user code status + # and code version to prevent dangling status + root_context = AuthedServiceContext( + credentials=context.node.verify_key, node=context.node + ) + + if code.status_link is not None: + _ = context.node.get_service("usercodestatusservice").remove( + root_context, code.status_link.object_uid + ) + return result result = self.stash.set(context.credentials, code) return result + @service_method( + path="code.update", + name="update", + roles=ADMIN_ROLE_LEVEL, + autosplat=["code_update"], + ) + def update( + self, + context: AuthedServiceContext, + code_update: UserCodeUpdate, + ) -> SyftSuccess | SyftError: + code = self.stash.get_by_uid(context.credentials, code_update.id) + if code.is_err(): + return SyftError(message=code.err()) + code = code.ok() + + result = self.stash.update(context.credentials, code) + if result.is_err(): + return SyftError(message=str(result.err())) + + if code_update.l0_deny_reason is not Empty: # type: ignore[comparison-overlap] + code.l0_deny_reason = code_update.l0_deny_reason + + result = self.stash.update(context.credentials, code) + + if result.is_ok(): + return result.ok() + return SyftError(message=str(result.err())) + @service_method(path="code.delete", name="delete", roles=ADMIN_ROLE_LEVEL) def delete( self, context: AuthedServiceContext, uid: UID @@ -101,58 +174,19 @@ def get_by_service_name( return SyftError(message=str(result.err())) return result.ok() - def _request_code_execution( - self, - context: AuthedServiceContext, - code: SubmitUserCode, - reason: str | None = "", - ) -> Request | SyftError: - user_code: UserCode = code.to(UserCode, context=context) - result = self._validate_request_code_execution(context, user_code) - if isinstance(result, SyftError): - # if the validation fails, we should remove the user code status - # and code version to prevent dangling status - root_context = AuthedServiceContext( - credentials=context.node.verify_key, node=context.node - ) - _ = context.node.get_service("usercodestatusservice").remove( - root_context, user_code.status_link.object_uid - ) - return result - result = self._request_code_execution_inner(context, user_code, reason) - return result - - def _validate_request_code_execution( + def _post_user_code_transform_ops( self, context: AuthedServiceContext, user_code: UserCode, - ) -> SyftSuccess | SyftError: + ) -> Result[UserCode, str]: if user_code.output_readers is None: - return SyftError( - message=f"there is no verified output readers for {user_code}" - ) + return Err(f"there is no verified output readers for {user_code}") if user_code.input_owner_verify_keys is None: - return SyftError( - message=f"there is no verified input owners for {user_code}" - ) + return Err(message=f"there is no verified input owners for {user_code}") if not all( x in user_code.input_owner_verify_keys for x in user_code.output_readers ): - raise ValueError("outputs can only be distributed to input owners") - - # check if the code with the same name and content already exists in the stash - - find_results = self.stash.get_by_code_hash( - context.credentials, code_hash=user_code.code_hash - ) - if find_results.is_err(): - return SyftError(message=str(find_results.err())) - find_results = find_results.ok() - - if find_results is not None: - return SyftError( - message="The code to be submitted (name and content) already exists" - ) + return Err("outputs can only be distributed to input owners") worker_pool_service = context.node.get_service("SyftWorkerPoolService") pool_result = worker_pool_service._get_worker_pool( @@ -161,26 +195,35 @@ def _validate_request_code_execution( ) if isinstance(pool_result, SyftError): - return pool_result - - result = self.stash.set(context.credentials, user_code) - if result.is_err(): - return SyftError(message=str(result.err())) + return Err(pool_result.message) # Create a code history code_history_service = context.node.get_service("codehistoryservice") result = code_history_service.submit_version(context=context, code=user_code) if isinstance(result, SyftError): - return result + return Err(result.message) - return SyftSuccess(message="") + return Ok(user_code) - def _request_code_execution_inner( + def _request_code_execution( self, context: AuthedServiceContext, user_code: UserCode, reason: str | None = "", ) -> Request | SyftError: + # Cannot make multiple requests for the same code + get_by_usercode_id = context.node.get_service_method( + RequestService.get_by_usercode_id + ) + existing_requests = get_by_usercode_id(context, user_code.id) + if isinstance(existing_requests, SyftError): + return existing_requests + if len(existing_requests) > 0: + return SyftError( + message=f"Request {existing_requests[0].id} already exists for this UserCode. " + f"Please use the existing request, or submit a new UserCode to create a new request." + ) + # Users that have access to the output also have access to the code item if user_code.output_readers is not None: self.stash.add_permissions( @@ -192,12 +235,20 @@ def _request_code_execution_inner( code_link = LinkedObject.from_obj(user_code, node_uid=context.node.id) - CODE_EXECUTE = UserCodeStatusChange( - value=UserCodeStatus.APPROVED, - linked_obj=user_code.status_link, - linked_user_code=code_link, - ) - changes = [CODE_EXECUTE] + # Requests made on low side are synced, and have their status computed instead of set manually. + if user_code.is_l0_deployment: + status_change = SyncedUserCodeStatusChange( + value=UserCodeStatus.APPROVED, + linked_obj=user_code.status_link, + linked_user_code=code_link, + ) + else: + status_change = UserCodeStatusChange( + value=UserCodeStatus.APPROVED, + linked_obj=user_code.status_link, + linked_user_code=code_link, + ) + changes = [status_change] request = SubmitRequest(changes=changes) method = context.node.get_service_method(RequestService.submit) @@ -206,6 +257,30 @@ def _request_code_execution_inner( # The Request service already returns either a SyftSuccess or SyftError return result + def _get_or_submit_user_code( + self, + context: AuthedServiceContext, + code: SubmitUserCode | UserCode, + ) -> Result[UserCode, str]: + """ + - If the code is a UserCode, check if it exists and return + - If the code is a SubmitUserCode and the same code hash exists, return the existing code + - If the code is a SubmitUserCode and the code hash does not exist, submit the code + """ + if isinstance(code, UserCode): + # Get existing UserCode + user_code_or_err = self.stash.get_by_uid(context.credentials, code.id) + if user_code_or_err.is_err(): + return user_code_or_err + user_code = user_code_or_err.ok() + if user_code is None: + return Err("UserCode not found on this node.") + return Ok(user_code) + else: # code: SubmitUserCode + # Submit new UserCode, or get existing UserCode with the same code hash + user_code_or_err = self._submit(context, code, exists_ok=True) # type: ignore + return user_code_or_err + @service_method( path="code.request_code_execution", name="request_code_execution", @@ -214,11 +289,22 @@ def _request_code_execution_inner( def request_code_execution( self, context: AuthedServiceContext, - code: SubmitUserCode, + code: SubmitUserCode | UserCode, reason: str | None = "", - ) -> SyftSuccess | SyftError: + ) -> Request | SyftError: """Request Code execution on user code""" - return self._request_code_execution(context=context, code=code, reason=reason) + + user_code_or_err = self._get_or_submit_user_code(context, code) + if user_code_or_err.is_err(): + return SyftError(message=user_code_or_err.err()) + user_code = user_code_or_err.ok() + + result = self._request_code_execution( + context, + user_code, + reason, + ) + return result @service_method(path="code.get_all", name="get_all", roles=GUEST_ROLE_LEVEL) def get_all(self, context: AuthedServiceContext) -> list[UserCode] | SyftError: @@ -274,70 +360,15 @@ def load_user_code(self, context: AuthedServiceContext) -> None: user_code_items = result.ok() load_approved_policy_code(user_code_items=user_code_items, context=context) - @service_method(path="code.get_results", name="get_results", roles=GUEST_ROLE_LEVEL) - def get_results( - self, context: AuthedServiceContext, inp: UID | UserCode - ) -> list[UserCode] | SyftError: - uid = inp.id if isinstance(inp, UserCode) else inp - code_result = self.stash.get_by_uid(context.credentials, uid=uid) - - if code_result.is_err(): - return SyftError(message=code_result.err()) - code = code_result.ok() - - if code.is_enclave_code: - # if the current node is not the enclave - if not context.node.node_type == NodeType.ENCLAVE: - connection = route_to_connection(code.enclave_metadata.route) - enclave_client = EnclaveClient( - connection=connection, - credentials=context.node.signing_key, - ) - if enclave_client.code is None: - return SyftError( - message=f"{enclave_client} can't access the user code api" - ) - outputs = enclave_client.code.get_results(code.id) - if isinstance(outputs, list): - for output in outputs: - output.syft_action_data # noqa: B018 - else: - outputs.syft_action_data # noqa: B018 - return outputs - - # if the current node is the enclave - else: - if not code.get_status(context.as_root_context()).approved: - return code.status.get_status_message() - - output_history = code.get_output_history( - context=context.as_root_context() - ) - if isinstance(output_history, SyftError): - return output_history - - if len(output_history) > 0: - res = resolve_outputs( - context=context, - output_ids=output_history[-1].output_ids, - ) - if res.is_err(): - return res - res = delist_if_single(res.ok()) - return Ok(res) - else: - return SyftError(message="No results available") - else: - return SyftError(message="Endpoint only supported for enclave code") - def is_execution_allowed( self, code: UserCode, context: AuthedServiceContext, output_policy: OutputPolicy | None, ) -> bool | SyftSuccess | SyftError | SyftNotReady: - if not code.get_status(context).approved: - return code.status.get_status_message() + status = code.get_status(context) + if not status.approved: + return status.get_status_message() # Check if the user has permission to execute the code. elif not (has_code_permission := self.has_code_permission(code, context)): return has_code_permission @@ -382,9 +413,29 @@ def keep_owned_kwargs( return mock_kwargs def is_execution_on_owned_args( - self, kwargs: dict[str, Any], context: AuthedServiceContext + self, + context: AuthedServiceContext, + user_code_id: UID, + passed_kwargs: dict[str, Any], ) -> bool: - return len(self.keep_owned_kwargs(kwargs, context)) == len(kwargs) + # Check if all kwargs are owned by the user + all_kwargs_are_owned = len( + self.keep_owned_kwargs(passed_kwargs, context) + ) == len(passed_kwargs) + if not all_kwargs_are_owned: + return False + + # Check if the kwargs match the code signature + code = self.stash.get_by_uid(context.credentials, user_code_id) + if code.is_err(): + return False + code = code.ok() + + # Skip the domain and context kwargs, they are passed by the backend + code_kwargs = set(code.signature.parameters.keys()) - {"domain", "context"} + + passed_kwarg_keys = set(passed_kwargs.keys()) + return passed_kwarg_keys == code_kwargs @service_method(path="code.call", name="call", roles=GUEST_ROLE_LEVEL) def call( @@ -427,15 +478,8 @@ def _call( return code_result code: UserCode = code_result.ok() - if not self.valid_worker_pool_for_context(context, code): - return Err( - value="You tried to run a syft function attached to a worker pool in blocking mode," - "which is currently not supported. Run your function with `blocking=False` to run" - " as a job on your worker pool" - ) - # Set Permissions - if self.is_execution_on_owned_args(kwargs, context): + if self.is_execution_on_owned_args(context, uid, kwargs): if self.is_execution_on_owned_args_allowed(context): # handles the case: if we have 1 or more owned args and execution permission # handles the case: if we have 0 owned args and execution permission @@ -460,24 +504,33 @@ def _call( kwarg2id = map_kwargs_to_id(kwargs) input_policy = code.get_input_policy(context) + # relative # Check output policy - output_policy = code.get_output_policy(context) if not override_execution_permission: output_history = code.get_output_history(context=context) if isinstance(output_history, SyftError): return Err(output_history.message) - can_execute = self.is_execution_allowed( + output_policy = code.get_output_policy(context) + + can_execute = output_policy and self.is_execution_allowed( code=code, context=context, output_policy=output_policy, ) if not can_execute: - if not code.is_output_policy_approved(context): - return Err( - "Execution denied: Your code is waiting for approval" - ) - if not (is_valid := output_policy._is_valid(context)): # type: ignore + # We check output policy only in l2 deployment. + # code is from low side (L0 setup) + status = code.get_status(context) + if not status.approved: + # return Err( + # "Execution denied: Your code is waiting for approval" + # ) + return Err(status.get_status_message().message) + is_valid = ( + output_policy._is_valid(context) if output_policy else False + ) + if not is_valid or code.is_l0_deployment: if len(output_history) > 0 and not skip_read_cache: last_executed_output = output_history[-1] # Check if the inputs of the last executed output match @@ -504,10 +557,15 @@ def _call( return result res = delist_if_single(result.ok()) + output_policy_message = "" + if code.is_l2_deployment: + # Skip output policy warning in L0 setup; + # admin overrides policy checks. + output_policy_message = is_valid.message return Ok( CachedSyftObject( result=res, - error_msg=is_valid.message, + error_msg=output_policy_message, ) ) else: @@ -515,9 +573,13 @@ def _call( return can_execute.to_result() # type: ignore # Execute the code item - + if not self.valid_worker_pool_for_context(context, code): + return Err( + value="You tried to run a syft function attached to a worker pool in blocking mode," + "which is currently not supported. Run your function with `blocking=False` to run" + " as a job on your worker pool" + ) action_service = context.node.get_service("actionservice") - result_action_object: Result[ActionObject | TwinObject, str] = ( action_service._user_code_execute( context, code, kwarg2id, result_id=result_id @@ -540,8 +602,10 @@ def _call( # this currently only works for nested syft_functions # and admins executing on high side (TODO, decide if we want to increment counter) - if not skip_fill_cache and output_policy is not None: - res = code.store_as_history( + # always store_execution_output on l0 setup + is_l0_request = context.role == ServiceRole.ADMIN and code.is_l0_deployment + if not skip_fill_cache and output_policy is not None or is_l0_request: + res = code.store_execution_output( context=context, outputs=result, job_id=context.job_id, @@ -596,9 +660,11 @@ def has_code_permission( return SyftSuccess(message="you have permission") @service_method( - path="code.store_as_history", name="store_as_history", roles=GUEST_ROLE_LEVEL + path="code.store_execution_output", + name="store_execution_output", + roles=GUEST_ROLE_LEVEL, ) - def store_as_history( + def store_execution_output( self, context: AuthedServiceContext, user_code_id: UID, @@ -610,11 +676,12 @@ def store_as_history( if code_result.is_err(): return SyftError(message=code_result.err()) + is_admin = context.role == ServiceRole.ADMIN code: UserCode = code_result.ok() - if not code.get_status(context).approved: - return SyftError(message="Code is not approved") + if not code.get_status(context).approved and not is_admin: + return SyftError(message="This UserCode is not approved") - res = code.store_as_history( + res = code.store_execution_output( context=context, outputs=outputs, job_id=job_id, diff --git a/packages/syft/src/syft/service/code_history/code_history.py b/packages/syft/src/syft/service/code_history/code_history.py index 139fd9e2d7a..e013ef22c34 100644 --- a/packages/syft/src/syft/service/code_history/code_history.py +++ b/packages/syft/src/syft/service/code_history/code_history.py @@ -8,17 +8,20 @@ from ...serde.serializable import serializable from ...service.user.user_roles import ServiceRole from ...types.syft_object import SYFT_OBJECT_VERSION_2 +from ...types.syft_object import SYFT_OBJECT_VERSION_3 from ...types.syft_object import SyftObject from ...types.syft_object import SyftVerifyKey from ...types.uid import UID -from ...util.notebook_ui.components.table_template import create_table_template +from ...util.notebook_ui.components.tabulator_template import ( + build_tabulator_table_with_data, +) from ...util.table import prepare_table_data from ..code.user_code import UserCode from ..response import SyftError @serializable() -class CodeHistory(SyftObject): +class CodeHistoryV2(SyftObject): # version __canonical_name__ = "CodeHistory" __version__ = SYFT_OBJECT_VERSION_2 @@ -31,6 +34,20 @@ class CodeHistory(SyftObject): service_func_name: str comment_history: list[str] = [] + +@serializable() +class CodeHistory(SyftObject): + # version + __canonical_name__ = "CodeHistory" + __version__ = SYFT_OBJECT_VERSION_3 + + id: UID + node_uid: UID + user_verify_key: SyftVerifyKey + user_code_history: list[UID] = [] + service_func_name: str + comment_history: list[str] = [] + __attr_searchable__ = ["user_verify_key", "service_func_name"] def add_code(self, code: UserCode, comment: str | None = None) -> None: @@ -54,9 +71,9 @@ class CodeHistoryView(SyftObject): def _coll_repr_(self) -> dict[str, int]: return {"Number of versions": len(self.user_code_history)} - def _repr_html_(self) -> str: - # TODO techdebt: move this to _coll_repr_ - rows, _ = prepare_table_data(self.user_code_history) + def _repr_html_(self) -> str | None: + rows, metadata = prepare_table_data(self.user_code_history) + for i, r in enumerate(rows): r["Version"] = f"v{i}" raw_code = self.user_code_history[i].raw_code @@ -64,8 +81,11 @@ def _repr_html_(self) -> str: if n_code_lines > 5: raw_code = "\n".join(raw_code.split("\n", 5)) r["Code"] = raw_code - # rows = sorted(rows, key=lambda x: x["Version"]) - return create_table_template(rows, "CodeHistory", icon=None) + + metadata["name"] = "Code History" + metadata["columns"] += ["Version", "Code"] + + return build_tabulator_table_with_data(rows, metadata) def __getitem__(self, index: int | str) -> UserCode | SyftError: if isinstance(index, str): @@ -75,7 +95,11 @@ def __getitem__(self, index: int | str) -> UserCode | SyftError: return SyftError( message=f"Can't access the api. You must login to {self.node_uid}" ) - if api.user_role.value >= ServiceRole.DATA_OWNER.value and index < 0: + if ( + api.user.get_current_user().role.value >= ServiceRole.DATA_OWNER.value + and index < 0 + ): + # negative index would dynamically resolve to a different version return SyftError( message="For security concerns we do not allow negative indexing. \ Try using absolute values when indexing" @@ -136,8 +160,14 @@ def __getitem__(self, key: str | int) -> CodeHistoriesDict | SyftError: ) return api.services.code_history.get_history_for_user(key) - def _repr_html_(self) -> str: - rows = [] - for user, funcs in self.user_dict.items(): - rows += [{"user": user, "UserCodes": funcs}] - return create_table_template(rows, "UserCodeHistory", icon=None) + def _repr_html_(self) -> str | None: + rows = [ + {"User": user, "UserCodes": ", ".join(funcs)} + for user, funcs in self.user_dict.items() + ] + metadata = { + "name": "UserCode Histories", + "columns": ["User", "UserCodes"], + "icon": None, + } + return build_tabulator_table_with_data(rows, metadata) diff --git a/packages/syft/src/syft/service/code_history/code_history_service.py b/packages/syft/src/syft/service/code_history/code_history_service.py index f32338ed936..41839045747 100644 --- a/packages/syft/src/syft/service/code_history/code_history_service.py +++ b/packages/syft/src/syft/service/code_history/code_history_service.py @@ -8,6 +8,7 @@ from ...util.telemetry import instrument from ..code.user_code import SubmitUserCode from ..code.user_code import UserCode +from ..code.user_code_service import UserCodeService from ..context import AuthedServiceContext from ..response import SyftError from ..response import SyftSuccess @@ -15,6 +16,7 @@ from ..service import service_method from ..user.user_roles import DATA_OWNER_ROLE_LEVEL from ..user.user_roles import DATA_SCIENTIST_ROLE_LEVEL +from ..user.user_roles import ServiceRole from .code_history import CodeHistoriesDict from .code_history import CodeHistory from .code_history import CodeHistoryView @@ -49,11 +51,6 @@ def submit_version( if result.is_err(): return SyftError(message=str(result.err())) code = result.ok() - elif isinstance(code, UserCode): # type: ignore[unreachable] - result = user_code_service.get_by_uid(context=context, uid=code.id) - if isinstance(result, SyftError): - return result - code = result result = self.stash.get_by_service_func_name_and_verify_key( credentials=context.credentials, @@ -120,23 +117,31 @@ def delete( def fetch_histories_for_user( self, context: AuthedServiceContext, user_verify_key: SyftVerifyKey ) -> CodeHistoriesDict | SyftError: - result = self.stash.get_by_verify_key( - credentials=context.credentials, user_verify_key=user_verify_key - ) + if context.role in [ServiceRole.DATA_OWNER, ServiceRole.ADMIN]: + result = self.stash.get_by_verify_key( + credentials=context.node.verify_key, user_verify_key=user_verify_key + ) + else: + result = self.stash.get_by_verify_key( + credentials=context.credentials, user_verify_key=user_verify_key + ) - user_code_service = context.node.get_service("usercodeservice") + user_code_service: UserCodeService = context.node.get_service("usercodeservice") # type: ignore def get_code(uid: UID) -> UserCode | SyftError: - return user_code_service.get_by_uid(context=context, uid=uid) + return user_code_service.stash.get_by_uid( + credentials=context.node.verify_key, + uid=uid, + ).ok() if result.is_ok(): code_histories = result.ok() code_versions_dict = {} for code_history in code_histories: - user_code_list = [] - for uid in code_history.user_code_history: - user_code_list.append(get_code(uid)) + user_code_list = [ + get_code(uid) for uid in code_history.user_code_history + ] code_versions = CodeHistoryView( user_code_history=user_code_list, service_func_name=code_history.service_func_name, @@ -186,7 +191,10 @@ def get_history_for_user( def get_histories_group_by_user( self, context: AuthedServiceContext ) -> UsersCodeHistoriesDict | SyftError: - result = self.stash.get_all(credentials=context.credentials) + if context.role in [ServiceRole.DATA_OWNER, ServiceRole.ADMIN]: + result = self.stash.get_all(context.credentials, has_permission=True) + else: + result = self.stash.get_all(context.credentials) if result.is_err(): return SyftError(message=result.err()) code_histories: list[CodeHistory] = result.ok() diff --git a/packages/syft/src/syft/service/data_subject/__init__.py b/packages/syft/src/syft/service/data_subject/__init__.py index f628bc5d753..f232044493c 100644 --- a/packages/syft/src/syft/service/data_subject/__init__.py +++ b/packages/syft/src/syft/service/data_subject/__init__.py @@ -1,2 +1,2 @@ # relative -from .data_subject import DataSubjectCreate # noqa: F401 +from .data_subject import DataSubjectCreate diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py index 687a2eb5f84..9dde84429c4 100644 --- a/packages/syft/src/syft/service/dataset/dataset.py +++ b/packages/syft/src/syft/service/dataset/dataset.py @@ -38,6 +38,8 @@ from ...util.notebook_ui.icons import Icon from ...util.notebook_ui.styles import FONT_CSS from ...util.notebook_ui.styles import ITABLES_CSS +from ..action.action_data_empty import ActionDataEmpty +from ..action.action_object import ActionObject from ..data_subject.data_subject import DataSubject from ..data_subject.data_subject import DataSubjectCreate from ..data_subject.data_subject_service import DataSubjectService @@ -320,6 +322,17 @@ def __mock_is_real_for_empty_mock_must_be_false(self) -> Self: return self + def contains_empty(self) -> bool: + if isinstance(self.mock, ActionObject) and isinstance( + self.mock.syft_action_data_cache, ActionDataEmpty + ): + return True + if isinstance(self.data, ActionObject) and isinstance( + self.data.syft_action_data_cache, ActionDataEmpty + ): + return True + return False + def add_data_subject(self, data_subject: DataSubject) -> None: self.data_subjects.append(data_subject) @@ -484,10 +497,7 @@ def _repr_html_(self) -> Any: """ def action_ids(self) -> list[UID]: - data = [] - for asset in self.asset_list: - if asset.action_id: - data.append(asset.action_id) + data = [asset.action_id for asset in self.asset_list if asset.action_id] return data @property @@ -688,17 +698,27 @@ def create_and_store_twin(context: TransformContext) -> TransformContext: if private_obj is None and mock_obj is None: raise ValueError("No data and no action_id means this asset has no data") + asset = context.obj # type: ignore + contains_empty = asset.contains_empty() # type: ignore twin = TwinObject( - private_obj=private_obj, - mock_obj=mock_obj, + private_obj=asset.data, # type: ignore + mock_obj=asset.mock, # type: ignore + syft_node_location=asset.syft_node_location, # type: ignore + syft_client_verify_key=asset.syft_client_verify_key, # type: ignore ) + res = twin._save_to_blob_storage(allow_empty=contains_empty) + if isinstance(res, SyftError): + raise ValueError(res.message) + + # TODO, upload to blob storage here if context.node is None: raise ValueError( "f{context}'s node is None, please log in. No trasformation happened" ) action_service = context.node.get_service("actionservice") - result = action_service.set( - context=context.to_node_context(), action_object=twin + result = action_service._set( + context=context.to_node_context(), + action_object=twin, ) if result.is_err(): raise RuntimeError(f"Failed to create and store twin. Error: {result}") diff --git a/packages/syft/src/syft/service/dataset/dataset_service.py b/packages/syft/src/syft/service/dataset/dataset_service.py index f791d6b9dc2..451746fa15a 100644 --- a/packages/syft/src/syft/service/dataset/dataset_service.py +++ b/packages/syft/src/syft/service/dataset/dataset_service.py @@ -187,16 +187,14 @@ def get_assets_by_action_id( ) -> list[Asset] | SyftError: """Get Assets by an Action ID""" datasets = self.get_by_action_id(context=context, uid=uid) - assets = [] - if isinstance(datasets, list): - for dataset in datasets: - for asset in dataset.asset_list: - if asset.action_id == uid: - assets.append(asset) - return assets - elif isinstance(datasets, SyftError): + if isinstance(datasets, SyftError): return datasets - return [] + return [ + asset + for dataset in datasets + for asset in dataset.asset_list + if asset.action_id == uid + ] @service_method( path="dataset.delete_by_uid", diff --git a/packages/syft/src/syft/service/enclave/enclave_service.py b/packages/syft/src/syft/service/enclave/enclave_service.py index be19b9c2659..0807ada8da0 100644 --- a/packages/syft/src/syft/service/enclave/enclave_service.py +++ b/packages/syft/src/syft/service/enclave/enclave_service.py @@ -1,145 +1,14 @@ # stdlib # relative -from ...client.enclave_client import EnclaveClient -from ...client.enclave_client import EnclaveMetadata from ...serde.serializable import serializable -from ...service.response import SyftError -from ...service.response import SyftSuccess -from ...service.user.user_roles import GUEST_ROLE_LEVEL from ...store.document_store import DocumentStore -from ...types.twin_object import TwinObject -from ...types.uid import UID -from ..action.action_object import ActionObject -from ..code.user_code import UserCode -from ..code.user_code import UserCodeStatus -from ..context import AuthedServiceContext -from ..context import ChangeContext -from ..network.routes import route_to_connection -from ..policy.policy import InputPolicy from ..service import AbstractService -from ..service import service_method -# TODO 🟣 Created a generic Enclave Service -# Currently it mainly works only for Azure @serializable() class EnclaveService(AbstractService): store: DocumentStore def __init__(self, store: DocumentStore) -> None: self.store = store - - @service_method( - path="enclave.send_user_code_inputs_to_enclave", - name="send_user_code_inputs_to_enclave", - roles=GUEST_ROLE_LEVEL, - ) - def send_user_code_inputs_to_enclave( - self, - context: AuthedServiceContext, - user_code_id: UID, - inputs: dict, - node_name: str, - node_id: UID, - ) -> SyftSuccess | SyftError: - if not context.node or not context.node.signing_key: - return SyftError(message=f"{type(context)} has no node") - - root_context = AuthedServiceContext( - credentials=context.node.verify_key, node=context.node - ) - - user_code_service = context.node.get_service("usercodeservice") - action_service = context.node.get_service("actionservice") - user_code = user_code_service.get_by_uid(context=root_context, uid=user_code_id) - if isinstance(user_code, SyftError): - return user_code - - reason: str = context.extra_kwargs.get("reason", "") - status_update = user_code.get_status(root_context).mutate( - value=(UserCodeStatus.APPROVED, reason), - node_name=node_name, - node_id=node_id, - verify_key=context.credentials, - ) - if isinstance(status_update, SyftError): - return status_update - - res = user_code.status_link.update_with_context(root_context, status_update) - if isinstance(res, SyftError): - return res - - root_context = context.as_root_context() - if not action_service.exists(context=context, obj_id=user_code_id): - dict_object = ActionObject.from_obj({}) - dict_object.id = user_code_id - dict_object[str(context.credentials)] = inputs - root_context.extra_kwargs = {"has_result_read_permission": True} - # TODO: Instead of using the action store, modify to - # use the action service directly to store objects - action_service.set(root_context, dict_object) - - else: - res = action_service.get(uid=user_code_id, context=root_context) - if res.is_ok(): - dict_object = res.ok() - dict_object[str(context.credentials)] = inputs - action_service.set(root_context, dict_object) - else: - return SyftError( - message=f"Error while fetching the object on Enclave: {res.err()}" - ) - - return SyftSuccess(message="Enclave Code Status Updated Successfully") - - -# Checks if the given user code would propogate value to enclave on acceptance -def propagate_inputs_to_enclave( - user_code: UserCode, context: ChangeContext -) -> SyftSuccess | SyftError: - if isinstance(user_code.enclave_metadata, EnclaveMetadata): - # TODO 🟣 Restructure url it work for local mode host.docker.internal - - connection = route_to_connection(user_code.enclave_metadata.route) - enclave_client = EnclaveClient( - connection=connection, - credentials=context.node.signing_key, - ) - - send_method = ( - enclave_client.api.services.enclave.send_user_code_inputs_to_enclave - ) - - else: - return SyftSuccess(message="Current Request does not require Enclave Transfer") - - input_policy: InputPolicy | None = user_code.get_input_policy( - context.to_service_ctx() - ) - if input_policy is None: - return SyftError(message=f"{user_code}'s input policy is None") - inputs = input_policy._inputs_for_context(context) - if isinstance(inputs, SyftError): - return inputs - - # Save inputs to blob store - for var_name, var_value in inputs.items(): - if isinstance(var_value, ActionObject | TwinObject): - # Set the obj location to enclave - var_value._set_obj_location_( - enclave_client.api.node_uid, - enclave_client.verify_key, - ) - var_value._save_to_blob_storage() - - inputs[var_name] = var_value - - # send data of the current node to enclave - res = send_method( - user_code_id=user_code.id, - inputs=inputs, - node_name=context.node.name, - node_id=context.node.id, - ) - return res diff --git a/packages/syft/src/syft/service/job/job_service.py b/packages/syft/src/syft/service/job/job_service.py index 323dff99ae9..368992ceaa5 100644 --- a/packages/syft/src/syft/service/job/job_service.py +++ b/packages/syft/src/syft/service/job/job_service.py @@ -11,6 +11,7 @@ from ...store.document_store import DocumentStore from ...types.uid import UID from ...util.telemetry import instrument +from ..action.action_object import ActionObject from ..action.action_permissions import ActionObjectPermission from ..action.action_permissions import ActionPermission from ..code.user_code import UserCode @@ -304,17 +305,27 @@ def add_read_permission_log_for_code_owner( roles=DATA_OWNER_ROLE_LEVEL, ) def create_job_for_user_code_id( - self, context: AuthedServiceContext, user_code_id: UID + self, + context: AuthedServiceContext, + user_code_id: UID, + result: ActionObject | None = None, + log_stdout: str = "", + log_stderr: str = "", + status: JobStatus = JobStatus.CREATED, + add_code_owner_read_permissions: bool = True, ) -> Job | SyftError: + is_resolved = status in [JobStatus.COMPLETED, JobStatus.ERRORED] job = Job( id=UID(), node_uid=context.node.id, action=None, - result_id=None, + result=result, + status=status, parent_id=None, log_id=UID(), job_pid=None, user_code_id=user_code_id, + resolved=is_resolved, ) user_code_service = context.node.get_service("usercodeservice") user_code = user_code_service.get_by_uid(context=context, uid=user_code_id) @@ -323,19 +334,21 @@ def create_job_for_user_code_id( # The owner of the code should be able to read the job self.stash.set(context.credentials, job) - self.add_read_permission_job_for_code_owner(context, job, user_code) log_service = context.node.get_service("logservice") - res = log_service.add(context, job.log_id, job.id) + res = log_service.add( + context, + job.log_id, + job.id, + stdout=log_stdout, + stderr=log_stderr, + ) if isinstance(res, SyftError): return res - # The owner of the code should be able to read the job log - self.add_read_permission_log_for_code_owner(context, job.log_id, user_code) - # log_service.stash.add_permission( - # ActionObjectPermission( - # job.log_id, ActionPermission.READ, user_code.user_verify_key - # ) - # ) + + if add_code_owner_read_permissions: + self.add_read_permission_job_for_code_owner(context, job, user_code) + self.add_read_permission_log_for_code_owner(context, job.log_id, user_code) return job diff --git a/packages/syft/src/syft/service/job/job_stash.py b/packages/syft/src/syft/service/job/job_stash.py index 9b9a1c623f7..e69c539cb5c 100644 --- a/packages/syft/src/syft/service/job/job_stash.py +++ b/packages/syft/src/syft/service/job/job_stash.py @@ -23,7 +23,7 @@ from ...serde.serializable import serializable from ...service.context import AuthedServiceContext from ...service.worker.worker_pool import SyftWorker -from ...store.document_store import BaseStash +from ...store.document_store import BaseUIDStoreStash from ...store.document_store import DocumentStore from ...store.document_store import PartitionKey from ...store.document_store import PartitionSettings @@ -317,7 +317,7 @@ def fetch(self) -> None: ) job: Job | None = api.make_call(call) if job is None: - return + return None self.resolved = job.resolved if job.resolved: self.result = job.result @@ -592,8 +592,9 @@ def _repr_html_(self) -> str: updated_at = str(self.updated_at)[:-7] if self.updated_at else "--" user_repr = "--" - if self.requested_by: - requesting_user = self.requesting_user + if self.requested_by and not isinstance( + requesting_user := self.requesting_user, SyftError + ): user_repr = f"{requesting_user.name} {requesting_user.email}" worker_attr = "" @@ -620,17 +621,12 @@ def _repr_html_(self) -> str: template = Template(job_repr_template) return template.substitute( - uid=str(UID()), - grid_template_columns=None, - grid_template_cell_columns=None, - cols=0, job_type=job_type, api_header=api_header, user_code_name=self.user_code_name, button_html=button_html, status=self.status.value.title(), creation_time=creation_time, - user_rerp=user_repr, updated_at=updated_at, worker_attr=worker_attr, no_subjobs=len(self.subjobs), @@ -644,7 +640,7 @@ def _repr_html_(self) -> str: def wait( self, job_only: bool = False, timeout: int | None = None - ) -> Any | SyftNotReady: + ) -> Any | SyftNotReady | SyftError: self.fetch() if self.resolved: return self.resolve @@ -653,28 +649,33 @@ def wait( node_uid=self.syft_node_location, user_verify_key=self.syft_client_verify_key, ) + if api is None: raise ValueError( - f"Can't access Syft API. You must login to {self.syft_node_location}" + f"Can't access Syft API. You must login to node with id '{self.syft_node_location}'" ) workers = api.services.worker.get_all() if not isinstance(workers, SyftError) and len(workers) == 0: return SyftError( - message="This node has no workers. " - "You need to start a worker to run jobs " - "by setting n_consumers > 0." + message=f"Node {self.syft_node_location} has no workers. " + f"You need to start a worker to run jobs " + f"by setting n_consumers > 0." ) - if not job_only and self.result is not None: - self.result.wait(timeout) - print_warning = True counter = 0 while True: self.fetch() + if self.resolved: + if isinstance(self.result, SyftError | Err) or self.status in [ # type: ignore[unreachable] + JobStatus.ERRORED, + JobStatus.INTERRUPTED, + ]: + return self.result + break if print_warning and self.result is not None: - result_obj = api.services.action.get( + result_obj = api.services.action.get( # type: ignore[unreachable] self.result.id, resolve_nested=False ) if result_obj.is_link and job_only: @@ -684,14 +685,20 @@ def wait( "Use job.wait().get() instead to wait for the linked result." ) print_warning = False + sleep(1) - if self.resolved: - break # type: ignore[unreachable] - # TODO: fix the mypy issue + if timeout is not None: counter += 1 if counter > timeout: return SyftError(message="Reached Timeout!") + + # if self.resolve returns self.result as error, then we + # return SyftError and not wait for the result + # otherwise if a job is resolved and not errored out, we wait for the result + if not job_only and self.result is not None: # type: ignore[unreachable] + self.result.wait(timeout) + return self.resolve # type: ignore[unreachable] @property @@ -820,7 +827,7 @@ def from_job( @instrument @serializable() -class JobStash(BaseStash): +class JobStash(BaseUIDStoreStash): object_type = Job settings: PartitionSettings = PartitionSettings( name=Job.__canonical_name__, object_type=Job @@ -863,29 +870,6 @@ def get_by_result_id( else: return Ok(res[0]) - def set_placeholder( - self, - credentials: SyftVerifyKey, - item: Job, - add_permissions: list[ActionObjectPermission] | None = None, - ) -> Result[Job, str]: - # 🟡 TODO 36: Needs distributed lock - if not item.resolved: - exists = self.get_by_uid(credentials, item.id) - if exists.is_ok() and exists.ok() is None: - valid = self.check_type(item, self.object_type) - if valid.is_err(): - return SyftError(message=valid.err()) - return super().set(credentials, item, add_permissions) - return item - - def get_by_uid( - self, credentials: SyftVerifyKey, uid: UID - ) -> Result[Job | None, str]: - qks = QueryKeys(qks=[UIDPartitionKey.with_obj(uid)]) - item = self.query_one(credentials=credentials, qks=qks) - return item - def get_by_parent_id( self, credentials: SyftVerifyKey, uid: UID ) -> Result[Job | None, str]: diff --git a/packages/syft/src/syft/service/log/log_service.py b/packages/syft/src/syft/service/log/log_service.py index 0e103e0d85e..3390171d1a4 100644 --- a/packages/syft/src/syft/service/log/log_service.py +++ b/packages/syft/src/syft/service/log/log_service.py @@ -28,9 +28,14 @@ def __init__(self, store: DocumentStore) -> None: @service_method(path="log.add", name="add", roles=DATA_SCIENTIST_ROLE_LEVEL) def add( - self, context: AuthedServiceContext, uid: UID, job_id: UID + self, + context: AuthedServiceContext, + uid: UID, + job_id: UID, + stdout: str = "", + stderr: str = "", ) -> SyftSuccess | SyftError: - new_log = SyftLog(id=uid, job_id=job_id) + new_log = SyftLog(id=uid, job_id=job_id, stdout=stdout, stderr=stderr) result = self.stash.set(context.credentials, new_log) if result.is_err(): return SyftError(message=str(result.err())) diff --git a/packages/syft/src/syft/service/network/network_service.py b/packages/syft/src/syft/service/network/network_service.py index b38d822c7f4..c32374ade31 100644 --- a/packages/syft/src/syft/service/network/network_service.py +++ b/packages/syft/src/syft/service/network/network_service.py @@ -1,11 +1,11 @@ # stdlib from collections.abc import Callable from enum import Enum +import logging import secrets from typing import Any # third party -from loguru import logger from result import Result # relative @@ -56,6 +56,8 @@ from .routes import NodeRouteType from .routes import PythonNodeRoute +logger = logging.getLogger(__name__) + VerifyKeyPartitionKey = PartitionKey(key="verify_key", type_=SyftVerifyKey) NodeTypePartitionKey = PartitionKey(key="node_type", type_=NodeType) OrderByNamePartitionKey = PartitionKey(key="name", type_=str) @@ -900,14 +902,15 @@ def _get_association_requests_by_peer_id( RequestService.get_all ) all_requests: list[Request] = request_get_all_method(context) - association_requests: list[Request] = [] - for request in all_requests: - for change in request.changes: - if ( - isinstance(change, AssociationRequestChange) - and change.remote_peer.id == peer_id - ): - association_requests.append(request) + association_requests: list[Request] = [ + request + for request in all_requests + if any( + isinstance(change, AssociationRequestChange) + and change.remote_peer.id == peer_id + for change in request.changes + ) + ] return sorted( association_requests, key=lambda request: request.request_time.utc_timestamp diff --git a/packages/syft/src/syft/service/network/node_peer.py b/packages/syft/src/syft/service/network/node_peer.py index c2db506ba23..5835cf7aa9e 100644 --- a/packages/syft/src/syft/service/network/node_peer.py +++ b/packages/syft/src/syft/service/network/node_peer.py @@ -1,6 +1,7 @@ # stdlib from collections.abc import Callable from enum import Enum +import logging # third party from result import Err @@ -35,6 +36,8 @@ from .routes import connection_to_route from .routes import route_to_connection +logger = logging.getLogger(__name__) + @serializable() class NodePeerConnectionStatus(Enum): @@ -245,7 +248,6 @@ def client_with_context( self, context: NodeServiceContext ) -> Result[type[SyftClient], str]: # third party - from loguru import logger if len(self.node_routes) < 1: raise ValueError(f"No routes to peer: {self}") @@ -255,12 +257,11 @@ def client_with_context( try: client_type = connection.get_client_type() except Exception as e: - logger.error( - f"Failed to establish a connection with {self.node_type} '{self.name}'. Exception: {e}" - ) - return Err( + msg = ( f"Failed to establish a connection with {self.node_type} '{self.name}'" ) + logger.error(msg, exc_info=e) + return Err(msg) if isinstance(client_type, SyftError): return Err(client_type.message) return Ok( diff --git a/packages/syft/src/syft/service/network/utils.py b/packages/syft/src/syft/service/network/utils.py index b03bc589d15..476411bc6e6 100644 --- a/packages/syft/src/syft/service/network/utils.py +++ b/packages/syft/src/syft/service/network/utils.py @@ -1,11 +1,9 @@ # stdlib +import logging import threading import time from typing import cast -# third party -from loguru import logger - # relative from ...serde.serializable import serializable from ...types.datetime import DateTime @@ -17,6 +15,8 @@ from .node_peer import NodePeerConnectionStatus from .node_peer import NodePeerUpdate +logger = logging.getLogger(__name__) + @serializable(without=["thread"]) class PeerHealthCheckTask: @@ -63,9 +63,7 @@ def peer_route_heathcheck(self, context: AuthedServiceContext) -> SyftError | No peer_update.ping_status = NodePeerConnectionStatus.TIMEOUT peer_client = None except Exception as e: - logger.error( - f"Failed to create client for peer: {peer} with exception {e}" - ) + logger.error(f"Failed to create client for peer: {peer}", exc_info=e) peer_update.ping_status = NodePeerConnectionStatus.TIMEOUT peer_client = None @@ -97,7 +95,7 @@ def peer_route_heathcheck(self, context: AuthedServiceContext) -> SyftError | No ) if result.is_err(): - logger.info(f"Failed to update peer in stash: {result.err()}") + logger.error(f"Failed to update peer in stash: {result.err()}") return None diff --git a/packages/syft/src/syft/service/notifier/notifier_service.py b/packages/syft/src/syft/service/notifier/notifier_service.py index aedb59b2e24..4c10708f0f0 100644 --- a/packages/syft/src/syft/service/notifier/notifier_service.py +++ b/packages/syft/src/syft/service/notifier/notifier_service.py @@ -2,6 +2,9 @@ # stdlib +# stdlib +import logging + # third party from pydantic import EmailStr from result import Err @@ -22,6 +25,8 @@ from .notifier_enums import NOTIFIERS from .notifier_stash import NotifierStash +logger = logging.getLogger(__name__) + @serializable() class NotifierService(AbstractService): @@ -109,7 +114,7 @@ def turn_on( message="You must provide both server and port to enable notifications." ) - print("[LOG] Got notifier from db") + logging.debug("Got notifier from db") # If no new credentials provided, check for existing ones if not (email_username and email_password): if not (notifier.email_username and notifier.email_password): @@ -119,10 +124,9 @@ def turn_on( + ".settings.enable_notifications(email=<>, password=<>)" ) else: - print("[LOG] No new credentials provided. Using existing ones.") + logging.debug("No new credentials provided. Using existing ones.") email_password = notifier.email_password email_username = notifier.email_username - print("[LOG] Validating credentials...") validation_result = notifier.validate_email_credentials( username=email_username, @@ -132,6 +136,7 @@ def turn_on( ) if validation_result.is_err(): + logging.error(f"Invalid SMTP credentials {validation_result.err()}") return SyftError( message="Invalid SMTP credentials. Please check your username and password." ) @@ -160,8 +165,8 @@ def turn_on( notifier.email_sender = email_sender notifier.active = True - print( - "[LOG] Email credentials are valid. Updating the notifier settings in the db." + logging.debug( + "Email credentials are valid. Updating the notifier settings in the db." ) result = self.stash.update(credentials=context.credentials, settings=notifier) @@ -260,9 +265,8 @@ def init_notifier( sender_not_set = not email_sender and not notifier.email_sender if validation_result.is_err() or sender_not_set: - print( - "Ops something went wrong while trying to setup your notification system.", - "Please check your credentials and configuration.", + logger.error( + f"Notifier validation error - {validation_result.err()}.", ) notifier.active = False else: diff --git a/packages/syft/src/syft/service/output/output_service.py b/packages/syft/src/syft/service/output/output_service.py index ef0b3e7b1f2..4efe75ec618 100644 --- a/packages/syft/src/syft/service/output/output_service.py +++ b/packages/syft/src/syft/service/output/output_service.py @@ -23,6 +23,7 @@ from ...types.uid import UID from ...util.telemetry import instrument from ..action.action_object import ActionObject +from ..action.action_permissions import ActionObjectREAD from ..context import AuthedServiceContext from ..response import SyftError from ..service import AbstractService @@ -297,6 +298,40 @@ def get_by_user_code_id( return result.ok() return SyftError(message=result.err()) + @service_method( + path="output.has_output_read_permissions", + name="has_output_read_permissions", + roles=GUEST_ROLE_LEVEL, + ) + def has_output_read_permissions( + self, + context: AuthedServiceContext, + user_code_id: UID, + user_verify_key: SyftVerifyKey, + ) -> bool | SyftError: + action_service = context.node.get_service("actionservice") + all_outputs = self.get_by_user_code_id(context, user_code_id) + if isinstance(all_outputs, SyftError): + return all_outputs + for output in all_outputs: + # TODO tech debt: unclear why code owner can see outputhistory without permissions. + # It is not a security issue (output history has no data) it is confusing for user + # if not self.stash.has_permission( + # ActionObjectREAD(uid=output.id, credentials=user_verify_key) + # ): + # continue + + # Check if all output ActionObjects have permissions + result_ids = output.output_id_list + permissions = [ + ActionObjectREAD(uid=_id.id, credentials=user_verify_key) + for _id in result_ids + ] + if action_service.store.has_permissions(permissions): + return True + + return False + @service_method( path="output.get_by_job_id", name="get_by_job_id", diff --git a/packages/syft/src/syft/service/policy/policy.py b/packages/syft/src/syft/service/policy/policy.py index 5bf5739d6fc..e4800f04d6e 100644 --- a/packages/syft/src/syft/service/policy/policy.py +++ b/packages/syft/src/syft/service/policy/policy.py @@ -14,9 +14,12 @@ import sys import types from typing import Any +from typing import ClassVar # third party from RestrictedPython import compile_restricted +from pydantic import field_validator +from pydantic import model_validator import requests from result import Err from result import Ok @@ -26,11 +29,13 @@ from ...abstract_node import NodeType from ...client.api import APIRegistry from ...client.api import NodeIdentity +from ...client.api import RemoteFunction from ...node.credentials import SyftVerifyKey from ...serde.recursive_primitives import recursive_serde_register_type from ...serde.serializable import serializable from ...store.document_store import PartitionKey from ...types.datetime import DateTime +from ...types.syft_object import SYFT_OBJECT_VERSION_1 from ...types.syft_object import SYFT_OBJECT_VERSION_2 from ...types.syft_object import SyftObject from ...types.transforms import TransformContext @@ -39,7 +44,10 @@ from ...types.twin_object import TwinObject from ...types.uid import UID from ...util.util import is_interpreter_jupyter +from ..action.action_endpoint import CustomEndpointActionObject from ..action.action_object import ActionObject +from ..action.action_permissions import ActionObjectPermission +from ..action.action_permissions import ActionPermission from ..code.code_parse import GlobalsVisitor from ..code.unparse import unparse from ..context import AuthedServiceContext @@ -68,13 +76,15 @@ def extract_uid(v: Any) -> UID: def filter_only_uids(results: Any) -> list[UID] | dict[str, UID] | UID: + # Prevent checking for __len__ on ActionObject (creates an Action) + if isinstance(results, ActionObject): + return extract_uid(results) + if not hasattr(results, "__len__"): results = [results] if isinstance(results, list): - output_list = [] - for v in results: - output_list.append(extract_uid(v)) + output_list = [extract_uid(v) for v in results] return output_list elif isinstance(results, dict): output_dict = {} @@ -88,6 +98,7 @@ class Policy(SyftObject): # version __canonical_name__: str = "Policy" __version__ = SYFT_OBJECT_VERSION_2 + has_safe_serde: ClassVar[bool] = True id: UID init_kwargs: dict[Any, Any] = {} @@ -173,6 +184,170 @@ def partition_by_node(kwargs: dict[str, Any]) -> dict[NodeIdentity, dict[str, UI return output_kwargs +@serializable() +class PolicyRule(SyftObject): + __canonical_name__ = "PolicyRule" + __version__ = SYFT_OBJECT_VERSION_1 + + kw: str + requires_input: bool = True + + def is_met( + self, context: AuthedServiceContext, action_object: ActionObject + ) -> bool: + return False + + +@serializable() +class CreatePolicyRule(SyftObject): + __canonical_name__ = "CreatePolicyRule" + __version__ = SYFT_OBJECT_VERSION_1 + + val: Any + + +@serializable() +class CreatePolicyRuleConstant(CreatePolicyRule): + __canonical_name__ = "CreatePolicyRuleConstant" + __version__ = SYFT_OBJECT_VERSION_1 + + val: Any + klass: None | type = None + + @model_validator(mode="before") + @classmethod + def set_klass(cls, data: Any) -> Any: + val = data["val"] + if isinstance(val, RemoteFunction): + klass = CustomEndpointActionObject + else: + klass = type(val) + data["klass"] = klass + return data + + @field_validator("val", mode="after") + @classmethod + def idify_endpoints(cls, value: str) -> str: + if isinstance(value, RemoteFunction): + return value.custom_function_actionobject_id() + return value + + def to_policy_rule(self, kw: Any) -> PolicyRule: + return Constant(kw=kw, val=self.val, klass=self.klass) + + +@serializable() +class Matches(PolicyRule): + __canonical_name__ = "Matches" + __version__ = SYFT_OBJECT_VERSION_1 + + val: UID + + def is_met( + self, context: AuthedServiceContext, action_object: ActionObject + ) -> bool: + return action_object.id == self.val + + +@serializable() +class Constant(PolicyRule): + __canonical_name__ = "PreFill" + __version__ = SYFT_OBJECT_VERSION_1 + + val: Any + klass: type + requires_input: bool = False + + def is_met(self, context: AuthedServiceContext, *args: Any, **kwargs: Any) -> bool: + return True + + def transform_kwarg( + self, context: AuthedServiceContext, val: Any + ) -> Result[Any, str]: + if isinstance(self.val, UID): + if issubclass(self.klass, CustomEndpointActionObject): + res = context.node.get_service("actionservice").get(context, self.val) + if res.is_err(): + return res + else: + obj = res.ok() + return Ok(obj.syft_action_data) + return Ok(self.val) + + +@serializable() +class UserOwned(PolicyRule): + __canonical_name__ = "UserOwned" + __version__ = SYFT_OBJECT_VERSION_1 + + # str, float, int, bool, dict, list, set, tuple + + type: ( + type[str] + | type[float] + | type[int] + | type[bool] + | type[dict] + | type[list] + | type[set] + | type[tuple] + | None + ) + + def is_owned( + self, context: AuthedServiceContext, action_object: ActionObject + ) -> bool: + action_store = context.node.get_service("actionservice").store + return action_store.has_permission( + ActionObjectPermission( + action_object.id, ActionPermission.OWNER, context.credentials + ) + ) + + def is_met( + self, context: AuthedServiceContext, action_object: ActionObject + ) -> bool: + return type(action_object.syft_action_data) == self.type and self.is_owned( + context, action_object + ) + + +def user_code_arg2id(arg: Any) -> UID: + if isinstance(arg, ActionObject): + uid = arg.id + elif isinstance(arg, TwinObject): + uid = arg.id + elif isinstance(arg, Asset): + uid = arg.action_id + elif isinstance(arg, RemoteFunction): + # TODO: Beach Fix + # why do we need another call to the server to get the UID? + uid = arg.custom_function_actionobject_id() + else: + uid = arg + return uid + + +def retrieve_item_from_db(id: UID, context: AuthedServiceContext) -> ActionObject: + # relative + from ...service.action.action_object import TwinMode + + action_service = context.node.get_service("actionservice") + root_context = AuthedServiceContext( + node=context.node, credentials=context.node.verify_key + ) + value = action_service._get( + context=root_context, + uid=id, + twin_mode=TwinMode.NONE, + has_permission=True, + ) + if value.is_err(): + return value + else: + return value.ok() + + class InputPolicy(Policy): __canonical_name__ = "InputPolicy" __version__ = SYFT_OBJECT_VERSION_2 @@ -229,6 +404,168 @@ def _inputs_for_context(self, context: ChangeContext) -> dict | SyftError: return inputs +@serializable() +class MixedInputPolicy(InputPolicy): + # version + __canonical_name__ = "MixedInputPolicy" + __version__ = SYFT_OBJECT_VERSION_1 + + kwarg_rules: dict[NodeIdentity, dict[str, PolicyRule]] + + def __init__( + self, init_kwargs: Any = None, client: Any = None, *args: Any, **kwargs: Any + ) -> None: + if init_kwargs is not None: + kwarg_rules = init_kwargs + kwargs = {} + else: + node_identity = self.find_node_identity(kwargs, client) + kwarg_rules_current_node = {} + for kw, arg in kwargs.items(): + if isinstance( + arg, UID | Asset | ActionObject | TwinObject | RemoteFunction + ): + kwarg_rules_current_node[kw] = Matches( + kw=kw, val=user_code_arg2id(arg) + ) + elif arg in [str, float, int, bool, dict, list, set, tuple]: + kwarg_rules_current_node[kw] = UserOwned(kw=kw, type=arg) + elif isinstance(arg, CreatePolicyRule): + kwarg_rules_current_node[kw] = arg.to_policy_rule(kw) + else: + raise ValueError("Incorrect argument") + kwarg_rules = {node_identity: kwarg_rules_current_node} + + super().__init__( + *args, kwarg_rules=kwarg_rules, init_kwargs=kwarg_rules, **kwargs + ) + + def transform_kwargs( + self, context: AuthedServiceContext, kwargs: dict[str, Any] + ) -> dict[str, Any]: + for _, rules in self.kwarg_rules.items(): + for kw, rule in rules.items(): + if hasattr(rule, "transform_kwarg"): + res_val = rule.transform_kwarg(context, kwargs.get(kw, None)) + if res_val.is_err(): + return res_val + else: + kwargs[kw] = res_val.ok() + return Ok(kwargs) + + def find_node_identity( + self, kwargs: dict[str, Any], client: Any = None + ) -> NodeIdentity: + if client is not None: + return NodeIdentity.from_api(client.api) + + apis = APIRegistry.get_all_api() + matches = set() + has_ids = False + for val in kwargs.values(): + # we mostly get the UID here because we don't want to store all those + # other objects, so we need to create a global UID obj lookup service + if isinstance( + val, UID | Asset | ActionObject | TwinObject | RemoteFunction + ): + has_ids = True + id = user_code_arg2id(val) + for api in apis: + # TODO: Beach Fix + # here be dragons, we need to refactor this since the existance + # depends on the type and service + # also the whole NodeIdentity needs to be removed + check_endpoints = [ + api.services.action.exists, + api.services.api.exists, + ] + for check_endpoint in check_endpoints: + result = check_endpoint(id) + if result: + break # stop looking + if result: + node_identity = NodeIdentity.from_api(api) + matches.add(node_identity) + + if len(matches) == 0: + if not has_ids: + if len(apis) == 1: + return NodeIdentity.from_api(api) + else: + raise ValueError( + "Multiple Node Identities, please only login to one client (for this policy) and try again" + ) + else: + raise ValueError("No Node Identities") + if len(matches) > 1: + # TODO: Beach Fix + raise ValueError("Multiple Node Identities") + # we need to fix this as its possible we could + # grab the wrong API and call a different user context in jupyter testing + pass # just grab the first one + return matches.pop() + + def filter_kwargs( + self, + kwargs: dict[str, UID], + context: AuthedServiceContext, + code_item_id: UID, + ) -> Result[dict[Any, Any], str]: + try: + res = {} + for _, rules in self.kwarg_rules.items(): + for kw, rule in rules.items(): + if rule.requires_input: + passed_id = kwargs[kw] + actionobject: ActionObject = retrieve_item_from_db( + passed_id, context + ) + rule_check_args = (actionobject,) + else: + rule_check_args = () # type: ignore + # TODO + actionobject = rule.value + if not rule.is_met(context, *rule_check_args): + raise ValueError(f"{rule} is not met") + else: + res[kw] = actionobject + except Exception as e: + return Err(str(e)) + return Ok(res) + + def _is_valid( + self, + context: AuthedServiceContext, + usr_input_kwargs: dict, + code_item_id: UID, + ) -> Result[bool, str]: + filtered_input_kwargs = self.filter_kwargs( + kwargs=usr_input_kwargs, + context=context, + code_item_id=code_item_id, + ) + + if filtered_input_kwargs.is_err(): + return filtered_input_kwargs + + filtered_input_kwargs = filtered_input_kwargs.ok() + + expected_input_kwargs = set() + for _inp_kwargs in self.inputs.values(): + for k in _inp_kwargs.keys(): + if k not in usr_input_kwargs: + return Err(f"Function missing required keyword argument: '{k}'") + expected_input_kwargs.update(_inp_kwargs.keys()) + + permitted_input_kwargs = list(filtered_input_kwargs.keys()) + not_approved_kwargs = set(expected_input_kwargs) - set(permitted_input_kwargs) + if len(not_approved_kwargs) > 0: + return Err( + f"Input arguments: {not_approved_kwargs} to the function are not approved yet." + ) + return Ok(True) + + def retrieve_from_db( code_item_id: UID, allowed_inputs: dict[str, UID], context: AuthedServiceContext ) -> Result[dict[str, Any], str]: @@ -257,14 +594,6 @@ def retrieve_from_db( if kwarg_value.is_err(): return Err(kwarg_value.err()) code_inputs[var_name] = kwarg_value.ok() - - elif context.node.node_type == NodeType.ENCLAVE: - dict_object = action_service.get(context=root_context, uid=code_item_id) - if dict_object.is_err(): - return Err(dict_object.err()) - for value in dict_object.ok().syft_action_data.values(): - code_inputs.update(value) - else: raise Exception( f"Invalid Node Type for Code Submission:{context.node.node_type}" @@ -276,7 +605,7 @@ def allowed_ids_only( allowed_inputs: dict[NodeIdentity, Any], kwargs: dict[str, Any], context: AuthedServiceContext, -) -> dict[str, UID]: +) -> dict[NodeIdentity, UID]: if context.node.node_type == NodeType.DOMAIN: node_identity = NodeIdentity( node_name=context.node.name, @@ -284,11 +613,6 @@ def allowed_ids_only( verify_key=context.node.signing_key.verify_key, ) allowed_inputs = allowed_inputs.get(node_identity, {}) - elif context.node.node_type == NodeType.ENCLAVE: - base_dict = {} - for key in allowed_inputs.values(): - base_dict.update(key) - allowed_inputs = base_dict else: raise Exception( f"Invalid Node Type for Code Submission:{context.node.node_type}" @@ -363,7 +687,7 @@ def _is_valid( not_approved_kwargs = set(expected_input_kwargs) - set(permitted_input_kwargs) if len(not_approved_kwargs) > 0: return Err( - f"Input arguments: {not_approved_kwargs} to the function are not approved yet." + f"Function arguments: {not_approved_kwargs} are not approved yet." ) return Ok(True) @@ -524,6 +848,7 @@ class CustomInputPolicy(metaclass=CustomPolicy): class UserPolicy(Policy): __canonical_name__: str = "UserPolicy" __version__ = SYFT_OBJECT_VERSION_2 + has_safe_serde: ClassVar[bool] = False id: UID node_uid: UID | None = None @@ -712,14 +1037,16 @@ def process_class_code(raw_code: str, class_name: str) -> str: "Tuple", "Type", ] - for typing_type in typing_types: - new_body.append( - ast.ImportFrom( - module="typing", - names=[ast.alias(name=typing_type, asname=typing_type)], - level=0, - ) + new_body.append( + ast.ImportFrom( + module="typing", + names=[ + ast.alias(name=typing_type, asname=typing_type) + for typing_type in typing_types + ], + level=0, ) + ) new_body.append(new_class) module = ast.Module(new_body, type_ignores=[]) try: diff --git a/packages/syft/src/syft/service/project/project.py b/packages/syft/src/syft/service/project/project.py index 981f7ff9192..e768154ed00 100644 --- a/packages/syft/src/syft/service/project/project.py +++ b/packages/syft/src/syft/service/project/project.py @@ -940,11 +940,11 @@ def create_code_request( ) def get_messages(self) -> list[ProjectMessage | ProjectThreadMessage]: - messages = [] - for event in self.events: - if isinstance(event, ProjectMessage | ProjectThreadMessage): - messages.append(event) - return messages + return [ + event + for event in self.events + if isinstance(event, (ProjectMessage | ProjectThreadMessage)) + ] @property def messages(self) -> str: diff --git a/packages/syft/src/syft/service/queue/queue.py b/packages/syft/src/syft/service/queue/queue.py index 968e4b7c975..c85b94468f3 100644 --- a/packages/syft/src/syft/service/queue/queue.py +++ b/packages/syft/src/syft/service/queue/queue.py @@ -1,13 +1,12 @@ # stdlib +import logging from multiprocessing import Process import threading from threading import Thread import time from typing import Any -from typing import cast # third party -from loguru import logger import psutil from result import Err from result import Ok @@ -34,6 +33,8 @@ from .queue_stash import QueueItem from .queue_stash import Status +logger = logging.getLogger(__name__) + class MonitorThread(threading.Thread): def __init__( @@ -168,13 +169,12 @@ def handle_message_multiprocessing( document_store_config=worker_settings.document_store_config, action_store_config=worker_settings.action_store_config, blob_storage_config=worker_settings.blob_store_config, + node_side_type=worker_settings.node_side_type, queue_config=queue_config, is_subprocess=True, migrate=False, ) - job_item = worker.job_stash.get_by_uid(credentials, queue_item.job_id).ok() - # Set monitor thread for this job. monitor_thread = MonitorThread(queue_item, worker, credentials) monitor_thread.start() @@ -184,7 +184,6 @@ def handle_message_multiprocessing( try: call_method = getattr(worker.get_service(queue_item.service), queue_item.method) - role = worker.get_role_for_credentials(credentials=credentials) context = AuthedServiceContext( @@ -205,7 +204,6 @@ def handle_message_multiprocessing( ) result: Any = call_method(context, *queue_item.args, **queue_item.kwargs) - status = Status.COMPLETED job_status = JobStatus.COMPLETED @@ -222,25 +220,19 @@ def handle_message_multiprocessing( else: raise Exception(f"Unknown result type: {type(result)}") - except Exception as e: # nosec + except Exception as e: status = Status.ERRORED job_status = JobStatus.ERRORED - # stdlib - - raise e - # result = SyftError( - # message=f"Failed with exception: {e}, {traceback.format_exc()}" - # ) - # print("HAD AN ERROR WHILE HANDLING MESSAGE", result.message) + logger.error("Unhandled error in handle_message_multiprocessing", exc_info=e) queue_item.result = result queue_item.resolved = True queue_item.status = status # get new job item to get latest iter status - job_item = worker.job_stash.get_by_uid(credentials, job_item.id).ok() - - # if result.is_ok(): + job_item = worker.job_stash.get_by_uid(credentials, queue_item.job_id).ok() + if job_item is None: + raise Exception(f"Job {queue_item.job_id} not found!") job_item.node_uid = worker.id job_item.result = result @@ -277,6 +269,7 @@ def handle_message(message: bytes, syft_worker_id: UID) -> None: document_store_config=worker_settings.document_store_config, action_store_config=worker_settings.action_store_config, blob_storage_config=worker_settings.blob_store_config, + node_side_type=worker_settings.node_side_type, queue_config=queue_config, is_subprocess=True, migrate=False, @@ -297,7 +290,7 @@ def handle_message(message: bytes, syft_worker_id: UID) -> None: queue_item.node_uid = worker.id job_item.status = JobStatus.PROCESSING - job_item.node_uid = cast(UID, worker.id) + job_item.node_uid = worker.id # type: ignore[assignment] job_item.updated_at = DateTime.now() if syft_worker_id is not None: @@ -311,6 +304,12 @@ def handle_message(message: bytes, syft_worker_id: UID) -> None: if isinstance(job_result, SyftError): raise Exception(f"{job_result.err()}") + logger.info( + f"Handling queue item: id={queue_item.id}, method={queue_item.method} " + f"args={queue_item.args}, kwargs={queue_item.kwargs} " + f"service={queue_item.service}, as_thread={queue_config.thread_workers}" + ) + if queue_config.thread_workers: thread = Thread( target=handle_message_multiprocessing, @@ -321,7 +320,6 @@ def handle_message(message: bytes, syft_worker_id: UID) -> None: else: # if psutil.pid_exists(job_item.job_pid): # psutil.Process(job_item.job_pid).terminate() - process = Process( target=handle_message_multiprocessing, args=(worker_settings, queue_item, credentials), diff --git a/packages/syft/src/syft/service/queue/zmq_queue.py b/packages/syft/src/syft/service/queue/zmq_queue.py index 3ad4b732f89..08ff386696e 100644 --- a/packages/syft/src/syft/service/queue/zmq_queue.py +++ b/packages/syft/src/syft/service/queue/zmq_queue.py @@ -2,14 +2,15 @@ from binascii import hexlify from collections import defaultdict import itertools +import logging import socketserver +import sys import threading import time from time import sleep from typing import Any # third party -from loguru import logger from pydantic import field_validator import zmq from zmq import Frame @@ -60,6 +61,8 @@ # Lock for working on ZMQ socket ZMQ_SOCKET_LOCK = threading.Lock() +logger = logging.getLogger(__name__) + class QueueMsgProtocol: W_WORKER = b"MDPW01" @@ -127,6 +130,13 @@ def get_expiry(self) -> float: def reset_expiry(self) -> None: self.expiry_t.reset() + def __str__(self) -> str: + svc = self.service.name if self.service else None + return ( + f"Worker(addr={self.address!r}, id={self.identity!r}, service={svc}, " + f"syft_worker_id={self.syft_worker_id!r})" + ) + @serializable() class ZMQProducer(QueueProducer): @@ -176,7 +186,7 @@ def close(self) -> None: try: self.poll_workers.unregister(self.socket) except Exception as e: - logger.exception("Failed to unregister poller. {}", e) + logger.exception("Failed to unregister poller.", exc_info=e) finally: if self.thread: self.thread.join(THREAD_TIMEOUT_SEC) @@ -231,7 +241,7 @@ def contains_unresolved_action_objects(self, arg: Any, recursion: int = 0) -> bo return True return value except Exception as e: - logger.exception("Failed to resolve action objects. {}", e) + logger.exception("Failed to resolve action objects.", exc_info=e) return True def unwrap_nested_actionobjects(self, data: Any) -> Any: @@ -251,91 +261,138 @@ def unwrap_nested_actionobjects(self, data: Any) -> Any: else: nested_res = res.syft_action_data if isinstance(nested_res, ActionObject): - nested_res.syft_node_location = res.syft_node_location - nested_res.syft_client_verify_key = res.syft_client_verify_key + raise ValueError( + "More than double nesting of ActionObjects is currently not supported" + ) return nested_res return data - def preprocess_action_arg(self, arg: Any) -> None: + def contains_nested_actionobjects(self, data: Any) -> bool: + """ + returns if this is a list/set/dict that contains ActionObjects + """ + + def unwrap_collection(col: set | dict | list) -> [Any]: # type: ignore + return_values = [] + if isinstance(col, dict): + values = list(col.values()) + list(col.keys()) + else: + values = list(col) + for v in values: + if isinstance(v, list | dict | set): + return_values += unwrap_collection(v) + else: + return_values.append(v) + return return_values + + if isinstance(data, list | dict | set): + values = unwrap_collection(data) + has_action_object = any(isinstance(x, ActionObject) for x in values) + return has_action_object + elif isinstance(data, ActionObject): + return True + return False + + def preprocess_action_arg(self, arg: UID) -> UID | None: + """ "If the argument is a collection (of collections) of ActionObjects, + We want to flatten the collection and upload a new ActionObject that contains + its values. E.g. [[ActionObject1, ActionObject2],[ActionObject3, ActionObject4]] + -> [[value1, value2],[value3, value4]] + """ res = self.action_service.get(context=self.auth_context, uid=arg) if res.is_err(): return arg action_object = res.ok() data = action_object.syft_action_data - new_data = self.unwrap_nested_actionobjects(data) - new_action_object = ActionObject.from_obj(new_data, id=action_object.id) - res = self.action_service.set( - context=self.auth_context, action_object=new_action_object - ) + if self.contains_nested_actionobjects(data): + new_data = self.unwrap_nested_actionobjects(data) + + new_action_object = ActionObject.from_obj( + new_data, + id=action_object.id, + syft_blob_storage_entry_id=action_object.syft_blob_storage_entry_id, + ) + res = self.action_service._set( + context=self.auth_context, action_object=new_action_object + ) + return None def read_items(self) -> None: while True: if self._stop.is_set(): break - sleep(1) - - # Items to be queued - items_to_queue = self.queue_stash.get_by_status( - self.queue_stash.partition.root_verify_key, - status=Status.CREATED, - ).ok() - - items_to_queue = [] if items_to_queue is None else items_to_queue - - # Queue Items that are in the processing state - items_processing = self.queue_stash.get_by_status( - self.queue_stash.partition.root_verify_key, - status=Status.PROCESSING, - ).ok() - - items_processing = [] if items_processing is None else items_processing - - for item in itertools.chain(items_to_queue, items_processing): - if item.status == Status.CREATED: - if isinstance(item, ActionQueueItem): - action = item.kwargs["action"] - if self.contains_unresolved_action_objects( - action.args - ) or self.contains_unresolved_action_objects(action.kwargs): + try: + sleep(1) + + # Items to be queued + items_to_queue = self.queue_stash.get_by_status( + self.queue_stash.partition.root_verify_key, + status=Status.CREATED, + ).ok() + + items_to_queue = [] if items_to_queue is None else items_to_queue + + # Queue Items that are in the processing state + items_processing = self.queue_stash.get_by_status( + self.queue_stash.partition.root_verify_key, + status=Status.PROCESSING, + ).ok() + + items_processing = [] if items_processing is None else items_processing + + for item in itertools.chain(items_to_queue, items_processing): + # TODO: if resolving fails, set queueitem to errored, and jobitem as well + if item.status == Status.CREATED: + if isinstance(item, ActionQueueItem): + action = item.kwargs["action"] + if self.contains_unresolved_action_objects( + action.args + ) or self.contains_unresolved_action_objects(action.kwargs): + continue + for arg in action.args: + self.preprocess_action_arg(arg) + for _, arg in action.kwargs.items(): + self.preprocess_action_arg(arg) + + msg_bytes = serialize(item, to_bytes=True) + worker_pool = item.worker_pool.resolve_with_context( + self.auth_context + ) + worker_pool = worker_pool.ok() + service_name = worker_pool.name + service: Service | None = self.services.get(service_name) + + # Skip adding message if corresponding service/pool + # is not registered. + if service is None: continue - for arg in action.args: - self.preprocess_action_arg(arg) - for _, arg in action.kwargs.items(): - self.preprocess_action_arg(arg) - - msg_bytes = serialize(item, to_bytes=True) - worker_pool = item.worker_pool.resolve_with_context( - self.auth_context - ) - worker_pool = worker_pool.ok() - service_name = worker_pool.name - service: Service | None = self.services.get(service_name) - # Skip adding message if corresponding service/pool - # is not registered. - if service is None: - continue + # append request message to the corresponding service + # This list is processed in dispatch method. - # append request message to the corresponding service - # This list is processed in dispatch method. - - # TODO: Logic to evaluate the CAN RUN Condition - service.requests.append(msg_bytes) - item.status = Status.PROCESSING - res = self.queue_stash.update(item.syft_client_verify_key, item) - if res.is_err(): - logger.error( - "Failed to update queue item={} error={}", - item, - res.err(), - ) - elif item.status == Status.PROCESSING: - # Evaluate Retry condition here - # If job running and timeout or job status is KILL - # or heartbeat fails - # or container id doesn't exists, kill process or container - # else decrease retry count and mark status as CREATED. - pass + # TODO: Logic to evaluate the CAN RUN Condition + service.requests.append(msg_bytes) + item.status = Status.PROCESSING + res = self.queue_stash.update(item.syft_client_verify_key, item) + if res.is_err(): + logger.error( + f"Failed to update queue item={item} error={res.err()}" + ) + elif item.status == Status.PROCESSING: + # Evaluate Retry condition here + # If job running and timeout or job status is KILL + # or heartbeat fails + # or container id doesn't exists, kill process or container + # else decrease retry count and mark status as CREATED. + pass + except Exception as e: + print(e, file=sys.stderr) + item.status = Status.ERRORED + res = self.queue_stash.update(item.syft_client_verify_key, item) + if res.is_err(): + logger.error( + f"Failed to update queue item={item} error={res.err()}" + ) def run(self) -> None: self.thread = threading.Thread(target=self._run) @@ -346,18 +403,18 @@ def run(self) -> None: def send(self, worker: bytes, message: bytes | list[bytes]) -> None: worker_obj = self.require_worker(worker) - self.send_to_worker(worker=worker_obj, msg=message) + self.send_to_worker(worker_obj, QueueMsgProtocol.W_REQUEST, message) def bind(self, endpoint: str) -> None: """Bind producer to endpoint.""" self.socket.bind(endpoint) - logger.info("Producer endpoint: {}", endpoint) + logger.info(f"ZMQProducer endpoint: {endpoint}") def send_heartbeats(self) -> None: """Send heartbeats to idle workers if it's time""" if self.heartbeat_t.has_expired(): for worker in self.waiting: - self.send_to_worker(worker, QueueMsgProtocol.W_HEARTBEAT, None, None) + self.send_to_worker(worker, QueueMsgProtocol.W_HEARTBEAT) self.heartbeat_t.reset() def purge_workers(self) -> None: @@ -368,22 +425,15 @@ def purge_workers(self) -> None: # work on a copy of the iterator for worker in list(self.waiting): if worker.has_expired(): - logger.info( - "Deleting expired Worker id={} uid={} expiry={} now={}", - worker.identity, - worker.syft_worker_id, - worker.get_expiry(), - Timeout.now(), - ) + logger.info(f"Deleting expired worker id={worker}") self.delete_worker(worker, False) def update_consumer_state_for_worker( self, syft_worker_id: UID, consumer_state: ConsumerState ) -> None: if self.worker_stash is None: - # TODO: fix the mypy issue logger.error( # type: ignore[unreachable] - f"Worker stash is not defined for ZMQProducer : {self.queue_name} - {self.id}" + f"ZMQProducer worker stash not defined for {self.queue_name} - {self.id}" ) return @@ -403,14 +453,13 @@ def update_consumer_state_for_worker( ) if res.is_err(): logger.error( - "Failed to update consumer state for worker id={} to state: {} error={}", - syft_worker_id, - consumer_state, - res.err(), + f"Failed to update consumer state for worker id={syft_worker_id} " + f"to state: {consumer_state} error={res.err()}", ) except Exception as e: logger.error( - f"Failed to update consumer state for worker id: {syft_worker_id} to state {consumer_state}. Error: {e}" + f"Failed to update consumer state for worker id: {syft_worker_id} to state {consumer_state}", + exc_info=e, ) def worker_waiting(self, worker: Worker) -> None: @@ -435,13 +484,12 @@ def dispatch(self, service: Service, msg: bytes) -> None: msg = service.requests.pop(0) worker = service.waiting.pop(0) self.waiting.remove(worker) - self.send_to_worker(worker, QueueMsgProtocol.W_REQUEST, None, msg) + self.send_to_worker(worker, QueueMsgProtocol.W_REQUEST, msg) def send_to_worker( self, worker: Worker, - command: bytes = QueueMsgProtocol.W_REQUEST, - option: bytes | None = None, + command: bytes, msg: bytes | list | None = None, ) -> None: """Send message to worker. @@ -458,50 +506,60 @@ def send_to_worker( elif not isinstance(msg, list): msg = [msg] - # Stack routing and protocol envelopes to start of message - # and routing envelope - if option is not None: - msg = [option] + msg - msg = [worker.address, b"", QueueMsgProtocol.W_WORKER, command] + msg + # ZMQProducer send frames: [address, empty, header, command, ...data] + core = [worker.address, b"", QueueMsgProtocol.W_WORKER, command] + msg = core + msg + + if command != QueueMsgProtocol.W_HEARTBEAT: + # log everything except the last frame which contains serialized data + logger.info(f"ZMQProducer send: {core}") - logger.debug("Send: {}", msg) with ZMQ_SOCKET_LOCK: try: self.socket.send_multipart(msg) except zmq.ZMQError as e: - logger.error("Failed to send message to producer. {}", e) + logger.error("ZMQProducer send error", exc_info=e) def _run(self) -> None: - while True: - if self._stop.is_set(): - return + try: + while True: + if self._stop.is_set(): + logger.info("ZMQProducer thread stopped") + return - for _, service in self.services.items(): - self.dispatch(service, None) + for service in self.services.values(): + self.dispatch(service, None) - items = None + items = None - try: - items = self.poll_workers.poll(ZMQ_POLLER_TIMEOUT_MSEC) - except Exception as e: - logger.exception("Failed to poll items: {}", e) + try: + items = self.poll_workers.poll(ZMQ_POLLER_TIMEOUT_MSEC) + except Exception as e: + logger.exception("ZMQProducer poll error", exc_info=e) + + if items: + msg = self.socket.recv_multipart() - if items: - msg = self.socket.recv_multipart() + if len(msg) < 3: + logger.error(f"ZMQProducer invalid recv: {msg}") + continue - logger.debug("Recieve: {}", msg) + # ZMQProducer recv frames: [address, empty, header, command, ...data] + (address, _, header, command, *data) = msg - address = msg.pop(0) - empty = msg.pop(0) # noqa: F841 - header = msg.pop(0) + if command != QueueMsgProtocol.W_HEARTBEAT: + # log everything except the last frame which contains serialized data + logger.info(f"ZMQProducer recv: {msg[:4]}") - if header == QueueMsgProtocol.W_WORKER: - self.process_worker(address, msg) - else: - logger.error("Invalid message header: {}", header) + if header == QueueMsgProtocol.W_WORKER: + self.process_worker(address, command, data) + else: + logger.error(f"Invalid message header: {header}") - self.send_heartbeats() - self.purge_workers() + self.send_heartbeats() + self.purge_workers() + except Exception as e: + logger.exception("ZMQProducer thread exception", exc_info=e) def require_worker(self, address: bytes) -> Worker: """Finds the worker (creates if necessary).""" @@ -512,16 +570,13 @@ def require_worker(self, address: bytes) -> Worker: self.workers[identity] = worker return worker - def process_worker(self, address: bytes, msg: list[bytes]) -> None: - command = msg.pop(0) - + def process_worker(self, address: bytes, command: bytes, data: list[bytes]) -> None: worker_ready = hexlify(address) in self.workers - worker = self.require_worker(address) if QueueMsgProtocol.W_READY == command: - service_name = msg.pop(0).decode() - syft_worker_id = msg.pop(0).decode() + service_name = data.pop(0).decode() + syft_worker_id = data.pop(0).decode() if worker_ready: # Not first command in session or Reserved service name # If worker was already present, then we disconnect it first @@ -537,18 +592,7 @@ def process_worker(self, address: bytes, msg: list[bytes]) -> None: self.services[service_name] = service if service is not None: worker.service = service - logger.info( - "New Worker service={}, id={}, uid={}", - service.name, - worker.identity, - worker.syft_worker_id, - ) - else: - logger.info( - "New Worker service=None, id={}, uid={}", - worker.identity, - worker.syft_worker_id, - ) + logger.info(f"New worker: {worker}") worker.syft_worker_id = UID(syft_worker_id) self.worker_waiting(worker) @@ -559,19 +603,18 @@ def process_worker(self, address: bytes, msg: list[bytes]) -> None: # if not already present self.worker_waiting(worker) else: - # extract the syft worker id and worker pool name from the message - # Get the corresponding worker pool and worker - # update the status to be unhealthy + logger.info(f"Got heartbeat, but worker not ready. {worker}") self.delete_worker(worker, True) elif QueueMsgProtocol.W_DISCONNECT == command: + logger.info(f"Removing disconnected worker: {worker}") self.delete_worker(worker, False) else: - logger.error("Invalid command: {}", command) + logger.error(f"Invalid command: {command!r}") def delete_worker(self, worker: Worker, disconnect: bool) -> None: """Deletes worker from all data structures, and deletes worker.""" if disconnect: - self.send_to_worker(worker, QueueMsgProtocol.W_DISCONNECT, None, None) + self.send_to_worker(worker, QueueMsgProtocol.W_DISCONNECT) if worker.service and worker in worker.service.waiting: worker.service.waiting.remove(worker) @@ -628,13 +671,12 @@ def reconnect_to_producer(self) -> None: self.socket.connect(self.address) self.poller.register(self.socket, zmq.POLLIN) - logger.info("Connecting Worker id={} to broker addr={}", self.id, self.address) + logger.info(f"Connecting Worker id={self.id} to broker addr={self.address}") # Register queue with the producer self.send_to_producer( QueueMsgProtocol.W_READY, - self.service_name.encode(), - [str(self.syft_worker_id).encode()], + [self.service_name.encode(), str(self.syft_worker_id).encode()], ) def post_init(self) -> None: @@ -652,7 +694,7 @@ def close(self) -> None: try: self.poller.unregister(self.socket) except Exception as e: - logger.exception("Failed to unregister worker. {}", e) + logger.exception("Failed to unregister worker.", exc_info=e) finally: if self.thread is not None: self.thread.join(timeout=THREAD_TIMEOUT_SEC) @@ -663,8 +705,7 @@ def close(self) -> None: def send_to_producer( self, - command: str, - option: bytes | None = None, + command: bytes, msg: bytes | list | None = None, ) -> None: """Send message to producer. @@ -680,23 +721,25 @@ def send_to_producer( elif not isinstance(msg, list): msg = [msg] - if option: - msg = [option] + msg + # ZMQConsumer send frames: [empty, header, command, ...data] + core = [b"", QueueMsgProtocol.W_WORKER, command] + msg = core + msg - msg = [b"", QueueMsgProtocol.W_WORKER, command] + msg - logger.debug("Send: msg={}", msg) + if command != QueueMsgProtocol.W_HEARTBEAT: + logger.info(f"ZMQ Consumer send: {core}") with ZMQ_SOCKET_LOCK: try: self.socket.send_multipart(msg) except zmq.ZMQError as e: - logger.error("Failed to send message to producer. {}", e) + logger.error("ZMQConsumer send error", exc_info=e) def _run(self) -> None: """Send reply, if any, to producer and wait for next request.""" try: while True: if self._stop.is_set(): + logger.info("ZMQConsumer thread stopped") return try: @@ -705,39 +748,38 @@ def _run(self) -> None: logger.info("Context terminated") return except Exception as e: - logger.error("Poll error={}", e) + logger.error("ZMQ poll error", exc_info=e) continue if items: - # Message format: - # [b"", "
    ", "", "", ""] msg = self.socket.recv_multipart() - logger.debug("Recieve: {}", msg) - # mark as alive self.set_producer_alive() if len(msg) < 3: - logger.error("Invalid message: {}", msg) + logger.error(f"ZMQConsumer invalid recv: {msg}") continue - empty = msg.pop(0) # noqa: F841 - header = msg.pop(0) # noqa: F841 + # Message frames recieved by consumer: + # [empty, header, command, ...data] + (_, _, command, *data) = msg - command = msg.pop(0) + if command != QueueMsgProtocol.W_HEARTBEAT: + # log everything except the last frame which contains serialized data + logger.info(f"ZMQConsumer recv: {msg[:-4]}") if command == QueueMsgProtocol.W_REQUEST: # Call Message Handler try: - message = msg.pop() + message = data.pop() self.associate_job(message) self.message_handler.handle_message( message=message, syft_worker_id=self.syft_worker_id, ) except Exception as e: - logger.exception("Error while handling message. {}", e) + logger.exception("Couldn't handle message", exc_info=e) finally: self.clear_job() elif command == QueueMsgProtocol.W_HEARTBEAT: @@ -745,7 +787,7 @@ def _run(self) -> None: elif command == QueueMsgProtocol.W_DISCONNECT: self.reconnect_to_producer() else: - logger.error("Invalid command: {}", command) + logger.error(f"ZMQConsumer invalid command: {command}") else: if not self.is_producer_alive(): logger.info("Producer check-alive timed out. Reconnecting.") @@ -756,12 +798,11 @@ def _run(self) -> None: except zmq.ZMQError as e: if e.errno == zmq.ETERM: - logger.info("Consumer connection terminated") + logger.info("zmq.ETERM") else: - logger.exception("Consumer error. {}", e) - raise e - - logger.info("Worker finished") + logger.exception("zmq.ZMQError", exc_info=e) + except Exception as e: + logger.exception("ZMQConsumer thread exception", exc_info=e) def set_producer_alive(self) -> None: self.producer_ping_t.reset() @@ -784,7 +825,7 @@ def associate_job(self, message: Frame) -> None: queue_item = _deserialize(message, from_bytes=True) self._set_worker_job(queue_item.job_id) except Exception as e: - logger.exception("Could not associate job. {}", e) + logger.exception("Could not associate job", exc_info=e) def clear_job(self) -> None: self._set_worker_job(None) @@ -928,12 +969,12 @@ def send_message( def close(self) -> SyftError | SyftSuccess: try: - for _, consumers in self.consumers.items(): + for consumers in self.consumers.values(): for consumer in consumers: # make sure look is stopped consumer.close() - for _, producer in self.producers.items(): + for producer in self.producers.values(): # make sure loop is stopped producer.close() # close existing connection. diff --git a/packages/syft/src/syft/service/request/request.py b/packages/syft/src/syft/service/request/request.py index 9f807ab5cb9..8c5687ac55e 100644 --- a/packages/syft/src/syft/service/request/request.py +++ b/packages/syft/src/syft/service/request/request.py @@ -3,6 +3,7 @@ from enum import Enum import hashlib import inspect +import logging from typing import Any # third party @@ -24,19 +25,23 @@ from ...serde.serialize import _serialize from ...store.linked_obj import LinkedObject from ...types.datetime import DateTime +from ...types.syft_migration import migrate from ...types.syft_object import SYFT_OBJECT_VERSION_2 from ...types.syft_object import SYFT_OBJECT_VERSION_3 from ...types.syft_object import SyftObject from ...types.syncable_object import SyncableSyftObject from ...types.transforms import TransformContext from ...types.transforms import add_node_uid_for_key +from ...types.transforms import drop from ...types.transforms import generate_id +from ...types.transforms import make_set_default from ...types.transforms import transform from ...types.twin_object import TwinObject from ...types.uid import LineageID from ...types.uid import UID from ...util import options from ...util.colors import SURFACE +from ...util.decorators import deprecated from ...util.markdown import markdown_as_class_with_fields from ...util.notebook_ui.icons import Icon from ...util.util import prompt_warning_message @@ -51,14 +56,14 @@ from ..context import AuthedServiceContext from ..context import ChangeContext from ..job.job_stash import Job -from ..job.job_stash import JobInfo from ..job.job_stash import JobStatus from ..notification.notifications import Notification -from ..policy.policy import UserPolicy from ..response import SyftError from ..response import SyftSuccess from ..user.user import UserView +logger = logging.getLogger(__name__) + @serializable() class RequestStatus(Enum): @@ -66,6 +71,15 @@ class RequestStatus(Enum): REJECTED = 1 APPROVED = 2 + @classmethod + def from_usercode_status(cls, status: UserCodeStatusCollection) -> "RequestStatus": + if status.approved: + return RequestStatus.APPROVED + elif status.denied: + return RequestStatus.REJECTED + else: + return RequestStatus.PENDING + @serializable() class Change(SyftObject): @@ -150,7 +164,7 @@ def _run( permission=self.apply_permission_type, ) if apply: - print( + logger.debug( "ADDING PERMISSION", requesting_permission_action_obj, id_action ) action_store.add_permission(requesting_permission_action_obj) @@ -174,7 +188,7 @@ def _run( ) return Ok(SyftSuccess(message=f"{type(self)} Success")) except Exception as e: - print(f"failed to apply {type(self)}", e) + logger.error(f"failed to apply {type(self)}", exc_info=e) return Err(SyftError(message=str(e))) def apply(self, context: ChangeContext) -> Result[SyftSuccess, SyftError]: @@ -379,7 +393,7 @@ class CreateCustomWorkerPoolChangeV2(Change): @serializable() -class Request(SyncableSyftObject): +class RequestV2(SyncableSyftObject): __canonical_name__ = "Request" __version__ = SYFT_OBJECT_VERSION_2 @@ -394,6 +408,48 @@ class Request(SyncableSyftObject): request_hash: str changes: list[Change] history: list[ChangeStatus] = [] + __table_coll_widths__ = [ + "min-content", + "auto", + "auto", + "auto", + "auto", + "auto", + ] + + __attr_searchable__ = [ + "requesting_user_verify_key", + "approving_user_verify_key", + ] + __attr_unique__ = ["request_hash"] + __repr_attrs__ = [ + "request_time", + "updated_at", + "status", + "changes", + "requesting_user_verify_key", + ] + __exclude_sync_diff_attrs__ = ["node_uid", "changes", "history"] + __table_sort_attr__ = "Request time" + + +@serializable() +class Request(SyncableSyftObject): + __canonical_name__ = "Request" + __version__ = SYFT_OBJECT_VERSION_3 + + requesting_user_verify_key: SyftVerifyKey + requesting_user_name: str = "" + requesting_user_email: str | None = "" + requesting_user_institution: str | None = "" + approving_user_verify_key: SyftVerifyKey | None = None + request_time: DateTime + updated_at: DateTime | None = None + node_uid: UID + request_hash: str + changes: list[Change] + history: list[ChangeStatus] = [] + tags: list[str] = [] __table_coll_widths__ = [ "min-content", @@ -416,7 +472,7 @@ class Request(SyncableSyftObject): "changes", "requesting_user_verify_key", ] - __exclude_sync_diff_attrs__ = ["node_uid"] + __exclude_sync_diff_attrs__ = ["node_uid", "changes", "history"] __table_sort_attr__ = "Request time" def _repr_html_(self) -> Any: @@ -444,7 +500,7 @@ def _repr_html_(self) -> Any: if self.code and len(self.code.output_readers) > 0: # owner_names = ["canada", "US"] owners_string = " and ".join( - [f"{x}" for x in self.code.output_reader_names] + [f"{x}" for x in self.code.output_reader_names] # type: ignore ) shared_with_line += ( f"

    Custom Policy: " @@ -539,8 +595,14 @@ def codes(self) -> Any: message="This type of request does not have code associated with it." ) + def get_user_code(self, context: AuthedServiceContext) -> UserCode | None: + for change in self.changes: + if isinstance(change, UserCodeStatusChange): + return change.get_user_code(context) + return None + @property - def code(self) -> Any: + def code(self) -> UserCode | SyftError: for change in self.changes: if isinstance(change, UserCodeStatusChange): return change.code @@ -548,9 +610,6 @@ def code(self) -> Any: message="This type of request does not have code associated with it." ) - def get_results(self) -> Any: - return self.code.get_results() - @property def current_change_state(self) -> dict[UID, bool]: change_applied_map = {} @@ -564,8 +623,14 @@ def current_change_state(self) -> dict[UID, bool]: def icon(self) -> str: return Icon.REQUEST.svg - @property - def status(self) -> RequestStatus: + def get_status(self, context: AuthedServiceContext | None = None) -> RequestStatus: + is_l0_deployment = ( + self.get_is_l0_deployment(context) if context else self.is_l0_deployment + ) + if is_l0_deployment: + code_status = self.code.get_status(context) if context else self.code.status + return RequestStatus.from_usercode_status(code_status) + if len(self.history) == 0: return RequestStatus.PENDING @@ -579,24 +644,30 @@ def status(self) -> RequestStatus: return request_status + @property + def status(self) -> RequestStatus: + return self.get_status() + def approve( self, disable_warnings: bool = False, approve_nested: bool = False, **kwargs: dict, ) -> Result[SyftSuccess, SyftError]: - api = APIRegistry.api_for( - self.node_uid, - self.syft_client_verify_key, - ) - if api is None: - return SyftError(message=f"api is None. You must login to {self.node_uid}") + api = self._get_api() + if isinstance(api, SyftError): + return api + + if self.is_l0_deployment: + return SyftError( + message="This request is a low-side request. Please sync your results to approve." + ) # TODO: Refactor so that object can also be passed to generate warnings if api.connection: metadata = api.connection.get_node_metadata(api.signing_key) else: metadata = None - message, is_enclave = None, False + message = None is_code_request = not isinstance(self.codes, SyftError) @@ -605,12 +676,7 @@ def approve( message="Multiple codes detected, please use approve_nested=True" ) - if self.code and not isinstance(self.code, SyftError): - is_enclave = getattr(self.code, "enclave_metadata", None) is not None - - if is_enclave: - message = "On approval, the result will be released to the enclave." - elif metadata and metadata.node_side_type == NodeSideType.HIGH_SIDE.value: + if metadata and metadata.node_side_type == NodeSideType.HIGH_SIDE.value: message = ( "You're approving a request on " f"{metadata.node_side_type} side {metadata.node_type} " @@ -634,15 +700,40 @@ def deny(self, reason: str) -> SyftSuccess | SyftError: Args: reason (str): Reason for which the request has been denied. """ - api = APIRegistry.api_for( - self.node_uid, - self.syft_client_verify_key, - ) - if api is None: - return SyftError(message=f"api is None. You must login to {self.node_uid}") + api = self._get_api() + if isinstance(api, SyftError): + return api + + if self.is_l0_deployment: + if self.status == RequestStatus.APPROVED: + prompt_warning_message( + "This request already has results published to the data scientist. " + "They will still be able to access those results." + ) + result = api.code.update(id=self.code_id, l0_deny_reason=reason) + if isinstance(result, SyftError): + return result + return SyftSuccess(message=f"Request denied with reason: {reason}") + return api.services.request.undo(uid=self.id, reason=reason) + @property + def is_l0_deployment(self) -> bool: + return bool(self.code) and self.code.is_l0_deployment + + def get_is_l0_deployment(self, context: AuthedServiceContext) -> bool: + code = self.get_user_code(context) + if code: + return code.is_l0_deployment + else: + return False + def approve_with_client(self, client: SyftClient) -> Result[SyftSuccess, SyftError]: + if self.is_l0_deployment: + return SyftError( + message="This request is a low-side request. Please sync your results to approve." + ) + print(f"Approving request for domain {client.name}") return client.api.services.request.apply(self.id) @@ -709,274 +800,144 @@ def save(self, context: AuthedServiceContext) -> Result[SyftSuccess, SyftError]: save_method = context.node.get_service_method(RequestService.save) return save_method(context=context, request=self) - def _get_latest_or_create_job(self) -> Job | SyftError: - """Get the latest job for this requests user_code, or creates one if no jobs exist""" - api = APIRegistry.api_for(self.node_uid, self.syft_client_verify_key) - if api is None: - return SyftError(message=f"api is None. You must login to {self.node_uid}") - job_service = api.services.job - - existing_jobs = job_service.get_by_user_code_id(self.code.id) - if isinstance(existing_jobs, SyftError): - return existing_jobs - - if len(existing_jobs) == 0: - print("Creating job for existing user code") - job = job_service.create_job_for_user_code_id(self.code.id) - else: - print("returning existing job") - print("setting permission") - job = existing_jobs[-1] - res = job_service.add_read_permission_job_for_code_owner(job, self.code) - print(res) - res = job_service.add_read_permission_log_for_code_owner( - job.log_id, self.code - ) - print(res) - - return job - - def accept_by_depositing_result( - self, result: Any, force: bool = False - ) -> SyftError | SyftSuccess: - api = APIRegistry.api_for(self.node_uid, self.syft_client_verify_key) - if not api: - raise Exception( - f"No access to Syft API. Please login to {self.node_uid} first." - ) - if api.signing_key is None: - raise ValueError(f"{api}'s signing key is None") - - # this code is extremely brittle because its a work around that relies on - # the type of request being very specifically tied to code which needs approving - - # Special case for results from Jobs (High-low side async) - if isinstance(result, JobInfo): - if result.user_code_id != self.code_id: - return SyftError( - message=f"JobInfo for user_code_id {result.user_code_id} does not match " - f"request's user_code_id {self.code_id}" - ) - job_info = result - if not job_info.includes_result: - return SyftError( - message="JobInfo should not include result. Use sync_job instead." - ) - result = job_info.result - elif isinstance(result, ActionObject): - # Do not allow accepting a result produced by a Job, - # This can cause an inconsistent Job state - job_service = api.services.job - action_object_job = job_service.get_by_result_id(result.id.id) - if action_object_job is not None: + def _create_action_object_for_deposited_result( + self, + result: Any, + ) -> ActionObject | SyftError: + api = self._get_api() + if isinstance(api, SyftError): + return api + + # Ensure result is an ActionObject + if isinstance(result, ActionObject): + existing_job = api.services.job.get_by_result_id(result.id.id) + if existing_job is not None: return SyftError( - message=f"This ActionObject is the result of existing Job {action_object_job.id}, " - f"please use the `Job.info` instead, or create a new ActionObject." - ) - else: - job_info = JobInfo( - user_code_id=self.code_id, - includes_metadata=True, - includes_result=True, - status=JobStatus.COMPLETED, - resolved=True, + message=f"This ActionObject is already the result of Job {existing_job.id}" ) + action_object = result else: - # NOTE result is added at the end of function (once ActionObject is created) - job_info = JobInfo( - user_code_id=self.code_id, - includes_metadata=True, - includes_result=True, - status=JobStatus.COMPLETED, - resolved=True, + action_object = ActionObject.from_obj( + result, + syft_client_verify_key=self.syft_client_verify_key, + syft_node_location=self.syft_node_location, ) - user_code_status_change: UserCodeStatusChange = self.changes[0] - code = user_code_status_change.code - output_history = code.output_history - if isinstance(output_history, SyftError): - return output_history - output_policy = code.output_policy - if isinstance(output_policy, SyftError): - return output_policy - if isinstance(user_code_status_change.code.output_policy_type, UserPolicy): - return SyftError( - message="UserCode uses an user-submitted custom policy. Please use .approve()" - ) + # Ensure ActionObject exists on this node + action_object_is_from_this_node = isinstance( + api.services.action.exists(action_object.id.id), SyftSuccess + ) + if ( + action_object.syft_blob_storage_entry_id is None + or not action_object_is_from_this_node + ): + action_object.reload_cache() + result = action_object._send(self.node_uid, self.syft_client_verify_key) + if isinstance(result, SyftError): + return result - if not user_code_status_change.change_object_is_type(UserCodeStatusCollection): - raise TypeError( - f"accept_by_depositing_result can only be run on {UserCodeStatusCollection} not " - f"{user_code_status_change.linked_obj.object_type}" - ) - if not type(user_code_status_change) == UserCodeStatusChange: - raise TypeError( - f"accept_by_depositing_result can only be run on {UserCodeStatusChange} not " - f"{type(user_code_status_change)}" - ) + return action_object - is_approved = user_code_status_change.approved + def _create_output_history_for_deposited_result( + self, job: Job, result: Any + ) -> SyftSuccess | SyftError: + code = self.code + if isinstance(code, SyftError): + return code + api = self._get_api() + if isinstance(api, SyftError): + return api + + input_ids = {} + input_policy = code.input_policy + if input_policy is not None: + for input_ in input_policy.inputs.values(): + input_ids.update(input_) + res = api.services.code.store_execution_output( + user_code_id=code.id, + outputs=result, + job_id=job.id, + input_ids=input_ids, + ) - permission_request = self.approve(approve_nested=True) - if isinstance(permission_request, SyftError): - return permission_request + return res - job = self._get_latest_or_create_job() - if isinstance(job, SyftError): - return job + def deposit_result( + self, + result: Any, + log_stdout: str = "", + log_stderr: str = "", + ) -> Job | SyftError: + """ + Adds a result to this Request: + - Create an ActionObject from the result (if not already an ActionObject) + - Ensure ActionObject exists on this node + - Create Job with new result and logs + - Update the output history - # This weird order is due to the fact that state is None before calling approve - # we could fix it in a future release - if is_approved: - if not force: - return SyftError( - message="Already approved, if you want to force updating the result use force=True" - ) - # TODO: this should overwrite the output history instead - action_obj_id = output_history[0].output_ids[0] # type: ignore - - if not isinstance(result, ActionObject): - action_object = ActionObject.from_obj( - result, - id=action_obj_id, - syft_client_verify_key=api.signing_key.verify_key, - syft_node_location=api.node_uid, - ) - else: - action_object = result - action_object_is_from_this_node = ( - self.syft_node_location == action_object.syft_node_location - ) - if ( - action_object.syft_blob_storage_entry_id is None - or not action_object_is_from_this_node - ): - action_object.reload_cache() - action_object.syft_node_location = self.syft_node_location - action_object.syft_client_verify_key = self.syft_client_verify_key - blob_store_result = action_object._save_to_blob_storage() - if isinstance(blob_store_result, SyftError): - return blob_store_result - result = api.services.action.set(action_object) - if isinstance(result, SyftError): - return result - else: - if not isinstance(result, ActionObject): - action_object = ActionObject.from_obj( - result, - syft_client_verify_key=api.signing_key.verify_key, - syft_node_location=api.node_uid, - ) - else: - action_object = result + Args: + result (Any): ActionObject or any object to be saved as an ActionObject. + logs (str | None, optional): Optional logs to be saved with the Job. Defaults to None. - # TODO: proper check for if actionobject is already uploaded - # we also need this for manualy syncing - action_object_is_from_this_node = ( - self.syft_node_location == action_object.syft_node_location - ) - if ( - action_object.syft_blob_storage_entry_id is None - or not action_object_is_from_this_node - ): - action_object.reload_cache() - action_object.syft_node_location = self.syft_node_location - action_object.syft_client_verify_key = self.syft_client_verify_key - blob_store_result = action_object._save_to_blob_storage() - if isinstance(blob_store_result, SyftError): - return blob_store_result - result = api.services.action.set(action_object) - if isinstance(result, SyftError): - return result - - # Do we still need this? - # policy_state_mutation = ObjectMutation( - # linked_obj=user_code_status_change.linked_obj, - # attr_name="output_policy", - # match_type=True, - # value=output_policy, - # ) - - action_object_link = LinkedObject.from_obj(result, node_uid=self.node_uid) - permission_change = ActionStoreChange( - linked_obj=action_object_link, - apply_permission_type=ActionPermission.READ, - ) + Returns: + Job | SyftError: Job object if successful, else SyftError. + """ - new_changes = [permission_change] - result_request = api.services.request.add_changes( - uid=self.id, changes=new_changes - ) - if isinstance(result_request, SyftError): - return result_request - self = result_request - - approved = self.approve(disable_warnings=True, approve_nested=True) - if isinstance(approved, SyftError): - return approved - - input_ids = {} - if code.input_policy is not None: - for inps in code.input_policy.inputs.values(): - input_ids.update(inps) - - res = api.services.code.store_as_history( - user_code_id=code.id, - outputs=result, - job_id=job.id, - input_ids=input_ids, - ) - if isinstance(res, SyftError): - return res + # TODO check if this is a low-side request. If not, SyftError - job_info.result = action_object - job_info.status = ( - JobStatus.ERRORED - if isinstance(action_object.syft_action_data, Err) - else JobStatus.COMPLETED - ) + api = self._get_api() + if isinstance(api, SyftError): + return api + code = self.code + if isinstance(code, SyftError): + return code + + if not self.is_l0_deployment: + return SyftError( + message="deposit_result is only available for low side code requests. " + "Please use request.approve() instead." + ) - existing_result = job.result.id if job.result is not None else None - print( - f"Job({job.id}) Setting new result {existing_result} -> {job_info.result.id}" + # Create ActionObject + action_object = self._create_action_object_for_deposited_result(result) + if isinstance(action_object, SyftError): + return action_object + + # Create Job + # NOTE code owner read permissions are added when syncing this Job + job = api.services.job.create_job_for_user_code_id( + code.id, + result=action_object, + log_stdout=log_stdout, + log_stderr=log_stderr, + status=JobStatus.COMPLETED, + add_code_owner_read_permissions=False, ) - job.apply_info(job_info) + if isinstance(job, SyftError): + return job - job_service = api.services.job - res = job_service.update(job) + # Add to output history + res = self._create_output_history_for_deposited_result(job, result) if isinstance(res, SyftError): return res - return SyftSuccess(message="Request submitted for updating result.") - - def sync_job( - self, job_info: JobInfo, **kwargs: Any - ) -> Result[SyftSuccess, SyftError]: - if job_info.includes_result: - return SyftError( - message="This JobInfo includes a Result. Please use Request.accept_by_depositing_result instead." - ) - - api = APIRegistry.api_for( - node_uid=self.node_uid, user_verify_key=self.syft_client_verify_key - ) - if api is None: - return SyftError(message=f"api is None. You must login to {self.node_uid}") - job_service = api.services.job + return job - job = self._get_latest_or_create_job() - job.apply_info(job_info) - return job_service.update(job) + @deprecated( + return_syfterror=True, + reason="accept_by_depositing_result has been removed. Use approve instead to " + "approve this request, or deposit_result to deposit a new result.", + ) + def accept_by_depositing_result(self, result: Any, force: bool = False) -> Any: + pass def get_sync_dependencies( self, context: AuthedServiceContext ) -> list[UID] | SyftError: dependencies = [] - code_id = self.code_id if isinstance(code_id, SyftError): return code_id - dependencies.append(code_id) return dependencies @@ -1243,13 +1204,19 @@ class UserCodeStatusChange(Change): @property def code(self) -> UserCode: + if self.linked_user_code._resolve_cache: + return self.linked_user_code._resolve_cache return self.linked_user_code.resolve + def get_user_code(self, context: AuthedServiceContext) -> UserCode: + resolve = self.linked_user_code.resolve_with_context(context) + return resolve.ok() + @property def codes(self) -> list[UserCode]: def recursive_code(node: Any) -> list: codes = [] - for _, (obj, new_node) in node.items(): + for obj, new_node in node.values(): codes.append(obj.resolve) codes.extend(recursive_code(new_node)) return codes @@ -1318,20 +1285,6 @@ def valid(self) -> SyftSuccess | SyftError: ) return SyftSuccess(message=f"{type(self)} valid") - # def get_nested_requests(self, context, code_tree: Dict[str: Tuple[LinkedObject, Dict]]): - # approved_nested_codes = {} - # for key, (linked_obj, new_code_tree) in code_tree.items(): - # code_obj = linked_obj.resolve_with_context(context).ok() - # approved_nested_codes[key] = code_obj.id - - # res = self.get_nested_requests(context, new_code_tree) - # if isinstance(res, SyftError): - # return res - # code_obj.nested_codes = res - # linked_obj.update_with_context(context, code_obj) - - # return approved_nested_codes - def mutate( self, status: UserCodeStatusCollection, @@ -1358,12 +1311,6 @@ def mutate( ) return res - def is_enclave_request(self, user_code: UserCode) -> bool: - return ( - user_code.is_enclave_code is not None - and self.value == UserCodeStatus.APPROVED - ) - def _run( self, context: ChangeContext, apply: bool ) -> Result[SyftSuccess, SyftError]: @@ -1387,26 +1334,16 @@ def _run( if isinstance(updated_status, SyftError): return Err(updated_status.message) - # relative - from ..enclave.enclave_service import propagate_inputs_to_enclave - self.linked_obj.update_with_context(context, updated_status) - if self.is_enclave_request(user_code): - enclave_res = propagate_inputs_to_enclave( - user_code=user_code, context=context - ) - if isinstance(enclave_res, SyftError): - return enclave_res else: updated_status = self.mutate(user_code_status, context, undo=True) if isinstance(updated_status, SyftError): return Err(updated_status.message) - # TODO: Handle Enclave approval. self.linked_obj.update_with_context(context, updated_status) return Ok(SyftSuccess(message=f"{type(self)} Success")) except Exception as e: - print(f"failed to apply {type(self)}. {e}") + logger.error(f"failed to apply {type(self)}", exc_info=e) return Err(SyftError(message=str(e))) def apply(self, context: ChangeContext) -> Result[SyftSuccess, SyftError]: @@ -1420,3 +1357,50 @@ def link(self) -> SyftObject | None: if self.linked_obj: return self.linked_obj.resolve return None + + +@serializable() +class SyncedUserCodeStatusChange(UserCodeStatusChange): + __canonical_name__ = "SyncedUserCodeStatusChange" + __version__ = SYFT_OBJECT_VERSION_3 + linked_obj: LinkedObject | None = None # type: ignore + + @property + def approved(self) -> bool: + return self.code.status.approved + + def mutate( + self, + status: UserCodeStatusCollection, + context: ChangeContext, + undo: bool, + ) -> UserCodeStatusCollection | SyftError: + return SyftError( + message="Synced UserCodes status is computed, and cannot be updated manually." + ) + + def _run( + self, context: ChangeContext, apply: bool + ) -> Result[SyftSuccess, SyftError]: + return Ok( + SyftError( + message="Synced UserCodes status is computed, and cannot be updated manually." + ) + ) + + def link(self) -> Any: # type: ignore + return self.code.status + + +@migrate(RequestV2, Request) +def migrate_request_v2_to_v3() -> list[Callable]: + return [ + make_set_default("tags", []), + ] + + +@migrate(Request, RequestV2) +def migrate_usercode_v5_to_v4() -> list[Callable]: + return [ + drop("tags"), + ] diff --git a/packages/syft/src/syft/service/request/request_service.py b/packages/syft/src/syft/service/request/request_service.py index ac166f0a32a..919effe4fcc 100644 --- a/packages/syft/src/syft/service/request/request_service.py +++ b/packages/syft/src/syft/service/request/request_service.py @@ -24,6 +24,7 @@ from ..service import TYPE_TO_SERVICE from ..service import service_method from ..user.user import UserView +from ..user.user_roles import ADMIN_ROLE_LEVEL from ..user.user_roles import DATA_SCIENTIST_ROLE_LEVEL from ..user.user_roles import GUEST_ROLE_LEVEL from ..user.user_service import UserService @@ -104,6 +105,17 @@ def submit( print("Failed to submit Request", e) raise e + @service_method( + path="request.get_by_uid", name="get_by_uid", roles=DATA_SCIENTIST_ROLE_LEVEL + ) + def get_by_uid( + self, context: AuthedServiceContext, uid: UID + ) -> Request | None | SyftError: + result = self.stash.get_by_uid(context.credentials, uid) + if result.is_err(): + return SyftError(message=str(result.err())) + return result.ok() + @service_method( path="request.get_all", name="get_all", roles=DATA_SCIENTIST_ROLE_LEVEL ) @@ -211,7 +223,7 @@ def apply( request_notification = filter_by_obj(context=context, obj_uid=uid) link = LinkedObject.with_context(request, context=context) - if not request.status == RequestStatus.PENDING: + if not request.get_status(context) == RequestStatus.PENDING: if request_notification is not None and not isinstance( request_notification, SyftError ): @@ -301,6 +313,36 @@ def delete_by_uid( return SyftError(message=str(result.err())) return SyftSuccess(message=f"Request with id {uid} deleted.") + @service_method( + path="request.set_tags", + name="set_tags", + roles=ADMIN_ROLE_LEVEL, + ) + def set_tags( + self, + context: AuthedServiceContext, + request: Request, + tags: list[str], + ) -> Request | SyftError: + request = self.stash.get_by_uid(context.credentials, request.id) + if request.is_err(): + return SyftError(message=str(request.err())) + if request.ok() is None: + return SyftError(message="Request does not exist.") + request = request.ok() + + request.tags = tags + return self.save(context, request) + + @service_method(path="request.get_by_usercode_id", name="get_by_usercode_id") + def get_by_usercode_id( + self, context: AuthedServiceContext, usercode_id: UID + ) -> list[Request] | SyftError: + result = self.stash.get_by_usercode_id(context.credentials, usercode_id) + if result.is_err(): + return SyftError(message=str(result.err())) + return result.ok() + TYPE_TO_SERVICE[Request] = RequestService SERVICE_TO_TYPES[RequestService].update({Request}) diff --git a/packages/syft/src/syft/service/request/request_stash.py b/packages/syft/src/syft/service/request/request_stash.py index 5b8fe3e08c5..dedee590357 100644 --- a/packages/syft/src/syft/service/request/request_stash.py +++ b/packages/syft/src/syft/service/request/request_stash.py @@ -1,6 +1,7 @@ # stdlib # third party +from result import Ok from result import Result # relative @@ -11,6 +12,7 @@ from ...store.document_store import PartitionSettings from ...store.document_store import QueryKeys from ...types.datetime import DateTime +from ...types.uid import UID from ...util.telemetry import instrument from .request import Request @@ -42,3 +44,14 @@ def get_all_for_verify_key( qks=qks, order_by=OrderByRequestTimeStampPartitionKey, ) + + def get_by_usercode_id( + self, credentials: SyftVerifyKey, user_code_id: UID + ) -> Result[list[Request], str]: + query = self.get_all(credentials=credentials) + if query.is_err(): + return query + + all_requests: list[Request] = query.ok() + results = [r for r in all_requests if r.code_id == user_code_id] + return Ok(results) diff --git a/packages/syft/src/syft/service/response.py b/packages/syft/src/syft/service/response.py index 06fefe4e50e..2908454096e 100644 --- a/packages/syft/src/syft/service/response.py +++ b/packages/syft/src/syft/service/response.py @@ -10,6 +10,7 @@ # relative from ..serde.serializable import serializable from ..types.base import SyftBaseModel +from ..util.util import sanitize_html class SyftResponseMessage(SyftBaseModel): @@ -64,7 +65,9 @@ def _repr_html_class_(self) -> str: def _repr_html_(self) -> str: return ( f'

    ' - + f"{type(self).__name__}: {self.message}

    " + f"{type(self).__name__}: " + f'
    '
    +            f"{sanitize_html(self.message)}

    " ) @@ -127,7 +130,7 @@ def _repr_html_class_(self) -> str: def _repr_html_(self) -> str: return ( f'
    ' - + f"{type(self).__name__}: {self.args}

    " + + f"{type(self).__name__}: {sanitize_html(self.args)}
    " ) @staticmethod diff --git a/packages/syft/src/syft/service/service.py b/packages/syft/src/syft/service/service.py index cda115cb8b4..c92695e2f6a 100644 --- a/packages/syft/src/syft/service/service.py +++ b/packages/syft/src/syft/service/service.py @@ -8,6 +8,7 @@ from functools import partial import inspect from inspect import Parameter +import logging from typing import Any from typing import TYPE_CHECKING @@ -43,6 +44,8 @@ from .user.user_roles import ServiceRole from .warnings import APIEndpointWarning +logger = logging.getLogger(__name__) + if TYPE_CHECKING: # relative from ..client.api import APIModule @@ -491,5 +494,5 @@ def from_api_or_context( ) return partial(service_method, node_context) else: - print("Could not get method from api or context") + logger.error("Could not get method from api or context") return None diff --git a/packages/syft/src/syft/service/settings/settings.py b/packages/syft/src/syft/service/settings/settings.py index 94adfbf307c..8898af36890 100644 --- a/packages/syft/src/syft/service/settings/settings.py +++ b/packages/syft/src/syft/service/settings/settings.py @@ -1,5 +1,6 @@ # stdlib from collections.abc import Callable +import logging from typing import Any # third party @@ -29,6 +30,8 @@ from ...util.schema import DEFAULT_WELCOME_MSG from ..response import SyftInfo +logger = logging.getLogger(__name__) + @serializable() class NodeSettingsUpdateV4(PartialSyftObject): @@ -54,8 +57,8 @@ def validate_node_side_type(cls, v: str) -> type[Empty]: as information might be leaked." try: display(SyftInfo(message=msg)) - except Exception: - print(SyftInfo(message=msg)) + except Exception as e: + logger.error(msg, exc_info=e) return Empty @@ -82,6 +85,7 @@ class NodeSettings(SyftObject): __repr_attrs__ = [ "name", "organization", + "description", "deployed_on", "signup_enabled", "admin_email", @@ -93,7 +97,7 @@ class NodeSettings(SyftObject): organization: str = "OpenMined" verify_key: SyftVerifyKey on_board: bool = True - description: str = "Text" + description: str = "This is the default description for a Domain Node." node_type: NodeType = NodeType.DOMAIN signup_enabled: bool admin_email: str @@ -116,6 +120,7 @@ def _repr_html_(self) -> Any:

    Id: {self.id}

    Name: {self.name}

    Organization: {self.organization}

    +

    Description: {self.description}

    Deployed on: {self.deployed_on}

    Signup enabled: {self.signup_enabled}

    Admin email: {self.admin_email}

    diff --git a/packages/syft/src/syft/service/settings/settings_service.py b/packages/syft/src/syft/service/settings/settings_service.py index 35ef9262860..f404c498662 100644 --- a/packages/syft/src/syft/service/settings/settings_service.py +++ b/packages/syft/src/syft/service/settings/settings_service.py @@ -353,6 +353,7 @@ def welcome_show( FONT_CSS=FONT_CSS, grid_symbol=load_png_base64("small-grid-symbol-logo.png"), domain_name=context.node.name, + description=context.node.metadata.description, # node_url='http://testing:8080', node_type=context.node.metadata.node_type.capitalize(), node_side_type=node_side_type, diff --git a/packages/syft/src/syft/service/sync/diff_state.py b/packages/syft/src/syft/service/sync/diff_state.py index d41974989d1..d5f8eb60caf 100644 --- a/packages/syft/src/syft/service/sync/diff_state.py +++ b/packages/syft/src/syft/service/sync/diff_state.py @@ -1,9 +1,11 @@ # stdlib from collections.abc import Callable +from collections.abc import Collection from collections.abc import Iterable from dataclasses import dataclass import enum import html +import logging import operator import textwrap from typing import Any @@ -12,9 +14,7 @@ from typing import TYPE_CHECKING # third party -from loguru import logger import pandas as pd -from pydantic import model_validator from rich import box from rich.console import Console from rich.console import Group @@ -61,6 +61,8 @@ from ..user.user import UserView from .sync_state import SyncState +logger = logging.getLogger(__name__) + if TYPE_CHECKING: # relative from .resolve_widget import PaginatedResolveWidget @@ -363,7 +365,8 @@ def repr_attr_diffstatus_dict(self) -> dict: def repr_attr_dict(self, side: str) -> dict[str, Any]: obj = self.low_obj if side == "low" else self.high_obj if isinstance(obj, ActionObject): - return {"value": obj.syft_action_data_cache} + # Only safe for ActionObjects created by data owners + return {"value": obj.syft_action_data_repr_} repr_attrs = getattr(obj, "__repr_attrs__", []) res = {} for attr in repr_attrs: @@ -509,7 +512,6 @@ def _repr_html_(self) -> str: obj_repr += diff.__repr__() + "
    " obj_repr = obj_repr.replace("\n", "
    ") - # print("New lines", res) attr_text = f"

    {self.object_type} ObjectDiff:

    \n{obj_repr}" return base_str + attr_text @@ -564,11 +566,11 @@ class ObjectDiffBatch(SyftObject): root_diff: ObjectDiff sync_direction: SyncDirection | None - def resolve(self) -> "ResolveWidget": + def resolve(self, build_state: bool = True) -> "ResolveWidget": # relative from .resolve_widget import ResolveWidget - return ResolveWidget(self) + return ResolveWidget(self, build_state=build_state) def walk_graph( self, @@ -676,7 +678,7 @@ def status(self) -> str: return "NEW" batch_statuses = [ - diff.status for diff in self.get_dependents(include_roots=False) + diff.status for diff in self.get_dependencies(include_roots=False) ] if all(status == "SAME" for status in batch_statuses): return "SAME" @@ -765,6 +767,7 @@ def from_dependencies( cls, root_uid: UID, obj_dependencies: dict[UID, list[UID]], + obj_dependents: dict[UID, list[UID]], obj_uid_to_diff: dict[UID, ObjectDiff], root_ids: list[UID], low_node_uid: UID, @@ -809,15 +812,13 @@ def _build_hierarchy_helper( levels = [level for _, level in batch_uids] batch_uids = {uid for uid, _ in batch_uids} # type: ignore - batch_dependencies = { - uid: [d for d in obj_dependencies.get(uid, []) if d in batch_uids] - for uid in batch_uids - } + return cls( global_diffs=obj_uid_to_diff, global_roots=root_ids, hierarchy_levels=levels, - dependencies=batch_dependencies, + dependencies=obj_dependencies, + dependents=obj_dependents, root_diff=obj_uid_to_diff[root_uid], low_node_uid=low_node_uid, high_node_uid=high_node_uid, @@ -910,15 +911,6 @@ def visual_hierarchy(self) -> tuple[type, dict]: else: raise ValueError(f"Unknown root type: {self.root.obj_type}") - @model_validator(mode="after") - def make_dependents(self) -> Self: - dependents: dict = {} - for parent, children in self.dependencies.items(): - for child in children: - dependents[child] = dependents.get(child, []) + [parent] - self.dependents = dependents - return self - @property def root(self) -> ObjectDiff: return self.root_diff @@ -1070,7 +1062,7 @@ def stage_change(self) -> None: other_batch.decision == SyncDecision.IGNORE and other_batch.root_id in required_dependencies ): - print(f"ignoring other batch ({other_batch.root_type.__name__})") + logger.debug(f"ignoring other batch ({other_batch.root_type.__name__})") other_batch.decision = None @@ -1150,14 +1142,16 @@ class NodeDiff(SyftObject): include_ignored: bool = False - def resolve(self) -> "PaginatedResolveWidget | SyftSuccess": + def resolve( + self, build_state: bool = True + ) -> "PaginatedResolveWidget | SyftSuccess": if len(self.batches) == 0: return SyftSuccess(message="No batches to resolve") # relative from .resolve_widget import PaginatedResolveWidget - return PaginatedResolveWidget(batches=self.batches) + return PaginatedResolveWidget(batches=self.batches, build_state=build_state) def __getitem__(self, idx: Any) -> ObjectDiffBatch: return self.batches[idx] @@ -1195,7 +1189,8 @@ def from_sync_state( include_ignored: bool = False, include_same: bool = False, filter_by_email: str | None = None, - filter_by_type: type | None = None, + include_types: Collection[type | str] | None = None, + exclude_types: Collection[type | str] | None = None, _include_node_status: bool = False, ) -> "NodeDiff": obj_uid_to_diff = {} @@ -1235,8 +1230,9 @@ def from_sync_state( ) obj_uid_to_diff[diff.object_id] = diff + # TODO move static methods to NodeDiff __init__ obj_dependencies = NodeDiff.dependencies_from_states(low_state, high_state) - all_batches = NodeDiff.hierarchies( + all_batches = NodeDiff._create_batches( low_state, high_state, obj_dependencies, @@ -1265,9 +1261,10 @@ def from_sync_state( res._filter( user_email=filter_by_email, - obj_type=filter_by_type, + include_types=include_types, include_ignored=include_ignored, include_same=include_same, + exclude_types=exclude_types, inplace=True, ) @@ -1289,7 +1286,7 @@ def apply_previous_ignore_state( if hash(batch) == batch_hash: batch.decision = SyncDecision.IGNORE else: - print( + logger.debug( f"""A batch with type {batch.root_type.__name__} was previously ignored but has changed It will be available for review again.""" ) @@ -1400,7 +1397,7 @@ def _sort_batches(hierarchies: list[ObjectDiffBatch]) -> list[ObjectDiffBatch]: return sorted_hierarchies @staticmethod - def hierarchies( + def _create_batches( low_sync_state: SyncState, high_sync_state: SyncState, obj_dependencies: dict[UID, list[UID]], @@ -1416,7 +1413,7 @@ def hierarchies( # TODO: Figure out nested user codes, do we even need that? root_ids.append(diff.object_id) # type: ignore - elif ( + elif ( # type: ignore[unreachable] isinstance(diff_obj, Job) # type: ignore and diff_obj.parent_job_id is None # ignore Job objects created by TwinAPIEndpoint @@ -1424,10 +1421,17 @@ def hierarchies( ): root_ids.append(diff.object_id) # type: ignore + # Dependents are the reverse edges of the dependency graph + obj_dependents = {} + for parent, children in obj_dependencies.items(): + for child in children: + obj_dependents[child] = obj_dependencies.get(child, []) + [parent] + for root_uid in root_ids: batch = ObjectDiffBatch.from_dependencies( root_uid, obj_dependencies, + obj_dependents, obj_uid_to_diff, root_ids, low_sync_state.node_uid, @@ -1483,9 +1487,10 @@ def _apply_filters( def _filter( self, user_email: str | None = None, - obj_type: str | type | None = None, include_ignored: bool = False, include_same: bool = False, + include_types: Collection[str | type] | None = None, + exclude_types: Collection[type | str] | None = None, inplace: bool = True, ) -> Self: new_filters = [] @@ -1493,12 +1498,6 @@ def _filter( new_filters.append( NodeDiffFilter(FilterProperty.USER, user_email, operator.eq) ) - if obj_type is not None: - if isinstance(obj_type, type): - obj_type = obj_type.__name__ - new_filters.append( - NodeDiffFilter(FilterProperty.TYPE, obj_type, operator.eq) - ) if not include_ignored: new_filters.append( NodeDiffFilter(FilterProperty.IGNORED, True, operator.ne) @@ -1507,6 +1506,20 @@ def _filter( new_filters.append( NodeDiffFilter(FilterProperty.STATUS, "SAME", operator.ne) ) + if include_types is not None: + include_types_ = { + t.__name__ if isinstance(t, type) else t for t in include_types + } + new_filters.append( + NodeDiffFilter(FilterProperty.TYPE, include_types_, operator.contains) + ) + if exclude_types: + for exclude_type in exclude_types: + if isinstance(exclude_type, type): + exclude_type = exclude_type.__name__ + new_filters.append( + NodeDiffFilter(FilterProperty.TYPE, exclude_type, operator.ne) + ) return self._apply_filters(new_filters, inplace=inplace) @@ -1543,7 +1556,10 @@ def from_batch_decision( if share_private_data: # or diff.object_type == "Job": if share_to_user is None: # job ran by another user - if not diff.object_type == "Job": + if ( + diff.object_type != "Job" + and diff.object_type != "ExecutionOutput" + ): raise ValueError( "share_to_user is required to share private data" ) @@ -1556,10 +1572,6 @@ def from_batch_decision( ) ] - # TODO move this to the widget - # if widget.has_unused_share_button: - # print("Share button was not used, so we will mockify the object") - # storage permissions new_storage_permissions = [] diff --git a/packages/syft/src/syft/service/sync/resolve_widget.py b/packages/syft/src/syft/service/sync/resolve_widget.py index 261ed28e075..4a868634df3 100644 --- a/packages/syft/src/syft/service/sync/resolve_widget.py +++ b/packages/syft/src/syft/service/sync/resolve_widget.py @@ -105,10 +105,19 @@ def __init__( direction: SyncDirection, with_box: bool = True, show_share_warning: bool = False, + build_state: bool = True, ): - self.low_properties = diff.repr_attr_dict("low") - self.high_properties = diff.repr_attr_dict("high") - self.statuses = diff.repr_attr_diffstatus_dict() + build_state = build_state + + if build_state: + self.low_properties = diff.repr_attr_dict("low") + self.high_properties = diff.repr_attr_dict("high") + self.statuses = diff.repr_attr_diffstatus_dict() + else: + self.low_properties = {} + self.high_properties = {} + self.statuses = {} + self.direction = direction self.diff: ObjectDiff = diff self.with_box = with_box @@ -126,11 +135,6 @@ def set_share_private_data(self) -> None: def mockify(self) -> bool: return not self.share_private_data - @property - def has_unused_share_button(self) -> bool: - # does not have share button - return False - @property def share_private_data(self) -> bool: # there are TwinAPIEndpoint.__private_sync_attr_mocks__ @@ -139,7 +143,7 @@ def share_private_data(self) -> bool: @property def warning_html(self) -> str: if isinstance(self.diff.non_empty_object, TwinAPIEndpoint): - message = "Only the private function of a TwinAPI will be synced to the public node." + message = "Only the public function of a TwinAPI will be synced to the public node." return Alert(message=message).to_html() elif self.show_share_warning: message = ( @@ -208,28 +212,27 @@ def __init__( self, diff: ObjectDiff, direction: SyncDirection, + build_state: bool = True, ): self.direction = direction + self.build_state = build_state self.share_private_data = False self.diff: ObjectDiff = diff self.sync: bool = False self.is_main_widget: bool = False + self.has_private_data = isinstance( + self.diff.non_empty_object, SyftLog | ActionObject | TwinAPIEndpoint + ) self.widget = self.build() self.set_and_disable_sync() @property def mockify(self) -> bool: - if isinstance(self.diff.non_empty_object, TwinAPIEndpoint): - return True - if self.has_unused_share_button: + if self.has_private_data and not self.share_private_data: return True else: return False - @property - def has_unused_share_button(self) -> bool: - return self.show_share_button and not self.share_private_data - @property def warning_html(self) -> str: if self.show_share_button: @@ -243,7 +246,7 @@ def warning_html(self) -> str: @property def show_share_button(self) -> bool: - return isinstance(self.diff.non_empty_object, SyftLog | ActionObject) + return self.has_private_data @property def title(self) -> str: @@ -282,6 +285,7 @@ def build(self) -> widgets.VBox: self.direction, with_box=False, show_share_warning=self.show_share_button, + build_state=self.build_state, ).widget accordion, share_private_checkbox, sync_checkbox = self.build_accordion( @@ -338,7 +342,7 @@ def create_accordion_css( def build_accordion( self, - accordion_body: widgets.Widget, + accordion_body: MainObjectDiffWidget, show_sync_checkbox: bool = True, show_share_private_checkbox: bool = True, ) -> VBox: @@ -375,8 +379,12 @@ def build_accordion( layout=Layout(flex="1"), ) + if isinstance(self.diff.non_empty_object, ActionObject): + share_data_description = "Share real data and approve" + else: + share_data_description = "Share real data" share_private_data_checkbox = Checkbox( - description="Sync Real Data", + description=share_data_description, layout=Layout(width="auto", margin="0 2px 0 0"), ) sync_checkbox = Checkbox( @@ -414,8 +422,12 @@ def _on_share_private_data_change(self, change: Any) -> None: class ResolveWidget: def __init__( - self, obj_diff_batch: ObjectDiffBatch, on_sync_callback: Callable | None = None + self, + obj_diff_batch: ObjectDiffBatch, + on_sync_callback: Callable | None = None, + build_state: bool = True, ): + self.build_state = build_state self.obj_diff_batch: ObjectDiffBatch = obj_diff_batch self.id2widget: dict[ UID, CollapsableObjectDiffWidget | MainObjectDiffWidget @@ -486,32 +498,36 @@ def batch_diff_widgets(self) -> list[CollapsableObjectDiffWidget]: CollapsableObjectDiffWidget( diff, direction=self.obj_diff_batch.sync_direction, + build_state=self.build_state, ) for diff in dependents ] return dependent_diff_widgets @property - def dependent_batch_diff_widgets(self) -> list[CollapsableObjectDiffWidget]: + def dependent_root_diff_widgets(self) -> list[CollapsableObjectDiffWidget]: dependencies = self.obj_diff_batch.get_dependencies( include_roots=True, include_batch_root=False ) other_roots = [ d for d in dependencies if d.object_id in self.obj_diff_batch.global_roots ] - dependent_root_diff_widgets = [ + widgets = [ CollapsableObjectDiffWidget( - diff, direction=self.obj_diff_batch.sync_direction + diff, + direction=self.obj_diff_batch.sync_direction, + build_state=self.build_state, ) for diff in other_roots ] - return dependent_root_diff_widgets + return widgets @property def main_object_diff_widget(self) -> MainObjectDiffWidget: obj_diff_widget = MainObjectDiffWidget( self.obj_diff_batch.root_diff, direction=self.obj_diff_batch.sync_direction, + build_state=self.build_state, ) return obj_diff_widget @@ -543,7 +559,7 @@ def build(self) -> VBox: self.id2widget = {} batch_diff_widgets = self.batch_diff_widgets - dependent_batch_diff_widgets = self.dependent_batch_diff_widgets + dependent_batch_diff_widgets = self.dependent_root_diff_widgets main_object_diff_widget = self.main_object_diff_widget self.id2widget[main_object_diff_widget.diff.object_id] = main_object_diff_widget @@ -579,7 +595,7 @@ def build(self) -> VBox: def sync_button(self) -> Button: sync_button = Button( - description="Sync Selected Changes", + description="Apply Selected Changes", style={ "text_color": "#464A91", "button_color": "transparent", @@ -695,7 +711,7 @@ def __getitem__(self, index: int) -> widgets.Widget: return self.children[index] def on_paginate(self, index: int) -> None: - self.container.children = [self.children[index]] + self.container.children = [self.children[index]] if self.children else [] if self.on_paginate_callback: self.on_paginate_callback(index) @@ -715,12 +731,14 @@ class PaginatedResolveWidget: paginated by a PaginationControl widget. """ - def __init__(self, batches: list[ObjectDiffBatch]): + def __init__(self, batches: list[ObjectDiffBatch], build_state: bool = True): + self.build_state = build_state self.batches = batches self.resolve_widgets: list[ResolveWidget] = [ ResolveWidget( batch, on_sync_callback=partial(self.on_click_sync, i), + build_state=build_state, ) for i, batch in enumerate(self.batches) ] diff --git a/packages/syft/src/syft/service/sync/sync_service.py b/packages/syft/src/syft/service/sync/sync_service.py index db50c2a7a61..62885742c5b 100644 --- a/packages/syft/src/syft/service/sync/sync_service.py +++ b/packages/syft/src/syft/service/sync/sync_service.py @@ -1,9 +1,9 @@ # stdlib from collections import defaultdict +import logging from typing import Any # third party -from loguru import logger from result import Err from result import Ok from result import Result @@ -24,6 +24,7 @@ from ..action.action_permissions import ActionPermission from ..action.action_permissions import StoragePermission from ..api.api import TwinAPIEndpoint +from ..api.api_service import APIService from ..code.user_code import UserCodeStatusCollection from ..context import AuthedServiceContext from ..job.job_stash import Job @@ -36,6 +37,8 @@ from .sync_stash import SyncStash from .sync_state import SyncState +logger = logging.getLogger(__name__) + def get_store(context: AuthedServiceContext, item: SyncableSyftObject) -> Any: if isinstance(item, ActionObject): @@ -156,11 +159,11 @@ def set_object( if isinstance(item, TwinAPIEndpoint): # we need the side effect of set function # to create an action object - res = context.node.get_service("apiservice").set( - context=context, endpoint=item - ) + apiservice: APIService = context.node.get_service("apiservice") # type: ignore + + res = apiservice.set(context=context, endpoint=item) if isinstance(res, SyftError): - return res + return Err(res.message) else: return Ok(item) diff --git a/packages/syft/src/syft/service/sync/sync_state.py b/packages/syft/src/syft/service/sync/sync_state.py index 85b876b485e..812f96fac19 100644 --- a/packages/syft/src/syft/service/sync/sync_state.py +++ b/packages/syft/src/syft/service/sync/sync_state.py @@ -242,9 +242,17 @@ def rows(self) -> list[SyncStateRow]: if previous_diff is None: raise ValueError("No previous state to compare to") for batch in previous_diff.batches: + # NOTE we re-use NodeDiff to compare to previous state, + # low_obj is previous state, high_obj is current state diff = batch.root_diff + + # If there is no high object, it means it was deleted + # and we don't need to show it in the table + if diff.high_obj is None: + continue if diff.object_id in ids: continue + ids.add(diff.object_id) row = SyncStateRow( object=diff.high_obj, diff --git a/packages/syft/src/syft/service/user/user_roles.py b/packages/syft/src/syft/service/user/user_roles.py index 6ed7f4a9796..34ab6d1ede6 100644 --- a/packages/syft/src/syft/service/user/user_roles.py +++ b/packages/syft/src/syft/service/user/user_roles.py @@ -34,9 +34,7 @@ class ServiceRole(Enum): # @property @classmethod def roles_descending(cls) -> list[tuple[int, Self]]: - tuples = [] - for x in cls: - tuples.append((x.value, x)) + tuples = [(x.value, x) for x in cls] return sorted(tuples, reverse=True) @classmethod diff --git a/packages/syft/src/syft/service/user/user_service.py b/packages/syft/src/syft/service/user/user_service.py index 63425f90103..25d85bccd9f 100644 --- a/packages/syft/src/syft/service/user/user_service.py +++ b/packages/syft/src/syft/service/user/user_service.py @@ -5,7 +5,6 @@ from ...exceptions.user import UserAlreadyExistsException from ...node.credentials import SyftSigningKey from ...node.credentials import SyftVerifyKey -from ...node.credentials import UserLoginCredentials from ...serde.serializable import serializable from ...store.document_store import DocumentStore from ...store.linked_obj import LinkedObject @@ -134,6 +133,23 @@ def get_all( # 🟡 TODO: No user exists will happen when result.ok() is empty list return SyftError(message="No users exists") + def signing_key_for_verify_key( + self, context: AuthedServiceContext, verify_key: SyftVerifyKey + ) -> UserPrivateKey | SyftError: + result = self.stash.get_by_verify_key( + credentials=self.admin_verify_key(), verify_key=verify_key + ) + if result.is_ok(): + user = result.ok() + if user is not None: + return user.to(UserPrivateKey) + + return SyftError(message=f"No user exists with {verify_key}.") + + return SyftError( + message=f"Failed to retrieve user with {verify_key} with error: {result.err()}" + ) + def get_role_for_credentials( self, credentials: SyftVerifyKey | SyftSigningKey ) -> ServiceRole | None | SyftError: @@ -395,7 +411,7 @@ def delete(self, context: AuthedServiceContext, uid: UID) -> bool | SyftError: def exchange_credentials( self, context: UnauthedServiceContext - ) -> UserLoginCredentials | SyftError: + ) -> UserPrivateKey | SyftError: """Verify user TODO: We might want to use a SyftObject instead """ diff --git a/packages/syft/src/syft/service/worker/utils.py b/packages/syft/src/syft/service/worker/utils.py index c952cbe8c13..f29c631b0c0 100644 --- a/packages/syft/src/syft/service/worker/utils.py +++ b/packages/syft/src/syft/service/worker/utils.py @@ -1,5 +1,6 @@ # stdlib import contextlib +import logging import os from pathlib import Path import socket @@ -34,6 +35,8 @@ from .worker_pool import WorkerOrchestrationType from .worker_pool import WorkerStatus +logger = logging.getLogger(__name__) + DEFAULT_WORKER_IMAGE_TAG = "openmined/default-worker-image-cpu:0.0.1" DEFAULT_WORKER_POOL_NAME = "default-pool" K8S_NODE_CREDS_NAME = "node-creds" @@ -261,9 +264,9 @@ def run_workers_in_threads( address=address, ) except Exception as e: - print( - "Failed to start consumer for " - f"pool={pool_name} worker={worker_name}. Error: {e}" + logger.error( + f"Failed to start consumer for pool={pool_name} worker={worker_name}", + exc_info=e, ) worker.status = WorkerStatus.STOPPED error = str(e) @@ -335,12 +338,7 @@ def create_kubernetes_pool( pool = None try: - print( - "Creating new pool " - f"name={pool_name} " - f"tag={tag} " - f"replicas={replicas}" - ) + logger.info(f"Creating new pool name={pool_name} tag={tag} replicas={replicas}") env_vars, mount_secrets = prepare_kubernetes_pool_env( runner, @@ -370,7 +368,12 @@ def create_kubernetes_pool( ) except Exception as e: if pool: - pool.delete() + try: + pool.delete() # this raises another exception if the pool never starts + except Exception as e2: + logger.error( + f"Failed to delete pool {pool_name} after failed creation. {e2}" + ) # stdlib import traceback @@ -391,7 +394,7 @@ def scale_kubernetes_pool( return SyftError(message=f"Pool does not exist. name={pool_name}") try: - print(f"Scaling pool name={pool_name} to replicas={replicas}") + logger.info(f"Scaling pool name={pool_name} to replicas={replicas}") runner.scale_pool(pool_name=pool_name, replicas=replicas) except Exception as e: return SyftError(message=f"Failed to scale workers {e}") @@ -520,7 +523,7 @@ def run_containers( if not worker_image.is_built: return SyftError(message="Image must be built before running it.") - print(f"Starting workers with start_idx={start_idx} count={number}") + logger.info(f"Starting workers with start_idx={start_idx} count={number}") if orchestration == WorkerOrchestrationType.DOCKER: with contextlib.closing(docker.from_env()) as client: diff --git a/packages/syft/src/syft/service/worker/worker_pool.py b/packages/syft/src/syft/service/worker/worker_pool.py index 24325ed2995..461c4f61b7d 100644 --- a/packages/syft/src/syft/service/worker/worker_pool.py +++ b/packages/syft/src/syft/service/worker/worker_pool.py @@ -180,10 +180,9 @@ def image(self) -> SyftWorkerImage | SyftError | None: @property def running_workers(self) -> list[SyftWorker] | SyftError: """Query the running workers using an API call to the server""" - _running_workers = [] - for worker in self.workers: - if worker.status == WorkerStatus.RUNNING: - _running_workers.append(worker) + _running_workers = [ + worker for worker in self.workers if worker.status == WorkerStatus.RUNNING + ] return _running_workers @@ -192,11 +191,11 @@ def healthy_workers(self) -> list[SyftWorker] | SyftError: """ Query the healthy workers using an API call to the server """ - _healthy_workers = [] - - for worker in self.workers: - if worker.healthcheck == WorkerHealth.HEALTHY: - _healthy_workers.append(worker) + _healthy_workers = [ + worker + for worker in self.workers + if worker.healthcheck == WorkerHealth.HEALTHY + ] return _healthy_workers diff --git a/packages/syft/src/syft/service/worker/worker_pool_service.py b/packages/syft/src/syft/service/worker/worker_pool_service.py index 9e7d02572c1..a44cf2e2d82 100644 --- a/packages/syft/src/syft/service/worker/worker_pool_service.py +++ b/packages/syft/src/syft/service/worker/worker_pool_service.py @@ -1,4 +1,5 @@ # stdlib +import logging from typing import Any # third party @@ -45,6 +46,8 @@ from .worker_service import WorkerService from .worker_stash import WorkerStash +logger = logging.getLogger(__name__) + @serializable() class SyftWorkerPoolService(AbstractService): @@ -362,9 +365,7 @@ def get_all( return SyftError(message=f"{result.err()}") worker_pools: list[WorkerPool] = result.ok() - res: list[tuple] = [] - for pool in worker_pools: - res.append((pool.name, pool)) + res = ((pool.name, pool) for pool in worker_pools) return DictTuple(res) @service_method( @@ -529,7 +530,7 @@ def scale( uid=worker.object_uid, ) if delete_result.is_err(): - print(f"Failed to delete worker: {worker.object_uid}") + logger.error(f"Failed to delete worker: {worker.object_uid}") # update worker_pool worker_pool.max_count = number diff --git a/packages/syft/src/syft/store/__init__.py b/packages/syft/src/syft/store/__init__.py index 2369be33ea4..9260d13f956 100644 --- a/packages/syft/src/syft/store/__init__.py +++ b/packages/syft/src/syft/store/__init__.py @@ -1,3 +1,3 @@ # relative -from .mongo_document_store import MongoDict # noqa: F401 -from .mongo_document_store import MongoStoreConfig # noqa: F401 +from .mongo_document_store import MongoDict +from .mongo_document_store import MongoStoreConfig diff --git a/packages/syft/src/syft/store/blob_storage/__init__.py b/packages/syft/src/syft/store/blob_storage/__init__.py index 663660e777c..b9677eda95b 100644 --- a/packages/syft/src/syft/store/blob_storage/__init__.py +++ b/packages/syft/src/syft/store/blob_storage/__init__.py @@ -44,6 +44,7 @@ from collections.abc import Callable from collections.abc import Generator from io import BytesIO +import logging from typing import Any # third party @@ -74,6 +75,8 @@ from ...types.transforms import make_set_default from ...types.uid import UID +logger = logging.getLogger(__name__) + DEFAULT_TIMEOUT = 10 MAX_RETRIES = 20 @@ -121,11 +124,11 @@ def syft_iter_content( max_retries: int = MAX_RETRIES, timeout: int = DEFAULT_TIMEOUT, ) -> Generator: - """custom iter content with smart retries (start from last byte read)""" + """Custom iter content with smart retries (start from last byte read)""" current_byte = 0 for attempt in range(max_retries): + headers = {"Range": f"bytes={current_byte}-"} try: - headers = {"Range": f"bytes={current_byte}-"} with requests.get( str(blob_url), stream=True, headers=headers, timeout=(timeout, timeout) ) as response: @@ -135,15 +138,14 @@ def syft_iter_content( ): current_byte += len(chunk) yield chunk - return - + return # If successful, exit the function except requests.exceptions.RequestException as e: if attempt < max_retries: - print( + logger.debug( f"Attempt {attempt}/{max_retries} failed: {e} at byte {current_byte}. Retrying..." ) else: - print(f"Max retries reached. Failed with error: {e}") + logger.error(f"Max retries reached - {e}") raise diff --git a/packages/syft/src/syft/store/blob_storage/seaweedfs.py b/packages/syft/src/syft/store/blob_storage/seaweedfs.py index 1d88fedda37..03c6f442c26 100644 --- a/packages/syft/src/syft/store/blob_storage/seaweedfs.py +++ b/packages/syft/src/syft/store/blob_storage/seaweedfs.py @@ -1,6 +1,7 @@ # stdlib from collections.abc import Generator from io import BytesIO +import logging import math from queue import Queue import threading @@ -40,6 +41,8 @@ from ...types.syft_object import SYFT_OBJECT_VERSION_3 from ...util.constants import DEFAULT_TIMEOUT +logger = logging.getLogger(__name__) + MAX_QUEUE_SIZE = 100 WRITE_EXPIRATION_TIME = 900 # seconds DEFAULT_FILE_PART_SIZE = 1024**3 # 1GB @@ -121,16 +124,17 @@ def add_chunks_to_queue( """Creates a data geneator for the part""" n = 0 - while n * chunk_size <= part_size: - try: + try: + while n * chunk_size <= part_size: chunk = data.read(chunk_size) + if not chunk: + break self.no_lines += chunk.count(b"\n") n += 1 queue.put(chunk) - except BlockingIOError: - # if end of file, stop - queue.put(0) - # if end of part, stop + except BlockingIOError: + pass + # if end of file or part, stop queue.put(0) gen = PartGenerator() @@ -148,7 +152,7 @@ def add_chunks_to_queue( etags.append({"ETag": etag, "PartNumber": part_no}) except requests.RequestException as e: - print(e) + logger.error(f"Failed to upload file to SeaweedFS - {e}") return SyftError(message=str(e)) mark_write_complete_method = from_api_or_context( diff --git a/packages/syft/src/syft/store/dict_document_store.py b/packages/syft/src/syft/store/dict_document_store.py index d0f6d9cb51f..71f0eced1f1 100644 --- a/packages/syft/src/syft/store/dict_document_store.py +++ b/packages/syft/src/syft/store/dict_document_store.py @@ -80,7 +80,7 @@ def __init__( ) def reset(self) -> None: - for _, partition in self.partitions.items(): + for partition in self.partitions.values(): partition.prune() diff --git a/packages/syft/src/syft/store/document_store.py b/packages/syft/src/syft/store/document_store.py index fea96e6d456..e71110667c3 100644 --- a/packages/syft/src/syft/store/document_store.py +++ b/packages/syft/src/syft/store/document_store.py @@ -350,7 +350,6 @@ def store_query_keys(self, objs: Any) -> QueryKeys: def _thread_safe_cbk(self, cbk: Callable, *args: Any, **kwargs: Any) -> Any | Err: locked = self.lock.acquire(blocking=True) if not locked: - print("FAILED TO LOCK") return Err( f"Failed to acquire lock for the operation {self.lock.lock_name} ({self.lock._lock})" ) @@ -640,7 +639,7 @@ def set( add_storage_permission: bool = True, ignore_duplicates: bool = False, ) -> Result[BaseStash.object_type, str]: - return self.partition.set( + res = self.partition.set( credentials=credentials, obj=obj, ignore_duplicates=ignore_duplicates, @@ -648,6 +647,8 @@ def set( add_storage_permission=add_storage_permission, ) + return res + def query_all( self, credentials: SyftVerifyKey, @@ -745,9 +746,10 @@ def update( has_permission: bool = False, ) -> Result[BaseStash.object_type, str]: qk = self.partition.store_query_key(obj) - return self.partition.update( + res = self.partition.update( credentials=credentials, qk=qk, obj=obj, has_permission=has_permission ) + return res @instrument @@ -764,8 +766,12 @@ def delete_by_uid( def get_by_uid( self, credentials: SyftVerifyKey, uid: UID ) -> Result[BaseUIDStoreStash.object_type | None, str]: - qks = QueryKeys(qks=[UIDPartitionKey.with_obj(uid)]) - return self.query_one(credentials=credentials, qks=qks) + res = self.partition.get(credentials=credentials, uid=uid) + + # NOTE Return Ok(None) when no results are found for backwards compatibility + if res.is_err(): + return Ok(None) + return res def set( self, diff --git a/packages/syft/src/syft/store/linked_obj.py b/packages/syft/src/syft/store/linked_obj.py index 93f63d1f8b4..6e76a799930 100644 --- a/packages/syft/src/syft/store/linked_obj.py +++ b/packages/syft/src/syft/store/linked_obj.py @@ -26,6 +26,8 @@ class LinkedObject(SyftObject): object_type: type[SyftObject] object_uid: UID + _resolve_cache: SyftObject | None = None + __exclude_sync_diff_attrs__ = ["node_uid"] def __str__(self) -> str: @@ -46,7 +48,9 @@ def resolve(self) -> SyftObject: if api is None: raise ValueError(f"api is None. You must login to {self.node_uid}") - return api.services.notifications.resolve_object(self) + resolve: SyftObject = api.services.notifications.resolve_object(self) + self._resolve_cache = resolve + return resolve def resolve_with_context(self, context: NodeServiceContext) -> Any: if context.node is None: diff --git a/packages/syft/src/syft/store/locks.py b/packages/syft/src/syft/store/locks.py index 6a29f6efdfb..48ae6ca1178 100644 --- a/packages/syft/src/syft/store/locks.py +++ b/packages/syft/src/syft/store/locks.py @@ -1,5 +1,6 @@ # stdlib from collections import defaultdict +import logging import threading import time from typing import Any @@ -11,6 +12,7 @@ # relative from ..serde.serializable import serializable +logger = logging.getLogger(__name__) THREAD_FILE_LOCKS: dict[int, dict[str, int]] = defaultdict(dict) @@ -190,7 +192,7 @@ def acquire(self, blocking: bool = True) -> bool: elapsed = time.time() - start_time else: return True - print( + logger.debug( f"Timeout elapsed after {self.timeout} seconds while trying to acquiring lock." ) # third party diff --git a/packages/syft/src/syft/store/mongo_document_store.py b/packages/syft/src/syft/store/mongo_document_store.py index 59d6799c2bb..9863cfcdeb2 100644 --- a/packages/syft/src/syft/store/mongo_document_store.py +++ b/packages/syft/src/syft/store/mongo_document_store.py @@ -398,6 +398,21 @@ def data(self) -> dict: values: list = self._all(credentials=None, has_permission=True).ok() return {v.id: v for v in values} + def _get( + self, + uid: UID, + credentials: SyftVerifyKey, + has_permission: bool | None = False, + ) -> Result[SyftObject, str]: + qks = QueryKeys.from_dict({"id": uid}) + res = self._get_all_from_store( + credentials, qks, order_by=None, has_permission=has_permission + ) + if res.is_err(): + return res + else: + return Ok(res.ok()[0]) + def _get_all_from_store( self, credentials: SyftVerifyKey, @@ -422,12 +437,12 @@ def _get_all_from_store( syft_objs.append(obj.to(self.settings.object_type, transform_context)) # TODO: maybe do this in loop before this - res = [] - for s in syft_objs: - if has_permission or self.has_permission( - ActionObjectREAD(uid=s.id, credentials=credentials) - ): - res.append(s) + res = [ + s + for s in syft_objs + if has_permission + or self.has_permission(ActionObjectREAD(uid=s.id, credentials=credentials)) + ] return Ok(res) def _delete( diff --git a/packages/syft/src/syft/store/sqlite_document_store.py b/packages/syft/src/syft/store/sqlite_document_store.py index 8ef1b2803a8..d992629b0b5 100644 --- a/packages/syft/src/syft/store/sqlite_document_store.py +++ b/packages/syft/src/syft/store/sqlite_document_store.py @@ -4,6 +4,7 @@ # stdlib from collections import defaultdict from copy import deepcopy +import logging from pathlib import Path import sqlite3 import tempfile @@ -33,6 +34,8 @@ from .locks import NoLockingConfig from .locks import SyftLock +logger = logging.getLogger(__name__) + # here we can create a single connection per cache_key # since pytest is concurrent processes, we need to isolate each connection # by its filename and optionally the thread that its running in @@ -165,7 +168,12 @@ def _close(self) -> None: if REF_COUNTS[cache_key(self.db_filename)] <= 0: # once you close it seems like other object references can't re-use the # same connection + self.db.close() + db_key = cache_key(self.db_filename) + if db_key in SQLITE_CONNECTION_POOL_CUR: + # NOTE if we don't remove the cursor, the cursor cache_key can clash with a future thread id + del SQLITE_CONNECTION_POOL_CUR[db_key] del SQLITE_CONNECTION_POOL_DB[cache_key(self.db_filename)] else: # don't close yet because another SQLiteBackingStore is probably still open @@ -266,7 +274,6 @@ def _get_all(self) -> Any: def _get_all_keys(self) -> Any: select_sql = f"select uid from {self.table_name} order by sqltime" # nosec - keys = [] res = self._execute(select_sql) if res.is_err(): @@ -277,8 +284,7 @@ def _get_all_keys(self) -> Any: if rows is None: return [] - for row in rows: - keys.append(UID(row[0])) + keys = [UID(row[0]) for row in rows] return keys def _delete(self, key: UID) -> None: @@ -352,7 +358,7 @@ def __del__(self) -> None: try: self._close() except Exception as e: - print(f"Could not close connection. Error: {e}") + logger.error("Could not close connection", exc_info=e) @serializable() diff --git a/packages/syft/src/syft/types/grid_url.py b/packages/syft/src/syft/types/grid_url.py index 91cf53e46d7..040969c2730 100644 --- a/packages/syft/src/syft/types/grid_url.py +++ b/packages/syft/src/syft/types/grid_url.py @@ -3,6 +3,7 @@ # stdlib import copy +import logging import os import re from urllib.parse import urlparse @@ -15,6 +16,8 @@ from ..serde.serializable import serializable from ..util.util import verify_tls +logger = logging.getLogger(__name__) + @serializable(attrs=["protocol", "host_or_ip", "port", "path", "query"]) class GridURL: @@ -43,7 +46,7 @@ def from_url(cls, url: str | GridURL) -> GridURL: query=getattr(parts, "query", ""), ) except Exception as e: - print(f"Failed to convert url: {url} to GridURL. {e}") + logger.error(f"Failed to convert url: {url} to GridURL. {e}") raise e def __init__( diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 3ec9c073165..10c975bc5f0 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -7,9 +7,11 @@ from collections.abc import Mapping from collections.abc import Sequence from collections.abc import Set +from functools import cache from hashlib import sha256 import inspect from inspect import Signature +import logging import types from types import NoneType from types import UnionType @@ -18,6 +20,7 @@ from typing import ClassVar from typing import Optional from typing import TYPE_CHECKING +from typing import TypeVar from typing import Union from typing import get_args from typing import get_origin @@ -37,6 +40,7 @@ from ..node.credentials import SyftVerifyKey from ..serde.recursive_primitives import recursive_serde_register_type from ..serde.serialize import _serialize as serialize +from ..service.response import SyftError from ..util.autoreload import autoreload_enabled from ..util.markdown import as_markdown_python_code from ..util.notebook_ui.components.tabulator_template import build_tabulator_table @@ -47,13 +51,17 @@ from .syft_metaclass import PartialModelMetaclass from .uid import UID +logger = logging.getLogger(__name__) + if TYPE_CHECKING: # relative + from ..client.api import SyftAPI from ..service.sync.diff_state import AttrDiff IntStr = int | str AbstractSetIntStr = Set[IntStr] MappingIntStrAny = Mapping[IntStr, Any] +T = TypeVar("T") SYFT_OBJECT_VERSION_1 = 1 @@ -224,6 +232,11 @@ def get_transform( ) +@cache +def cached_get_type_hints(cls: type) -> dict[str, Any]: + return typing.get_type_hints(cls) + + class SyftMigrationRegistry: __migration_version_registry__: dict[str, dict[int, str]] = {} __migration_transform_registry__: dict[str, dict[str, Callable]] = {} @@ -545,7 +558,7 @@ def _upgrade_version(self, latest: bool = True) -> "SyftObject": return upgraded # transform from one supported type to another - def to(self, projection: type, context: Context | None = None) -> Any: + def to(self, projection: type[T], context: Context | None = None) -> T: # 🟡 TODO 19: Could we do an mro style inheritence conversion? Risky? transform = SyftObjectRegistry.get_transform(type(self), projection) return transform(self, context) @@ -573,7 +586,7 @@ def _syft_set_validate_private_attrs_(self, **kwargs: Any) -> None: return # Validate and set private attributes # https://github.com/pydantic/pydantic/issues/2105 - annotations = typing.get_type_hints(self.__class__) + annotations = cached_get_type_hints(self.__class__) for attr, decl in self.__private_attributes__.items(): value = kwargs.get(attr, decl.get_default()) var_annotation = annotations.get(attr) @@ -609,8 +622,9 @@ def _syft_keys_types_dict(cls, attr_name: str) -> dict[str, type]: if isinstance(method, types.FunctionType): type_ = method.__annotations__["return"] except Exception as e: - print( - f"Failed to get attribute from key {key} type for {cls} storage. {e}" + logger.error( + f"Failed to get attribute from key {key} type for {cls} storage.", + exc_info=e, ) raise e # EmailStr seems to be lost every time the value is set even with a validator @@ -678,6 +692,15 @@ def syft_get_diffs(self, ext_obj: Self) -> list["AttrDiff"]: obj_attr = getattr(self, attr) ext_obj_attr = getattr(ext_obj, attr) + if (obj_attr is None) ^ (ext_obj_attr is None): + # If either attr is None, but not both, we have a diff + # NOTE This clause is needed because attr.__eq__ is not implemented for None, and will eval to True + diff_attr = AttrDiff( + attr_name=attr, + low_attr=obj_attr, + high_attr=ext_obj_attr, + ) + diff_attrs.append(diff_attr) if isinstance(obj_attr, list) and isinstance(ext_obj_attr, list): list_diff = ListDiff.from_lists( attr_name=attr, low_list=obj_attr, high_list=ext_obj_attr @@ -700,6 +723,15 @@ def syft_get_diffs(self, ext_obj: Self) -> list["AttrDiff"]: diff_attrs.append(diff_attr) return diff_attrs + def _get_api(self) -> "SyftAPI | SyftError": + # relative + from ..client.api import APIRegistry + + api = APIRegistry.api_for(self.syft_node_location, self.syft_client_verify_key) + if api is None: + return SyftError(f"Can't access the api. You must login to {self.node_uid}") + return api + ## OVERRIDING pydantic.BaseModel.__getattr__ ## return super().__getattribute__(item) -> return self.__getattribute__(item) ## so that ActionObject.__getattribute__ works properly, diff --git a/packages/syft/src/syft/types/twin_object.py b/packages/syft/src/syft/types/twin_object.py index 458c69c0923..f94d75744d6 100644 --- a/packages/syft/src/syft/types/twin_object.py +++ b/packages/syft/src/syft/types/twin_object.py @@ -11,6 +11,7 @@ from typing_extensions import Self # relative +from ..client.client import SyftClient from ..serde.serializable import serializable from ..service.action.action_object import ActionObject from ..service.action.action_object import TwinMode @@ -81,15 +82,25 @@ def mock(self) -> ActionObject: mock.id = twin_id return mock - def _save_to_blob_storage(self) -> SyftError | None: + def _save_to_blob_storage(self, allow_empty: bool = False) -> SyftError | None: # Set node location and verify key self.private_obj._set_obj_location_( self.syft_node_location, self.syft_client_verify_key, ) - # self.mock_obj._set_obj_location_( - # self.syft_node_location, - # self.syft_client_verify_key, - # ) - return self.private_obj._save_to_blob_storage() - # self.mock_obj._save_to_blob_storage() + self.mock_obj._set_obj_location_( + self.syft_node_location, + self.syft_client_verify_key, + ) + mock_store_res = self.mock_obj._save_to_blob_storage(allow_empty=allow_empty) + if isinstance(mock_store_res, SyftError): + return mock_store_res + return self.private_obj._save_to_blob_storage(allow_empty=allow_empty) + + def send(self, client: SyftClient, add_storage_permission: bool = True) -> Any: + self._set_obj_location_(client.id, client.verify_key) + self._save_to_blob_storage() + res = client.api.services.action.set( + self, add_storage_permission=add_storage_permission + ) + return res diff --git a/packages/syft/src/syft/types/uid.py b/packages/syft/src/syft/types/uid.py index b4aab67302e..cd3a0dafba5 100644 --- a/packages/syft/src/syft/types/uid.py +++ b/packages/syft/src/syft/types/uid.py @@ -5,6 +5,7 @@ from collections.abc import Callable from collections.abc import Sequence import hashlib +import logging from typing import Any import uuid from uuid import UUID as uuid_type @@ -14,8 +15,8 @@ # relative from ..serde.serializable import serializable -from ..util.logger import critical -from ..util.logger import traceback_and_raise + +logger = logging.getLogger(__name__) @serializable(attrs=["value"]) @@ -81,9 +82,8 @@ def from_string(value: str) -> UID: try: return UID(value=uuid.UUID(value)) except ValueError as e: - critical(f"Unable to convert {value} to UUID. {e}") - traceback_and_raise(e) - raise + logger.critical(f"Unable to convert {value} to UUID. {e}") + raise e @staticmethod def with_seed(value: str) -> UID: diff --git a/packages/syft/src/syft/util/__init__.py b/packages/syft/src/syft/util/__init__.py index f6394760c7b..aec1f392faf 100644 --- a/packages/syft/src/syft/util/__init__.py +++ b/packages/syft/src/syft/util/__init__.py @@ -1,2 +1,2 @@ # relative -from .schema import generate_json_schemas # noqa: F401 +from .schema import generate_json_schemas diff --git a/packages/syft/src/syft/util/logger.py b/packages/syft/src/syft/util/logger.py deleted file mode 100644 index d9f0611a6c6..00000000000 --- a/packages/syft/src/syft/util/logger.py +++ /dev/null @@ -1,134 +0,0 @@ -# stdlib -from collections.abc import Callable -import logging -import os -import sys -from typing import Any -from typing import NoReturn -from typing import TextIO - -# third party -from loguru import logger - -LOG_FORMAT = "[{time}][{level}][{module}]][{process.id}] {message}" - -logger.remove() -DEFAULT_SINK = "syft_{time}.log" - - -def remove() -> None: - logger.remove() - - -def add( - sink: None | str | os.PathLike | TextIO | logging.Handler = None, - level: str = "ERROR", -) -> None: - sink = DEFAULT_SINK if sink is None else sink - try: - logger.add( - sink=sink, - format=LOG_FORMAT, - enqueue=True, - colorize=False, - diagnose=True, - backtrace=True, - rotation="10 MB", - retention="1 day", - level=level, - ) - except BaseException: - logger.add( - sink=sink, - format=LOG_FORMAT, - colorize=False, - diagnose=True, - backtrace=True, - level=level, - ) - - -def start() -> None: - add(sink=sys.stderr, level="CRITICAL") - - -def stop() -> None: - logger.stop() - - -def traceback_and_raise(e: Any, verbose: bool = False) -> NoReturn: - try: - if verbose: - logger.opt(lazy=True).exception(e) - else: - logger.opt(lazy=True).critical(e) - except BaseException as ex: - logger.debug("failed to print exception", ex) - if not issubclass(type(e), Exception): - e = Exception(e) - raise e - - -def create_log_and_print_function(level: str) -> Callable: - def log_and_print(*args: Any, **kwargs: Any) -> None: - try: - method = getattr(logger.opt(lazy=True), level, None) - if "print" in kwargs and kwargs["print"] is True: - del kwargs["print"] - print(*args, **kwargs) - if "end" in kwargs: - # clean up extra end for printinga - del kwargs["end"] - - if method is not None: - method(*args, **kwargs) - else: - raise Exception(f"no method {level} on logger") - except BaseException as e: - msg = f"failed to log exception. {e}" - try: - logger.debug(msg) - - except Exception as e: - print(f"{msg}. {e}") - - return log_and_print - - -def traceback(*args: Any, **kwargs: Any) -> None: - # caller = inspect.getframeinfo(inspect.stack()[1][0]) - # print(f"traceback:{caller.filename}:{caller.function}:{caller.lineno}") - return create_log_and_print_function(level="exception")(*args, **kwargs) - - -def critical(*args: Any, **kwargs: Any) -> None: - # caller = inspect.getframeinfo(inspect.stack()[1][0]) - # print(f"critical:{caller.filename}:{caller.function}:{caller.lineno}:{args}") - return create_log_and_print_function(level="critical")(*args, **kwargs) - - -def error(*args: Any, **kwargs: Any) -> None: - # caller = inspect.getframeinfo(inspect.stack()[1][0]) - # print(f"error:{caller.filename}:{caller.function}:{caller.lineno}") - return create_log_and_print_function(level="error")(*args, **kwargs) - - -def warning(*args: Any, **kwargs: Any) -> None: - return create_log_and_print_function(level="warning")(*args, **kwargs) - - -def info(*args: Any, **kwargs: Any) -> None: - return create_log_and_print_function(level="info")(*args, **kwargs) - - -def debug(*args: Any) -> None: - debug_msg = " ".join([str(a) for a in args]) - return logger.debug(debug_msg) - - -def _debug(*args: Any, **kwargs: Any) -> None: - return create_log_and_print_function(level="debug")(*args, **kwargs) - - -def trace(*args: Any, **kwargs: Any) -> None: - return create_log_and_print_function(level="trace")(*args, **kwargs) diff --git a/packages/syft/src/syft/util/notebook_ui/components/table_template.py b/packages/syft/src/syft/util/notebook_ui/components/table_template.py deleted file mode 100644 index 7de891af6a9..00000000000 --- a/packages/syft/src/syft/util/notebook_ui/components/table_template.py +++ /dev/null @@ -1,330 +0,0 @@ -# stdlib -from collections.abc import Sequence -import json -from string import Template - -# relative -from ....types.uid import UID -from ..icons import Icon -from ..styles import CSS_CODE - -TABLE_INDEX_KEY = "_table_repr_index" - -custom_code = """ - - -
    -
    -
    ${icon}
    -

    ${list_name}

    -
    - -
    -
    -
    -
    -
    - -
    - -
    - -
    - -

    0

    -
    -
    - -
    -
    - -
    -
    -""" - - -def create_table_template( - table_data: Sequence, - name: str, - rows: int = 5, - icon: str | None = None, - grid_template_columns: str | None = None, - grid_template_cell_columns: str | None = None, - **kwargs: dict, -) -> str: - if icon is None: - icon = Icon.TABLE.svg - if grid_template_columns is None: - grid_template_columns = "1fr repeat({cols}, 1fr)" - if grid_template_cell_columns is None: - grid_template_cell_columns = "span 4" - - items_dict = json.dumps(table_data) - code = CSS_CODE + custom_code - template = Template(code) - rows = min(len(table_data), rows) - if len(table_data) == 0: - cols = 0 - else: - col_names = [k for k in table_data[0].keys() if k != TABLE_INDEX_KEY] - cols = (len(col_names)) * 4 - if "{cols}" in grid_template_columns: - grid_template_columns = grid_template_columns.format(cols=cols) - final_html = template.substitute( - uid=str(UID()), - element=items_dict, - list_name=name, - cols=cols, - rows=rows, - icon=icon, - searchIcon=Icon.SEARCH.svg, - clipboardIconEscaped=Icon.CLIPBOARD.js_escaped_svg, - grid_template_columns=grid_template_columns, - grid_template_cell_columns=grid_template_cell_columns, - ) - return final_html diff --git a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py index ee0576cc206..24b65f0b8f0 100644 --- a/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py +++ b/packages/syft/src/syft/util/notebook_ui/components/tabulator_template.py @@ -1,5 +1,6 @@ # stdlib import json +import logging import secrets from typing import Any @@ -7,15 +8,17 @@ from IPython.display import HTML from IPython.display import display import jinja2 -from loguru import logger # relative from ...assets import load_css from ...assets import load_js from ...table import TABLE_INDEX_KEY from ...table import prepare_table_data +from ...util import sanitize_html from ..icons import Icon +logger = logging.getLogger(__name__) + DEFAULT_ID_WIDTH = 110 env = jinja2.Environment(loader=jinja2.PackageLoader("syft", "assets/jinja")) # nosec @@ -68,8 +71,7 @@ def format_dict(data: Any) -> str: return data if set(data.keys()) != {"type", "value"}: - return str(data) - + return sanitize_html(str(data)) if "badge" in data["type"]: return Badge(value=data["value"], badge_class=data["type"]).to_html() elif "label" in data["type"]: @@ -77,7 +79,7 @@ def format_dict(data: Any) -> str: if "clipboard" in data["type"]: return CopyButton(copy_text=data["value"]).to_html() - return str(data) + return sanitize_html(str(data)) def format_table_data(table_data: list[dict[str, Any]]) -> list[dict[str, str]]: @@ -86,7 +88,7 @@ def format_table_data(table_data: list[dict[str, Any]]) -> list[dict[str, str]]: row_formatted: dict[str, str] = {} for k, v in row.items(): if isinstance(v, str): - row_formatted[k] = v.replace("\n", "
    ") + row_formatted[k] = sanitize_html(v.replace("\n", "
    ")) continue v_formatted = format_dict(v) row_formatted[k] = v_formatted @@ -94,6 +96,71 @@ def format_table_data(table_data: list[dict[str, Any]]) -> list[dict[str, str]]: return formatted +def _render_tabulator_table( + uid: str, + table_data: list[dict], + table_metadata: dict, + max_height: int | None, + pagination: bool, + header_sort: bool, +) -> str: + table_template = env.get_template("table.jinja2") + tabulator_js = load_js("tabulator.min.js") + tabulator_css = load_css("tabulator_pysyft.min.css") + js = load_js("table.js") + css = load_css("style.css") + + # Add tabulator as a named module for VSCode compatibility + tabulator_js = tabulator_js.replace( + "define(t)", "define('tabulator-tables', [], t)" + ) + + icon = table_metadata.get("icon", None) + if icon is None: + icon = Icon.TABLE.svg + + column_data, row_header = create_tabulator_columns( + table_metadata["columns"], header_sort=header_sort + ) + table_data = format_table_data(table_data) + table_html = table_template.render( + uid=uid, + columns=json.dumps(column_data), + row_header=json.dumps(row_header), + data=json.dumps(table_data), + css=css, + js=js, + index_field_name=TABLE_INDEX_KEY, + icon=icon, + name=table_metadata["name"], + tabulator_js=tabulator_js, + tabulator_css=tabulator_css, + max_height=json.dumps(max_height), + pagination=json.dumps(pagination), + header_sort=json.dumps(header_sort), + ) + + return table_html + + +def build_tabulator_table_with_data( + table_data: list[dict], + table_metadata: dict, + uid: str | None = None, + max_height: int | None = None, + pagination: bool = True, + header_sort: bool = True, +) -> str | None: + try: + uid = uid if uid is not None else secrets.token_hex(4) + return _render_tabulator_table( + uid, table_data, table_metadata, max_height, pagination, header_sort + ) + except Exception as e: + logger.debug("error building table", e) + return None + + def build_tabulator_table( obj: Any, uid: str | None = None, @@ -105,49 +172,13 @@ def build_tabulator_table( table_data, table_metadata = prepare_table_data(obj) if len(table_data) == 0: return obj.__repr__() - - table_template = env.get_template("table.jinja2") - tabulator_js = load_js("tabulator.min.js") - tabulator_css = load_css("tabulator_pysyft.min.css") - js = load_js("table.js") - css = load_css("style.css") - - # Add tabulator as a named module for VSCode compatibility - tabulator_js = tabulator_js.replace( - "define(t)", "define('tabulator-tables', [], t)" - ) - - icon = table_metadata.get("icon", None) - if icon is None: - icon = Icon.TABLE.svg - uid = uid if uid is not None else secrets.token_hex(4) - column_data, row_header = create_tabulator_columns( - table_metadata["columns"], header_sort=header_sort - ) - table_data = format_table_data(table_data) - table_html = table_template.render( - uid=uid, - columns=json.dumps(column_data), - row_header=json.dumps(row_header), - data=json.dumps(table_data), - css=css, - js=js, - index_field_name=TABLE_INDEX_KEY, - icon=icon, - name=table_metadata["name"], - tabulator_js=tabulator_js, - tabulator_css=tabulator_css, - max_height=json.dumps(max_height), - pagination=json.dumps(pagination), - header_sort=json.dumps(header_sort), + return _render_tabulator_table( + uid, table_data, table_metadata, max_height, pagination, header_sort ) - - return table_html except Exception as e: - logger.debug("error building table", e) - - return None + logger.debug("error building table", exc_info=e) + return None def show_table(obj: Any) -> None: diff --git a/packages/syft/src/syft/util/notebook_ui/styles.py b/packages/syft/src/syft/util/notebook_ui/styles.py index a250c20a7dc..2e780394687 100644 --- a/packages/syft/src/syft/util/notebook_ui/styles.py +++ b/packages/syft/src/syft/util/notebook_ui/styles.py @@ -28,6 +28,6 @@ CSS_CODE = f""" """ diff --git a/packages/syft/src/syft/util/patch_ipython.py b/packages/syft/src/syft/util/patch_ipython.py new file mode 100644 index 00000000000..4910eb77d52 --- /dev/null +++ b/packages/syft/src/syft/util/patch_ipython.py @@ -0,0 +1,187 @@ +# stdlib +import re +from types import MethodType +from typing import Any + +# relative +from ..types.dicttuple import DictTuple +from ..types.syft_object import SyftObject +from .util import sanitize_html + + +def _patch_ipython_sanitization() -> None: + try: + # third party + from IPython import get_ipython + except ImportError: + return + + ip = get_ipython() + if ip is None: + return + + # stdlib + from importlib import resources + + # relative + from .assets import load_css + from .assets import load_js + from .notebook_ui.components.sync import ALERT_CSS + from .notebook_ui.components.sync import COPY_CSS + from .notebook_ui.styles import CSS_CODE + from .notebook_ui.styles import FONT_CSS + from .notebook_ui.styles import ITABLES_CSS + from .notebook_ui.styles import JS_DOWNLOAD_FONTS + + tabulator_js = load_js("tabulator.min.js") + tabulator_js = tabulator_js.replace( + "define(t)", "define('tabulator-tables', [], t)" + ) + + SKIP_SANITIZE = [ + FONT_CSS, + ITABLES_CSS, + CSS_CODE, + JS_DOWNLOAD_FONTS, + tabulator_js, + load_css("tabulator_pysyft.min.css"), + load_js("table.js"), + ] + + css_reinsert = f""" + + +{JS_DOWNLOAD_FONTS} +{CSS_CODE} + + +""" + + escaped_js_css = re.compile( + "|".join(re.escape(substr) for substr in SKIP_SANITIZE), + re.IGNORECASE | re.MULTILINE, + ) + + table_template = ( + resources.files("syft.assets.jinja").joinpath("table.jinja2").read_text() + ) + table_template = table_template.strip() + table_template = re.sub(r"\\{\\{.*?\\}\\}", ".*?", re.escape(table_template)) + escaped_template = re.compile(table_template, re.DOTALL | re.VERBOSE) + + jobs_repr_template = ( + r"(.*?)" + ) + jobs_pattern = re.compile(jobs_repr_template, re.DOTALL) + + def display_sanitized_html(obj: SyftObject | DictTuple) -> str | None: + if callable(getattr(obj, "_repr_html_", None)): + html_str = obj._repr_html_() + if html_str is not None: + matching_table = escaped_template.findall(html_str) + matching_jobs = jobs_pattern.findall(html_str) + template = "\n".join(matching_table + matching_jobs) + sanitized_str = escaped_template.sub("", html_str) + sanitized_str = escaped_js_css.sub("", sanitized_str) + sanitized_str = jobs_pattern.sub("", html_str) + sanitized_str = sanitize_html(sanitized_str) + return f"{css_reinsert} {sanitized_str} {template}" + return None + + def display_sanitized_md(obj: SyftObject) -> str | None: + if callable(getattr(obj, "_repr_markdown_", None)): + md = obj._repr_markdown_() + if md is not None: + return sanitize_html(md) + return None + + ip.display_formatter.formatters["text/html"].for_type( + SyftObject, display_sanitized_html + ) + ip.display_formatter.formatters["text/html"].for_type( + DictTuple, display_sanitized_html + ) + ip.display_formatter.formatters["text/markdown"].for_type( + SyftObject, display_sanitized_md + ) + + +def _patch_ipython_autocompletion() -> None: + try: + # third party + from IPython import get_ipython + from IPython.core.guarded_eval import EVALUATION_POLICIES + except ImportError: + return + + ipython = get_ipython() + if ipython is None: + return + + try: + # this allows property getters to be used in nested autocomplete + ipython.Completer.evaluation = "limited" + ipython.Completer.use_jedi = False + policy = EVALUATION_POLICIES["limited"] + + policy.allowed_getattr_external.update( + [ + ("syft.client.api", "APIModule"), + ("syft.client.api", "SyftAPI"), + ] + ) + original_can_get_attr = policy.can_get_attr + + def patched_can_get_attr(value: Any, attr: str) -> bool: + attr_name = "__syft_allow_autocomplete__" + # first check if exist to prevent side effects + if hasattr(value, attr_name) and attr in getattr(value, attr_name, []): + if attr in dir(value): + return True + else: + return False + else: + return original_can_get_attr(value, attr) + + policy.can_get_attr = patched_can_get_attr + except Exception: + print("Failed to patch ipython autocompletion for syft property getters") + + try: + # this constraints the completions for autocomplete. + # if __syft_dir__ is defined we only autocomplete those properties + original_attr_matches = ipython.Completer.attr_matches + + def patched_attr_matches(self, text: str) -> list[str]: # type: ignore + res = original_attr_matches(text) + m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer) + if not m2: + return res + expr, _ = m2.group(1, 2) + obj = self._evaluate_expr(expr) + if isinstance(obj, SyftObject) and hasattr(obj, "__syft_dir__"): + # here we filter all autocomplete results to only contain those + # defined in __syft_dir__, however the original autocomplete prefixes + # have the full path, while __syft_dir__ only defines the attr + attrs = set(obj.__syft_dir__()) + new_res = [] + for r in res: + splitted = r.split(".") + if len(splitted) > 1: + attr_name = splitted[-1] + if attr_name in attrs: + new_res.append(r) + return new_res + else: + return res + + ipython.Completer.attr_matches = MethodType( + patched_attr_matches, ipython.Completer + ) + except Exception: + print("Failed to patch syft autocompletion for __syft_dir__") + + +def patch_ipython() -> None: + _patch_ipython_sanitization() + _patch_ipython_autocompletion() diff --git a/packages/syft/src/syft/util/schema.py b/packages/syft/src/syft/util/schema.py index f918ed0d4af..859efa796b5 100644 --- a/packages/syft/src/syft/util/schema.py +++ b/packages/syft/src/syft/util/schema.py @@ -59,6 +59,7 @@

    Welcome to $domain_name

    URL: $node_url
    + Node Description: $description
    Node Type: $node_type
    Node Side Type:$node_side_type
    Syft Version: $node_version
    @@ -179,7 +180,7 @@ def process_type_bank(type_bank: dict[str, tuple[Any, ...]]) -> dict[str, dict]: def resolve_references(json_mappings: dict[str, dict]) -> dict[str, dict]: # track second pass generated types new_types = {} - for _, json_schema in json_mappings.items(): + for json_schema in json_mappings.values(): replace_types = {} for attribute, config in json_schema["properties"].items(): if "type" in config: diff --git a/packages/syft/src/syft/util/table.py b/packages/syft/src/syft/util/table.py index 998e022bdbd..611ca5e33a2 100644 --- a/packages/syft/src/syft/util/table.py +++ b/packages/syft/src/syft/util/table.py @@ -3,16 +3,17 @@ from collections.abc import Iterable from collections.abc import Mapping from collections.abc import Set +import logging import re from typing import Any -# third party -from loguru import logger - # relative -from .notebook_ui.components.table_template import TABLE_INDEX_KEY -from .notebook_ui.components.table_template import create_table_template from .util import full_name_with_qualname +from .util import sanitize_html + +TABLE_INDEX_KEY = "_table_repr_index" + +logger = logging.getLogger(__name__) def _syft_in_mro(self: Any, item: Any) -> bool: @@ -134,7 +135,7 @@ def _create_table_rows( except Exception as e: print(e) value = None - cols[field].append(str(value)) + cols[field].append(sanitize_html(str(value))) col_lengths = {len(cols[col]) for col in cols.keys()} if len(col_lengths) != 1: @@ -238,21 +239,3 @@ def prepare_table_data( } return table_data, table_metadata - - -def list_dict_repr_html(self: Mapping | Set | Iterable) -> str | None: - try: - table_data, table_metadata = prepare_table_data(self) - if len(table_data) == 0: - # TODO cleanup tech debt: _repr_html_ is used in syft without `None` fallback. - return self.__repr__() - return create_table_template( - table_data=table_data, - **table_metadata, - ) - - except Exception as e: - logger.debug(f"Could not create table: {e}") - - # _repr_html_ returns None -> fallback to default repr - return None diff --git a/packages/syft/src/syft/util/telemetry.py b/packages/syft/src/syft/util/telemetry.py index 32a57dd0534..d03f240a1de 100644 --- a/packages/syft/src/syft/util/telemetry.py +++ b/packages/syft/src/syft/util/telemetry.py @@ -1,9 +1,12 @@ # stdlib from collections.abc import Callable +import logging import os from typing import Any from typing import TypeVar +logger = logging.getLogger(__name__) + def str_to_bool(bool_str: str | None) -> bool: result = False @@ -27,7 +30,6 @@ def noop(__func_or_class: T, /, *args: Any, **kwargs: Any) -> T: instrument = noop else: try: - print("OpenTelemetry Tracing enabled") service_name = os.environ.get("SERVICE_NAME", "client") jaeger_host = os.environ.get("JAEGER_HOST", "localhost") jaeger_port = int(os.environ.get("JAEGER_PORT", "14268")) @@ -74,6 +76,6 @@ def noop(__func_or_class: T, /, *args: Any, **kwargs: Any) -> T: from .trace_decorator import instrument as _instrument instrument = _instrument - except Exception: # nosec - print("Failed to import opentelemetry") + except Exception as e: + logger.error("Failed to import opentelemetry", exc_info=e) instrument = noop diff --git a/packages/syft/src/syft/util/util.py b/packages/syft/src/syft/util/util.py index b0affa2b1a0..22f003fefaf 100644 --- a/packages/syft/src/syft/util/util.py +++ b/packages/syft/src/syft/util/util.py @@ -7,9 +7,13 @@ from concurrent.futures import ProcessPoolExecutor from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager +from copy import deepcopy +from datetime import datetime import functools import hashlib from itertools import repeat +import json +import logging import multiprocessing import multiprocessing as mp from multiprocessing import set_start_method @@ -35,13 +39,10 @@ from forbiddenfruit import curse from nacl.signing import SigningKey from nacl.signing import VerifyKey +import nh3 import requests -# relative -from .logger import critical -from .logger import debug -from .logger import error -from .logger import traceback_and_raise +logger = logging.getLogger(__name__) DATASETS_URL = "https://raw.githubusercontent.com/OpenMined/datasets/main" PANDAS_DATA = f"{DATASETS_URL}/pandas_cookbook" @@ -57,9 +58,9 @@ def full_name_with_qualname(klass: type) -> str: if not hasattr(klass, "__module__"): return f"builtins.{get_qualname_for(klass)}" return f"{klass.__module__}.{get_qualname_for(klass)}" - except Exception: + except Exception as e: # try name as backup - print("Failed to get FQN for:", klass, type(klass)) + logger.error(f"Failed to get FQN for: {klass} {type(klass)}", exc_info=e) return full_name_with_name(klass=klass) @@ -70,7 +71,7 @@ def full_name_with_name(klass: type) -> str: return f"builtins.{get_name_for(klass)}" return f"{klass.__module__}.{get_name_for(klass)}" except Exception as e: - print("Failed to get FQN for:", klass, type(klass)) + logger.error(f"Failed to get FQN for: {klass} {type(klass)}", exc_info=e) raise e @@ -107,7 +108,7 @@ def extract_name(klass: type) -> str: return fqn.split(".")[-1] return fqn except Exception as e: - print(f"Failed to get klass name {klass}") + logger.error(f"Failed to get klass name {klass}", exc_info=e) raise e else: raise ValueError(f"Failed to match regex for klass {klass}") @@ -117,9 +118,7 @@ def validate_type(_object: object, _type: type, optional: bool = False) -> Any: if isinstance(_object, _type) or (optional and (_object is None)): return _object - traceback_and_raise( - f"Object {_object} should've been of type {_type}, not {_object}." - ) + raise Exception(f"Object {_object} should've been of type {_type}, not {_object}.") def validate_field(_object: object, _field: str) -> Any: @@ -128,7 +127,7 @@ def validate_field(_object: object, _field: str) -> Any: if object is not None: return object - traceback_and_raise(f"Object {_object} has no {_field} field set.") + raise Exception(f"Object {_object} has no {_field} field set.") def get_fully_qualified_name(obj: object) -> str: @@ -150,7 +149,7 @@ def get_fully_qualified_name(obj: object) -> str: try: fqn += "." + obj.__class__.__name__ except Exception as e: - error(f"Failed to get FQN: {e}") + logger.error(f"Failed to get FQN: {e}") return fqn @@ -175,7 +174,7 @@ def key_emoji(key: object) -> str: hex_chars = bytes(key).hex()[-8:] return char_emoji(hex_chars=hex_chars) except Exception as e: - error(f"Fail to get key emoji: {e}") + logger.error(f"Fail to get key emoji: {e}") pass return "ALL" @@ -332,7 +331,7 @@ def find_available_port( sock.close() except Exception as e: - print(f"Failed to check port {port}. {e}") + logger.error(f"Failed to check port {port}. {e}") sock.close() if search is False and port_available is False: @@ -446,7 +445,7 @@ def obj2pointer_type(obj: object | None = None, fqn: str | None = None) -> type: except Exception as e: # sometimes the object doesn't have a __module__ so you need to use the type # like: collections.OrderedDict - debug( + logger.debug( f"Unable to get get_fully_qualified_name of {type(obj)} trying type. {e}" ) fqn = get_fully_qualified_name(obj=type(obj)) @@ -457,10 +456,8 @@ def obj2pointer_type(obj: object | None = None, fqn: str | None = None) -> type: try: ref = get_loaded_syft().lib_ast.query(fqn, obj_type=type(obj)) - except Exception as e: - log = f"Cannot find {type(obj)} {fqn} in lib_ast. {e}" - critical(log) - raise Exception(log) + except Exception: + raise Exception(f"Cannot find {type(obj)} {fqn} in lib_ast.") return ref.pointer_type @@ -919,3 +916,71 @@ def get_queue_address(port: int) -> str: def get_dev_mode() -> bool: return str_to_bool(os.getenv("DEV_MODE", "False")) + + +def sanitize_html(html: str) -> str: + policy = { + "tags": ["svg", "strong", "rect", "path", "circle"], + "attributes": { + "*": {"class", "style"}, + "svg": { + "class", + "style", + "xmlns", + "width", + "height", + "viewBox", + "fill", + "stroke", + "stroke-width", + }, + "path": {"d", "fill", "stroke", "stroke-width"}, + "rect": {"x", "y", "width", "height", "fill", "stroke", "stroke-width"}, + "circle": {"cx", "cy", "r", "fill", "stroke", "stroke-width"}, + }, + "remove": {"script", "style"}, + } + + tags = nh3.ALLOWED_TAGS + for tag in policy["tags"]: + tags.add(tag) + + _attributes = deepcopy(nh3.ALLOWED_ATTRIBUTES) + attributes = {**_attributes, **policy["attributes"]} # type: ignore + + return nh3.clean( + html, + tags=tags, + clean_content_tags=policy["remove"], + attributes=attributes, + ) + + +def parse_iso8601_date(date_string: str) -> datetime: + # Handle variable length of microseconds by trimming to 6 digits + if "." in date_string: + base_date, microseconds = date_string.split(".") + microseconds = microseconds.rstrip("Z") # Remove trailing 'Z' + microseconds = microseconds[:6] # Trim to 6 digits + date_string = f"{base_date}.{microseconds}Z" + return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S.%fZ") + + +def get_latest_tag(registry: str, repo: str) -> str | None: + repo_url = f"http://{registry}/v2/{repo}" + res = requests.get(url=f"{repo_url}/tags/list", timeout=5) + tags = res.json().get("tags", []) + + tag_times = [] + for tag in tags: + manifest_response = requests.get(f"{repo_url}/manifests/{tag}", timeout=5) + manifest = manifest_response.json() + created_time = json.loads(manifest["history"][0]["v1Compatibility"])["created"] + created_datetime = parse_iso8601_date(created_time) + tag_times.append((tag, created_datetime)) + + # sort tags by datetime + tag_times.sort(key=lambda x: x[1], reverse=True) + if len(tag_times) > 0: + return tag_times[0][0] + return None diff --git a/packages/syft/tests/conftest.py b/packages/syft/tests/conftest.py index c160034b532..2d781f817d7 100644 --- a/packages/syft/tests/conftest.py +++ b/packages/syft/tests/conftest.py @@ -25,19 +25,19 @@ # relative # our version of mongomock that has a fix for CodecOptions and custom TypeRegistry Support from .mongomock.mongo_client import MongoClient -from .syft.stores.store_fixtures_test import dict_action_store # noqa: F401 -from .syft.stores.store_fixtures_test import dict_document_store # noqa: F401 -from .syft.stores.store_fixtures_test import dict_queue_stash # noqa: F401 -from .syft.stores.store_fixtures_test import dict_store_partition # noqa: F401 -from .syft.stores.store_fixtures_test import mongo_action_store # noqa: F401 -from .syft.stores.store_fixtures_test import mongo_document_store # noqa: F401 -from .syft.stores.store_fixtures_test import mongo_queue_stash # noqa: F401 -from .syft.stores.store_fixtures_test import mongo_store_partition # noqa: F401 -from .syft.stores.store_fixtures_test import sqlite_action_store # noqa: F401 -from .syft.stores.store_fixtures_test import sqlite_document_store # noqa: F401 -from .syft.stores.store_fixtures_test import sqlite_queue_stash # noqa: F401 -from .syft.stores.store_fixtures_test import sqlite_store_partition # noqa: F401 -from .syft.stores.store_fixtures_test import sqlite_workspace # noqa: F401 +from .syft.stores.store_fixtures_test import dict_action_store +from .syft.stores.store_fixtures_test import dict_document_store +from .syft.stores.store_fixtures_test import dict_queue_stash +from .syft.stores.store_fixtures_test import dict_store_partition +from .syft.stores.store_fixtures_test import mongo_action_store +from .syft.stores.store_fixtures_test import mongo_document_store +from .syft.stores.store_fixtures_test import mongo_queue_stash +from .syft.stores.store_fixtures_test import mongo_store_partition +from .syft.stores.store_fixtures_test import sqlite_action_store +from .syft.stores.store_fixtures_test import sqlite_document_store +from .syft.stores.store_fixtures_test import sqlite_queue_stash +from .syft.stores.store_fixtures_test import sqlite_store_partition +from .syft.stores.store_fixtures_test import sqlite_workspace def patch_protocol_file(filepath: Path): diff --git a/packages/syft/tests/syft/action_test.py b/packages/syft/tests/syft/action_test.py index c28f5c31615..1f4dc0cc36b 100644 --- a/packages/syft/tests/syft/action_test.py +++ b/packages/syft/tests/syft/action_test.py @@ -17,7 +17,7 @@ def test_actionobject_method(worker): assert root_domain_client.settings.enable_eager_execution(enable=True) action_store = worker.get_service("actionservice").store obj = ActionObject.from_obj("abc") - pointer = root_domain_client.api.services.action.set(obj) + pointer = obj.send(root_domain_client) assert len(action_store.data) == 1 res = pointer.capitalize() assert len(action_store.data) == 2 diff --git a/packages/syft/tests/syft/dataset/fixtures.py b/packages/syft/tests/syft/dataset/fixtures.py index 7d92e1104bd..ca57f7e9220 100644 --- a/packages/syft/tests/syft/dataset/fixtures.py +++ b/packages/syft/tests/syft/dataset/fixtures.py @@ -47,6 +47,8 @@ def mock_asset(worker, root_domain_client) -> Asset: node_uid=worker.id, uploader=uploader, contributors=[uploader], + syft_node_location=worker.id, + syft_client_verify_key=root_domain_client.credentials.verify_key, ) node_transform_context = TransformContext( node=worker, diff --git a/packages/syft/tests/syft/eager_test.py b/packages/syft/tests/syft/eager_test.py index a597a79b2d6..7640e295cdc 100644 --- a/packages/syft/tests/syft/eager_test.py +++ b/packages/syft/tests/syft/eager_test.py @@ -20,11 +20,11 @@ def test_eager_permissions(worker, guest_client): mock_obj=np.array([[1, 1, 1], [1, 1, 1]]), ) - input_ptr = root_domain_client.api.services.action.set(input_obj) + input_ptr = input_obj.send(root_domain_client) pointer = guest_client.api.services.action.get_pointer(input_ptr.id) - input_ptr = root_domain_client.api.services.action.set(input_obj) + input_ptr = input_obj.send(root_domain_client) pointer = guest_client.api.services.action.get_pointer(input_ptr.id) @@ -53,8 +53,9 @@ def my_plan(x=np.array([[2, 2, 2], [2, 2, 2]])): # noqa: B008 mock_obj=np.array([[1, 1, 1], [1, 1, 1]]), ) - input_obj = root_domain_client.api.services.action.set(input_obj) - pointer = guest_client.api.services.action.get_pointer(input_obj.id) + input_ptr = input_obj.send(root_domain_client) + + pointer = guest_client.api.services.action.get_pointer(input_ptr.id) res_ptr = plan_ptr(x=pointer) # guest cannot access result @@ -95,7 +96,7 @@ def my_plan(x=np.array([[2, 2, 2], [2, 2, 2]])): # noqa: B008 mock_obj=np.array([[1, 1, 1], [1, 1, 1]]), ) - input_obj = root_domain_client.api.services.action.set(input_obj) + input_obj = input_obj.send(root_domain_client) pointer = guest_client.api.services.action.get_pointer(input_obj.id) res_ptr = plan_ptr(x=pointer) @@ -117,7 +118,7 @@ def my_plan(x=np.array([1, 2, 3, 4, 5, 6])): # noqa: B008 private_obj=np.array([1, 2, 3, 4, 5, 6]), mock_obj=np.array([1, 1, 1, 1, 1, 1]) ) - _id = root_domain_client.api.services.action.set(input_obj).id + _id = input_obj.send(root_domain_client).id pointer = guest_client.api.services.action.get_pointer(_id) res_ptr = plan_ptr(x=pointer) @@ -142,7 +143,7 @@ def test_setattribute(worker, guest_client): assert private_data.dtype != np.int32 - obj_pointer = root_domain_client.api.services.action.set(obj) + obj_pointer = obj.send(root_domain_client) obj_pointer = guest_client.api.services.action.get_pointer(obj_pointer.id) original_id = obj_pointer.id @@ -177,7 +178,7 @@ def test_getattribute(worker, guest_client): mock_obj=np.array([[1, 1, 1], [1, 1, 1]]), ) - obj_pointer = root_domain_client.api.services.action.set(obj) + obj_pointer = obj.send(root_domain_client) obj_pointer = guest_client.api.services.action.get_pointer(obj_pointer.id) size_pointer = obj_pointer.size @@ -196,7 +197,7 @@ def test_eager_method(worker, guest_client): mock_obj=np.array([[1, 1, 1], [1, 1, 1]]), ) - obj_pointer = root_domain_client.api.services.action.set(obj) + obj_pointer = obj.send(root_domain_client) obj_pointer = guest_client.api.services.action.get_pointer(obj_pointer.id) flat_pointer = obj_pointer.flatten() @@ -219,7 +220,7 @@ def test_eager_dunder_method(worker, guest_client): mock_obj=np.array([[1, 1, 1], [1, 1, 1]]), ) - obj_pointer = root_domain_client.api.services.action.set(obj) + obj_pointer = obj.send(root_domain_client) obj_pointer = guest_client.api.services.action.get_pointer(obj_pointer.id) first_row_pointer = obj_pointer[0] diff --git a/packages/syft/tests/syft/request/request_code_accept_deny_test.py b/packages/syft/tests/syft/request/request_code_accept_deny_test.py index e84f0360b12..b21a06579d1 100644 --- a/packages/syft/tests/syft/request/request_code_accept_deny_test.py +++ b/packages/syft/tests/syft/request/request_code_accept_deny_test.py @@ -80,7 +80,7 @@ def test_action_store_change(faker: Faker, worker: Worker): root_client = worker.root_client dummy_data = [1, 2, 3] data = ActionObject.from_obj(dummy_data) - action_obj = root_client.api.services.action.set(data) + action_obj = data.send(root_client) assert action_obj.get() == dummy_data @@ -120,7 +120,7 @@ def test_user_code_status_change(faker: Faker, worker: Worker): root_client = worker.root_client dummy_data = [1, 2, 3] data = ActionObject.from_obj(dummy_data) - action_obj = root_client.api.services.action.set(data) + action_obj = data.send(root_client) ds_client = get_ds_client(faker, root_client, worker.guest_client) @@ -168,7 +168,7 @@ def test_code_accept_deny(faker: Faker, worker: Worker): root_client = worker.root_client dummy_data = [1, 2, 3] data = ActionObject.from_obj(dummy_data) - action_obj = root_client.api.services.action.set(data) + action_obj = data.send(root_client) ds_client = get_ds_client(faker, root_client, worker.guest_client) @@ -183,13 +183,13 @@ def simple_function(data): assert not isinstance(result, SyftError) request = root_client.requests.get_all()[0] - result = request.accept_by_depositing_result(result=10) + result = request.approve() assert isinstance(result, SyftSuccess) request = root_client.requests.get_all()[0] assert request.status == RequestStatus.APPROVED result = ds_client.code.simple_function(data=action_obj) - assert result.get() == 10 + assert result.get() == sum(dummy_data) result = request.deny(reason="Function output needs differential privacy !!") assert isinstance(result, SyftSuccess) @@ -202,4 +202,4 @@ def simple_function(data): result = ds_client.code.simple_function(data=action_obj) assert isinstance(result, SyftError) - assert "Execution denied" in result.message + assert "DENIED" in result.message diff --git a/packages/syft/tests/syft/service/action/action_object_test.py b/packages/syft/tests/syft/service/action/action_object_test.py index a595fdd0e8d..dd7351f78e4 100644 --- a/packages/syft/tests/syft/service/action/action_object_test.py +++ b/packages/syft/tests/syft/service/action/action_object_test.py @@ -33,8 +33,8 @@ def helper_make_action_obj(orig_obj: Any): def helper_make_action_pointers(worker, obj, *args, **kwargs): root_domain_client = worker.root_client - root_domain_client.api.services.action.set(obj) - obj_pointer = root_domain_client.api.services.action.get_pointer(obj.id) + res = obj.send(root_domain_client) + obj_pointer = root_domain_client.api.services.action.get_pointer(res.id) # The args and kwargs should automatically be pointerized by obj_pointer return obj_pointer, args, kwargs diff --git a/packages/syft/tests/syft/service/action/action_service_test.py b/packages/syft/tests/syft/service/action/action_service_test.py index e4d9b663500..0c262f839a0 100644 --- a/packages/syft/tests/syft/service/action/action_service_test.py +++ b/packages/syft/tests/syft/service/action/action_service_test.py @@ -15,10 +15,10 @@ def get_auth_ctx(worker): def test_action_service_sanity(worker): service = worker.get_service("actionservice") + root_domain_client = worker.root_client obj = ActionObject.from_obj("abc") - - pointer = service.set(get_auth_ctx(worker), obj).ok() + pointer = obj.send(root_domain_client) assert len(service.store.data) == 1 res = pointer.capitalize() diff --git a/packages/syft/tests/syft/service/dataset/dataset_service_test.py b/packages/syft/tests/syft/service/dataset/dataset_service_test.py index a60bc653c13..1aa85c07865 100644 --- a/packages/syft/tests/syft/service/dataset/dataset_service_test.py +++ b/packages/syft/tests/syft/service/dataset/dataset_service_test.py @@ -5,8 +5,10 @@ # third party import numpy as np +import pandas as pd from pydantic import ValidationError import pytest +import torch # syft absolute import syft as sy @@ -253,3 +255,49 @@ def test_adding_contributors_with_duplicate_email(): assert isinstance(res3, SyftSuccess) assert isinstance(res4, SyftError) assert len(asset.contributors) == 1 + + +@pytest.fixture( + params=[ + 1, + "hello", + {"key": "value"}, + {1, 2, 3}, + np.array([1, 2, 3]), + pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}), + torch.Tensor([1, 2, 3]), + ] +) +def different_data_types( + request, +) -> int | str | dict | set | np.ndarray | pd.DataFrame | torch.Tensor: + return request.param + + +def test_upload_dataset_with_assets_of_different_data_types( + worker: Worker, + different_data_types: int + | str + | dict + | set + | np.ndarray + | pd.DataFrame + | torch.Tensor, +) -> None: + asset = sy.Asset( + name=random_hash(), + data=different_data_types, + mock=different_data_types, + ) + dataset = Dataset(name=random_hash()) + dataset.add_asset(asset) + root_domain_client = worker.root_client + res = root_domain_client.upload_dataset(dataset) + assert isinstance(res, SyftSuccess) + assert len(root_domain_client.api.services.dataset.get_all()) == 1 + assert type(root_domain_client.datasets[0].assets[0].data) == type( + different_data_types + ) + assert type(root_domain_client.datasets[0].assets[0].mock) == type( + different_data_types + ) diff --git a/packages/syft/tests/syft/service/sync/sync_resolve_single_test.py b/packages/syft/tests/syft/service/sync/sync_resolve_single_test.py index bced841db4a..83fba7fd168 100644 --- a/packages/syft/tests/syft/service/sync/sync_resolve_single_test.py +++ b/packages/syft/tests/syft/service/sync/sync_resolve_single_test.py @@ -1,5 +1,7 @@ # third party -from result import Err + +# third party +import numpy as np # syft absolute import syft @@ -8,7 +10,8 @@ from syft.client.sync_decision import SyncDecision from syft.client.syncing import compare_clients from syft.client.syncing import resolve -from syft.service.code.user_code import UserCode +from syft.service.job.job_stash import Job +from syft.service.request.request import RequestStatus from syft.service.response import SyftError from syft.service.response import SyftSuccess from syft.service.sync.resolve_widget import ResolveWidget @@ -22,6 +25,9 @@ def handle_decision( return widget.obj_diff_batch.ignore() elif decision in [SyncDecision.LOW, SyncDecision.HIGH]: return widget.click_sync() + elif decision == SyncDecision.SKIP: + # Skip is no-op + return SyftSuccess(message="skipped") else: raise ValueError(f"Unknown decision {decision}") @@ -32,6 +38,7 @@ def compare_and_resolve( to_client: DomainClient, decision: SyncDecision = SyncDecision.LOW, decision_callback: callable = None, + share_private_data: bool = True, ): diff_state_before = compare_clients(from_client, to_client) for obj_diff_batch in diff_state_before.active_batches: @@ -40,7 +47,8 @@ def compare_and_resolve( ) if decision_callback: decision = decision_callback(obj_diff_batch) - widget.click_share_all_private_data() + if share_private_data: + widget.click_share_all_private_data() res = handle_decision(widget, decision) assert isinstance(res, SyftSuccess) from_client.refresh() @@ -49,10 +57,32 @@ def compare_and_resolve( return diff_state_before, diff_state_after -def run_and_accept_result(client): - job_high = client.code.compute(blocking=True) - client.requests[0].accept_by_depositing_result(job_high) - return job_high +def run_and_deposit_result(client): + result = client.code.compute(blocking=True) + job = client.requests[0].deposit_result(result) + return job + + +def create_dataset(client): + mock = np.random.random(5) + private = np.random.random(5) + + dataset = sy.Dataset( + name=sy.util.util.random_name().lower(), + description="Lorem ipsum dolor sit amet, consectetur adipiscing elit", + asset_list=[ + sy.Asset( + name="numpy-data", + mock=mock, + data=private, + shape=private.shape, + mock_is_real=True, + ) + ], + ) + + client.upload_dataset(dataset) + return dataset @syft.syft_function_single_use() @@ -89,7 +119,7 @@ def compute() -> int: assert diff_state_after.is_same - run_and_accept_result(high_client) + run_and_deposit_result(high_client) diff_state_before, diff_state_after = compare_and_resolve( from_client=high_client, to_client=low_client ) @@ -105,6 +135,60 @@ def compute() -> int: assert res == compute(syft_no_node=True) +def test_diff_state_with_dataset(low_worker, high_worker): + low_client: DomainClient = low_worker.root_client + client_low_ds = get_ds_client(low_client) + high_client: DomainClient = high_worker.root_client + + _ = create_dataset(high_client) + _ = create_dataset(low_client) + + @sy.syft_function_single_use() + def compute_mean(data) -> int: + return data.mean() + + _ = client_low_ds.code.request_code_execution(compute_mean) + + result = client_low_ds.code.compute_mean(blocking=False) + assert isinstance(result, SyftError), "DS cannot start a job on low side" + + diff_state_before, diff_state_after = compare_and_resolve( + from_client=low_client, to_client=high_client + ) + + assert not diff_state_before.is_same + + assert diff_state_after.is_same + + # run_and_deposit_result(high_client) + data_high = high_client.datasets[0].assets[0] + result = high_client.code.compute_mean(data=data_high, blocking=True) + high_client.requests[0].deposit_result(result) + + diff_state_before, diff_state_after = compare_and_resolve( + from_client=high_client, to_client=low_client + ) + + high_state = high_client.get_sync_state() + low_state = high_client.get_sync_state() + assert high_state.get_previous_state_diff().is_same + assert low_state.get_previous_state_diff().is_same + assert diff_state_after.is_same + + client_low_ds.refresh() + + # check loading results for both blocking and non-blocking case + res_blocking = client_low_ds.code.compute_mean(blocking=True) + res_non_blocking = client_low_ds.code.compute_mean(blocking=False).wait() + + # expected_result = compute_mean(syft_no_node=True, data=) + assert ( + res_blocking + == res_non_blocking + == high_client.datasets[0].assets[0].data.mean() + ) + + def test_sync_with_error(low_worker, high_worker): """Check syncing with an error in a syft function""" low_client: DomainClient = low_worker.root_client @@ -126,7 +210,7 @@ def compute() -> int: assert diff_state_after.is_same - run_and_accept_result(high_client) + run_and_deposit_result(high_client) diff_state_before, diff_state_after = compare_and_resolve( from_client=high_client, to_client=low_client ) @@ -136,7 +220,7 @@ def compute() -> int: client_low_ds.refresh() res = client_low_ds.code.compute(blocking=True) - assert isinstance(res.get(), Err) + assert isinstance(res.get(), SyftError) def test_ignore_unignore_single(low_worker, high_worker): @@ -150,7 +234,7 @@ def compute() -> int: _ = client_low_ds.code.request_code_execution(compute) - diff = compare_clients(low_client, high_client) + diff = compare_clients(low_client, high_client, hide_usercode=False) assert len(diff.batches) == 2 # Request + UserCode assert len(diff.ignored_batches) == 0 @@ -159,7 +243,7 @@ def compute() -> int: res = diff[0].ignore() assert isinstance(res, SyftSuccess) - diff = compare_clients(low_client, high_client) + diff = compare_clients(low_client, high_client, hide_usercode=False) assert len(diff.batches) == 0 assert len(diff.ignored_batches) == 2 assert len(diff.all_batches) == 2 @@ -168,67 +252,119 @@ def compute() -> int: res = diff.ignored_batches[0].unignore() assert isinstance(res, SyftSuccess) - diff = compare_clients(low_client, high_client) + diff = compare_clients(low_client, high_client, hide_usercode=False) assert len(diff.batches) == 1 assert len(diff.ignored_batches) == 1 assert len(diff.all_batches) == 2 -def test_forget_usercode(low_worker, high_worker): +def test_request_code_execution_multiple(low_worker, high_worker): low_client = low_worker.root_client client_low_ds = low_worker.guest_client high_client = high_worker.root_client @sy.syft_function_single_use() def compute() -> int: - print("computing...") return 42 + @sy.syft_function_single_use() + def compute_twice() -> int: + return 42 * 2 + + @sy.syft_function_single_use() + def compute_thrice() -> int: + return 42 * 3 + _ = client_low_ds.code.request_code_execution(compute) + _ = client_low_ds.code.request_code_execution(compute_twice) diff_before, diff_after = compare_and_resolve( from_client=low_client, to_client=high_client ) - run_and_accept_result(high_client) - - def skip_if_user_code(diff): - if diff.root_type is UserCode: - return SyncDecision.IGNORE + assert not diff_before.is_same + assert diff_after.is_same - raise ValueError( - f"Should not reach here after ignoring user code, got {diff.root_type}" - ) + _ = client_low_ds.code.request_code_execution(compute_thrice) diff_before, diff_after = compare_and_resolve( - from_client=low_client, - to_client=high_client, - decision_callback=skip_if_user_code, + from_client=low_client, to_client=high_client ) + assert not diff_before.is_same - assert len(diff_after.batches) == 0 + assert diff_after.is_same -def test_request_code_execution_multiple(low_worker, high_worker): +def test_approve_request_on_sync_blocking(low_worker, high_worker): low_client = low_worker.root_client - client_low_ds = low_worker.guest_client + client_low_ds = get_ds_client(low_client) high_client = high_worker.root_client @sy.syft_function_single_use() def compute() -> int: return 42 - @sy.syft_function_single_use() - def compute_twice() -> int: - return 42 * 2 + _ = client_low_ds.code.request_code_execution(compute) + + # No execute permissions + result_error = client_low_ds.code.compute(blocking=True) + assert isinstance(result_error, SyftError) + assert low_client.requests[0].status == RequestStatus.PENDING + + # Sync request to high side + diff_before, diff_after = compare_and_resolve( + from_client=low_client, to_client=high_client + ) + + assert not diff_before.is_same + assert diff_after.is_same + + # Execute on high side + job = run_and_deposit_result(high_client) + assert job.result.get() == 42 + + assert high_client.requests[0].status == RequestStatus.PENDING + + # Sync back to low side, share private data + diff_before, diff_after = compare_and_resolve( + from_client=high_client, to_client=low_client, share_private_data=True + ) + assert len(diff_before.batches) == 1 and diff_before.batches[0].root_type is Job + assert low_client.requests[0].status == RequestStatus.APPROVED + + assert client_low_ds.code.compute().get() == 42 + assert len(client_low_ds.code.compute.jobs) == 1 + # check if user retrieved from cache, instead of re-executing + assert len(client_low_ds.requests[0].code.output_history) == 1 + + +def test_deny_and_sync(low_worker, high_worker): + low_client = low_worker.root_client + client_low_ds = get_ds_client(low_client) + high_client = high_worker.root_client @sy.syft_function_single_use() - def compute_thrice() -> int: - return 42 * 3 + def compute() -> int: + return 42 _ = client_low_ds.code.request_code_execution(compute) - _ = client_low_ds.code.request_code_execution(compute_twice) + # No execute permissions + result_error = client_low_ds.code.compute(blocking=True) + assert isinstance(result_error, SyftError) + assert low_client.requests[0].status == RequestStatus.PENDING + + # Deny on low side + request_low = low_client.requests[0] + res = request_low.deny(reason="bad request") + print(res) + assert low_client.requests[0].status == RequestStatus.REJECTED + + # Un-deny. NOTE: not supported by current UX, this is just used to re-deny on high side + low_client.api.code.update(id=request_low.code_id, l0_deny_reason=None) + assert low_client.requests[0].status == RequestStatus.PENDING + + # Sync request to high side diff_before, diff_after = compare_and_resolve( from_client=low_client, to_client=high_client ) @@ -236,11 +372,14 @@ def compute_thrice() -> int: assert not diff_before.is_same assert diff_after.is_same - _ = client_low_ds.code.request_code_execution(compute_thrice) + # Deny on high side + high_client.requests[0].deny(reason="bad request") + assert high_client.requests[0].status == RequestStatus.REJECTED diff_before, diff_after = compare_and_resolve( - from_client=low_client, to_client=high_client + from_client=high_client, to_client=low_client ) - assert not diff_before.is_same assert diff_after.is_same + + assert low_client.requests[0].status == RequestStatus.REJECTED diff --git a/packages/syft/tests/syft/types/dicttuple_test.py b/packages/syft/tests/syft/types/dicttuple_test.py index de32f2545bc..43beb3116c2 100644 --- a/packages/syft/tests/syft/types/dicttuple_test.py +++ b/packages/syft/tests/syft/types/dicttuple_test.py @@ -41,7 +41,7 @@ def test_dict_tuple_not_subclassing_mapping(): def test_should_iter_over_value(dict_tuple: DictTuple) -> None: values = [] for v in dict_tuple: - values.append(v) + values.append(v) # noqa: PERF402 assert values == [1, 2] diff --git a/packages/syft/tests/syft/users/user_code_test.py b/packages/syft/tests/syft/users/user_code_test.py index 53758e3c451..69f69ab76d4 100644 --- a/packages/syft/tests/syft/users/user_code_test.py +++ b/packages/syft/tests/syft/users/user_code_test.py @@ -4,6 +4,8 @@ # third party from faker import Faker import numpy as np +from pydantic import ValidationError +import pytest # syft absolute import syft as sy @@ -12,6 +14,7 @@ from syft.service.request.request import Request from syft.service.request.request import UserCodeStatusChange from syft.service.response import SyftError +from syft.service.response import SyftSuccess from syft.service.user.user import User @@ -59,7 +62,7 @@ def test_user_code(worker) -> None: request = message.link user_code = request.changes[0].code result = user_code.unsafe_function() - request.accept_by_depositing_result(result) + request.approve() result = guest_client.api.services.code.mock_syft_func() assert isinstance(result, ActionObject) @@ -74,23 +77,50 @@ def test_user_code(worker) -> None: assert multi_call_res.get() == result.get() -def test_duplicated_user_code(worker, guest_client: User) -> None: +def test_duplicated_user_code(worker) -> None: + worker.root_client.register( + name="Jane Doe", + email="jane@caltech.edu", + password="abc123", + password_verify="abc123", + institution="Caltech", + website="https://www.caltech.edu/", + ) + ds_client = worker.root_client.login( + email="jane@caltech.edu", + password="abc123", + ) + # mock_syft_func() - result = guest_client.api.services.code.request_code_execution(mock_syft_func) + result = ds_client.api.services.code.request_code_execution(mock_syft_func) assert isinstance(result, Request) - assert len(guest_client.code.get_all()) == 1 + assert len(ds_client.code.get_all()) == 1 # request the exact same code should return an error - result = guest_client.api.services.code.request_code_execution(mock_syft_func) + result = ds_client.api.services.code.request_code_execution(mock_syft_func) assert isinstance(result, SyftError) - assert len(guest_client.code.get_all()) == 1 + assert len(ds_client.code.get_all()) == 1 # request the a different function name but same content will also succeed # flaky if not blocking mock_syft_func_2(syft_no_node=True) - result = guest_client.api.services.code.request_code_execution(mock_syft_func_2) + result = ds_client.api.services.code.request_code_execution(mock_syft_func_2) assert isinstance(result, Request) - assert len(guest_client.code.get_all()) == 2 + assert len(ds_client.code.get_all()) == 2 + + code_history = ds_client.code_history + assert code_history.code_versions, "No code version found." + + code_histories = worker.root_client.code_histories + user_code_history = code_histories[ds_client.logged_in_user] + assert not isinstance(code_histories, SyftError) + assert not isinstance(user_code_history, SyftError) + assert user_code_history.code_versions, "No code version found." + assert user_code_history.mock_syft_func.user_code_history[0].status is not None + assert user_code_history.mock_syft_func[0]._repr_markdown_(), "repr markdown failed" + + result = user_code_history.mock_syft_func_2[0]() + assert result.get() == 1 def random_hash() -> str: @@ -310,22 +340,165 @@ def compute_sum(): ds_client.api.services.code.request_code_execution(compute_sum) - # no accept_by_depositing_result, no mock execution + # not approved, no mock execution result = ds_client.api.services.code.compute_sum() assert isinstance(result, SyftError) - # no accept_by_depositing_result, mock execution + # not approved, mock execution users[-1].allow_mock_execution() result = ds_client.api.services.code.compute_sum() + assert result, result assert result.get() == 1 - # accept_by_depositing_result, no mock execution + # approved, no mock execution users[-1].allow_mock_execution(allow=False) message = root_domain_client.notifications[-1] request = message.link user_code = request.changes[0].code result = user_code.unsafe_function() - request.accept_by_depositing_result(result) + request.approve() result = ds_client.api.services.code.compute_sum() + assert result, result assert result.get() == 1 + + +def test_submit_invalid_name(worker) -> None: + client = worker.root_client + + @sy.syft_function_single_use() + def valid_name(): + pass + + res = client.code.submit(valid_name) + assert isinstance(res, SyftSuccess) + + @sy.syft_function_single_use() + def get_all(): + pass + + assert isinstance(get_all, SyftError) + + @sy.syft_function_single_use() + def _(): + pass + + assert isinstance(_, SyftError) + + # overwrite valid function name before submit, fail on serde + @sy.syft_function_single_use() + def valid_name_2(): + pass + + valid_name_2.func_name = "get_all" + with pytest.raises(ValidationError): + client.code.submit(valid_name_2) + + +def test_request_existing_usercodesubmit(worker) -> None: + root_domain_client = worker.root_client + + root_domain_client.register( + name="data-scientist", + email="test_user@openmined.org", + password="0000", + password_verify="0000", + ) + ds_client = root_domain_client.login( + email="test_user@openmined.org", + password="0000", + ) + + @sy.syft_function_single_use() + def my_func(): + return 42 + + res_submit = ds_client.api.services.code.submit(my_func) + assert isinstance(res_submit, SyftSuccess) + res_request = ds_client.api.services.code.request_code_execution(my_func) + assert isinstance(res_request, Request) + + # Second request fails, cannot have multiple requests for the same code + res_request = ds_client.api.services.code.request_code_execution(my_func) + assert isinstance(res_request, SyftError) + + assert len(ds_client.code.get_all()) == 1 + assert len(ds_client.requests.get_all()) == 1 + + +def test_request_existing_usercode(worker) -> None: + root_domain_client = worker.root_client + + root_domain_client.register( + name="data-scientist", + email="test_user@openmined.org", + password="0000", + password_verify="0000", + ) + ds_client = root_domain_client.login( + email="test_user@openmined.org", + password="0000", + ) + + @sy.syft_function_single_use() + def my_func(): + return 42 + + res_submit = ds_client.api.services.code.submit(my_func) + assert isinstance(res_submit, SyftSuccess) + + code = ds_client.code.get_all()[0] + res_request = ds_client.api.services.code.request_code_execution(my_func) + assert isinstance(res_request, Request) + + # Second request fails, cannot have multiple requests for the same code + res_request = ds_client.api.services.code.request_code_execution(code) + assert isinstance(res_request, SyftError) + + assert len(ds_client.code.get_all()) == 1 + assert len(ds_client.requests.get_all()) == 1 + + +def test_submit_existing_code_different_user(worker): + root_domain_client = worker.root_client + + root_domain_client.register( + name="data-scientist", + email="test_user@openmined.org", + password="0000", + password_verify="0000", + ) + ds_client_1 = root_domain_client.login( + email="test_user@openmined.org", + password="0000", + ) + + root_domain_client.register( + name="data-scientist-2", + email="test_user_2@openmined.org", + password="0000", + password_verify="0000", + ) + ds_client_2 = root_domain_client.login( + email="test_user_2@openmined.org", + password="0000", + ) + + @sy.syft_function_single_use() + def my_func(): + return 42 + + res_submit = ds_client_1.api.services.code.submit(my_func) + assert isinstance(res_submit, SyftSuccess) + res_resubmit = ds_client_1.api.services.code.submit(my_func) + assert isinstance(res_resubmit, SyftError) + + # Resubmit with different user + res_submit = ds_client_2.api.services.code.submit(my_func) + assert isinstance(res_submit, SyftSuccess) + res_resubmit = ds_client_2.api.services.code.submit(my_func) + assert isinstance(res_resubmit, SyftError) + + assert len(ds_client_1.code.get_all()) == 1 + assert len(ds_client_2.code.get_all()) == 1 + assert len(root_domain_client.code.get_all()) == 2 diff --git a/packages/syft/tests/syft/users/user_service_test.py b/packages/syft/tests/syft/users/user_service_test.py index 7c0cc32562a..cd6d0aeefd4 100644 --- a/packages/syft/tests/syft/users/user_service_test.py +++ b/packages/syft/tests/syft/users/user_service_test.py @@ -219,7 +219,7 @@ def test_userservice_search( guest_user: User, ) -> None: def mock_find_all(credentials: SyftVerifyKey, **kwargs) -> Ok | Err: - for key, _ in kwargs.items(): + for key in kwargs.keys(): if hasattr(guest_user, key): return Ok([guest_user]) return Err("Invalid kwargs") diff --git a/packages/syftcli/manifest.yml b/packages/syftcli/manifest.yml index a9e200f513d..966706f9b11 100644 --- a/packages/syftcli/manifest.yml +++ b/packages/syftcli/manifest.yml @@ -1,11 +1,11 @@ manifestVersion: 1.0 -syftVersion: 0.8.7-beta.10 -dockerTag: 0.8.7-beta.10 +syftVersion: 0.8.7-beta.13 +dockerTag: 0.8.7-beta.13 images: - - docker.io/openmined/grid-frontend:0.8.7-beta.10 - - docker.io/openmined/grid-backend:0.8.7-beta.10 + - docker.io/openmined/grid-frontend:0.8.7-beta.13 + - docker.io/openmined/grid-backend:0.8.7-beta.13 - docker.io/library/mongo:7.0.4 - docker.io/traefik:v2.11.0 diff --git a/ruff.toml b/ruff.toml index 6d8e8a2f93a..bdf2c46b9cf 100644 --- a/ruff.toml +++ b/ruff.toml @@ -14,6 +14,7 @@ select = [ "F", # pyflake "B", # flake8-bugbear "C4", # flake8-comprehensions + # "PERF", # perflint "UP", # pyupgrade ] ignore = [ @@ -23,6 +24,7 @@ ignore = [ [lint.per-file-ignores] "*.ipynb" = ["E402"] +"__init__.py" = ["F401"] [lint.pycodestyle] max-line-length = 120 diff --git a/tests/integration/container_workload/pool_image_test.py b/tests/integration/container_workload/pool_image_test.py index a7fcd7d691a..23fa1d6e8fc 100644 --- a/tests/integration/container_workload/pool_image_test.py +++ b/tests/integration/container_workload/pool_image_test.py @@ -5,7 +5,6 @@ # third party import numpy as np import pytest -import requests # syft absolute import syft as sy @@ -19,13 +18,13 @@ from syft.service.worker.worker_pool import SyftWorker from syft.service.worker.worker_pool import WorkerPool from syft.types.uid import UID +from syft.util.util import get_latest_tag registry = os.getenv("SYFT_BASE_IMAGE_REGISTRY", "docker.io") repo = "openmined/grid-backend" if "k3d" in registry: - res = requests.get(url=f"http://{registry}/v2/{repo}/tags/list") - tag = res.json()["tags"][0] + tag = get_latest_tag(registry, repo) else: tag = sy.__version__ @@ -105,7 +104,8 @@ def test_image_build(domain_1_port: int, external_registry_uid: UID) -> None: @pytest.mark.container_workload -@pytest.mark.parametrize("prebuilt", [True, False]) +# @pytest.mark.parametrize("prebuilt", [True, False]) +@pytest.mark.parametrize("prebuilt", [False]) def test_pool_launch( domain_1_port: int, external_registry_uid: UID, prebuilt: bool ) -> None: @@ -114,6 +114,7 @@ def test_pool_launch( ) # Submit Worker Image + # nginx is intended to cause the startupProbe and livenessProbe to fail worker_config, docker_tag = ( (PrebuiltWorkerConfig(tag="docker.io/library/nginx:latest"), None) if prebuilt @@ -152,40 +153,51 @@ def test_pool_launch( worker_pool_res = domain_client.api.services.worker_pool.launch( pool_name=worker_pool_name, image_uid=worker_image.id, - num_workers=3, + num_workers=2, ) - assert not isinstance(worker_pool_res, SyftError) - assert all(worker.error is None for worker in worker_pool_res) + # TODO: we need to refactor this because the test is broken + if prebuilt: + # if the container has no liveness probe like nginx then _create_stateful_set + # will timeout with CREATE_POOL_TIMEOUT_SEC + # however this is currently longer than the blocking api call so we just see + # assert "timeout" in str(worker_pool_res).lower() + # if we lower the timout we get an exception here + # assert "Failed to start workers" in str(worker_pool_res) + pass + else: + assert not isinstance(worker_pool_res, SyftError) - worker_pool = domain_client.worker_pools[worker_pool_name] - assert len(worker_pool.worker_list) == 3 + assert all(worker.error is None for worker in worker_pool_res) - workers = worker_pool.workers - assert len(workers) == 3 + worker_pool = domain_client.worker_pools[worker_pool_name] + assert len(worker_pool.worker_list) == 2 - for worker in workers: - assert worker.worker_pool_name == worker_pool_name - assert worker.image.id == worker_image.id + workers = worker_pool.workers + assert len(workers) == 2 - assert len(worker_pool.healthy_workers) == 3 + for worker in workers: + assert worker.worker_pool_name == worker_pool_name + assert worker.image.id == worker_image.id - # Grab the first worker - first_worker = workers[0] + assert len(worker_pool.healthy_workers) == 2 - # Check worker Logs - logs = domain_client.api.services.worker.logs(uid=first_worker.id) - assert not isinstance(logs, sy.SyftError) + # Grab the first worker + first_worker = workers[0] - # Check for worker status - status_res = domain_client.api.services.worker.status(uid=first_worker.id) - assert not isinstance(status_res, sy.SyftError) - assert isinstance(status_res, tuple) + # Check worker Logs + logs = domain_client.api.services.worker.logs(uid=first_worker.id) + assert not isinstance(logs, sy.SyftError) - # Delete the pool's workers - for worker in worker_pool.workers: - res = domain_client.api.services.worker.delete(uid=worker.id, force=True) - assert isinstance(res, sy.SyftSuccess) + # Check for worker status + status_res = domain_client.api.services.worker.status(uid=first_worker.id) + assert not isinstance(status_res, sy.SyftError) + assert isinstance(status_res, tuple) + + # Delete the pool's workers + for worker in worker_pool.workers: + res = domain_client.api.services.worker.delete(uid=worker.id, force=True) + assert isinstance(res, sy.SyftSuccess) # TODO: delete the launched pool @@ -274,7 +286,7 @@ def test_pool_image_creation_job_requests( # Dataset data = np.array([1, 2, 3]) data_action_obj = sy.ActionObject.from_obj(data) - data_pointer = ds_client.api.services.action.set(data_action_obj) + data_pointer = data_action_obj.send(ds_client) # Function @sy.syft_function( diff --git a/tests/integration/local/request_multiple_nodes_test.py b/tests/integration/local/request_multiple_nodes_test.py index e81f75b57d6..4b0a2950c66 100644 --- a/tests/integration/local/request_multiple_nodes_test.py +++ b/tests/integration/local/request_multiple_nodes_test.py @@ -7,8 +7,6 @@ # syft absolute import syft as sy -from syft.service.job.job_stash import Job -from syft.service.job.job_stash import JobStatus @pytest.fixture(scope="function") @@ -107,94 +105,3 @@ def dataset_2(client_do_2): client_do_2.upload_dataset(dataset) return client_do_2.datasets[0].assets[0] - - -@pytest.mark.flaky(reruns=3, reruns_delay=3) -@pytest.mark.local_node -def test_transfer_request_blocking( - client_ds_1, client_do_1, client_do_2, dataset_1, dataset_2 -): - @sy.syft_function_single_use(data=dataset_1) - def compute_sum(data) -> float: - return data.mean() - - client_ds_1.code.request_code_execution(compute_sum) - - # Submit + execute on second node - request_1_do = client_do_1.requests[0] - client_do_2.sync_code_from_request(request_1_do) - - # DO executes + syncs - client_do_2._fetch_api(client_do_2.credentials) - result_2 = client_do_2.code.compute_sum(data=dataset_2).get() - assert result_2 == dataset_2.data.mean() - res = request_1_do.accept_by_depositing_result(result_2) - assert isinstance(res, sy.SyftSuccess) - - # DS gets result blocking + nonblocking - result_ds_blocking = client_ds_1.code.compute_sum( - data=dataset_1, blocking=True - ).get() - - job_1_ds = client_ds_1.code.compute_sum(data=dataset_1, blocking=False) - assert isinstance(job_1_ds, Job) - assert job_1_ds == client_ds_1.code.compute_sum.jobs[-1] - assert job_1_ds.status == JobStatus.COMPLETED - - result_ds_nonblocking = job_1_ds.wait().get() - - assert result_ds_blocking == result_ds_nonblocking == dataset_2.data.mean() - - -@pytest.mark.flaky(reruns=3, reruns_delay=3) -@pytest.mark.local_node -def test_transfer_request_nonblocking( - client_ds_1, client_do_1, client_do_2, dataset_1, dataset_2 -): - @sy.syft_function_single_use(data=dataset_1) - def compute_mean(data) -> float: - return data.mean() - - client_ds_1.code.request_code_execution(compute_mean) - - # Submit + execute on second node - request_1_do = client_do_1.requests[0] - client_do_2.sync_code_from_request(request_1_do) - - client_do_2._fetch_api(client_do_2.credentials) - job_2 = client_do_2.code.compute_mean(data=dataset_2, blocking=False) - assert isinstance(job_2, Job) - - # Transfer back Job Info - job_2_info = job_2.info() - assert job_2_info.result is None - assert job_2_info.status is not None - res = request_1_do.sync_job(job_2_info) - assert isinstance(res, sy.SyftSuccess) - - # DS checks job info - job_1_ds = client_ds_1.code.compute_mean.jobs[-1] - assert job_1_ds.status == job_2.status - - # DO finishes + syncs job result - result = job_2.wait().get() - assert result == dataset_2.data.mean() - assert job_2.status == JobStatus.COMPLETED - - job_2_info_with_result = job_2.info(result=True) - res = request_1_do.accept_by_depositing_result(job_2_info_with_result) - assert isinstance(res, sy.SyftSuccess) - - # DS gets result blocking + nonblocking - result_ds_blocking = client_ds_1.code.compute_mean( - data=dataset_1, blocking=True - ).get() - - job_1_ds = client_ds_1.code.compute_mean(data=dataset_1, blocking=False) - assert isinstance(job_1_ds, Job) - assert job_1_ds == client_ds_1.code.compute_mean.jobs[-1] - assert job_1_ds.status == JobStatus.COMPLETED - - result_ds_nonblocking = job_1_ds.wait().get() - - assert result_ds_blocking == result_ds_nonblocking == dataset_2.data.mean() diff --git a/tests/integration/local/twin_api_sync_test.py b/tests/integration/local/twin_api_sync_test.py index e09c82001d1..b30b4e50382 100644 --- a/tests/integration/local/twin_api_sync_test.py +++ b/tests/integration/local/twin_api_sync_test.py @@ -3,7 +3,6 @@ # third party import pytest -from result import Err # syft absolute import syft @@ -11,6 +10,7 @@ from syft.client.domain_client import DomainClient from syft.client.syncing import compare_clients from syft.client.syncing import resolve +from syft.service.action.action_object import ActionObject from syft.service.job.job_stash import JobStatus from syft.service.response import SyftError from syft.service.response import SyftSuccess @@ -100,7 +100,7 @@ def compute(query): ) job_high = high_client.code.compute(query=high_client.api.services.testapi.query) - high_client.requests[0].accept_by_depositing_result(job_high) + high_client.requests[0].deposit_result(job_high) diff_before, diff_after = compare_and_resolve( from_client=high_client, to_client=low_client ) @@ -121,6 +121,37 @@ def compute(query): private_res, SyftError ), "Should not be able to access private function on low side." + # verify updating twin api endpoint works + + timeout_before = ( + full_low_worker.python_node.get_service("apiservice") + .stash.get_all( + credentials=full_low_worker.client.credentials, has_permission=True + ) + .ok()[0] + .endpoint_timeout + ) + expected_timeout_after = timeout_before + 1 + + high_client.custom_api.update( + endpoint_path="testapi.query", endpoint_timeout=expected_timeout_after + ) + widget = sy.sync(from_client=high_client, to_client=low_client) + result = widget[0].click_sync() + assert result, result + + timeout_after = ( + full_low_worker.python_node.get_service("apiservice") + .stash.get_all( + credentials=full_low_worker.client.credentials, has_permission=True + ) + .ok()[0] + .endpoint_timeout + ) + assert ( + timeout_after == expected_timeout_after + ), "Timeout should be updated on low side." + @pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") @pytest.mark.local_node @@ -149,9 +180,11 @@ def compute_sum(): users[-1].allow_mock_execution() result = ds_client.api.services.code.compute_sum(blocking=True) - assert isinstance(result.get(), Err) + assert isinstance(result, ActionObject) + assert isinstance(result.get(), SyftError) job_info = ds_client.api.services.code.compute_sum(blocking=False) result = job_info.wait(timeout=10) - assert isinstance(result.get(), Err) + assert isinstance(result, ActionObject) + assert isinstance(result.get(), SyftError) assert job_info.status == JobStatus.ERRORED diff --git a/tox.ini b/tox.ini index 3f8d1601474..d15fd50c6da 100644 --- a/tox.ini +++ b/tox.ini @@ -207,8 +207,14 @@ deps = {[testenv:syft]deps} jupyter jupyterlab +allowlist_externals = + bash commands = - jupyter lab --ip 0.0.0.0 --ServerApp.token={posargs} + bash -c 'if [ -z "{posargs}" ]; then \ + jupyter lab --ip 0.0.0.0; \ + else \ + jupyter lab --ip 0.0.0.0 --ServerApp.token={posargs}; \ + fi' [testenv:syft.protocol.check] description = Syft Protocol Check
  • \n", - " \n", - " #\n", - " \n", - " \n", - " \n", - " Message\n", - " \n", - "
    \n", - "
    \n", - " 0\n", - "
    \n", - "
    \n", - "
    \n", - " Subjob Iter 0\n", - "
    \n", - "
    \n", - "
    \n", - " 1\n", - "
    \n", - "
    \n", - "
    \n", - " Subjob Iter 1\n", - "
    \n", - "
    \n", - "
    \n", - " 2\n", - "
    \n", - "
    \n", - "
    \n", - " \n", - "
    \n", - "