merge dev

.gitignore

@@ -73,5 +73,7 @@ nohup.out
 
 # notebook data
 notebooks/helm/scenario_data.jsonl
+
 # tox syft.build.helm generated file
-out.txt
+out.*
+.git-blame-ignore-revs

packages/grid/devspace.yaml

@@ -22,9 +22,10 @@ pipelines:
       create_deployments --all
 
 vars:
-  DEVSPACE_ENV_FILE: "default.env"
+  DOCKER_IMAGE_BACKEND: openmined/grid-backend
+  DOCKER_IMAGE_FRONTEND: openmined/grid-frontend
+  DOCKER_IMAGE_SEAWEEDFS: openmined/grid-seaweedfs
   CONTAINER_REGISTRY: "docker.io"
-  NODE_NAME: "mynode"
   VERSION: "0.8.7-beta.1"
   PLATFORM: $(uname -m | grep -q 'arm64' && echo "arm64" || echo "amd64")
 
@@ -68,7 +69,7 @@ deployments:
           registry: ${CONTAINER_REGISTRY}
           version: dev-${DEVSPACE_TIMESTAMP}
         node:
-          name: ${NODE_NAME}
+          type: domain # required for the gateway profile
       # anything that does not need devspace $env vars should go in values.dev.yaml
       valuesFiles:
         - ./helm/values.dev.yaml
@@ -117,3 +118,24 @@ profiles:
         path: images.seaweedfs
       - op: remove
         path: dev.seaweedfs
+
+  - name: gcp
+    patches:
+      - op: replace
+        path: deployments.syft.helm.valuesFiles
+        value:
+          - ./helm/examples/gcp/gcp.high.yaml
+
+  - name: gcp-low
+    patches:
+      - op: replace
+        path: deployments.syft.helm.valuesFiles
+        value:
+          - ./helm/examples/gcp/gcp.low.yaml
+
+  - name: azure
+    patches:
+      - op: replace
+        path: deployments.syft.helm.valuesFiles
+        value:
+          - ./helm/examples/azure/azure.high.yaml

packages/grid/helm/examples/azure/azure.high.yaml

@@ -0,0 +1,34 @@
+# ===================================================
+# Deploy Syft High Side Node to an AKS Cluster
+# DOES NOT sync seaweedfs to a Azure bucket
+# ===================================================
+
+node:
+  name: syft-azure
+  side: high
+  rootEmail: [email protected]
+  defaultWorkerPoolCount: 1
+  resourcesPreset: 2xlarge
+
+ingress:
+  # Make sure cluster is created with --enable-app-routing
+  # az aks create -g group-name -n cluster-name -l region --enable-app-routing
+  className: webapprouting.kubernetes.azure.com
+  # or, has azure-application-gateway setup through dashboard or CLI
+  # className: azure-application-gateway
+
+
+seaweedfs:
+  resourcesPreset: 2xlarge
+
+proxy:
+  resourcesPreset: medium
+
+registry:
+  resourcesPreset: medium
+
+frontend:
+  resourcesPreset: medium
+
+mongo:
+  resourcesPreset: large

packages/grid/helm/examples/gcp/README.md

@@ -0,0 +1,41 @@
+# GCS Deployment
+
+## Resource Links
+* [Autopilot Overview](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-overview)
+* [Autopilot Resource Limits Defaults](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-resource-requests#compute-class-defaults)
+* [AutoPilot Resource Limits Min/Max](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-resource-requests#min-max-requests)
+* [Compute Classes](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-compute-classes)
+* [Performance Pods](https://cloud.google.com/kubernetes-engine/docs/how-to/performance-pods)
+
+## Setup
+
+Helm `values.yaml` for high & low side deployments
+- [`gcp.high.yaml`](./gcp.high.yaml)
+- [`gcp.low.yaml`](./gcp.low.yaml)
+
+Deployment on GKE with SeaweedFS sync to GCS requires:
+1. A GCS bucket in the same project where the cluster will be deployed
+    * `syft-bucket-high` (for high side deployment)
+    * `syft-bucket-low` (for low side deployment)
+
+2. An IAM service account with sufficient permissions to read/write/delete object to these buckets
+
+## For Autopilot Cluster
+
+Uncomment the `nodeSelector` to use a specific compute class + machine family.
+
+Set resource limits in-line with the machine family
+
+```yaml
+  nodeSelector:
+    cloud.google.com/compute-class: Performance
+    cloud.google.com/machine-family: c3
+
+  resources:
+    requests:
+      cpu: 2
+      memory: "8Gi"
+    limits:
+      cpu: 4
+      memory: "16Gi"
+```

packages/grid/helm/examples/gcp/gcp.high.yaml

@@ -0,0 +1,81 @@
+# ===================================================
+# Deploy Syft High Side Node to a GKE Cluster
+# Syncs seaweedfs to a GCS bucket "syft-bucket-high"
+# ===================================================
+
+node:
+  name: syft-gcp
+  side: high
+
+  # Force backend to write results to this bucket
+  # should be same as mountApi.mounts.local_bucket
+  defaultBucketName: syft-bucket-high-gcs
+
+  # For autopilot clusters with GKE 1.28+, uncomment this
+  # nodeSelector:
+  #   cloud.google.com/compute-class: Performance
+  #   cloud.google.com/machine-family: c3
+
+  # Resources set inline with c3-standard-4 machine type
+  resources:
+    requests:
+      cpu: 2
+      memory: "8Gi"
+    limits:
+      cpu: 4
+      memory: "16Gi"
+
+seaweedfs:
+  storageSize: 100Gi
+
+  mountApi:
+    # Mount remote GCS bucket named "syft-bucket-high" to seaweedfs bucket named "syft-bucket-high-gcs
+    mounts:
+      - local_bucket: syft-bucket-high-gcs
+        remote_bucket:
+          type: gcs
+          bucket_name: syft-bucket-high
+          # filename should be as secret.mountCredentials.name
+          # directory will always be /run/secrets/
+          creds: /run/secrets/gcs_creds.json
+
+  secret:
+    mountCredentials:
+      # remove this comment and paste the Service Account JSON Here
+      # TODO: move to workload identity
+      - name: gcs_creds.json
+        value: |
+          {
+            "": ""
+          }
+
+  # For autopilot clusters with GKE 1.28+, uncomment this
+  # nodeSelector:
+  #   cloud.google.com/compute-class: Performance
+  #   cloud.google.com/machine-family: c3
+
+  # Resources set inline with c3-standard-4 machine type
+  resources:
+    requests:
+      cpu: 2
+      memory: "8Gi"
+    limits:
+      cpu: 4
+      memory: "16Gi"
+
+ingress:
+  # GKE 1.28+ need ingressClassName
+  className: gce
+  class: gce
+
+proxy:
+  resourcesPreset: medium
+
+registry:
+  resourcesPreset: medium
+
+frontend:
+  resourcesPreset: medium
+
+mongo:
+  resourcesPreset: large

packages/grid/helm/examples/gcp/gcp.low.yaml

@@ -0,0 +1,82 @@
+# ===================================================
+# Deploy Syft Low Side Node to a GKE Cluster
+# Syncs seaweedfs to a GCS bucket "syft-bucket-low"
+# ===================================================
+
+node:
+  name: syft-gcp
+  side: low
+
+  # Force backend to write results to this bucket
+  # should be same as mountApi.mounts.local_bucket
+  defaultBucketName: syft-bucket-low-gcs
+
+  # For autopilot clusters with GKE 1.28+, uncomment this
+  # nodeSelector:
+  #   cloud.google.com/compute-class: Performance
+  #   cloud.google.com/machine-family: c3
+
+  # Resources set inline with c3-standard-4 machine type
+  resources:
+    requests:
+      cpu: 2
+      memory: "8Gi"
+    limits:
+      cpu: 4
+      memory: "16Gi"
+
+seaweedfs:
+  storageSize: 100Gi
+
+  mountApi:
+    # Mount remote GCS bucket named "syft-bucket-low" to seaweedfs bucket named "syft-bucket-low-gcs
+    mounts:
+      - local_bucket: syft-bucket-low-gcs
+        remote_bucket:
+          type: gcs
+          bucket_name: syft-bucket-low
+          # filename should be as secret.mountCredentials.name
+          # directory will always be /run/secrets/
+          creds: /run/secrets/gcs_creds.json
+
+  secret:
+    mountCredentials:
+      # remove this comment and paste the Service Account JSON Here
+      # TODO: move to workload identity
+      - name: gcs_creds.json
+        value: |
+          {
+            "": ""
+          }
+
+  # For autopilot clusters with GKE 1.28+, uncomment this
+  # nodeSelector:
+  #   cloud.google.com/compute-class: Performance
+  #   cloud.google.com/machine-family: c3
+
+  # Resources set inline with c3-standard-4 machine type
+  resources:
+    requests:
+      cpu: 2
+      memory: "8Gi"
+    limits:
+      cpu: 4
+      memory: "16Gi"
+
+ingress:
+  # GKE 1.28+ need ingressClassName
+  className: gce
+  class: gce
+
+proxy:
+  resourcesPreset: medium
+
+registry:
+  resourcesPreset: medium
+  storageSize: 50Gi
+
+frontend:
+  resourcesPreset: medium
+
+mongo:
+  resourcesPreset: large

packages/grid/helm/examples/gcp/gcp.nosync.yaml

@@ -0,0 +1,47 @@
+# ===================================================
+# Deploy Syft High Side Node to a GKE Cluster
+# DOES NOT sync seaweedfs to a GCS bucket
+# ===================================================
+
+node:
+  name: syft-gcp
+  side: high
+
+  # Resources set inline with c3-standard-4 machine type
+  resources:
+    requests:
+      cpu: 2
+      memory: "8Gi"
+    limits:
+      cpu: 4
+      memory: "16Gi"
+
+seaweedfs:
+  storageSize: 100Gi
+
+  # Resources set inline with c3-standard-4 machine type
+  resources:
+    requests:
+      cpu: 2
+      memory: "8Gi"
+    limits:
+      cpu: 4
+      memory: "16Gi"
+
+ingress:
+  # GKE 1.28+ need ingressClassName
+  className: gce
+  class: gce
+
+proxy:
+  resourcesPreset: medium
+
+registry:
+  resourcesPreset: medium
+  storageSize: 100Gi
+
+frontend:
+  resourcesPreset: medium
+
+mongo:
+  resourcesPreset: large

packages/grid/helm/syft/templates/backend/backend-statefulset.yaml

@@ -20,7 +20,16 @@ spec:
       labels:
         {{- include "common.labels" . | nindent 8 }}
         app.kubernetes.io/component: backend
+        {{- if .Values.node.podLabels }}
+        {{- toYaml .Values.node.podLabels | nindent 8 }}
+        {{- end }}
+      {{- if .Values.node.podAnnotations }}
+      annotations: {{- toYaml .Values.node.podAnnotations | nindent 8 }}
+      {{- end }}
     spec:
+      {{- if .Values.node.nodeSelector }}
+      nodeSelector: {{- .Values.node.nodeSelector | toYaml | nindent 8 }}
+      {{- end }}
       containers:
         - name: backend-container
           image: {{ .Values.global.registry }}/openmined/grid-backend:{{ .Values.global.version }}
@@ -42,7 +51,7 @@ spec:
             - name: NODE_TYPE
               value: {{ .Values.node.type | default "domain" | quote }}
             - name: NODE_NAME
-              value: {{ .Values.node.name | default .Release.Name | quote }}
+              value: {{ .Values.node.name | default (printf "%s-node" .Release.Name) | quote }}
             - name: NODE_SIDE_TYPE
               value: {{ .Values.node.side | quote }}
             - name: DEFAULT_ROOT_EMAIL
@@ -104,8 +113,6 @@ spec:
                 secretKeyRef:
                     name: {{ .Values.seaweedfs.secretKeyName | required "seaweedfs.secretKeyName is required" }}
                     key: s3RootPassword
-            - name: SEAWEED_MOUNT_PORT
-              value: {{ .Values.seaweedfs.mountApi.port | quote }}
             {{- end }}
             # Tracing
             - name: TRACE

packages/grid/helm/syft/templates/frontend/frontend-deployment.yaml

@@ -18,7 +18,16 @@ spec:
       labels:
         {{- include "common.labels" . | nindent 8 }}
         app.kubernetes.io/component: frontend
+        {{- if .Values.frontend.podLabels }}
+        {{- toYaml .Values.frontend.podLabels | nindent 8 }}
+        {{- end }}
+      {{- if .Values.frontend.podAnnotations }}
+      annotations: {{- toYaml .Values.frontend.podAnnotations | nindent 8 }}
+      {{- end }}
     spec:
+      {{- if .Values.frontend.nodeSelector }}
+      nodeSelector: {{- .Values.frontend.nodeSelector | toYaml | nindent 8 }}
+      {{- end }}
       containers:
         - name: frontend-container
           image: {{ .Values.global.registry }}/openmined/grid-frontend:{{ .Values.global.version }}