Skip to content

Commit

Permalink
merge dev
Browse files Browse the repository at this point in the history
  • Loading branch information
koenvanderveen committed Apr 18, 2024
2 parents f2a6ab8 + 74b6410 commit 91ab7a0
Show file tree
Hide file tree
Showing 26 changed files with 807 additions and 139 deletions.
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -73,5 +73,7 @@ nohup.out

# notebook data
notebooks/helm/scenario_data.jsonl

# tox syft.build.helm generated file
out.txt
out.*
.git-blame-ignore-revs
28 changes: 25 additions & 3 deletions packages/grid/devspace.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,10 @@ pipelines:
create_deployments --all
vars:
DEVSPACE_ENV_FILE: "default.env"
DOCKER_IMAGE_BACKEND: openmined/grid-backend
DOCKER_IMAGE_FRONTEND: openmined/grid-frontend
DOCKER_IMAGE_SEAWEEDFS: openmined/grid-seaweedfs
CONTAINER_REGISTRY: "docker.io"
NODE_NAME: "mynode"
VERSION: "0.8.7-beta.1"
PLATFORM: $(uname -m | grep -q 'arm64' && echo "arm64" || echo "amd64")

Expand Down Expand Up @@ -68,7 +69,7 @@ deployments:
registry: ${CONTAINER_REGISTRY}
version: dev-${DEVSPACE_TIMESTAMP}
node:
name: ${NODE_NAME}
type: domain # required for the gateway profile
# anything that does not need devspace $env vars should go in values.dev.yaml
valuesFiles:
- ./helm/values.dev.yaml
Expand Down Expand Up @@ -117,3 +118,24 @@ profiles:
path: images.seaweedfs
- op: remove
path: dev.seaweedfs

- name: gcp
patches:
- op: replace
path: deployments.syft.helm.valuesFiles
value:
- ./helm/examples/gcp/gcp.high.yaml

- name: gcp-low
patches:
- op: replace
path: deployments.syft.helm.valuesFiles
value:
- ./helm/examples/gcp/gcp.low.yaml

- name: azure
patches:
- op: replace
path: deployments.syft.helm.valuesFiles
value:
- ./helm/examples/azure/azure.high.yaml
34 changes: 34 additions & 0 deletions packages/grid/helm/examples/azure/azure.high.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# ===================================================
# Deploy Syft High Side Node to an AKS Cluster
# DOES NOT sync seaweedfs to a Azure bucket
# ===================================================

node:
name: syft-azure
side: high
rootEmail: [email protected]
defaultWorkerPoolCount: 1
resourcesPreset: 2xlarge

ingress:
# Make sure cluster is created with --enable-app-routing
# az aks create -g group-name -n cluster-name -l region --enable-app-routing
className: webapprouting.kubernetes.azure.com
# or, has azure-application-gateway setup through dashboard or CLI
# className: azure-application-gateway


seaweedfs:
resourcesPreset: 2xlarge

proxy:
resourcesPreset: medium

registry:
resourcesPreset: medium

frontend:
resourcesPreset: medium

mongo:
resourcesPreset: large
41 changes: 41 additions & 0 deletions packages/grid/helm/examples/gcp/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# GCS Deployment

## Resource Links
* [Autopilot Overview](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-overview)
* [Autopilot Resource Limits Defaults](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-resource-requests#compute-class-defaults)
* [AutoPilot Resource Limits Min/Max](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-resource-requests#min-max-requests)
* [Compute Classes](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-compute-classes)
* [Performance Pods](https://cloud.google.com/kubernetes-engine/docs/how-to/performance-pods)

## Setup

Helm `values.yaml` for high & low side deployments
- [`gcp.high.yaml`](./gcp.high.yaml)
- [`gcp.low.yaml`](./gcp.low.yaml)

Deployment on GKE with SeaweedFS sync to GCS requires:
1. A GCS bucket in the same project where the cluster will be deployed
* `syft-bucket-high` (for high side deployment)
* `syft-bucket-low` (for low side deployment)

2. An IAM service account with sufficient permissions to read/write/delete object to these buckets

## For Autopilot Cluster

Uncomment the `nodeSelector` to use a specific compute class + machine family.

Set resource limits in-line with the machine family

```yaml
nodeSelector:
cloud.google.com/compute-class: Performance
cloud.google.com/machine-family: c3

resources:
requests:
cpu: 2
memory: "8Gi"
limits:
cpu: 4
memory: "16Gi"
```
81 changes: 81 additions & 0 deletions packages/grid/helm/examples/gcp/gcp.high.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
# ===================================================
# Deploy Syft High Side Node to a GKE Cluster
# Syncs seaweedfs to a GCS bucket "syft-bucket-high"
# ===================================================

node:
name: syft-gcp
side: high

# Force backend to write results to this bucket
# should be same as mountApi.mounts.local_bucket
defaultBucketName: syft-bucket-high-gcs

# For autopilot clusters with GKE 1.28+, uncomment this
# nodeSelector:
# cloud.google.com/compute-class: Performance
# cloud.google.com/machine-family: c3

# Resources set inline with c3-standard-4 machine type
resources:
requests:
cpu: 2
memory: "8Gi"
limits:
cpu: 4
memory: "16Gi"

seaweedfs:
storageSize: 100Gi

mountApi:
# Mount remote GCS bucket named "syft-bucket-high" to seaweedfs bucket named "syft-bucket-high-gcs
mounts:
- local_bucket: syft-bucket-high-gcs
remote_bucket:
type: gcs
bucket_name: syft-bucket-high
# filename should be as secret.mountCredentials.name
# directory will always be /run/secrets/
creds: /run/secrets/gcs_creds.json

secret:
mountCredentials:
# remove this comment and paste the Service Account JSON Here
# TODO: move to workload identity
- name: gcs_creds.json
value: |
{
"": ""
}
# For autopilot clusters with GKE 1.28+, uncomment this
# nodeSelector:
# cloud.google.com/compute-class: Performance
# cloud.google.com/machine-family: c3

# Resources set inline with c3-standard-4 machine type
resources:
requests:
cpu: 2
memory: "8Gi"
limits:
cpu: 4
memory: "16Gi"

ingress:
# GKE 1.28+ need ingressClassName
className: gce
class: gce

proxy:
resourcesPreset: medium

registry:
resourcesPreset: medium

frontend:
resourcesPreset: medium

mongo:
resourcesPreset: large
82 changes: 82 additions & 0 deletions packages/grid/helm/examples/gcp/gcp.low.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
# ===================================================
# Deploy Syft Low Side Node to a GKE Cluster
# Syncs seaweedfs to a GCS bucket "syft-bucket-low"
# ===================================================

node:
name: syft-gcp
side: low

# Force backend to write results to this bucket
# should be same as mountApi.mounts.local_bucket
defaultBucketName: syft-bucket-low-gcs

# For autopilot clusters with GKE 1.28+, uncomment this
# nodeSelector:
# cloud.google.com/compute-class: Performance
# cloud.google.com/machine-family: c3

# Resources set inline with c3-standard-4 machine type
resources:
requests:
cpu: 2
memory: "8Gi"
limits:
cpu: 4
memory: "16Gi"

seaweedfs:
storageSize: 100Gi

mountApi:
# Mount remote GCS bucket named "syft-bucket-low" to seaweedfs bucket named "syft-bucket-low-gcs
mounts:
- local_bucket: syft-bucket-low-gcs
remote_bucket:
type: gcs
bucket_name: syft-bucket-low
# filename should be as secret.mountCredentials.name
# directory will always be /run/secrets/
creds: /run/secrets/gcs_creds.json

secret:
mountCredentials:
# remove this comment and paste the Service Account JSON Here
# TODO: move to workload identity
- name: gcs_creds.json
value: |
{
"": ""
}
# For autopilot clusters with GKE 1.28+, uncomment this
# nodeSelector:
# cloud.google.com/compute-class: Performance
# cloud.google.com/machine-family: c3

# Resources set inline with c3-standard-4 machine type
resources:
requests:
cpu: 2
memory: "8Gi"
limits:
cpu: 4
memory: "16Gi"

ingress:
# GKE 1.28+ need ingressClassName
className: gce
class: gce

proxy:
resourcesPreset: medium

registry:
resourcesPreset: medium
storageSize: 50Gi

frontend:
resourcesPreset: medium

mongo:
resourcesPreset: large
47 changes: 47 additions & 0 deletions packages/grid/helm/examples/gcp/gcp.nosync.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# ===================================================
# Deploy Syft High Side Node to a GKE Cluster
# DOES NOT sync seaweedfs to a GCS bucket
# ===================================================

node:
name: syft-gcp
side: high

# Resources set inline with c3-standard-4 machine type
resources:
requests:
cpu: 2
memory: "8Gi"
limits:
cpu: 4
memory: "16Gi"

seaweedfs:
storageSize: 100Gi

# Resources set inline with c3-standard-4 machine type
resources:
requests:
cpu: 2
memory: "8Gi"
limits:
cpu: 4
memory: "16Gi"

ingress:
# GKE 1.28+ need ingressClassName
className: gce
class: gce

proxy:
resourcesPreset: medium

registry:
resourcesPreset: medium
storageSize: 100Gi

frontend:
resourcesPreset: medium

mongo:
resourcesPreset: large
13 changes: 10 additions & 3 deletions packages/grid/helm/syft/templates/backend/backend-statefulset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,16 @@ spec:
labels:
{{- include "common.labels" . | nindent 8 }}
app.kubernetes.io/component: backend
{{- if .Values.node.podLabels }}
{{- toYaml .Values.node.podLabels | nindent 8 }}
{{- end }}
{{- if .Values.node.podAnnotations }}
annotations: {{- toYaml .Values.node.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.node.nodeSelector }}
nodeSelector: {{- .Values.node.nodeSelector | toYaml | nindent 8 }}
{{- end }}
containers:
- name: backend-container
image: {{ .Values.global.registry }}/openmined/grid-backend:{{ .Values.global.version }}
Expand All @@ -42,7 +51,7 @@ spec:
- name: NODE_TYPE
value: {{ .Values.node.type | default "domain" | quote }}
- name: NODE_NAME
value: {{ .Values.node.name | default .Release.Name | quote }}
value: {{ .Values.node.name | default (printf "%s-node" .Release.Name) | quote }}
- name: NODE_SIDE_TYPE
value: {{ .Values.node.side | quote }}
- name: DEFAULT_ROOT_EMAIL
Expand Down Expand Up @@ -104,8 +113,6 @@ spec:
secretKeyRef:
name: {{ .Values.seaweedfs.secretKeyName | required "seaweedfs.secretKeyName is required" }}
key: s3RootPassword
- name: SEAWEED_MOUNT_PORT
value: {{ .Values.seaweedfs.mountApi.port | quote }}
{{- end }}
# Tracing
- name: TRACE
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,16 @@ spec:
labels:
{{- include "common.labels" . | nindent 8 }}
app.kubernetes.io/component: frontend
{{- if .Values.frontend.podLabels }}
{{- toYaml .Values.frontend.podLabels | nindent 8 }}
{{- end }}
{{- if .Values.frontend.podAnnotations }}
annotations: {{- toYaml .Values.frontend.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.frontend.nodeSelector }}
nodeSelector: {{- .Values.frontend.nodeSelector | toYaml | nindent 8 }}
{{- end }}
containers:
- name: frontend-container
image: {{ .Values.global.registry }}/openmined/grid-frontend:{{ .Values.global.version }}
Expand Down
Loading

0 comments on commit 91ab7a0

Please sign in to comment.