From 3cd131c3e23f6ebd8c313aba44c0447de8e2e688 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 6 Aug 2024 17:10:10 +0530 Subject: [PATCH 1/2] k8s: fix selector for pod name in kubernetes job - Add a property to ImageResult to derive failure based on exit code --- packages/syft/src/syft/custom_worker/builder_k8s.py | 6 +++--- packages/syft/src/syft/custom_worker/builder_types.py | 4 ++++ packages/syft/src/syft/service/worker/utils.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/custom_worker/builder_k8s.py b/packages/syft/src/syft/custom_worker/builder_k8s.py index b799b35f7b1..9d53c34c9d9 100644 --- a/packages/syft/src/syft/custom_worker/builder_k8s.py +++ b/packages/syft/src/syft/custom_worker/builder_k8s.py @@ -173,17 +173,17 @@ def _get_tag_hash(self, tag: str) -> str: return sha256(tag.encode()).hexdigest() def _get_image_digest(self, job: Job) -> str | None: - selector = {"batch.kubernetes.io/job-name": job.metadata.name} + selector = {"job-name": job.metadata.name} pods = self.client.get("pods", label_selector=selector) return KubeUtils.get_container_exit_message(pods) def _get_exit_code(self, job: Job) -> list[int]: - selector = {"batch.kubernetes.io/job-name": job.metadata.name} + selector = {"job-name": job.metadata.name} pods = self.client.get("pods", label_selector=selector) return KubeUtils.get_container_exit_code(pods) def _get_logs(self, job: Job) -> str: - selector = {"batch.kubernetes.io/job-name": job.metadata.name} + selector = {"job-name": job.metadata.name} pods = self.client.get("pods", label_selector=selector) return KubeUtils.get_logs(pods) diff --git a/packages/syft/src/syft/custom_worker/builder_types.py b/packages/syft/src/syft/custom_worker/builder_types.py index 2c6b1529adc..c7f34ec395d 100644 --- a/packages/syft/src/syft/custom_worker/builder_types.py +++ b/packages/syft/src/syft/custom_worker/builder_types.py @@ -29,6 +29,10 @@ class ImagePushResult(BaseModel): logs: str exit_code: int + @property + def has_failed(self) -> bool: + return self.exit_code != 0 + class BuilderBase(ABC): @abstractmethod diff --git a/packages/syft/src/syft/service/worker/utils.py b/packages/syft/src/syft/service/worker/utils.py index 55d608c2964..f444965182f 100644 --- a/packages/syft/src/syft/service/worker/utils.py +++ b/packages/syft/src/syft/service/worker/utils.py @@ -650,7 +650,7 @@ def image_push( password=password, ) - if "error" in result.logs.lower() or result.exit_code: + if "error" in result.logs.lower() or result.has_failed: return SyftError( message=f"Failed to push {full_tag}. " f"Exit code: {result.exit_code}. " From 92ab9b76e0630e0d526004434004ec984985768b Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 6 Aug 2024 17:26:53 +0530 Subject: [PATCH 2/2] pass/set kaniko version via env vars --- packages/grid/default.env | 2 ++ .../helm/syft/templates/backend/backend-statefulset.yaml | 2 ++ packages/grid/helm/syft/values.yaml | 5 +++++ packages/syft/src/syft/custom_worker/builder_k8s.py | 3 ++- packages/syft/src/syft/custom_worker/k8s.py | 3 +++ 5 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/grid/default.env b/packages/grid/default.env index e3edfdc38df..3018a4c2ce2 100644 --- a/packages/grid/default.env +++ b/packages/grid/default.env @@ -71,6 +71,8 @@ S3_REGION="us-east-1" #not-using S3_PRESIGNED_TIMEOUT_SECS=1800 +# Kaniko +KANIKO_VERSION="v1.23.2" # Jax JAX_ENABLE_X64=True diff --git a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml index 226c8ab1dec..3dcefcd0f6b 100644 --- a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml @@ -79,6 +79,8 @@ spec: value: {{ .Values.server.defaultWorkerPool.podAnnotations | toJson | quote }} - name: USE_INTERNAL_REGISTRY value: {{ .Values.server.useInternalRegistry | quote }} + - name: KANIKO_VERSION + value: {{ .Values.global.kaniko.version | quote }} {{- if .Values.server.defaultBucketName }} - name: DEFAULT_BUCKET_NAME value: {{ .Values.server.defaultBucketName | quote }} diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 46922bd8f7b..ef201447b02 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -6,6 +6,11 @@ global: # Force default secret values for development. DO NOT SET THIS TO FALSE IN PRODUCTION randomizedSecrets: true + # Kaniko version + kaniko: + version: "v1.23.2" + + # ================================================================================= mongo: diff --git a/packages/syft/src/syft/custom_worker/builder_k8s.py b/packages/syft/src/syft/custom_worker/builder_k8s.py index 9d53c34c9d9..0c5d54102ff 100644 --- a/packages/syft/src/syft/custom_worker/builder_k8s.py +++ b/packages/syft/src/syft/custom_worker/builder_k8s.py @@ -16,6 +16,7 @@ from .builder_types import ImagePushResult from .builder_types import PUSH_IMAGE_TIMEOUT_SEC from .k8s import INTERNAL_REGISTRY_HOST +from .k8s import KANIKO_VERSION from .k8s import KUBERNETES_NAMESPACE from .k8s import KubeUtils from .k8s import USE_INTERNAL_REGISTRY @@ -241,7 +242,7 @@ def _create_kaniko_build_job( "containers": [ { "name": "kaniko", - "image": "gcr.io/kaniko-project/executor:latest", + "image": f"gcr.io/kaniko-project/executor:{KANIKO_VERSION}", "args": [ "--dockerfile=Dockerfile", "--context=dir:///workspace", diff --git a/packages/syft/src/syft/custom_worker/k8s.py b/packages/syft/src/syft/custom_worker/k8s.py index c3d047a0daa..319b691db50 100644 --- a/packages/syft/src/syft/custom_worker/k8s.py +++ b/packages/syft/src/syft/custom_worker/k8s.py @@ -25,6 +25,9 @@ # skip pushing to internal registry USE_INTERNAL_REGISTRY = os.getenv("USE_INTERNAL_REGISTRY", "true").lower() == "true" +# Kaniko version +KANIKO_VERSION = os.getenv("KANIKO_VERSION", "latest") + # Internal registry URL DEFAULT_INTERNAL_REGISTRY = f"registry.{KUBERNETES_NAMESPACE}.svc.cluster.local" INTERNAL_REGISTRY_HOST = os.getenv("INTERNAL_REGISTRY_HOST", DEFAULT_INTERNAL_REGISTRY)