Merge pull request #9139 from OpenMined/shubham/fix-kaniko-build

Fix worker build and push failing in K8s
OpenMined · Aug 12, 2024 · d81d7fe · d81d7fe
2 parents f315b98 + 2710621
commit d81d7fe
Show file tree

Hide file tree

Showing 7 changed files with 22 additions and 5 deletions.
diff --git a/packages/grid/default.env b/packages/grid/default.env
@@ -71,6 +71,8 @@ S3_REGION="us-east-1"
  #not-using
 S3_PRESIGNED_TIMEOUT_SECS=1800
 
+# Kaniko
+KANIKO_VERSION="v1.23.2"
 
 # Jax
 JAX_ENABLE_X64=True

diff --git a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml
@@ -79,6 +79,8 @@ spec:
               value: {{ .Values.server.defaultWorkerPool.podAnnotations | toJson | quote }}
             - name: USE_INTERNAL_REGISTRY
               value: {{ .Values.server.useInternalRegistry | quote }}
+            - name: KANIKO_VERSION
+              value: {{ .Values.global.kaniko.version | quote }}
             {{- if .Values.server.defaultBucketName }}
             - name: DEFAULT_BUCKET_NAME
               value: {{ .Values.server.defaultBucketName | quote }}

diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml
@@ -6,6 +6,11 @@ global:
   # Force default secret values for development. DO NOT SET THIS TO FALSE IN PRODUCTION
   randomizedSecrets: true
 
+  # Kaniko version
+  kaniko:
+    version: "v1.23.2"
+
+
 # =================================================================================
 
 mongo:

diff --git a/packages/syft/src/syft/custom_worker/builder_k8s.py b/packages/syft/src/syft/custom_worker/builder_k8s.py
@@ -16,6 +16,7 @@
 from .builder_types import ImagePushResult
 from .builder_types import PUSH_IMAGE_TIMEOUT_SEC
 from .k8s import INTERNAL_REGISTRY_HOST
+from .k8s import KANIKO_VERSION
 from .k8s import KUBERNETES_NAMESPACE
 from .k8s import KubeUtils
 from .k8s import USE_INTERNAL_REGISTRY
@@ -173,17 +174,17 @@ def _get_tag_hash(self, tag: str) -> str:
         return sha256(tag.encode()).hexdigest()
 
     def _get_image_digest(self, job: Job) -> str | None:
-        selector = {"batch.kubernetes.io/job-name": job.metadata.name}
+        selector = {"job-name": job.metadata.name}
         pods = self.client.get("pods", label_selector=selector)
         return KubeUtils.get_container_exit_message(pods)
 
     def _get_exit_code(self, job: Job) -> list[int]:
-        selector = {"batch.kubernetes.io/job-name": job.metadata.name}
+        selector = {"job-name": job.metadata.name}
         pods = self.client.get("pods", label_selector=selector)
         return KubeUtils.get_container_exit_code(pods)
 
     def _get_logs(self, job: Job) -> str:
-        selector = {"batch.kubernetes.io/job-name": job.metadata.name}
+        selector = {"job-name": job.metadata.name}
         pods = self.client.get("pods", label_selector=selector)
         return KubeUtils.get_logs(pods)
 
@@ -241,7 +242,7 @@ def _create_kaniko_build_job(
                             "containers": [
                                 {
                                     "name": "kaniko",
-                                    "image": "gcr.io/kaniko-project/executor:latest",
+                                    "image": f"gcr.io/kaniko-project/executor:{KANIKO_VERSION}",
                                     "args": [
                                         "--dockerfile=Dockerfile",
                                         "--context=dir:///workspace",

diff --git a/packages/syft/src/syft/custom_worker/builder_types.py b/packages/syft/src/syft/custom_worker/builder_types.py
@@ -29,6 +29,10 @@ class ImagePushResult(BaseModel):
     logs: str
     exit_code: int
 
+    @property
+    def has_failed(self) -> bool:
+        return self.exit_code != 0
+
 
 class BuilderBase(ABC):
     @abstractmethod

diff --git a/packages/syft/src/syft/custom_worker/k8s.py b/packages/syft/src/syft/custom_worker/k8s.py
@@ -25,6 +25,9 @@
 # skip pushing to internal registry
 USE_INTERNAL_REGISTRY = os.getenv("USE_INTERNAL_REGISTRY", "true").lower() == "true"
 
+# Kaniko version
+KANIKO_VERSION = os.getenv("KANIKO_VERSION", "latest")
+
 # Internal registry URL
 DEFAULT_INTERNAL_REGISTRY = f"registry.{KUBERNETES_NAMESPACE}.svc.cluster.local"
 INTERNAL_REGISTRY_HOST = os.getenv("INTERNAL_REGISTRY_HOST", DEFAULT_INTERNAL_REGISTRY)

diff --git a/packages/syft/src/syft/service/worker/utils.py b/packages/syft/src/syft/service/worker/utils.py
@@ -650,7 +650,7 @@ def image_push(
                 password=password,
             )
 
-            if "error" in result.logs.lower() or result.exit_code:
+            if "error" in result.logs.lower() or result.has_failed:
                 return SyftError(
                     message=f"Failed to push {full_tag}. "
                     f"Exit code: {result.exit_code}. "