Skip to content

Commit

Permalink
[pre-commit.ci] auto fixes from pre-commit.com hooks
Browse files Browse the repository at this point in the history
for more information, see https://pre-commit.ci
  • Loading branch information
pre-commit-ci[bot] committed Jun 2, 2024
1 parent eff70e1 commit 6982637
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 15 deletions.
8 changes: 4 additions & 4 deletions enterprise_gateway/services/kernels/remotemanager.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,21 @@
from __future__ import annotations

import asyncio
import json
import os
import re
import signal
import time
import uuid
import json
from typing import Any, ClassVar

from jupyter_client.ioloop.manager import AsyncIOLoopKernelManager
from jupyter_client.kernelspec import KernelSpec
from jupyter_server.services.kernels.kernelmanager import AsyncMappingKernelManager
from tornado import web
from traitlets import directional_link, default
from traitlets import log as traitlets_log
from traitlets import List as ListTrait
from traitlets import default, directional_link
from traitlets import log as traitlets_log
from zmq import IO_THREADS, MAX_SOCKETS, Context

from enterprise_gateway.mixins import EnterpriseGatewayConfigMixin
Expand Down Expand Up @@ -171,7 +171,7 @@ class RemoteMappingKernelManager(AsyncMappingKernelManager):
help="""Comma-separated list of dictionaries, each describing an event by `type`, `reason`,
and `timeout_in_seconds` (e.g. [{"type": "Warning", "reason": "FailedMount", "timeout_in_seconds": 0}]).
Kernel pod events will be sampled during startup, and if an event described in this list is detected,
the kernel launch will be terminated after the set timeout. Only available for container kernels. """
the kernel launch will be terminated after the set timeout. Only available for container kernels. """,
)

@default("kernel_launch_terminate_on_events")
Expand Down
28 changes: 18 additions & 10 deletions enterprise_gateway/services/processproxies/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,9 @@ def _initialize_kernel_launch_terminate_on_events(self):
"""
self.kernel_launch_terminate_on_events = defaultdict(dict)
for configuration in self.kernel_manager.parent.kernel_launch_terminate_on_events:
self.kernel_launch_terminate_on_events[
configuration["type"]
][configuration["reason"]] = configuration["timeout_in_seconds"]
self.kernel_launch_terminate_on_events[configuration["type"]][
configuration["reason"]
] = configuration["timeout_in_seconds"]

def _determine_kernel_images(self, **kwargs: dict[str, Any] | None) -> None:
"""
Expand Down Expand Up @@ -237,18 +237,26 @@ def _handle_pending_kernel(self):
self.log.debug("Sampling kernel container events")
kernel_pod_events = self.get_container_events()
for event in kernel_pod_events:
if event.type in self.kernel_launch_terminate_on_events and event.reason in self.kernel_launch_terminate_on_events[event.type]:
if (
event.type in self.kernel_launch_terminate_on_events
and event.reason in self.kernel_launch_terminate_on_events[event.type]
):
event_key = f"{event.type}{event.reason}"
if event_key not in self.kernel_events_to_occurrence_time:
self.kernel_events_to_occurrence_time[event_key] = RemoteProcessProxy.get_current_time()
if RemoteProcessProxy.get_time_diff(
RemoteProcessProxy.get_current_time(),
self.kernel_events_to_occurrence_time[event_key]
) >= self.kernel_launch_terminate_on_events[event.type][event.reason]:
self.kernel_events_to_occurrence_time[event_key] = (
RemoteProcessProxy.get_current_time()
)
if (
RemoteProcessProxy.get_time_diff(
RemoteProcessProxy.get_current_time(),
self.kernel_events_to_occurrence_time[event_key],
)
>= self.kernel_launch_terminate_on_events[event.type][event.reason]
):
self.kill()
self.log_and_raise(
http_status_code=409,
reason=f"Error starting kernel container; The container encountered an event which may cause a longer than usual startup: '{event.reason} - {event.message[:64]}'"
reason=f"Error starting kernel container; The container encountered an event which may cause a longer than usual startup: '{event.reason} - {event.message[:64]}'",
)

def get_process_info(self) -> dict[str, Any]:
Expand Down
3 changes: 2 additions & 1 deletion enterprise_gateway/services/processproxies/k8s.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,8 @@ def get_container_events(self) -> list:
core_v1_api = client.CoreV1Api()
if self.container_name:
ret = core_v1_api.list_namespaced_event(
namespace=self.kernel_namespace, field_selector=f"involvedObject.name={self.container_name}"
namespace=self.kernel_namespace,
field_selector=f"involvedObject.name={self.container_name}",
)
if ret and ret.items:
pod_events = ret.items
Expand Down

0 comments on commit 6982637

Please sign in to comment.