config.yaml.j2

proxy:
  secretToken: "{{ os.stdout }}"
  https:
    enabled: true
    hosts:
    - {{ ingress_host }}
    type: offload
    #letsencrypt:
    #  contactEmail: help@dandiarchive.org
  service:
    annotations:
      # Certificate ARN
      service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "{{ aws_certificate_arn }}"
      # The protocol to use on the backend, we use TCP since we're using websockets
      service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "tcp"
      # Which ports should use SSL
      service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "https"
      service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "3600"
hub:
  extraConfig:
    myConfig: |
      from kubernetes_asyncio import client
      def modify_pod_hook(spawner, pod):
          pod.spec.containers[0].security_context = client.V1SecurityContext(
              privileged=True
          )
          return pod
      c.KubeSpawner.modify_pod_hook = modify_pod_hook

      # Based on <https://github.com/jupyterhub/oauthenticator/blob/master/examples/auth_state/jupyterhub_config.py>:
      import os
      import warnings

      from oauthenticator.github import GitHubOAuthenticator
      from tornado.httpclient import HTTPRequest, HTTPClientError, AsyncHTTPClient
      import json

      # define our OAuthenticator with `.pre_spawn_start`
      # for passing auth_state into the user environment

      class GitHubEnvAuthenticator(GitHubOAuthenticator):

          async def check_allowed(self, username, auth_model):
              """
              Overrides the OAuthenticator.check_allowed to check dandi registered users
              """
              if auth_model["auth_state"].get("scope", []):
                 scopes = []
                 for val in auth_model["auth_state"]["scope"]:
                     scopes.extend(val.split(","))
                 auth_model["auth_state"]["scope"] = scopes
              auth_model = await self.update_auth_model(auth_model)
              # print("check_allowed:", username, auth_model)
              if await super().check_allowed(username, auth_model):
                  return True
              req = HTTPRequest(
                          f"https://api.dandiarchive.org/api/users/search/?username={username}",
                          method="GET",
                          headers={"Accept": "application/json",
                                   "User-Agent": "JupyterHub",
                                   "Authorization": "token {{ danditoken }}"},
                          validate_cert=self.validate_server_cert,
                     )
              try:
                  client = AsyncHTTPClient()
                  resp = await client.fetch(req)
              except HTTPClientError:
                  return False
              else:
                  if resp.body:
                      resp_json = json.loads(resp.body.decode('utf8', 'replace'))
                      for val in resp_json:
                          if val["username"].lower() == username.lower():
                              return True

              # users should be explicitly allowed via config, otherwise they aren't
              return False

          async def pre_spawn_start(self, user, spawner):
              auth_state = await user.get_auth_state()
              if not auth_state:
                  # user has no auth state
                  return
              # define some environment variables from auth_state
              spawner.environment['GITHUB_TOKEN'] = auth_state['access_token']
              spawner.environment['GITHUB_USER'] = auth_state['github_user']['login']
              spawner.environment['GITHUB_EMAIL'] = auth_state['github_user']['email']

      c.JupyterHub.authenticator_class = GitHubEnvAuthenticator

      # enable authentication state
      c.GitHubOAuthenticator.enable_auth_state = True

      # c.JupyterHub.authenticator_class = "dummy"
      # c.DummyAuthenticator.password = "{{ dummypass }}"

  config:
    Authenticator:
      admin_users:
        - "satra"
        - "yarikoptic"
        - "dandibot"
    GitHubOAuthenticator:
      client_id: "{{ github_client_id }}"
      client_secret: "{{ github_client_secret }}"
      oauth_callback_url: "{{ ingress }}/hub/oauth_callback"
      scope:
        - read:user
        - gist
        - user:email

scheduling:
  userScheduler:
    enabled: true
  podPriority:
    enabled: true
  userPlaceholder:
    enabled: false
    replicas: 4

cull:
  enabled: true
  timeout: 3600
  every: 300

singleuser:
  defaultUrl: "/lab"
  image:
    name: {{ singleuser_image_repo }}
    tag: {{ singleuser_image_tag }}
  memory:
    limit: 16G
    guarantee: 1G
  cpu:
    limit: 12
    guarantee: 0.5
  startTimeout: 2400
  profileList:
    - display_name: "Tiny. Useful for many quick things"
      description: "0.5 CPU / 1 GB"
      kubespawner_override:
        image: '{{ singleuser_image_repo }}:{{ singleuser_image_tag }}'
        image_pull_policy: Always
        cpu_limit: 2
        cpu_guarantee: 0.25
        mem_limit: 2G
        mem_guarantee: 0.5G
    - display_name: "Base"
      description: "6 CPU / 16 GB upto 12C/32G. May take up to 15 mins to start."
      default: true
      kubespawner_override:
        image: '{{ singleuser_image_repo }}:{{ singleuser_image_tag }}'
        image_pull_policy: Always
        cpu_limit: 12
        cpu_guarantee: 6
        mem_limit: 32G
        mem_guarantee: 16G
    - display_name: "Medium"
      description: "12C/32G upto 24C/64G. May take up to 15 mins to start."
      kubespawner_override:
        image: '{{ singleuser_image_repo }}:{{ singleuser_image_tag }}'
        image_pull_policy: Always
        cpu_limit: 24
        cpu_guarantee: 12
        mem_limit: 64G
        mem_guarantee: 32G
    - display_name: "Large"
      description: "24C/64G upto 48C/96G. May take up to 15 mins to start."
      kubespawner_override:
        image: '{{ singleuser_image_repo }}:{{ singleuser_image_tag }}'
        image_pull_policy: Always
        cpu_limit: 48
        cpu_guarantee: 24
        mem_limit: 96G
        mem_guarantee: 64G
    - display_name: "T4 GPU for inference"
      description: "8 CPU / 30 GB / 1 T4 GPU. May take up to 15 mins to start."
      kubespawner_override:
        image: '{{ singleuser_image_repo }}:{{ singleuser_image_tag }}-gpu'
        image_pull_policy: Always
        cpu_limit: 8
        cpu_guarantee: 6
        mem_limit: 31G
        mem_guarantee: 30G
        extra_resource_limits:
          nvidia.com/gpu: "1"
        extra_pod_config:
          runtimeClassName: nvidia
    - display_name: "Base (MATLAB)"
      description: "6 CPU / 16 GB upto 12C/32G. May take up to 15 mins to start. This requires your own license."
      kubespawner_override:
        image: '{{ singleuser_image_repo }}:{{ singleuser_image_tag }}-matlab'
        image_pull_policy: Always
        cpu_limit: 12
        cpu_guarantee: 6
        mem_limit: 32G
        mem_guarantee: 16G
    - display_name: "T4 GPU for inference (MATLAB)"
      description: "8 CPU / 30 GB / 1 T4 GPU. May take up to 15 mins to start. This requires your own license."
      kubespawner_override:
        image: '{{ singleuser_image_repo }}:{{ singleuser_image_tag }}-gpu-matlab'
        image_pull_policy: Always
        cpu_limit: 8
        cpu_guarantee: 6
        mem_limit: 31G
        mem_guarantee: 30G
        extra_resource_limits:
          nvidia.com/gpu: "1"
        extra_pod_config:
          runtimeClassName: nvidia
  storage:
    type: none
    extraVolumes:
      - name: fuse
        hostPath:
          path: /dev/fuse
      - name: shm-volume
        emptyDir:
          medium: Memory
      - name: persistent-storage
        persistentVolumeClaim:
          claimName: efs-claim
    extraVolumeMounts:
      - name: fuse
        mountPath: /dev/fuse
      - name: shm-volume
        mountPath: /dev/shm
      - name: persistent-storage
        mountPath: '/home/jovyan'
        subPath: 'home/{username}'
      - name: persistent-storage
        mountPath: '/shared'
        subPath: 'shared'
      - name: persistent-storage
        mountPath: '/readonly'
        readOnly: true
        subPath: 'readonly'
  initContainers:
    - name: nfs-fixer
      image: alpine
      securityContext:
        runAsUser: 0
      volumeMounts:
      - name: persistent-storage
        mountPath: /nfs
        subPath: 'home/{username}'
      - name: persistent-storage
        mountPath: /shared
        subPath: 'shared'
      - name: persistent-storage
        mountPath: /readonly
        subPath: 'readonly'
      command:
      - sh
      - -c
      - >
        chmod 0775 /nfs;
        chown 1000:100 /nfs;
        chmod 0775 /shared;
        chown 1000:100 /shared;
        chmod 0555 /readonly
  cmd: "start-singleuser.sh"
  lifecycleHooks:
    postStart:
      exec:
        command:
          - "sh"
          - "-c"
          - >
            gitpuller https://github.com/dandi/example-notebooks master dandi-notebooks;
            chown -R jovyan:users dandi-notebooks;
            datalad install https://github.com/dandi/dandisets;
            /opt/conda/envs/allen/bin/python -m ipykernel install --user --name allen --display-name "Python (Allen SDK)";
            /opt/conda/bin/pip install --upgrade dandi;
            git config --global user.email "${GITHUB_EMAIL}";
            git config --global user.name "${GITHUB_USER}"