update base - fix sync issues, improve provisioning, preload TCMalloc…

…, first iteration of serverless mode

build/COPY_ROOT/etc/supervisor/supervisord/conf.d/comfyui.conf

@@ -2,7 +2,7 @@
 command=/opt/ai-dock/bin/supervisor-comfyui.sh
 process_name=%(program_name)s
 numprocs=1
-directory=/root
+directory=/opt/ComfyUI
 priority=1500
 autostart=true
 startsecs=0

build/COPY_ROOT/opt/ai-dock/bin/build/layer0/common.sh

@@ -13,13 +13,16 @@ main() {
 
 create_env() {
     apt-get update
-    $APT_INSTALL libgl1
+    $APT_INSTALL libgl1 \
+        libgoogle-perftools4
+
+    ln -sf $(ldconfig -p | grep -Po "libtcmalloc.so.\d" | head -n 1) \
+        /lib/x86_64-linux-gnu/libtcmalloc.so
+
     # A new pytorch env costs ~ 300Mb
     exported_env=/tmp/${MAMBA_DEFAULT_ENV}.yaml
     micromamba env export -n ${MAMBA_DEFAULT_ENV} > "${exported_env}"
     $MAMBA_CREATE -n comfyui --file "${exported_env}"
-    $MAMBA_INSTALL -n comfyui -c conda-forge -y \
-        gperftools
 }
 
 install_jupyter_kernels() {

build/COPY_ROOT/opt/ai-dock/bin/preflight.sh

@@ -5,11 +5,13 @@
 function preflight_main() {
     preflight_copy_notebook
     preflight_update_comfyui
+    printf "%s" "${COMFYUI_FLAGS}" > /etc/comfyui_flags.conf
 }
 
 function preflight_serverless() {
-  printf "Refusing to update ComfyUI in serverless mode\n \
-    Nothing to do. \n"
+  printf "Refusing to update ComfyUI in serverless mode\n"
+  printf "%s" "${COMFYUI_FLAGS}" > /etc/comfyui_flags.conf
+
 }
 
 function preflight_copy_notebook() {
@@ -21,7 +23,11 @@ function preflight_copy_notebook() {
 }
 
 function preflight_update_comfyui() {
-    /opt/ai-dock/bin/update-comfyui.sh
+    if [[ ${AUTO_UPDATE,,} != "false" ]]; then
+        /opt/ai-dock/bin/update-comfyui.sh
+    else
+        printf "Skipping auto update (AUTO_UPDATE=false)"
+    fi
 }
 
 if [[ ${SERVERLESS,,} != "true" ]]; then

build/COPY_ROOT/opt/ai-dock/bin/supervisor-comfyui.sh

@@ -36,13 +36,13 @@ function start() {
         PLATFORM_FLAGS="--cpu"
     fi
 
-    BASE_FLAGS="--listen 127.0.0.1 --port ${LISTEN_PORT} --disable-auto-launch"
+    BASE_FLAGS="--listen 127.0.0.1 --disable-auto-launch"
 
     # Delay launch until micromamba is ready
     if [[ -f /run/workspace_sync || -f /run/container_config ]]; then
         if [[ ${SERVERLESS,,} != "true" ]]; then
             printf "Waiting for workspace sync...\n"
-            kill -9 $(lsof -t -i:$LISTEN_PORT) > /dev/null 2>&1 &
+            kill $(lsof -t -i:$LISTEN_PORT) > /dev/null 2>&1 &
             wait -n
             /usr/bin/python3 /opt/ai-dock/fastapi/logviewer/main.py \
                 -p $LISTEN_PORT \
@@ -60,19 +60,22 @@ function start() {
         else
             printf "Waiting for workspace symlinks and pre-flight checks...\n"
             while [[ -f /run/workspace_sync || -f /run/container_config ]]; do
-                sleep 0.1s
+                sleep 1
             done
         fi
     fi
 
     printf "Starting %s...\n" ${SERVICE_NAME}
-    cd /opt/ComfyUI
-    kill -9 $(lsof -t -i:$LISTEN_PORT) > /dev/null 2>&1 &
+
+    kill $(lsof -t -i:$LISTEN_PORT) > /dev/null 2>&1 &
     wait -n
-    micromamba run -n comfyui python main.py \
-        ${PLATFORM_FLAGS} \
-        ${BASE_FLAGS} \
-        ${COMFYUI_FLAGS}
+
+    FLAGS_COMBINED="${PLATFORM_FLAGS} ${BASE_FLAGS} $(cat /etc/comfyui_flags.conf)"
+    printf "Starting %s...\n" "${SERVICE_NAME}"
+
+    cd /opt/ComfyUI && \
+    micromamba run -n comfyui -e LD_PRELOAD=libtcmalloc.so python main.py \
+        ${FLAGS_COMBINED} --port ${LISTEN_PORT}
 }
 
 start 2>&1

build/COPY_ROOT/opt/serverless/handlers/basehandler.py

@@ -132,11 +132,11 @@ def get_result(self, job_id):
 
         self.job_time_completed = datetime.datetime.now()
         self.result["timings"] = {
-            "job_time_received": self.job_time_received,
-            "job_time_queued": self.job_time_queued,
-            "job_time_processed": self.job_time_processed,
-            "job_time_completed": self.job_time_completed,
-            "job_time_total": (self.job_time_completed - self.job_time_received.seconds
+            "job_time_received": self.job_time_received.ctime(),
+            "job_time_queued": self.job_time_queued.ctime(),
+            "job_time_processed": self.job_time_processed.ctime(),
+            "job_time_completed": self.job_time_completed.ctime(),
+            "job_time_total": (self.job_time_completed - self.job_time_received).seconds
         }
 
         return self.result

build/COPY_ROOT/opt/serverless/providers/runpod/test_input.json

@@ -4,6 +4,6 @@
         "aws_bucket_name": "ai-dock",
         "ckpt_name": "v1-5-pruned-emaonly.ckpt",
         "include_text": "A scuba diver exploring an ancient shipwreck",
-        "exclude_text": "https://cdn.pixabay.com/photo/2014/06/03/19/38/board-361516_1280.jpg"
+        "exclude_text": "steel, modern, cartoon"
         }
 }

config/provisioning/default.sh

@@ -3,85 +3,137 @@
 # This file will be sourced in init.sh
 
 # https://raw.githubusercontent.com/ai-dock/comfyui/main/config/provisioning/default.sh
-printf "\n##############################################\n#                                            #\n#          Provisioning container            #\n#                                            #\n#         This will take some time           #\n#                                            #\n# Your container will be ready on completion #\n#                                            #\n##############################################\n\n"
-function download() {
-    wget -q --show-progress -e dotbytes="${3:-4M}" -O "$2" "$1"
+
+
+NODES=(
+    "https://github.com/ltdrdata/ComfyUI-Manager"
+)
+
+CHECKPOINT_MODELS=(
+    "https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt"
+    #"https://huggingface.co/stabilityai/stable-diffusion-2-1/resolve/main/v2-1_768-ema-pruned.ckpt"
+    "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/sd_xl_base_1.0.safetensors"
+    "https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-1.0/resolve/main/sd_xl_refiner_1.0.safetensors"
+)
+
+LORA_MODELS=(
+    #"https://civitai.com/api/download/models/16576"
+)
+
+VAE_MODELS=(
+    "https://huggingface.co/stabilityai/sd-vae-ft-ema-original/resolve/main/vae-ft-ema-560000-ema-pruned.safetensors"
+    "https://huggingface.co/stabilityai/sd-vae-ft-mse-original/resolve/main/vae-ft-mse-840000-ema-pruned.safetensors"
+    "https://huggingface.co/stabilityai/sdxl-vae/resolve/main/sdxl_vae.safetensors"
+)
+
+ESRGAN_MODELS=(
+    "https://huggingface.co/ai-forever/Real-ESRGAN/resolve/main/RealESRGAN_x4.pth"
+    "https://huggingface.co/FacehugmanIII/4x_foolhardy_Remacri/resolve/main/4x_foolhardy_Remacri.pth"
+    "https://huggingface.co/Akumetsu971/SD_Anime_Futuristic_Armor/resolve/main/4x_NMKD-Siax_200k.pth"
+)
+
+CONTROLNET_MODELS=(
+    "https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_canny-fp16.safetensors"
+    #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_depth-fp16.safetensors"
+    #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_hed-fp16.safetensors"
+    #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_mlsd-fp16.safetensors"
+    #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_normal-fp16.safetensors"
+    "https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_openpose-fp16.safetensors"
+    #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_scribble-fp16.safetensors"
+    #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_seg-fp16.safetensors"
+    "https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_canny-fp16.safetensors"
+    #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_color-fp16.safetensors"
+    #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_depth-fp16.safetensors"
+    #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_keypose-fp16.safetensors"
+    "https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_openpose-fp16.safetensors"
+    #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_seg-fp16.safetensors"
+    #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_sketch-fp16.safetensors"
+    #"https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/t2iadapter_style-fp16.safetensors"
+)
+
+### DO NOT EDIT BELOW HERE UNLESS YOU KNOW WHAT YOU ARE DOING ###
+
+function provisioning_start() {
+    DISK_GB_AVAILABLE=$(($(df --output=avail -m "${WORKSPACE}" | tail -n1) / 1000))
+    DISK_GB_USED=$(($(df --output=used -m "${WORKSPACE}" | tail -n1) / 1000))
+    DISK_GB_ALLOCATED=$(($DISK_GB_AVAILABLE + $DISK_GB_USED))
+    provisioning_print_header
+    provisioning_get_nodes
+    provisioning_get_models \
+        "/opt/ComfyUI/models/checkpoints" \
+        "${CHECKPOINT_MODELS[@]}"
+    provisioning_get_models \
+        "/opt/ComfyUI/models/loras" \
+        "${LORA_MODELS[@]}"
+    provisioning_get_models \
+        "/opt/ComfyUI/models/controlnet" \
+        "${CONTROLNET_MODELS[@]}"
+    provisioning_get_models \
+        "/opt/ComfyUI/models/vae" \
+        "${VAE_MODELS[@]}"
+    provisioning_get_models \
+        "/opt/ComfyUI/models/upscale_models" \
+        "${ESRGAN_MODELS[@]}"
+    provisioning_print_end
+}
+
+function provisioning_get_nodes() {
+    for repo in "${NODES[@]}"; do
+        dir="${repo##*/}"
+        path="/opt/ComfyUI/custom_nodes/${dir}"
+        requirements="${path}/requirements.txt"
+        if [[ -d $path ]]; then
+            if [[ ${AUTO_UPDATE,,} != "false" ]]; then
+                printf "Updating node: %s...\n" "${repo}"
+                ( cd "$path" && git pull )
+                if [[ -e $requirements ]]; then
+                    micromamba -n comfyui run ${PIP_INSTALL} -r "$requirements"
+                fi
+            fi
+        else
+            printf "Downloading node: %s...\n" "${repo}"
+            git clone "${repo}" "${path}"
+            if [[ -e $requirements ]]; then
+                micromamba -n comfyui run ${PIP_INSTALL} -r "${requirements}"
+            fi
+        fi
+    done
+}
+
+function provisioning_get_models() {
+    if [[ -z $2 ]]; then return 1; fi
+    dir="$1"
+    mkdir -p "$dir"
+    shift
+    if [[ $DISK_GB_ALLOCATED -ge $DISK_GB_REQUIRED ]]; then
+        arr=("$@")
+    else
+        printf "WARNING: Low disk space allocation - Only the first model will be downloaded!\n"
+        arr=("$1")
+    fi
+
+    printf "Downloading %s model(s) to %s...\n" "${#arr[@]}" "$dir"
+    for url in "${arr[@]}"; do
+        printf "Downloading: %s\n" "${url}"
+        provisioning_download "${url}" "${dir}"
+        printf "\n"
+    done
+}
+
+function provisioning_print_header() {
+    printf "\n##############################################\n#                                            #\n#          Provisioning container            #\n#                                            #\n#         This will take some time           #\n#                                            #\n# Your container will be ready on completion #\n#                                            #\n##############################################\n\n"
+    if [[ $DISK_GB_ALLOCATED -lt $DISK_GB_REQUIRED ]]; then
+        printf "WARNING: Your allocated disk size (%sGB) is below the recommended %sGB - Some models will not be downloaded\n" "$DISK_GB_ALLOCATED" "$DISK_GB_REQUIRED"
+    fi
+}
+
+function provisioning_print_end() {
+    printf "\nProvisioning complete:  Web UI will start now\n\n"
+}
+
+# Download from $1 URL to $2 file path
+function provisioning_download() {
+    wget -qnc --content-disposition --show-progress -e dotbytes="${3:-4M}" -P "$2" "$1"
 }
 
-## Set paths
-nodes_dir=/opt/ComfyUI/custom_nodes
-models_dir=/opt/ComfyUI/models
-checkpoints_dir=${models_dir}/checkpoints
-vae_dir=${models_dir}/vae
-controlnet_dir=${models_dir}/controlnet
-loras_dir=${models_dir}/loras
-upscale_dir=${models_dir}/upscale_models
-
-## Install custom nodes
-
-# ComfyUI-Manager
-comfyui_manager_dir=${nodes_dir}/ComfyUI-Manager
-if [[ ! -d $comfyui_manager_dir ]]; then
-    git clone https://github.com/ltdrdata/ComfyUI-Manager $comfyui_manager_dir
-else
-    (cd $comfyui_manager_dir && git pull)
-fi
-
-## Download checkpoints
-
-# v1-5-pruned-emaonly
-model_file=${checkpoints_dir}/v1-5-pruned-emaonly.ckpt
-model_url=https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.ckpt
-
-if [[ ! -e ${model_file} ]]; then
-    printf "Downloading Stable Diffusion 1.5...\n"
-    download ${model_url} ${model_file}
-fi
-
-# v2-1_768-ema-pruned
-model_file=${checkpoints_dir}/v2-1_768-ema-pruned.ckpt
-model_url=https://huggingface.co/stabilityai/stable-diffusion-2-1/resolve/main/v2-1_768-ema-pruned.ckpt
-
-if [[ ! -e ${model_file} ]]; then
-    printf "Downloading Stable Diffusion 2.1...\n"
-    download ${model_url} ${model_file}
-fi
-
-# sd_xl_base_1
-model_file=${checkpoints_dir}/sd_xl_base_1.0.safetensors
-model_url=https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/sd_xl_base_1.0.safetensors
-
-if [[ ! -e ${model_file} ]]; then
-    printf "Downloading Stable Diffusion XL base...\n"
-    download ${model_url} ${model_file}
-fi
-
-# sd_xl_refiner_1
-model_file=${checkpoints_dir}/sd_xl_refiner_1.0.safetensors
-model_url=https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-1.0/resolve/main/sd_xl_refiner_1.0.safetensors
-
-if [[ ! -e ${model_file} ]]; then
-    printf "Downloading Stable Diffusion XL refiner...\n"
-    download ${model_url} ${model_file}
-fi
-
-## Download controlnet
-
-model_file=${controlnet_dir}/control_canny-fp16.safetensors
-model_url=https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_canny-fp16.safetensors
-
-if [[ ! -e ${model_file} ]]; then
-    printf "Downloading Canny...\n"
-    download ${model_url} ${model_file}
-fi
-
-## Download loras
-
-# example
-#model_file=${loras_dir}/epi_noiseoffset2.safetensors
-#model_url=https://civitai.com/api/download/models/16576
-
-#if [[ ! -e ${model_file} ]]; then
-#    printf "Downloading epi_noiseoffset2 lora...\n"
-#    download ${model_url} ${model_file}
-#fi
+provisioning_start