gradio-app.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/envs/dev11/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import gradio as gr\n",
    "import os\n",
    "import json\n",
    "import qdrant_client\n",
    "from typing import cast\n",
    "from glob import glob\n",
    "from pathlib import Path\n",
    "from src.preprocess import (\n",
    "    VideoIngestionPipeline,\n",
    "    AppConfig,\n",
    "    RunConfig,\n",
    "    get_yt_id_from_url,\n",
    ")\n",
    "from llama_index.core.base.response.schema import RESPONSE_TYPE, Response\n",
    "from llama_index.core.schema import QueryBundle\n",
    "from llama_index.core.schema import ImageNode\n",
    "from src.video_to_index import video_to_index, chat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_config = RunConfig(seconds_per_frame=5)\n",
    "# Create a local Qdrant vector store\n",
    "db_client = qdrant_client.QdrantClient(path=\"qdrant_mm_db\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "demo = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def chat_response_with_context(response: Response):\n",
    "    context_response = str(response) + \"\\n\\n\"\n",
    "    source_nodes = response.source_nodes\n",
    "\n",
    "    context_response += \"<sub>Sources: </sub>\"\n",
    "    for node in source_nodes:\n",
    "        if isinstance(node.node, ImageNode):\n",
    "            context_response += f\"<sub>{str(Path(node.node.image_path).name)}</sub>, \"\n",
    "    return context_response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on local URL:  http://127.0.0.1:8000\n",
      "\n",
      "To create a public link, set `share=True` in `launch()`.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"http://127.0.0.1:8000/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Parsing nodes: 100%|██████████| 48/48 [00:00<00:00, 4053.77it/s]\n",
      "Generating embeddings: 100%|██████████| 1/1 [00:00<00:00,  1.77it/s]\n",
      "Generating image embeddings: 100%|██████████| 47/47 [00:14<00:00,  3.26it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Retrieving using text...\n",
      "Retrieving using image...\n",
      "Query:\n",
      "Give the timestamp of this scene.\n",
      " Assistant:\n",
      " The scene is at 00:15.\n",
      "Query:\n",
      "What is this scene about?\n",
      " Assistant:\n",
      " The scene is about Casey Neistat talking about how he got started on YouTube and how he became successful. He talks about how he was told that he couldn't do it, but he didn't listen and he kept going. He encourages others to do the same and to never give up on their dreams.\n",
      "Retrieving using text...\n",
      "Retrieving using image...\n",
      "Query:\n",
      "How about the timestamp of this scene?\n",
      " Assistant:\n",
      " The first image is a screenshot of a man sitting on a couch, wearing sunglasses and a black shirt. He is sitting in front of a bookshelf. The other images are also screenshots from the video. The first image is at the timestamp of 01:05, and the second image is at the timestamp of 01:15. The video transcription shows that the man in the video is talking about how he doesn't listen to anyone and how he does what he wants. He also talks about how he doesn't need anyone's permission to do what he wants.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Parsing nodes: 100%|██████████| 48/48 [00:00<00:00, 3804.36it/s]\n",
      "Generating embeddings: 100%|██████████| 1/1 [00:01<00:00,  1.62s/it]\n",
      "Generating image embeddings: 100%|██████████| 47/47 [00:14<00:00,  3.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Retrieving using text...\n",
      "Retrieving using image...\n",
      "Query:\n",
      "What is the timestamp of this scene?\n",
      " Assistant:\n",
      " The scene is at 03:10.\n",
      "Retrieving using text...\n",
      "Retrieving using image...\n",
      "Query:\n",
      "In which timestamp does this scene appear?\n",
      " Assistant:\n",
      " The scene appears at 00:14 in the video.\n",
      "Retrieving using text...\n",
      "Retrieving using image...\n",
      "Query:\n",
      "Who is this man?\n",
      " Assistant:\n",
      " The man in the video is Casey Neistat. He is a popular YouTuber who is known for his vlogs and his work with the Beme app. He is also the co-founder of the multimedia company 368.\n",
      "[youtube] Extracting URL: https://www.youtube.com/watch?v=PznJqxon4zE\n",
      "[youtube] PznJqxon4zE: Downloading webpage\n",
      "[youtube] PznJqxon4zE: Downloading ios player API JSON\n",
      "[youtube] PznJqxon4zE: Downloading android player API JSON\n",
      "[youtube] PznJqxon4zE: Downloading m3u8 information\n",
      "[info] PznJqxon4zE: Downloading 1 format(s): 18\n",
      "[download] Destination: data/PznJqxon4zE/video/Steve Jobs passion in work.mp4\n",
      "[download] 100% of    3.97MiB in 00:00:00 at 5.14MiB/s   \n",
      "[ExtractAudio] Destination: data/PznJqxon4zE/video/Steve Jobs passion in work.wav\n",
      "Moviepy - Writing frames data/PznJqxon4zE/img/steve-jobs-passion-in-work-frame-%04d.png.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Moviepy - Done writing frames data/PznJqxon4zE/img/steve-jobs-passion-in-work-frame-%04d.png.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Parsing nodes: 100%|██████████| 20/20 [00:00<00:00, 1002.86it/s]\n",
      "Generating embeddings: 100%|██████████| 1/1 [00:02<00:00,  2.05s/it]\n",
      "Generating image embeddings: 100%|██████████| 19/19 [00:05<00:00,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query:\n",
      "Who are in the videos?\n",
      " Assistant:\n",
      " Steve Jobs.\n",
      "Query:\n",
      "Who are in the videos?\n",
      " Assistant:\n",
      " Steve Jobs.\n",
      "steve-jobs-passion-in-work /private/var/folders/8r/8d2_blbs1md53x1nyd_6f9m00000gp/T/gradio/66e57a1aaec0cbe1f4320844295265346f658507/steve-jobs-passion-in-work.mp4\n",
      "<moviepy.audio.io.AudioFileClip.AudioFileClip object at 0x1eb283d10>\n",
      "MoviePy - Writing audio in data/steve-jobs-passion-in-work/audio/steve-jobs-passion-in-work.wav\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                      \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MoviePy - Done.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Moviepy - Writing frames data/steve-jobs-passion-in-work/img/steve-jobs-passion-in-work-frame-%04d.png.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Moviepy - Done writing frames data/steve-jobs-passion-in-work/img/steve-jobs-passion-in-work-frame-%04d.png.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Parsing nodes: 100%|██████████| 20/20 [00:00<00:00, 783.14it/s]\n",
      "Generating embeddings: 100%|██████████| 1/1 [00:08<00:00,  8.05s/it]\n",
      "Generating image embeddings: 100%|██████████| 19/19 [00:05<00:00,  3.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query:\n",
      "Summarize the video\n",
      " Assistant:\n",
      " Steve Jobs talked about the importance of passion in work. He said that if you don't love what you do, you'll never be successful. He also said that you need to be a good talent scout and surround yourself with great people.\n",
      "Query:\n",
      "What did Steve Jobs wear in the talk?\n",
      " Assistant:\n",
      " Steve Jobs wore a black turtleneck, glasses, and dark pants.\n",
      "Query:\n",
      "What is the color of his chair?\n",
      " Assistant:\n",
      " The color of the chair is red.\n",
      "Query:\n",
      "Did he sit next to anybody?\n",
      " Assistant:\n",
      " Yes, there was a woman sitting next to him.\n",
      "Query:\n",
      "Who is she/he?\n",
      " Assistant:\n",
      " The person in the video is Steve Jobs, the co-founder of Apple Inc.\n",
      "Query:\n",
      "In which timestamp did Steve Jobs mention about \"talent scout\"?\n",
      " Assistant:\n",
      " Steve Jobs mentioned about \"talent scout\" at 59.2.\n"
     ]
    }
   ],
   "source": [
    "from src.preprocess import VideoMetadata\n",
    "\n",
    "\n",
    "def ingest_and_indexing(input_path_or_url: str, save_dir: Path):\n",
    "    if not os.path.exists(save_dir):\n",
    "        vid_pipe = VideoIngestionPipeline(AppConfig(save_dir=save_dir))\n",
    "        video_metadata = vid_pipe.run(input_path_or_url, run_config)\n",
    "    else:\n",
    "        metadata_file = glob(f\"{save_dir}/metadata/*.json\")\n",
    "        video_metadata = cast(VideoMetadata, json.load(open(metadata_file[0], \"r\")))\n",
    "    # TODO: load if index are saved.\n",
    "    db_client.delete_collection(f\"{video_metadata['id']}_text_collection\")\n",
    "    db_client.delete_collection(f\"{video_metadata['id']}_image_collection\")\n",
    "    index = video_to_index(save_dir, db_client, video_metadata[\"id\"])\n",
    "    return video_metadata, index\n",
    "\n",
    "\n",
    "if demo:\n",
    "    demo.close()\n",
    "\n",
    "try:\n",
    "    with gr.Blocks(fill_height=True, theme=\"ParityError/Anime\") as demo:\n",
    "        index_state = gr.State(None)\n",
    "        video_metadata_state = gr.State(None)\n",
    "        with gr.Row(variant=\"panel\"):\n",
    "            with gr.Column(scale=3):\n",
    "                with gr.Tab(\"Youtube video\"):\n",
    "                    yt_video_output = gr.Video(sources=[\"upload\"], height=\"50%\")\n",
    "                    url_input = gr.Textbox(\n",
    "                        label=\"Copy & paste a Youtube URL\",\n",
    "                        show_copy_button=True,\n",
    "                        placeholder=\"Youtube video URL\",\n",
    "                    )\n",
    "                    fetch_btn = gr.Button(\"Fetch and build index\", variant=\"primary\")\n",
    "\n",
    "                with gr.Tab(\"Your video\"):\n",
    "                    # gr.Markdown(\"Currently not working.\")\n",
    "                    video_output = gr.Video(sources=[\"upload\"], height=\"50%\")\n",
    "                    video_upload = gr.UploadButton(\n",
    "                        label=\"Select video\", file_types=[\"video\"], size=\"sm\"\n",
    "                    )\n",
    "\n",
    "            with gr.Column(scale=7):\n",
    "\n",
    "                def send_message(message, history, index, video_metadata):\n",
    "                    text = message[\"text\"]\n",
    "\n",
    "                    file = None\n",
    "                    if len(message[\"files\"]):\n",
    "                        file = message[\"files\"][0][\"path\"]\n",
    "\n",
    "                    response = chat(\n",
    "                        QueryBundle(query_str=text, image_path=file),\n",
    "                        history,\n",
    "                        index,\n",
    "                        video_metadata,\n",
    "                    )\n",
    "\n",
    "                    # return str(response)\n",
    "                    return chat_response_with_context(response)\n",
    "\n",
    "                bot = gr.Chatbot(height=\"500px\", render=False)\n",
    "                itf = gr.ChatInterface(\n",
    "                    send_message,\n",
    "                    chatbot=bot,\n",
    "                    title=\"Youtube QA\",\n",
    "                    description=\"Paste a youtube link to start chat\",\n",
    "                    multimodal=True,\n",
    "                    additional_inputs=[index_state, video_metadata_state],\n",
    "                )\n",
    "\n",
    "        def on_url_submit(youtube_url: str):\n",
    "            youtube_id = get_yt_id_from_url(youtube_url)\n",
    "            save_dir = Path(f\"data/{youtube_id}\")\n",
    "            video_metadata, index = ingest_and_indexing(youtube_url, save_dir)\n",
    "            video_title = video_metadata[\"title\"]\n",
    "\n",
    "            return {\n",
    "                yt_video_output: video_metadata[\"video_file\"],\n",
    "                index_state: index,\n",
    "                video_metadata_state: video_metadata,\n",
    "                bot: [\n",
    "                    [\n",
    "                        None,\n",
    "                        f\"🎉Loaded the video! If you have any question about {video_title}, feel free to ask.\",\n",
    "                    ]\n",
    "                ],\n",
    "            }\n",
    "\n",
    "        fetch_btn.click(\n",
    "            on_url_submit,\n",
    "            inputs=[url_input],\n",
    "            outputs=[yt_video_output, index_state, video_metadata_state, bot],\n",
    "        )\n",
    "\n",
    "        def on_video_upload(filepath: str):\n",
    "            save_dir = Path(f\"data/{Path(filepath).stem}\")\n",
    "            video_metadata, index = ingest_and_indexing(filepath, save_dir)\n",
    "\n",
    "            return {\n",
    "                video_output: video_metadata[\"video_file\"],\n",
    "                index_state: index,\n",
    "                video_metadata_state: video_metadata,\n",
    "            }\n",
    "\n",
    "        video_upload.upload(\n",
    "            on_video_upload,\n",
    "            inputs=[video_upload],\n",
    "            outputs=[video_output, index_state, video_metadata_state],\n",
    "        )\n",
    "        # video_upload.upload()\n",
    "\n",
    "    demo.launch(share=False, server_port=8000)\n",
    "except Exception as e:\n",
    "    demo.close()\n",
    "    raise e"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}