notebooks/01_bm25_elastic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e87968a1-46d4-4446-ac03-efdd1c97b2f6",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Token Retrieval using BM25\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "143347f8-ae24-4d7e-94b0-25caf11e817b",
   "metadata": {
    "tags": []
   },
   "source": [
    "# About"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30a2d4e5-efe9-470f-bb5c-aa70e789f9ef",
   "metadata": {},
   "source": [
    "In this notebook we will do follows:\n",
    "- Create a text corpus by using descriptions of images\n",
    "- Explore the built in `analyzers` in ElasticSearch\n",
    "- Explore ES apis\n",
    "- Explore the different results of bm25 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "904fc916-2fd9-42c2-9ac4-063f24863fc1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "fd222dd6-7620-40e0-92dc-3a37aaca7fe0",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "358cd5a4-0e74-49c7-8017-f908113866df",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:28:49.560746Z",
     "iopub.status.busy": "2023-04-25T19:28:49.559868Z",
     "iopub.status.idle": "2023-04-25T19:28:49.566129Z",
     "shell.execute_reply": "2023-04-25T19:28:49.564996Z",
     "shell.execute_reply.started": "2023-04-25T19:28:49.560715Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import datasets\n",
    "\n",
    "from IPython.display import Image, JSON\n",
    "from IPython.core.display import HTML\n",
    "import rich\n",
    "import re\n",
    "\n",
    "import requests\n",
    "import tqdm.auto\n",
    "import ipywidgets as widgets\n",
    "from ipywidgets import interact\n",
    "\n",
    "import ipyplot\n",
    "\n",
    "from elasticsearch import Elasticsearch\n",
    "from elasticsearch.helpers import streaming_bulk\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1f90274-c89e-4587-b3eb-cd7a380fea6c",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "790d9b8b-c8b5-4ef2-b3ce-432a54503790",
   "metadata": {},
   "source": [
    "##### We load the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "1fb8e04c-56de-4ec5-a296-e7aba0d75750",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:23:37.398388Z",
     "iopub.status.busy": "2023-04-25T19:23:37.397447Z",
     "iopub.status.idle": "2023-04-25T19:23:37.409931Z",
     "shell.execute_reply": "2023-04-25T19:23:37.409188Z",
     "shell.execute_reply.started": "2023-04-25T19:23:37.398351Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "dset = datasets.load_from_disk(\"../data/processed\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d5c6a5c-fcea-4130-bb20-ca34a9c8191f",
   "metadata": {},
   "source": [
    "##### Dataset consists of 24995 rows and each row has columns that contains information about the photo like \"photo description\", \"size\", etc.."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "3b125e2a-b54d-44fc-b068-3de279d7d2c7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:23:38.438694Z",
     "iopub.status.busy": "2023-04-25T19:23:38.437666Z",
     "iopub.status.idle": "2023-04-25T19:23:38.443999Z",
     "shell.execute_reply": "2023-04-25T19:23:38.443052Z",
     "shell.execute_reply.started": "2023-04-25T19:23:38.438658Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['photo_id', 'photo_url', 'photo_image_url', 'photo_submitted_at', 'photo_featured', 'photo_width', 'photo_height', 'photo_aspect_ratio', 'photo_description', 'photographer_username', 'photographer_first_name', 'photographer_last_name', 'exif_camera_make', 'exif_camera_model', 'exif_iso', 'exif_aperture_value', 'exif_focal_length', 'exif_exposure_time', 'photo_location_name', 'photo_location_latitude', 'photo_location_longitude', 'photo_location_country', 'photo_location_city', 'stats_views', 'stats_downloads', 'ai_description', 'ai_primary_landmark_name', 'ai_primary_landmark_latitude', 'ai_primary_landmark_longitude', 'ai_primary_landmark_confidence', 'blur_hash', 'description_final', 'image', 'image_path_local'],\n",
       "    num_rows: 24992\n",
       "})"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2766a7bf-4279-4561-911d-3b24a85b695c",
   "metadata": {},
   "source": [
    "##### We use \"description_final\" field of the photos to create a text corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "072fd640-9393-4cfb-8eb2-cab705ec6c01",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:23:40.305071Z",
     "iopub.status.busy": "2023-04-25T19:23:40.304118Z",
     "iopub.status.idle": "2023-04-25T19:23:40.335366Z",
     "shell.execute_reply": "2023-04-25T19:23:40.334494Z",
     "shell.execute_reply.started": "2023-04-25T19:23:40.305036Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Woman exploring a forest',\n",
       " 'Succulents in a terrarium',\n",
       " 'Rural winter mountainside',\n",
       " 'Poppy seeds and flowers',\n",
       " 'Silhouette near dark trees']"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dset['description_final'][:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b982dc8d-fba8-4f68-b45e-3f630e4777b1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "62f2f898-bad8-45b5-bed0-b4d8109fe4b5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:23:41.203262Z",
     "iopub.status.busy": "2023-04-25T19:23:41.202559Z",
     "iopub.status.idle": "2023-04-25T19:23:41.216398Z",
     "shell.execute_reply": "2023-04-25T19:23:41.215455Z",
     "shell.execute_reply.started": "2023-04-25T19:23:41.203229Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'photo_id': 'XMyPniM9LF0',\n",
       " 'photo_url': 'https://unsplash.com/photos/XMyPniM9LF0',\n",
       " 'photo_image_url': 'https://images.unsplash.com/uploads/14119492946973137ce46/f1f2ebf3',\n",
       " 'photo_submitted_at': '2014-09-29 00:08:38.594364',\n",
       " 'photo_featured': 't',\n",
       " 'photo_width': 4272,\n",
       " 'photo_height': 2848,\n",
       " 'photo_aspect_ratio': 1.5,\n",
       " 'photo_description': 'Woman exploring a forest',\n",
       " 'photographer_username': 'michellespencer77',\n",
       " 'photographer_first_name': 'Michelle',\n",
       " 'photographer_last_name': 'Spencer',\n",
       " 'exif_camera_make': 'Canon',\n",
       " 'exif_camera_model': 'Canon EOS REBEL T3',\n",
       " 'exif_iso': 400.0,\n",
       " 'exif_aperture_value': '1.8',\n",
       " 'exif_focal_length': '50.0',\n",
       " 'exif_exposure_time': '1/100',\n",
       " 'photo_location_name': None,\n",
       " 'photo_location_latitude': None,\n",
       " 'photo_location_longitude': None,\n",
       " 'photo_location_country': None,\n",
       " 'photo_location_city': None,\n",
       " 'stats_views': 2375421,\n",
       " 'stats_downloads': 6967,\n",
       " 'ai_description': 'woman walking in the middle of forest',\n",
       " 'ai_primary_landmark_name': None,\n",
       " 'ai_primary_landmark_latitude': None,\n",
       " 'ai_primary_landmark_longitude': None,\n",
       " 'ai_primary_landmark_confidence': None,\n",
       " 'blur_hash': 'L56bVcRRIWMh.gVunlS4SMbsRRxr',\n",
       " 'description_final': 'Woman exploring a forest',\n",
       " 'image': <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x427>,\n",
       " 'image_path_local': '../data/raw/images/XMyPniM9LF0.jpg'}"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81af1a68-e3ad-4281-a547-5f9c96e7ca8d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9db8ce66-e522-4567-ade2-8c22cc573d81",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "bb7d7329-959c-4d1c-80ae-3fac0656d3ab",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Elastic Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "7dbf4773-4f90-43f0-b732-8d6edc8e93bb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:23:43.270663Z",
     "iopub.status.busy": "2023-04-25T19:23:43.269694Z",
     "iopub.status.idle": "2023-04-25T19:23:43.274521Z",
     "shell.execute_reply": "2023-04-25T19:23:43.273598Z",
     "shell.execute_reply.started": "2023-04-25T19:23:43.270624Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "ELASTIC_HOST=\"localhost\"\n",
    "ELASTIC_INDEX=\"unsplash\"\n",
    "ELASTIC_PORT=9200\n",
    "\n",
    "ELASTIC_FULL_URL =f\"http://{ELASTIC_HOST}:{ELASTIC_PORT}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9de785ae-1d53-491c-a1cd-6bf04e52f46e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "220a4932-8bb0-4678-b7dc-5948f39b63b6",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Elastic Search Default Analyzers "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b98c88a6-f24a-4db5-996b-84081e9f2196",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Elastic Search Analyzer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19fbee4d-a2ce-4b86-9f1f-38de3460b4c3",
   "metadata": {},
   "source": [
    "ElasticSearch has many default analyzer.\n",
    "\n",
    "Analyzers are composed of `tokenizers` and `normalizers`.\n",
    "\n",
    "tokenization: breaking a text down into smaller chunks \n",
    "\n",
    "normalizers: format the token\n",
    "\n",
    "[ElasticDoc](https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-overview.html)\n",
    "\n",
    "[Documentation for analyzers](https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-analyzers.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3be4bca-480b-4127-8066-12675a171e2e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8b7fd6e1-53c2-402a-b4c9-319a2ce52433",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Analyzers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "2a08a9a4-d02a-472c-8df3-68d05c8ae221",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:23:47.498540Z",
     "iopub.status.busy": "2023-04-25T19:23:47.497730Z",
     "iopub.status.idle": "2023-04-25T19:23:47.503250Z",
     "shell.execute_reply": "2023-04-25T19:23:47.502233Z",
     "shell.execute_reply.started": "2023-04-25T19:23:47.498500Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def elastic_analyze(analyzer,  text, url = ELASTIC_FULL_URL+\"/_analyze\"):\n",
    "    r =requests.post(url, \n",
    "              json =\n",
    "                    {\n",
    "                      \"analyzer\": analyzer ,\n",
    "                      \"text\": text\n",
    "                    }\n",
    "        )\n",
    "\n",
    "    rich.print (r.json() )\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "80c956d7-c919-4c8f-9ad5-222e7f8bb12f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:23:47.839839Z",
     "iopub.status.busy": "2023-04-25T19:23:47.838946Z",
     "iopub.status.idle": "2023-04-25T19:23:47.843440Z",
     "shell.execute_reply": "2023-04-25T19:23:47.842463Z",
     "shell.execute_reply.started": "2023-04-25T19:23:47.839806Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sentence = \"Two dogs walking in the snow. ❄️ 雪\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a95c07c1-40f9-4459-83d9-428ab1d316b4",
   "metadata": {},
   "source": [
    "**whitespace analyzer**\n",
    "\n",
    "The whitespace analyzer breaks text into terms whenever it encounters a whitespace character.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "a2dc2105-626a-440e-b7c6-e2faaf84f108",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:23:49.712332Z",
     "iopub.status.busy": "2023-04-25T19:23:49.711663Z",
     "iopub.status.idle": "2023-04-25T19:23:49.727717Z",
     "shell.execute_reply": "2023-04-25T19:23:49.726826Z",
     "shell.execute_reply.started": "2023-04-25T19:23:49.712298Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'tokens'</span>: <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'Two'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'dogs'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'walking'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'in'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">17</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">19</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'the'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">23</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'snow.'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">24</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">29</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'❄️'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">30</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">32</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'雪'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">33</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span><span style=\"font-weight: bold\">}</span>\n",
       "    <span style=\"font-weight: bold\">]</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'tokens'\u001b[0m: \u001b[1m[\u001b[0m\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'Two'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m3\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m0\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'dogs'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m4\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m8\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'walking'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m9\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m16\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m2\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'in'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m17\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m19\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m3\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'the'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m20\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m23\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m4\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'snow.'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m24\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m29\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m5\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'❄️'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m30\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m32\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m6\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'雪'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m33\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m34\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m7\u001b[0m\u001b[1m}\u001b[0m\n",
       "    \u001b[1m]\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "elastic_analyze(analyzer = \"whitespace\", text = sentence )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a46e729c-16f3-4e95-aa89-bbeaceeda1f6",
   "metadata": {},
   "source": [
    "**stop analyzer**\n",
    "\n",
    "changes uppercase to lowercase.  \n",
    "also uses _english_ stop words.  \n",
    "breaks text into tokens at any non-letter character    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "4ea1b281-23f0-4dc4-96dd-f5c4e077af56",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:23:50.866627Z",
     "iopub.status.busy": "2023-04-25T19:23:50.865758Z",
     "iopub.status.idle": "2023-04-25T19:23:50.879949Z",
     "shell.execute_reply": "2023-04-25T19:23:50.878978Z",
     "shell.execute_reply.started": "2023-04-25T19:23:50.866592Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'tokens'</span>: <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'two'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'dogs'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'walking'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'snow'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">24</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">28</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'雪'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">33</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span><span style=\"font-weight: bold\">}</span>\n",
       "    <span style=\"font-weight: bold\">]</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'tokens'\u001b[0m: \u001b[1m[\u001b[0m\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'two'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m3\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m0\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'dogs'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m4\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m8\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'walking'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m9\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m16\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m2\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'snow'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m24\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m28\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m5\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'雪'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m33\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m34\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m6\u001b[0m\u001b[1m}\u001b[0m\n",
       "    \u001b[1m]\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "elastic_analyze(analyzer = \"stop\", text = sentence )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8d541c7-2025-49d6-80a9-bfe177aa9906",
   "metadata": {},
   "source": [
    "**standard analyzer**\n",
    "\n",
    "default analyzer       \n",
    "grammar based tokenization\n",
    "stopword disabled\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "ef1b97ac-b33f-4f93-97cd-f8cb16d4e77b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:23:51.902922Z",
     "iopub.status.busy": "2023-04-25T19:23:51.902155Z",
     "iopub.status.idle": "2023-04-25T19:23:51.918497Z",
     "shell.execute_reply": "2023-04-25T19:23:51.917554Z",
     "shell.execute_reply.started": "2023-04-25T19:23:51.902881Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'tokens'</span>: <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'two'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'dogs'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'walking'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'in'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">17</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">19</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'the'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">23</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'snow'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">24</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">28</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'❄️'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">30</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">32</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;EMOJI&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'雪'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">33</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;IDEOGRAPHIC&gt;'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span><span style=\"font-weight: bold\">}</span>\n",
       "    <span style=\"font-weight: bold\">]</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'tokens'\u001b[0m: \u001b[1m[\u001b[0m\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'two'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m3\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m<\u001b[0m\u001b[32mALPHANUM\u001b[0m\u001b[32m>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'dogs'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m4\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m8\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'walking'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m9\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m16\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'in'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m17\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m19\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'the'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m20\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m23\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'snow'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m24\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m28\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'❄️'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m30\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m32\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<EMOJI>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m6\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'雪'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m33\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m34\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<IDEOGRAPHIC\u001b[0m\u001b[32m>\u001b[0m\u001b[32m'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m7\u001b[0m\u001b[1m}\u001b[0m\n",
       "    \u001b[1m]\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "elastic_analyze(analyzer = \"standard\", text = sentence )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ffd3135-e083-4bad-b3af-5a4f168c7914",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6197cf69-f484-4aeb-9689-d9c1f059bc62",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "bb9a708f-122c-4b33-88c3-2ddc39befc7a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:23:53.736451Z",
     "iopub.status.busy": "2023-04-25T19:23:53.735506Z",
     "iopub.status.idle": "2023-04-25T19:23:53.750809Z",
     "shell.execute_reply": "2023-04-25T19:23:53.749863Z",
     "shell.execute_reply.started": "2023-04-25T19:23:53.736409Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'tokens'</span>: <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'two'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'dogs'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'walking'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'in'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">17</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">19</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'the'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">23</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'snow'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">24</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">28</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'雪'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">33</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span><span style=\"font-weight: bold\">}</span>\n",
       "    <span style=\"font-weight: bold\">]</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'tokens'\u001b[0m: \u001b[1m[\u001b[0m\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'two'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m3\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m0\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'dogs'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m4\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m8\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'walking'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m9\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m16\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m2\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'in'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m17\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m19\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m3\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'the'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m20\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m23\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m4\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'snow'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m24\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m28\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m5\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'雪'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m33\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m34\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m6\u001b[0m\u001b[1m}\u001b[0m\n",
       "    \u001b[1m]\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "elastic_analyze(analyzer = \"simple\", text = sentence )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "e7791f11-7669-480b-b224-9549baa741d3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:23:53.869783Z",
     "iopub.status.busy": "2023-04-25T19:23:53.869159Z",
     "iopub.status.idle": "2023-04-25T19:23:53.883168Z",
     "shell.execute_reply": "2023-04-25T19:23:53.882383Z",
     "shell.execute_reply.started": "2023-04-25T19:23:53.869752Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'tokens'</span>: <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'two'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'dog'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'walk'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'snow'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">24</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">28</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'❄️'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">30</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">32</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;EMOJI&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'雪'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">33</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;IDEOGRAPHIC&gt;'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span><span style=\"font-weight: bold\">}</span>\n",
       "    <span style=\"font-weight: bold\">]</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'tokens'\u001b[0m: \u001b[1m[\u001b[0m\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'two'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m3\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m<\u001b[0m\u001b[32mALPHANUM\u001b[0m\u001b[32m>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'dog'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m4\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m8\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'walk'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m9\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m16\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'snow'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m24\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m28\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'❄️'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m30\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m32\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<EMOJI>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m6\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'雪'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m33\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m34\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<IDEOGRAPHIC\u001b[0m\u001b[32m>\u001b[0m\u001b[32m'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m7\u001b[0m\u001b[1m}\u001b[0m\n",
       "    \u001b[1m]\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "elastic_analyze(analyzer = \"english\", text = sentence )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37908019-95b1-4177-a5db-361a43b3487c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16c1da24-1a44-4d98-97da-e4792403c28b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7630993-e6e1-404d-82a6-b4fba2e78adf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8d2e53c5-ed4f-469c-a9c0-fd49d82b907f",
   "metadata": {},
   "source": [
    "## Index Documents\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0955e4d2-fe38-43c2-b3e7-baf9a2740e59",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "34537a31-96e4-4845-82a9-eb2b0c5f0d69",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:23:55.856762Z",
     "iopub.status.busy": "2023-04-25T19:23:55.855993Z",
     "iopub.status.idle": "2023-04-25T19:23:55.861702Z",
     "shell.execute_reply": "2023-04-25T19:23:55.860669Z",
     "shell.execute_reply.started": "2023-04-25T19:23:55.856730Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "client = Elasticsearch(\n",
    "    [ELASTIC_FULL_URL]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "0548ee61-fff2-4994-959d-0d9b2d072e34",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:20.899652Z",
     "iopub.status.busy": "2023-04-25T19:24:20.898763Z",
     "iopub.status.idle": "2023-04-25T19:24:20.908218Z",
     "shell.execute_reply": "2023-04-25T19:24:20.907511Z",
     "shell.execute_reply.started": "2023-04-25T19:24:20.899620Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_index(client,index:str, num_shards=1):\n",
    "    \"\"\"Creates an index in Elasticsearch. Delete old index.\"\"\"\n",
    "    \n",
    "    if client.indices.exists(index=index):\n",
    "        client.indices.delete(index=index)\n",
    "    \n",
    "    client.indices.create(\n",
    "        index=index\n",
    "        ,settings = {\"number_of_shards\": num_shards}\n",
    "        ,mappings= {\n",
    "            \n",
    "            \"properties\": {\n",
    "                        \"description_final\": {\"type\": \"text\"}\n",
    "                        ,\"photo_image_url\": {\"type\": \"text\" ,\"index\":False}\n",
    "                        ,\"photo_id\": {\"type\": \"keyword\" ,\"index\":False}\n",
    "                \n",
    "                   }\n",
    "            }\n",
    "       \n",
    "        #,ignore=400\n",
    "    )\n",
    "\n",
    "\n",
    "def generate_docs(df:pd.DataFrame):\n",
    "    \"\"\"\n",
    "    Given a datframe containing posts data, yields a generator of dicitionary \n",
    "    \"\"\"\n",
    "    \n",
    "    df = df[['photo_id','description_final','photo_image_url']]\n",
    "    \n",
    "    # iterate over dataframe contains posts with metadata\n",
    "    for index, row in df.iterrows():\n",
    "        doc = {**row} \n",
    "        \n",
    "        # use PostId as document id\n",
    "        doc['_id'] = doc[\"photo_id\"]\n",
    "        \n",
    "        for k in list(doc.keys()):\n",
    "            # don't insert nan fields\n",
    "            if type(doc[k]) !=list and (doc[k] ==None or  ( pd.isna( doc[k] )  )) :\n",
    "                del doc[k]\n",
    "        \n",
    "        yield doc\n",
    "        \n",
    "\n",
    "\n",
    "def fetch_results(client:Elasticsearch, query:str,  num_hits=5, fields = [\"description_final\"], analyzer =\"stop\"):\n",
    "    \"\"\"\n",
    "    With the passed elastic search client, return documents that contain the passed `query` in the fields specified by `fields`\n",
    "\n",
    "    If the fields is empty, it will search all text fields\n",
    "    \n",
    "    We are using mult-match, which by default uses `or`\n",
    "    https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-match-query.html\n",
    "    \"\"\"\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "    resp = client.search(\n",
    "        query= {\n",
    "                \"multi_match\": {\n",
    "                    \"query\": query,\n",
    "                    \"fields\": fields,\n",
    "                     \"analyzer\": analyzer\n",
    "                   # \"operator\": \"and\" \n",
    "                },\n",
    "               \n",
    "            }\n",
    "        ,size = num_hits\n",
    "    )\n",
    "    \n",
    "    return resp\n",
    "    \n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01c7fd33-0b84-4f30-875e-f74927c2a01a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8567da35-60fd-46dc-993e-a5aa05597f2e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c84f7e4-5ec4-49cc-96ee-84d770821761",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "16aa9e7c-fafa-4313-90ef-78b3ce454ec6",
   "metadata": {},
   "source": [
    "tell elastic search to create an index     \n",
    "An ES index is a collection of documents. \n",
    "\n",
    "ES suports inferring the documents without specifying the schema before hand "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "c053ab03-dc02-4082-b2a1-2d823643129b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:24.131703Z",
     "iopub.status.busy": "2023-04-25T19:24:24.130975Z",
     "iopub.status.idle": "2023-04-25T19:24:24.476199Z",
     "shell.execute_reply": "2023-04-25T19:24:24.475384Z",
     "shell.execute_reply.started": "2023-04-25T19:24:24.131670Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "create_index(client, index= ELASTIC_INDEX, num_shards=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b70400ce-b0b0-49b2-8e8a-6635c4c1a157",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "642685c2-cbea-4936-a40e-901421bfecb0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "d04f11cb-1024-4879-a328-bac28d0d6d31",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:25.664428Z",
     "iopub.status.busy": "2023-04-25T19:24:25.663680Z",
     "iopub.status.idle": "2023-04-25T19:24:25.673946Z",
     "shell.execute_reply": "2023-04-25T19:24:25.673154Z",
     "shell.execute_reply.started": "2023-04-25T19:24:25.664394Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[0;31mSignature:\u001b[0m\n",
       "\u001b[0mclient\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindices\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\n",
       "\u001b[0;34m\u001b[0m    \u001b[0;34m*\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
       "\u001b[0;34m\u001b[0m    \u001b[0mindex\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
       "\u001b[0;34m\u001b[0m    \u001b[0maliases\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mMapping\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mMapping\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNoneType\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
       "\u001b[0;34m\u001b[0m    \u001b[0merror_trace\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNoneType\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
       "\u001b[0;34m\u001b[0m    \u001b[0mfilter_path\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTuple\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m...\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNoneType\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
       "\u001b[0;34m\u001b[0m    \u001b[0mhuman\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNoneType\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
       "\u001b[0;34m\u001b[0m    \u001b[0mmappings\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mMapping\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNoneType\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
       "\u001b[0;34m\u001b[0m    \u001b[0mmaster_timeout\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mForwardRef\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m't.Literal[-1]'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mForwardRef\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m't.Literal[0]'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNoneType\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
       "\u001b[0;34m\u001b[0m    \u001b[0mpretty\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNoneType\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
       "\u001b[0;34m\u001b[0m    \u001b[0msettings\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mMapping\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNoneType\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
       "\u001b[0;34m\u001b[0m    \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mForwardRef\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m't.Literal[-1]'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mForwardRef\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m't.Literal[0]'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNoneType\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
       "\u001b[0;34m\u001b[0m    \u001b[0mwait_for_active_shards\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mForwardRef\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"t.Literal['all', 'index-setting']\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNoneType\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n",
       "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0melastic_transport\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mObjectApiResponse\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtyping\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAny\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;31mDocstring:\u001b[0m\n",
       "Creates an index with optional settings and mappings.\n",
       "\n",
       "`<https://www.elastic.co/guide/en/elasticsearch/reference/8.7/indices-create-index.html>`_\n",
       "\n",
       ":param index: The name of the index\n",
       ":param aliases:\n",
       ":param mappings: Mapping for fields in the index. If specified, this mapping\n",
       "    can include: - Field names - Field data types - Mapping parameters\n",
       ":param master_timeout: Specify timeout for connection to master\n",
       ":param settings:\n",
       ":param timeout: Explicit operation timeout\n",
       ":param wait_for_active_shards: Set the number of active shards to wait for before\n",
       "    the operation returns.\n",
       "\u001b[0;31mFile:\u001b[0m      /opt/conda/envs/workshop/lib/python3.7/site-packages/elasticsearch/_sync/client/indices.py\n",
       "\u001b[0;31mType:\u001b[0m      method\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "?client.indices.create"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "37cebde7-107c-4294-bb1a-b0113f23283f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:26.293628Z",
     "iopub.status.busy": "2023-04-25T19:24:26.292864Z",
     "iopub.status.idle": "2023-04-25T19:24:26.304186Z",
     "shell.execute_reply": "2023-04-25T19:24:26.303400Z",
     "shell.execute_reply.started": "2023-04-25T19:24:26.293592Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'unsplash': {'settings': {'index': {'routing': {'allocation': {'include': {'_tier_preference': 'data_content'}}},\n",
       "    'number_of_shards': '1',\n",
       "    'provided_name': 'unsplash',\n",
       "    'creation_date': '1682450664263',\n",
       "    'number_of_replicas': '1',\n",
       "    'uuid': '7gKPz7FiTE2FXojjGpABgA',\n",
       "    'version': {'created': '8070099'}}}}}"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "requests.get(f\"{ELASTIC_FULL_URL}/_all/_settings\").json()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d19a623-ad97-4576-b2f4-fb949528e0f4",
   "metadata": {},
   "source": [
    "The index we created is composed of `1` shards and `1` replica.   \n",
    "\n",
    "When searching , ES queries each shard independantly and combines it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "9ee355a2-e370-4ea6-86c1-d0592a4e5bf3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:27.246904Z",
     "iopub.status.busy": "2023-04-25T19:24:27.246160Z",
     "iopub.status.idle": "2023-04-25T19:24:27.252375Z",
     "shell.execute_reply": "2023-04-25T19:24:27.251456Z",
     "shell.execute_reply.started": "2023-04-25T19:24:27.246870Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24992"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "ee2a36b9-d2db-4cd9-9ae2-bf259b9d0dd6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:27.479207Z",
     "iopub.status.busy": "2023-04-25T19:24:27.478581Z",
     "iopub.status.idle": "2023-04-25T19:24:27.587702Z",
     "shell.execute_reply": "2023-04-25T19:24:27.586783Z",
     "shell.execute_reply.started": "2023-04-25T19:24:27.479173Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_subset = dset.to_pandas()\n",
    "number_of_docs = len(df_subset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d241790-bb82-4efa-b4b1-40007bf6b91e",
   "metadata": {},
   "source": [
    "Bulk insert all of our documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "bc893ce3-6a3b-4f43-b8b5-476bca5a87aa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:28.351039Z",
     "iopub.status.busy": "2023-04-25T19:24:28.350063Z",
     "iopub.status.idle": "2023-04-25T19:24:28.357333Z",
     "shell.execute_reply": "2023-04-25T19:24:28.356595Z",
     "shell.execute_reply.started": "2023-04-25T19:24:28.351003Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'photo_id': 'XMyPniM9LF0',\n",
       " 'photo_url': 'https://unsplash.com/photos/XMyPniM9LF0',\n",
       " 'photo_image_url': 'https://images.unsplash.com/uploads/14119492946973137ce46/f1f2ebf3',\n",
       " 'photo_submitted_at': '2014-09-29 00:08:38.594364',\n",
       " 'photo_featured': 't',\n",
       " 'photo_width': 4272,\n",
       " 'photo_height': 2848,\n",
       " 'photo_aspect_ratio': 1.5,\n",
       " 'photo_description': 'Woman exploring a forest',\n",
       " 'photographer_username': 'michellespencer77',\n",
       " 'photographer_first_name': 'Michelle',\n",
       " 'photographer_last_name': 'Spencer',\n",
       " 'exif_camera_make': 'Canon',\n",
       " 'exif_camera_model': 'Canon EOS REBEL T3',\n",
       " 'exif_iso': 400.0,\n",
       " 'exif_aperture_value': '1.8',\n",
       " 'exif_focal_length': '50.0',\n",
       " 'exif_exposure_time': '1/100',\n",
       " 'photo_location_name': None,\n",
       " 'photo_location_latitude': nan,\n",
       " 'photo_location_longitude': nan,\n",
       " 'photo_location_country': None,\n",
       " 'photo_location_city': None,\n",
       " 'stats_views': 2375421,\n",
       " 'stats_downloads': 6967,\n",
       " 'ai_description': 'woman walking in the middle of forest',\n",
       " 'ai_primary_landmark_name': None,\n",
       " 'ai_primary_landmark_latitude': nan,\n",
       " 'ai_primary_landmark_longitude': nan,\n",
       " 'ai_primary_landmark_confidence': nan,\n",
       " 'blur_hash': 'L56bVcRRIWMh.gVunlS4SMbsRRxr',\n",
       " 'description_final': 'Woman exploring a forest',\n",
       " 'image': {'bytes': None, 'path': '../data/raw/images/XMyPniM9LF0.jpg'},\n",
       " 'image_path_local': '../data/raw/images/XMyPniM9LF0.jpg'}"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_subset.iloc[0].to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "91490c0d-a49c-4b21-bf27-29ce7d259693",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:28.894544Z",
     "iopub.status.busy": "2023-04-25T19:24:28.893774Z",
     "iopub.status.idle": "2023-04-25T19:24:32.246032Z",
     "shell.execute_reply": "2023-04-25T19:24:32.245162Z",
     "shell.execute_reply.started": "2023-04-25T19:24:28.894510Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c599e02a80cf4bc88384e164ead65301",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/24992 [00:00<?, ?docs/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with tqdm.auto.tqdm(total=number_of_docs , unit=\"docs\" ) as pbar:\n",
    "    successes = 0\n",
    "\n",
    "\n",
    "    for ok, action in streaming_bulk(\n",
    "            client=client, index=ELASTIC_INDEX, actions=generate_docs(df_subset) ,\n",
    "        ):\n",
    "        pbar.update(1)\n",
    "        successes += ok\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a02fb180-d608-4ee8-8610-ecea82fd628d",
   "metadata": {},
   "source": [
    "Inserting `20k` documents at `3000` docs/sec on a single node is pretty good"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "08b189f0-481a-495d-a4f0-04b93b486c9b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:32.247660Z",
     "iopub.status.busy": "2023-04-25T19:24:32.247412Z",
     "iopub.status.idle": "2023-04-25T19:24:32.263011Z",
     "shell.execute_reply": "2023-04-25T19:24:32.262318Z",
     "shell.execute_reply.started": "2023-04-25T19:24:32.247638Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">ip         heap.percent ram.percent cpu load_1m load_5m load_15m node.role   master name\n",
       "<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-weight: bold\">172.18.0.2</span>            <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>          <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span>   <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.02</span>    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.07</span>     <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.51</span> cdfhilmrstw *      734d6d950b5c\n",
       "\n",
       "</pre>\n"
      ],
      "text/plain": [
       "ip         heap.percent ram.percent cpu load_1m load_5m load_15m node.role   master name\n",
       "\u001b[1;92m172.18.0.2\u001b[0m            \u001b[1;36m3\u001b[0m          \u001b[1;36m67\u001b[0m   \u001b[1;36m5\u001b[0m    \u001b[1;36m1.02\u001b[0m    \u001b[1;36m1.07\u001b[0m     \u001b[1;36m0.51\u001b[0m cdfhilmrstw *      734d6d950b5c\n",
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "rich.print (\n",
    "    requests.get(f\"{ELASTIC_FULL_URL}/_cat/nodes?v=true\").content.decode()\n",
    "    \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "712315c7-1cf6-4d77-8575-df429f8e155b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:32.264412Z",
     "iopub.status.busy": "2023-04-25T19:24:32.263931Z",
     "iopub.status.idle": "2023-04-25T19:24:32.275801Z",
     "shell.execute_reply": "2023-04-25T19:24:32.274901Z",
     "shell.execute_reply.started": "2023-04-25T19:24:32.264385Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'unsplash'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'mappings'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "            <span style=\"color: #008000; text-decoration-color: #008000\">'properties'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "                <span style=\"color: #008000; text-decoration-color: #008000\">'description_final'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'text'</span><span style=\"font-weight: bold\">}</span>,\n",
       "                <span style=\"color: #008000; text-decoration-color: #008000\">'photo_id'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'keyword'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'index'</span>: <span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">}</span>,\n",
       "                <span style=\"color: #008000; text-decoration-color: #008000\">'photo_image_url'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'text'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'index'</span>: <span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">}</span>\n",
       "            <span style=\"font-weight: bold\">}</span>\n",
       "        <span style=\"font-weight: bold\">}</span>\n",
       "    <span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'unsplash'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "        \u001b[32m'mappings'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "            \u001b[32m'properties'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "                \u001b[32m'description_final'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'type'\u001b[0m: \u001b[32m'text'\u001b[0m\u001b[1m}\u001b[0m,\n",
       "                \u001b[32m'photo_id'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'type'\u001b[0m: \u001b[32m'keyword'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[3;91mFalse\u001b[0m\u001b[1m}\u001b[0m,\n",
       "                \u001b[32m'photo_image_url'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'type'\u001b[0m: \u001b[32m'text'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[3;91mFalse\u001b[0m\u001b[1m}\u001b[0m\n",
       "            \u001b[1m}\u001b[0m\n",
       "        \u001b[1m}\u001b[0m\n",
       "    \u001b[1m}\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "rich.print (\n",
    "    requests.get(f\"{ELASTIC_FULL_URL}/{ELASTIC_INDEX}/_mapping\").json()\n",
    "    \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "a8aeb431-c5e1-4f0b-b9dc-84e2dde45bcb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:32.277859Z",
     "iopub.status.busy": "2023-04-25T19:24:32.277616Z",
     "iopub.status.idle": "2023-04-25T19:24:32.287493Z",
     "shell.execute_reply": "2023-04-25T19:24:32.286819Z",
     "shell.execute_reply.started": "2023-04-25T19:24:32.277835Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">index    shard prirep state       docs   store ip         node\n",
       "unsplash <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>     p      STARTED    <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">19000</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">785.</span>6kb <span style=\"color: #00ff00; text-decoration-color: #00ff00; font-weight: bold\">172.18.0.2</span> 734d6d950b5c\n",
       "unsplash <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>     r      UNASSIGNED                          \n",
       "\n",
       "</pre>\n"
      ],
      "text/plain": [
       "index    shard prirep state       docs   store ip         node\n",
       "unsplash \u001b[1;36m0\u001b[0m     p      STARTED    \u001b[1;36m19000\u001b[0m \u001b[1;36m785.\u001b[0m6kb \u001b[1;92m172.18.0.2\u001b[0m 734d6d950b5c\n",
       "unsplash \u001b[1;36m0\u001b[0m     r      UNASSIGNED                          \n",
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "rich.print (\n",
    "    requests.get(f\"{ELASTIC_FULL_URL}/_cat/shards/{ELASTIC_INDEX}?v=true\").content.decode()\n",
    "    \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee3c814b-a6d4-41ed-8499-c71851d31d02",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "de93a098-26ae-4af5-8411-f889121a5823",
   "metadata": {},
   "source": [
    "getting a specific document by their id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "1926dcec-bcd1-41ca-99e8-8781ca0f5c29",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:33.081594Z",
     "iopub.status.busy": "2023-04-25T19:24:33.080762Z",
     "iopub.status.idle": "2023-04-25T19:24:33.091417Z",
     "shell.execute_reply": "2023-04-25T19:24:33.090413Z",
     "shell.execute_reply.started": "2023-04-25T19:24:33.081560Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'_index': 'unsplash',\n",
       " '_id': 'XMyPniM9LF0',\n",
       " '_version': 1,\n",
       " '_seq_no': 0,\n",
       " '_primary_term': 1,\n",
       " 'found': True,\n",
       " '_source': {'photo_id': 'XMyPniM9LF0',\n",
       "  'description_final': 'Woman exploring a forest',\n",
       "  'photo_image_url': 'https://images.unsplash.com/uploads/14119492946973137ce46/f1f2ebf3'}}"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resp = client.get(index=ELASTIC_INDEX, id=\"XMyPniM9LF0\")\n",
    "resp.body"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3bdfbff-0b73-4fc2-ac4c-8cd485c1b099",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32af85d1-e6cc-4690-af43-c352add1dd9e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "77bda2d9-f179-497b-9383-dd0e4de489bb",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Evaluate"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5dba76ce-8926-47ed-9c45-680a85450a0e",
   "metadata": {},
   "source": [
    "retrieve a document with a query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16ee2249-8ea0-44ac-a841-5fc39e3b342d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "c6ae0b74-83b9-4649-b2df-08cae31f6831",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:37.097233Z",
     "iopub.status.busy": "2023-04-25T19:24:37.096333Z",
     "iopub.status.idle": "2023-04-25T19:24:37.100506Z",
     "shell.execute_reply": "2023-04-25T19:24:37.099808Z",
     "shell.execute_reply.started": "2023-04-25T19:24:37.097191Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "query = \"Two dogs playing in the snow\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "6d5dcaa3-9093-458a-b777-7a86d5f770d9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:37.371758Z",
     "iopub.status.busy": "2023-04-25T19:24:37.370828Z",
     "iopub.status.idle": "2023-04-25T19:24:37.382034Z",
     "shell.execute_reply": "2023-04-25T19:24:37.381197Z",
     "shell.execute_reply.started": "2023-04-25T19:24:37.371722Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/34147471/elasticsearch-how-to-search-for-a-value-in-any-field-across-all-types-in-one\n",
    "\n",
    "\n",
    "resp = client.search(\n",
    "    query = {\n",
    "            \"multi_match\": {\n",
    "                \"query\": query,\n",
    "                # \"fields\": [\"Title\", \"QuestionBody\"],\n",
    "                            }\n",
    "            }\n",
    "    , size=5\n",
    "    , explain=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "605e49ae-b070-4c93-b025-6c1b6bd7a0d4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:37.596508Z",
     "iopub.status.busy": "2023-04-25T19:24:37.595636Z",
     "iopub.status.idle": "2023-04-25T19:24:37.601098Z",
     "shell.execute_reply": "2023-04-25T19:24:37.600327Z",
     "shell.execute_reply.started": "2023-04-25T19:24:37.596476Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/json": {
       "_shards": {
        "failed": 0,
        "skipped": 0,
        "successful": 1,
        "total": 1
       },
       "hits": {
        "hits": [
         {
          "_id": "slIAqQrcIXk",
          "_index": "unsplash",
          "_score": 14.7062645,
          "_source": {
           "description_final": "two white dogs",
           "photo_id": "slIAqQrcIXk",
           "photo_image_url": "https://images.unsplash.com/photo-1556526588-a0bd9b5a42c3"
          }
         },
         {
          "_id": "wsK4zgu293Q",
          "_index": "unsplash",
          "_score": 13.708565,
          "_source": {
           "description_final": "two dogs near mailbox",
           "photo_id": "wsK4zgu293Q",
           "photo_image_url": "https://images.unsplash.com/photo-1545024552-97cf43d274c0"
          }
         },
         {
          "_id": "Hhh30PbAIig",
          "_index": "unsplash",
          "_score": 13.670029,
          "_source": {
           "description_final": "Two baby elephants playing",
           "photo_id": "Hhh30PbAIig",
           "photo_image_url": "https://images.unsplash.com/photo-1586674757927-e8e76353311b"
          }
         },
         {
          "_id": "5AyUHeSnxz8",
          "_index": "unsplash",
          "_score": 13.254602,
          "_source": {
           "description_final": "#dog #dogs #snow",
           "photo_id": "5AyUHeSnxz8",
           "photo_image_url": "https://images.unsplash.com/photo-1584754269526-83ca2b7f1302"
          }
         },
         {
          "_id": "3X-QSL5DHug",
          "_index": "unsplash",
          "_score": 12.837636,
          "_source": {
           "description_final": "two short-coated gray dogs",
           "photo_id": "3X-QSL5DHug",
           "photo_image_url": "https://images.unsplash.com/photo-1565886345796-bad3185da648"
          }
         }
        ],
        "max_score": 14.7062645,
        "total": {
         "relation": "eq",
         "value": 6241
        }
       },
       "timed_out": false,
       "took": 4
      },
      "text/plain": [
       "<IPython.core.display.JSON object>"
      ]
     },
     "execution_count": 112,
     "metadata": {
      "application/json": {
       "expanded": true,
       "root": "root"
      }
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "JSON(resp.body, expanded = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b40330f1-3135-477d-81cb-617a97a39761",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "cce80547-c969-4525-957b-cf45e88115e6",
   "metadata": {},
   "source": [
    "## Explain the score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "843d016d-008e-4111-a6a1-e772550da704",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:38.758658Z",
     "iopub.status.busy": "2023-04-25T19:24:38.757869Z",
     "iopub.status.idle": "2023-04-25T19:24:38.763517Z",
     "shell.execute_reply": "2023-04-25T19:24:38.762672Z",
     "shell.execute_reply.started": "2023-04-25T19:24:38.758623Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Two dogs playing in the snow'"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "7c257c23-542c-4663-8447-79aa989a11dc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:38.983622Z",
     "iopub.status.busy": "2023-04-25T19:24:38.982889Z",
     "iopub.status.idle": "2023-04-25T19:24:38.994582Z",
     "shell.execute_reply": "2023-04-25T19:24:38.993783Z",
     "shell.execute_reply.started": "2023-04-25T19:24:38.983591Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "resp = client.search(\n",
    "    query = {\n",
    "            \"multi_match\": {\n",
    "                \"query\": query,\n",
    "                 \"fields\": [\"description_final\"],\n",
    "                            }\n",
    "            }\n",
    "    , size=2\n",
    "    , explain=True\n",
    "    , source = [\"description_final\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "7b41936d-7609-4909-af31-713b267e57a6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:39.231701Z",
     "iopub.status.busy": "2023-04-25T19:24:39.230965Z",
     "iopub.status.idle": "2023-04-25T19:24:39.236563Z",
     "shell.execute_reply": "2023-04-25T19:24:39.235823Z",
     "shell.execute_reply.started": "2023-04-25T19:24:39.231669Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/json": {
       "_shards": {
        "failed": 0,
        "skipped": 0,
        "successful": 1,
        "total": 1
       },
       "hits": {
        "hits": [
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:two in 3735) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 276
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 4.4725995
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 3
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.6012784
                }
               ],
               "value": 5.9164104
              }
             ],
             "value": 5.9164104
            },
            {
             "description": "weight(description_final:dogs in 3735) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 31
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 6.6448226
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 3
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.6012784
                }
               ],
               "value": 8.789854
              }
             ],
             "value": 8.789854
            }
           ],
           "value": 14.7062645
          },
          "_id": "slIAqQrcIXk",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 14.7062645,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "two white dogs"
          }
         },
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:two in 1041) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 276
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 4.4725995
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 4
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.56048656
                }
               ],
               "value": 5.5150304
              }
             ],
             "value": 5.5150304
            },
            {
             "description": "weight(description_final:dogs in 1041) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 31
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 6.6448226
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 4
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.56048656
                }
               ],
               "value": 8.193535
              }
             ],
             "value": 8.193535
            }
           ],
           "value": 13.708565
          },
          "_id": "wsK4zgu293Q",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 13.708565,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "two dogs near mailbox"
          }
         }
        ],
        "max_score": 14.7062645,
        "total": {
         "relation": "eq",
         "value": 6241
        }
       },
       "timed_out": false,
       "took": 4
      },
      "text/plain": [
       "<IPython.core.display.JSON object>"
      ]
     },
     "execution_count": 115,
     "metadata": {
      "application/json": {
       "expanded": true,
       "root": "root"
      }
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "JSON (resp.body , expanded=True)\n",
    "\n",
    "#print ( json.dumps(resp.body, indent=2) )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dac42ccd-585a-44ee-91a0-bbc8e863b79a",
   "metadata": {},
   "source": [
    "in the `hits.hits['idx']['_expanation']` , we see individual score computed for each of the components that make BM25\n",
    "```\n",
    "weight(Title:pandas in 35543) [PerFieldSimilarity], result of:\"\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b870b5e-04a9-4949-a034-55ac14b94502",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d3b70b7-3742-4a3f-9036-c54d6d57a6e7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "299354bc-0c33-46d3-9915-1fe138c77979",
   "metadata": {
    "tags": []
   },
   "source": [
    "# More Examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "939fc5fd-134c-4e5a-84be-7b0002749549",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e822094-cfd7-46ef-b83b-d9351e46ce42",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "65ee6a3a-2278-4e4f-a245-754faad7252a",
   "metadata": {},
   "source": [
    "##### Lets go over the method below. It gets the search query and k value that is the recall limit.\n",
    "- Stop, stem and tokenize the query\n",
    "- Get bm25 scores of the documents\n",
    "- Sort the documents by bm25 scores and get top k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "5bb6762f-d339-49dd-bc18-0a18f9de5880",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:24:44.987373Z",
     "iopub.status.busy": "2023-04-25T19:24:44.986550Z",
     "iopub.status.idle": "2023-04-25T19:24:44.992769Z",
     "shell.execute_reply": "2023-04-25T19:24:44.991999Z",
     "shell.execute_reply.started": "2023-04-25T19:24:44.987341Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def fetch_results(client:Elasticsearch, query:str,  num_hits=5, fields_search = [\"description_final\"],  analyzer =\"stop\", explain=False\n",
    "                  , fields_metadata=[\"photographer_username\",\"photographer_first_name\",\"photographer_last_name\",\"photo_image_url\"]):\n",
    "    \"\"\"\n",
    "    With the passed elastic search client, return documents that contain the passed `query` in the fields specified by `fields`\n",
    "\n",
    "    If the fields is empty, it will search all text fields\n",
    "    \n",
    "    We are using mult-match, which by default uses `or`\n",
    "    https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-match-query.html\n",
    "    \"\"\"\n",
    "\n",
    "\n",
    "    \n",
    "    resp = client.search(\n",
    "        query= {\n",
    "                \"multi_match\": {\n",
    "                    \"query\": query,\n",
    "                    \"fields\": fields_search,\n",
    "                     \"analyzer\": analyzer,\n",
    "                     \n",
    "                    #  \"operator\": \"and\" \n",
    "                },\n",
    "               \n",
    "            }\n",
    "        ,size = num_hits\n",
    "        ,explain = explain\n",
    "        , source = fields_search + fields_metadata\n",
    "    )\n",
    "    \n",
    "    return resp\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "4b9a00bb-9c3a-4e4d-8107-ef4493d3aeab",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:30:42.163571Z",
     "iopub.status.busy": "2023-04-25T19:30:42.162717Z",
     "iopub.status.idle": "2023-04-25T19:30:42.170401Z",
     "shell.execute_reply": "2023-04-25T19:30:42.169383Z",
     "shell.execute_reply.started": "2023-04-25T19:30:42.163538Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def find_results(query:str , k =5, analyzer=\"english\", explain=False,image_path_local_dir = \"../data/raw/images\"):\n",
    "    \n",
    "    \n",
    "    top_items = fetch_results(client,query=query,num_hits=k, analyzer=analyzer, explain=explain)\n",
    "    \n",
    "    \n",
    "\n",
    "    display(HTML(f\"<h3>Query: {query} </h3>\"))\n",
    "    \n",
    "    elastic_analyze(  analyzer = analyzer, text =query)\n",
    "\n",
    "    images = []\n",
    "    labels = []\n",
    "    \n",
    "    # Iterate over the top k results\n",
    "    for hit in top_items['hits']['hits']:\n",
    "        doc_id = hit['_id']\n",
    "        \n",
    "        photo_data = hit[\"_source\"]\n",
    "        \n",
    "        # Display the photo\n",
    "        \n",
    "        image_path_local = f\"{image_path_local_dir}/{doc_id}.jpg\"\n",
    "        if os.path.exists(image_path_local):\n",
    "            image_path = image_path_local\n",
    "        else:\n",
    "            image_path = photo_data[\"photo_image_url\"]\n",
    "        images.append(image_path)\n",
    "        score = \"{:.2f}\".format(hit['_score'])\n",
    "        \n",
    "        labels.append (f\"\"\"\n",
    "                     Photo title: {photo_data[\"description_final\"]}   <br/>\n",
    "                     Distance: {score}\n",
    "            \n",
    "                     \"\"\")\n",
    "        \n",
    "        #display(Image(url=photo_data[\"photo_image_url\"] + \"?w=200\"))\n",
    "\n",
    "        # # Display the attribution text\n",
    "        # display(HTML(f\"\"\"\n",
    "        #              Photo title: {photo_data[\"description_final\"]}   <br/>\n",
    "        #              Photo by <a href=\"https://unsplash.com/@{photo_data[\"photographer_username\"]}?utm_source=SearchWorkshop&utm_medium=referral\">{photo_data[\"photographer_first_name\"]} {photo_data.get(\"photographer_last_name\",\"\")}</a> on <a href=\"https://unsplash.com/?utm_source=SearchWorkshop&utm_medium=referral\">Unsplash</a> <br/>\n",
    "        #              Distance: {hit['_score']}\n",
    "        #              \"\"\"\n",
    "        #                                 ))\n",
    "        \n",
    "        \n",
    "    ipyplot.plot_images(images=images, labels=labels, img_width=200)\n",
    "        \n",
    "        \n",
    "    if explain:\n",
    "        return JSON (top_items.body , expanded=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0de51515-7432-467c-a7b5-c30e7db5ef24",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "dfc7042f-b829-4fdd-9e87-0aecd36ed7de",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:30:42.664161Z",
     "iopub.status.busy": "2023-04-25T19:30:42.663436Z",
     "iopub.status.idle": "2023-04-25T19:30:42.667761Z",
     "shell.execute_reply": "2023-04-25T19:30:42.666845Z",
     "shell.execute_reply.started": "2023-04-25T19:30:42.664128Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "query = \"Two dogs playing in the snow\"\n",
    "analyzer = \"english\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "cede1452-8590-4138-ac6a-cb9e999473e4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:30:43.413570Z",
     "iopub.status.busy": "2023-04-25T19:30:43.412647Z",
     "iopub.status.idle": "2023-04-25T19:30:43.450968Z",
     "shell.execute_reply": "2023-04-25T19:30:43.450232Z",
     "shell.execute_reply.started": "2023-04-25T19:30:43.413535Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h3>Query: Two dogs playing in the snow </h3>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'tokens'</span>: <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'two'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'dog'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'plai'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'snow'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">24</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">28</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span><span style=\"font-weight: bold\">}</span>\n",
       "    <span style=\"font-weight: bold\">]</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'tokens'\u001b[0m: \u001b[1m[\u001b[0m\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'two'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m3\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m<\u001b[0m\u001b[32mALPHANUM\u001b[0m\u001b[32m>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'dog'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m4\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m8\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'plai'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m9\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m16\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'snow'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m24\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m28\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM\u001b[0m\u001b[32m>\u001b[0m\u001b[32m'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m5\u001b[0m\u001b[1m}\u001b[0m\n",
       "    \u001b[1m]\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <style>\n",
       "        #ipyplot-html-viewer-toggle-bopFRB49knSRrc4TVZ4pCJ {\n",
       "            position: absolute;\n",
       "            top: -9999px;\n",
       "            left: -9999px;\n",
       "            visibility: hidden;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-label-bopFRB49knSRrc4TVZ4pCJ { \n",
       "            position: relative;\n",
       "            display: inline-block;\n",
       "            cursor: pointer;\n",
       "            color: blue;\n",
       "            text-decoration: underline;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-textarea-bopFRB49knSRrc4TVZ4pCJ {\n",
       "            background: lightgrey;\n",
       "            width: 100%;\n",
       "            height: 0px;\n",
       "            display: none;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-toggle-bopFRB49knSRrc4TVZ4pCJ:checked ~ #ipyplot-html-viewer-textarea-bopFRB49knSRrc4TVZ4pCJ {\n",
       "            height: 200px;\n",
       "            display: block;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-toggle-bopFRB49knSRrc4TVZ4pCJ:checked + #ipyplot-html-viewer-label-bopFRB49knSRrc4TVZ4pCJ:after {\n",
       "            content: \"hide html\";\n",
       "            position: absolute;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "            right: 0;\n",
       "            bottom: 0;\n",
       "            background: white;\n",
       "            cursor: pointer;\n",
       "            color: blue;\n",
       "            text-decoration: underline;\n",
       "        }\n",
       "    </style>\n",
       "    <div>\n",
       "        <input type=\"checkbox\" id=\"ipyplot-html-viewer-toggle-bopFRB49knSRrc4TVZ4pCJ\">\n",
       "        <label id=\"ipyplot-html-viewer-label-bopFRB49knSRrc4TVZ4pCJ\" for=\"ipyplot-html-viewer-toggle-bopFRB49knSRrc4TVZ4pCJ\">show html</label>\n",
       "        <textarea id=\"ipyplot-html-viewer-textarea-bopFRB49knSRrc4TVZ4pCJ\" readonly>\n",
       "            \n",
       "        <style>\n",
       "        #ipyplot-imgs-container-div-ExDYgyzTHccQGbP3Di2CHG {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            margin: 0%;\n",
       "            overflow: auto;\n",
       "            position: relative;\n",
       "            overflow-y: scroll;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-placeholder-div-ExDYgyzTHccQGbP3Di2CHG {\n",
       "            width: 200px;\n",
       "            display: inline-block;\n",
       "            margin: 3px;\n",
       "            position: relative;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG {\n",
       "            width: 200px;\n",
       "            background: white;\n",
       "            display: inline-block;\n",
       "            vertical-align: top;\n",
       "            text-align: center;\n",
       "            position: relative;\n",
       "            border: 2px solid #ddd;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG span.ipyplot-img-close {\n",
       "            display: none;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG span {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            position: absolute;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG img {\n",
       "            width: 200px;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG span.ipyplot-img-close:hover {\n",
       "            cursor: zoom-out;\n",
       "        }\n",
       "        div.ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG span.ipyplot-img-expand:hover {\n",
       "            cursor: zoom-in;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG]:target {\n",
       "            transform: scale(2.5);\n",
       "            transform-origin: left top;\n",
       "            z-index: 5000;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "            position: absolute;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG]:target span.ipyplot-img-close {\n",
       "            display: block;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG]:target span.ipyplot-img-expand {\n",
       "            display: none;\n",
       "        }\n",
       "        </style>\n",
       "    <div id=\"ipyplot-imgs-container-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "    <div class=\"ipyplot-placeholder-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "        <div id=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-ZxNCiQjMQeYEUCvBuLYhRd\" class=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: #dog #dogs #snow   <br/>\n",
       "                     Distance: 9.53\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/5AyUHeSnxz8.jpg</h4><img src=\"../data/raw/images/5AyUHeSnxz8.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-ZxNCiQjMQeYEUCvBuLYhRd\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "        <div id=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-dgFfLq74fY5FBNyyUvy5aE\" class=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: Dog in snow   <br/>\n",
       "                     Distance: 9.53\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/jtnH16_HfPE.jpg</h4><img src=\"../data/raw/images/jtnH16_HfPE.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-dgFfLq74fY5FBNyyUvy5aE\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "        <div id=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-AJMfiKVvLLovnF4C8trpZe\" class=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: two cabins covered with snow   <br/>\n",
       "                     Distance: 9.06\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/T3WsIW4QwDU.jpg</h4><img src=\"../data/raw/images/T3WsIW4QwDU.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-AJMfiKVvLLovnF4C8trpZe\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "        <div id=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-m7jzsK9CPnepPNYpfwj6HV\" class=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: Two mountain peaks with snow   <br/>\n",
       "                     Distance: 9.06\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/oQgZrmpNwoY.jpg</h4><img src=\"../data/raw/images/oQgZrmpNwoY.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-m7jzsK9CPnepPNYpfwj6HV\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "        <div id=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-9bJauBkPQzerddPYawCXd3\" class=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: tan dog playing on snow   <br/>\n",
       "                     Distance: 8.32\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/AVUX8QXnj4Y.jpg</h4><img src=\"../data/raw/images/AVUX8QXnj4Y.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-9bJauBkPQzerddPYawCXd3\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    </div>\n",
       "        </textarea>\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <style>\n",
       "        #ipyplot-imgs-container-div-ExDYgyzTHccQGbP3Di2CHG {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            margin: 0%;\n",
       "            overflow: auto;\n",
       "            position: relative;\n",
       "            overflow-y: scroll;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-placeholder-div-ExDYgyzTHccQGbP3Di2CHG {\n",
       "            width: 200px;\n",
       "            display: inline-block;\n",
       "            margin: 3px;\n",
       "            position: relative;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG {\n",
       "            width: 200px;\n",
       "            background: white;\n",
       "            display: inline-block;\n",
       "            vertical-align: top;\n",
       "            text-align: center;\n",
       "            position: relative;\n",
       "            border: 2px solid #ddd;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG span.ipyplot-img-close {\n",
       "            display: none;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG span {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            position: absolute;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG img {\n",
       "            width: 200px;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG span.ipyplot-img-close:hover {\n",
       "            cursor: zoom-out;\n",
       "        }\n",
       "        div.ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG span.ipyplot-img-expand:hover {\n",
       "            cursor: zoom-in;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG]:target {\n",
       "            transform: scale(2.5);\n",
       "            transform-origin: left top;\n",
       "            z-index: 5000;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "            position: absolute;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG]:target span.ipyplot-img-close {\n",
       "            display: block;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG]:target span.ipyplot-img-expand {\n",
       "            display: none;\n",
       "        }\n",
       "        </style>\n",
       "    <div id=\"ipyplot-imgs-container-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "    <div class=\"ipyplot-placeholder-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "        <div id=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-ZxNCiQjMQeYEUCvBuLYhRd\" class=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: #dog #dogs #snow   <br/>\n",
       "                     Distance: 9.53\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/5AyUHeSnxz8.jpg</h4><img src=\"../data/raw/images/5AyUHeSnxz8.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-ZxNCiQjMQeYEUCvBuLYhRd\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "        <div id=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-dgFfLq74fY5FBNyyUvy5aE\" class=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: Dog in snow   <br/>\n",
       "                     Distance: 9.53\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/jtnH16_HfPE.jpg</h4><img src=\"../data/raw/images/jtnH16_HfPE.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-dgFfLq74fY5FBNyyUvy5aE\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "        <div id=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-AJMfiKVvLLovnF4C8trpZe\" class=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: two cabins covered with snow   <br/>\n",
       "                     Distance: 9.06\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/T3WsIW4QwDU.jpg</h4><img src=\"../data/raw/images/T3WsIW4QwDU.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-AJMfiKVvLLovnF4C8trpZe\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "        <div id=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-m7jzsK9CPnepPNYpfwj6HV\" class=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: Two mountain peaks with snow   <br/>\n",
       "                     Distance: 9.06\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/oQgZrmpNwoY.jpg</h4><img src=\"../data/raw/images/oQgZrmpNwoY.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-m7jzsK9CPnepPNYpfwj6HV\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "        <div id=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-9bJauBkPQzerddPYawCXd3\" class=\"ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: tan dog playing on snow   <br/>\n",
       "                     Distance: 8.32\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/AVUX8QXnj4Y.jpg</h4><img src=\"../data/raw/images/AVUX8QXnj4Y.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-ExDYgyzTHccQGbP3Di2CHG-9bJauBkPQzerddPYawCXd3\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "_shards": {
        "failed": 0,
        "skipped": 0,
        "successful": 1,
        "total": 1
       },
       "hits": {
        "hits": [
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:dog in 1756) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 524
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 3.8323648
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 3
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.6012784
                }
               ],
               "value": 5.0695
              }
             ],
             "value": 5.0695
            },
            {
             "description": "weight(description_final:snow in 1756) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 828
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 3.3751934
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 3
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.6012784
                }
               ],
               "value": 4.464748
              }
             ],
             "value": 4.464748
            }
           ],
           "value": 9.534248
          },
          "_id": "5AyUHeSnxz8",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 9.534248,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "#dog #dogs #snow",
           "photo_image_url": "https://images.unsplash.com/photo-1584754269526-83ca2b7f1302"
          }
         },
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:dog in 985) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 524
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 3.8323648
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 3
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.6012784
                }
               ],
               "value": 5.0695
              }
             ],
             "value": 5.0695
            },
            {
             "description": "weight(description_final:snow in 985) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 828
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 3.3751934
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 3
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.6012784
                }
               ],
               "value": 4.464748
              }
             ],
             "value": 4.464748
            }
           ],
           "value": 9.534248
          },
          "_id": "jtnH16_HfPE",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 9.534248,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "Dog in snow",
           "photo_image_url": "https://images.unsplash.com/photo-1545080511-0a837d1e2ac4"
          }
         },
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:two in 88) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 276
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 4.4725995
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 5
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.5248779
                }
               ],
               "value": 5.164651
              }
             ],
             "value": 5.164651
            },
            {
             "description": "weight(description_final:snow in 88) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 828
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 3.3751934
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 5
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.5248779
                }
               ],
               "value": 3.8974416
              }
             ],
             "value": 3.8974416
            }
           ],
           "value": 9.062093
          },
          "_id": "T3WsIW4QwDU",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 9.062093,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "two cabins covered with snow",
           "photo_image_url": "https://images.unsplash.com/photo-1517259227355-de4b20b3fe46"
          }
         },
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:two in 4516) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 276
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 4.4725995
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 5
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.5248779
                }
               ],
               "value": 5.164651
              }
             ],
             "value": 5.164651
            },
            {
             "description": "weight(description_final:snow in 4516) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 828
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 3.3751934
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 5
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.5248779
                }
               ],
               "value": 3.8974416
              }
             ],
             "value": 3.8974416
            }
           ],
           "value": 9.062093
          },
          "_id": "oQgZrmpNwoY",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 9.062093,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "Two mountain peaks with snow",
           "photo_image_url": "https://images.unsplash.com/photo-1455753723382-4febf29162e0"
          }
         },
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:dog in 1974) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 524
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 3.8323648
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 5
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.5248779
                }
               ],
               "value": 4.4253516
              }
             ],
             "value": 4.4253516
            },
            {
             "description": "weight(description_final:snow in 1974) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 828
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 3.3751934
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 5
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.5248779
                }
               ],
               "value": 3.8974416
              }
             ],
             "value": 3.8974416
            }
           ],
           "value": 8.322793
          },
          "_id": "AVUX8QXnj4Y",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 8.322793,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "tan dog playing on snow",
           "photo_image_url": "https://images.unsplash.com/photo-1546717689-5955401cd6b2"
          }
         }
        ],
        "max_score": 9.534248,
        "total": {
         "relation": "eq",
         "value": 1590
        }
       },
       "timed_out": false,
       "took": 3
      },
      "text/plain": [
       "<IPython.core.display.JSON object>"
      ]
     },
     "execution_count": 136,
     "metadata": {
      "application/json": {
       "expanded": false,
       "root": "root"
      }
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_results( query, analyzer=analyzer, explain=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38612188-40c0-4a73-aa0f-e4cc47e9c091",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41427cd9-b8ba-498b-a1b1-d4cac800f34c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e37f28be-5c92-49ea-8e99-c95e8d7d722a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "dd0c1c6f-87d7-4fb2-93dd-ae43f5f132ae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:38:43.426397Z",
     "iopub.status.busy": "2023-04-25T19:38:43.425657Z",
     "iopub.status.idle": "2023-04-25T19:38:43.450910Z",
     "shell.execute_reply": "2023-04-25T19:38:43.450030Z",
     "shell.execute_reply.started": "2023-04-25T19:38:43.426365Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h3>Query: the boy and girl on a beach </h3>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'tokens'</span>: <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'boy'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'girl'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'beach'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">22</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">27</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'word'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span><span style=\"font-weight: bold\">}</span>\n",
       "    <span style=\"font-weight: bold\">]</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'tokens'\u001b[0m: \u001b[1m[\u001b[0m\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'boy'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m4\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m7\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'girl'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m12\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m16\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m3\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'beach'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m22\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m27\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'word'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m6\u001b[0m\u001b[1m}\u001b[0m\n",
       "    \u001b[1m]\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <style>\n",
       "        #ipyplot-html-viewer-toggle-7d27Ppo8q9SojvXeKryemV {\n",
       "            position: absolute;\n",
       "            top: -9999px;\n",
       "            left: -9999px;\n",
       "            visibility: hidden;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-label-7d27Ppo8q9SojvXeKryemV { \n",
       "            position: relative;\n",
       "            display: inline-block;\n",
       "            cursor: pointer;\n",
       "            color: blue;\n",
       "            text-decoration: underline;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-textarea-7d27Ppo8q9SojvXeKryemV {\n",
       "            background: lightgrey;\n",
       "            width: 100%;\n",
       "            height: 0px;\n",
       "            display: none;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-toggle-7d27Ppo8q9SojvXeKryemV:checked ~ #ipyplot-html-viewer-textarea-7d27Ppo8q9SojvXeKryemV {\n",
       "            height: 200px;\n",
       "            display: block;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-toggle-7d27Ppo8q9SojvXeKryemV:checked + #ipyplot-html-viewer-label-7d27Ppo8q9SojvXeKryemV:after {\n",
       "            content: \"hide html\";\n",
       "            position: absolute;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "            right: 0;\n",
       "            bottom: 0;\n",
       "            background: white;\n",
       "            cursor: pointer;\n",
       "            color: blue;\n",
       "            text-decoration: underline;\n",
       "        }\n",
       "    </style>\n",
       "    <div>\n",
       "        <input type=\"checkbox\" id=\"ipyplot-html-viewer-toggle-7d27Ppo8q9SojvXeKryemV\">\n",
       "        <label id=\"ipyplot-html-viewer-label-7d27Ppo8q9SojvXeKryemV\" for=\"ipyplot-html-viewer-toggle-7d27Ppo8q9SojvXeKryemV\">show html</label>\n",
       "        <textarea id=\"ipyplot-html-viewer-textarea-7d27Ppo8q9SojvXeKryemV\" readonly>\n",
       "            \n",
       "        <style>\n",
       "        #ipyplot-imgs-container-div-KwpHY3DecPp5WKc6MGwhzp {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            margin: 0%;\n",
       "            overflow: auto;\n",
       "            position: relative;\n",
       "            overflow-y: scroll;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-placeholder-div-KwpHY3DecPp5WKc6MGwhzp {\n",
       "            width: 200px;\n",
       "            display: inline-block;\n",
       "            margin: 3px;\n",
       "            position: relative;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp {\n",
       "            width: 200px;\n",
       "            background: white;\n",
       "            display: inline-block;\n",
       "            vertical-align: top;\n",
       "            text-align: center;\n",
       "            position: relative;\n",
       "            border: 2px solid #ddd;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp span.ipyplot-img-close {\n",
       "            display: none;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp span {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            position: absolute;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp img {\n",
       "            width: 200px;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp span.ipyplot-img-close:hover {\n",
       "            cursor: zoom-out;\n",
       "        }\n",
       "        div.ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp span.ipyplot-img-expand:hover {\n",
       "            cursor: zoom-in;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp]:target {\n",
       "            transform: scale(2.5);\n",
       "            transform-origin: left top;\n",
       "            z-index: 5000;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "            position: absolute;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp]:target span.ipyplot-img-close {\n",
       "            display: block;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp]:target span.ipyplot-img-expand {\n",
       "            display: none;\n",
       "        }\n",
       "        </style>\n",
       "    <div id=\"ipyplot-imgs-container-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "    <div class=\"ipyplot-placeholder-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "        <div id=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-XEZohJZLYW7YLGJMnNizfs\" class=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: Winter Girl   <br/>\n",
       "                     Distance: 8.75\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/rGL9bjHDqNM.jpg</h4><img src=\"../data/raw/images/rGL9bjHDqNM.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-XEZohJZLYW7YLGJMnNizfs\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "        <div id=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-ii4nALgXBgx3nMLzpArF2H\" class=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: The girl   <br/>\n",
       "                     Distance: 8.75\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/p5NcxQ6TE8I.jpg</h4><img src=\"../data/raw/images/p5NcxQ6TE8I.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-ii4nALgXBgx3nMLzpArF2H\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "        <div id=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-j7bHKGLhZdRETDdCHs2Y3c\" class=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: boy wearing goggles underwater   <br/>\n",
       "                     Distance: 8.55\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/0O0Cia89YLU.jpg</h4><img src=\"../data/raw/images/0O0Cia89YLU.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-j7bHKGLhZdRETDdCHs2Y3c\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "        <div id=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-FKBpEv4v4fUQvSntSJoYiG\" class=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: boy stands under trees   <br/>\n",
       "                     Distance: 8.55\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/vJNe-sw36WU.jpg</h4><img src=\"../data/raw/images/vJNe-sw36WU.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-FKBpEv4v4fUQvSntSJoYiG\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "        <div id=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-dhPUnNTjiR2auLnjBZ3fSy\" class=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: boy standing near stone   <br/>\n",
       "                     Distance: 8.55\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/hSY4lkFR_E0.jpg</h4><img src=\"../data/raw/images/hSY4lkFR_E0.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-dhPUnNTjiR2auLnjBZ3fSy\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    </div>\n",
       "        </textarea>\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <style>\n",
       "        #ipyplot-imgs-container-div-KwpHY3DecPp5WKc6MGwhzp {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            margin: 0%;\n",
       "            overflow: auto;\n",
       "            position: relative;\n",
       "            overflow-y: scroll;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-placeholder-div-KwpHY3DecPp5WKc6MGwhzp {\n",
       "            width: 200px;\n",
       "            display: inline-block;\n",
       "            margin: 3px;\n",
       "            position: relative;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp {\n",
       "            width: 200px;\n",
       "            background: white;\n",
       "            display: inline-block;\n",
       "            vertical-align: top;\n",
       "            text-align: center;\n",
       "            position: relative;\n",
       "            border: 2px solid #ddd;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp span.ipyplot-img-close {\n",
       "            display: none;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp span {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            position: absolute;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp img {\n",
       "            width: 200px;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp span.ipyplot-img-close:hover {\n",
       "            cursor: zoom-out;\n",
       "        }\n",
       "        div.ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp span.ipyplot-img-expand:hover {\n",
       "            cursor: zoom-in;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp]:target {\n",
       "            transform: scale(2.5);\n",
       "            transform-origin: left top;\n",
       "            z-index: 5000;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "            position: absolute;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp]:target span.ipyplot-img-close {\n",
       "            display: block;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp]:target span.ipyplot-img-expand {\n",
       "            display: none;\n",
       "        }\n",
       "        </style>\n",
       "    <div id=\"ipyplot-imgs-container-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "    <div class=\"ipyplot-placeholder-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "        <div id=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-XEZohJZLYW7YLGJMnNizfs\" class=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: Winter Girl   <br/>\n",
       "                     Distance: 8.75\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/rGL9bjHDqNM.jpg</h4><img src=\"../data/raw/images/rGL9bjHDqNM.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-XEZohJZLYW7YLGJMnNizfs\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "        <div id=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-ii4nALgXBgx3nMLzpArF2H\" class=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: The girl   <br/>\n",
       "                     Distance: 8.75\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/p5NcxQ6TE8I.jpg</h4><img src=\"../data/raw/images/p5NcxQ6TE8I.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-ii4nALgXBgx3nMLzpArF2H\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "        <div id=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-j7bHKGLhZdRETDdCHs2Y3c\" class=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: boy wearing goggles underwater   <br/>\n",
       "                     Distance: 8.55\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/0O0Cia89YLU.jpg</h4><img src=\"../data/raw/images/0O0Cia89YLU.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-j7bHKGLhZdRETDdCHs2Y3c\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "        <div id=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-FKBpEv4v4fUQvSntSJoYiG\" class=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: boy stands under trees   <br/>\n",
       "                     Distance: 8.55\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/vJNe-sw36WU.jpg</h4><img src=\"../data/raw/images/vJNe-sw36WU.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-FKBpEv4v4fUQvSntSJoYiG\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "        <div id=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-dhPUnNTjiR2auLnjBZ3fSy\" class=\"ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: boy standing near stone   <br/>\n",
       "                     Distance: 8.55\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/hSY4lkFR_E0.jpg</h4><img src=\"../data/raw/images/hSY4lkFR_E0.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-KwpHY3DecPp5WKc6MGwhzp-dhPUnNTjiR2auLnjBZ3fSy\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "_shards": {
        "failed": 0,
        "skipped": 0,
        "successful": 1,
        "total": 1
       },
       "hits": {
        "hits": [
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:girl in 732) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 52
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 6.133997
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 2
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.648474
                }
               ],
               "value": 8.751022
              }
             ],
             "value": 8.751022
            }
           ],
           "value": 8.751022
          },
          "_id": "rGL9bjHDqNM",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 8.751022,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "Winter Girl",
           "photo_image_url": "https://images.unsplash.com/photo-1544457783-a62dba938262"
          }
         },
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:girl in 4943) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 52
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 6.133997
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 2
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.648474
                }
               ],
               "value": 8.751022
              }
             ],
             "value": 8.751022
            }
           ],
           "value": 8.751022
          },
          "_id": "p5NcxQ6TE8I",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 8.751022,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "The girl",
           "photo_image_url": "https://images.unsplash.com/photo-1575087107694-90097df5ce68"
          }
         },
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:boy in 1898) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 23
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 6.93781
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 4
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.56048656
                }
               ],
               "value": 8.554809
              }
             ],
             "value": 8.554809
            }
           ],
           "value": 8.554809
          },
          "_id": "0O0Cia89YLU",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 8.554809,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "boy wearing goggles underwater",
           "photo_image_url": "https://images.unsplash.com/photo-1567878543260-a729b5c79706"
          }
         },
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:boy in 2807) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 23
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 6.93781
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 4
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.56048656
                }
               ],
               "value": 8.554809
              }
             ],
             "value": 8.554809
            }
           ],
           "value": 8.554809
          },
          "_id": "vJNe-sw36WU",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 8.554809,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "boy stands under trees",
           "photo_image_url": "https://images.unsplash.com/photo-1546880882-8dcc122653a6"
          }
         },
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:boy in 7783) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 23
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 6.93781
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 4
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.56048656
                }
               ],
               "value": 8.554809
              }
             ],
             "value": 8.554809
            }
           ],
           "value": 8.554809
          },
          "_id": "hSY4lkFR_E0",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 8.554809,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "boy standing near stone",
           "photo_image_url": "https://images.unsplash.com/photo-1517206771387-88a0ebd29880"
          }
         }
        ],
        "max_score": 8.751022,
        "total": {
         "relation": "eq",
         "value": 587
        }
       },
       "timed_out": false,
       "took": 2
      },
      "text/plain": [
       "<IPython.core.display.JSON object>"
      ]
     },
     "execution_count": 147,
     "metadata": {
      "application/json": {
       "expanded": false,
       "root": "root"
      }
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_results( \"the boy and girl on a beach\", analyzer=\"stop\", explain=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d07b6862-0fb4-415e-8748-c85db7047cf3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5f7f71c-3eb5-4bc2-baff-f28e743ea856",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "ef78dbad-956e-486b-83c9-0241bbcdec1c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:31:20.808228Z",
     "iopub.status.busy": "2023-04-25T19:31:20.807479Z",
     "iopub.status.idle": "2023-04-25T19:31:20.833781Z",
     "shell.execute_reply": "2023-04-25T19:31:20.832672Z",
     "shell.execute_reply.started": "2023-04-25T19:31:20.808195Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h3>Query: image of a man in a desert </h3>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'tokens'</span>: <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'imag'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'man'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'desert'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">26</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span><span style=\"font-weight: bold\">}</span>\n",
       "    <span style=\"font-weight: bold\">]</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'tokens'\u001b[0m: \u001b[1m[\u001b[0m\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'imag'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m<\u001b[0m\u001b[32mALPHANUM\u001b[0m\u001b[32m>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'man'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m11\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m14\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'desert'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m20\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m26\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM\u001b[0m\u001b[32m>\u001b[0m\u001b[32m'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m6\u001b[0m\u001b[1m}\u001b[0m\n",
       "    \u001b[1m]\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <style>\n",
       "        #ipyplot-html-viewer-toggle-jVZrHgXZWuynZLe2YLEwnK {\n",
       "            position: absolute;\n",
       "            top: -9999px;\n",
       "            left: -9999px;\n",
       "            visibility: hidden;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-label-jVZrHgXZWuynZLe2YLEwnK { \n",
       "            position: relative;\n",
       "            display: inline-block;\n",
       "            cursor: pointer;\n",
       "            color: blue;\n",
       "            text-decoration: underline;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-textarea-jVZrHgXZWuynZLe2YLEwnK {\n",
       "            background: lightgrey;\n",
       "            width: 100%;\n",
       "            height: 0px;\n",
       "            display: none;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-toggle-jVZrHgXZWuynZLe2YLEwnK:checked ~ #ipyplot-html-viewer-textarea-jVZrHgXZWuynZLe2YLEwnK {\n",
       "            height: 200px;\n",
       "            display: block;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-toggle-jVZrHgXZWuynZLe2YLEwnK:checked + #ipyplot-html-viewer-label-jVZrHgXZWuynZLe2YLEwnK:after {\n",
       "            content: \"hide html\";\n",
       "            position: absolute;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "            right: 0;\n",
       "            bottom: 0;\n",
       "            background: white;\n",
       "            cursor: pointer;\n",
       "            color: blue;\n",
       "            text-decoration: underline;\n",
       "        }\n",
       "    </style>\n",
       "    <div>\n",
       "        <input type=\"checkbox\" id=\"ipyplot-html-viewer-toggle-jVZrHgXZWuynZLe2YLEwnK\">\n",
       "        <label id=\"ipyplot-html-viewer-label-jVZrHgXZWuynZLe2YLEwnK\" for=\"ipyplot-html-viewer-toggle-jVZrHgXZWuynZLe2YLEwnK\">show html</label>\n",
       "        <textarea id=\"ipyplot-html-viewer-textarea-jVZrHgXZWuynZLe2YLEwnK\" readonly>\n",
       "            \n",
       "        <style>\n",
       "        #ipyplot-imgs-container-div-B6odnqQhJLpueWqQgJxVSH {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            margin: 0%;\n",
       "            overflow: auto;\n",
       "            position: relative;\n",
       "            overflow-y: scroll;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-placeholder-div-B6odnqQhJLpueWqQgJxVSH {\n",
       "            width: 200px;\n",
       "            display: inline-block;\n",
       "            margin: 3px;\n",
       "            position: relative;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH {\n",
       "            width: 200px;\n",
       "            background: white;\n",
       "            display: inline-block;\n",
       "            vertical-align: top;\n",
       "            text-align: center;\n",
       "            position: relative;\n",
       "            border: 2px solid #ddd;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH span.ipyplot-img-close {\n",
       "            display: none;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH span {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            position: absolute;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH img {\n",
       "            width: 200px;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH span.ipyplot-img-close:hover {\n",
       "            cursor: zoom-out;\n",
       "        }\n",
       "        div.ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH span.ipyplot-img-expand:hover {\n",
       "            cursor: zoom-in;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH]:target {\n",
       "            transform: scale(2.5);\n",
       "            transform-origin: left top;\n",
       "            z-index: 5000;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "            position: absolute;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH]:target span.ipyplot-img-close {\n",
       "            display: block;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH]:target span.ipyplot-img-expand {\n",
       "            display: none;\n",
       "        }\n",
       "        </style>\n",
       "    <div id=\"ipyplot-imgs-container-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "    <div class=\"ipyplot-placeholder-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "        <div id=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-h5xYr8Tr6neZyTxpAzcA5a\" class=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: man standing at desert   <br/>\n",
       "                     Distance: 10.47\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/eUSfRwB_qY8.jpg</h4><img src=\"../data/raw/images/eUSfRwB_qY8.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-h5xYr8Tr6neZyTxpAzcA5a\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "        <div id=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-8TW59DANECPvGQ3Cn579We\" class=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: man jumping on desert   <br/>\n",
       "                     Distance: 10.47\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/rCPE_BjGaBo.jpg</h4><img src=\"../data/raw/images/rCPE_BjGaBo.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-8TW59DANECPvGQ3Cn579We\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "        <div id=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-abBQWLHWmazNFWkypgEbWR\" class=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: man standing while facing desert during daytime   <br/>\n",
       "                     Distance: 8.70\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/BEwrVP6o0yQ.jpg</h4><img src=\"../data/raw/images/BEwrVP6o0yQ.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-abBQWLHWmazNFWkypgEbWR\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "        <div id=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-is4CZ4eecFBhGtPLWFcfr2\" class=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: man and woman walking on desert during daytime   <br/>\n",
       "                     Distance: 8.23\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/MasnAA5KqDY.jpg</h4><img src=\"../data/raw/images/MasnAA5KqDY.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-is4CZ4eecFBhGtPLWFcfr2\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "        <div id=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-8ve3Z34ZjULLCP8GetzxTY\" class=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: man in black shirt standing on desert lang   <br/>\n",
       "                     Distance: 8.23\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/L3HrRyVyINo.jpg</h4><img src=\"../data/raw/images/L3HrRyVyINo.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-8ve3Z34ZjULLCP8GetzxTY\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    </div>\n",
       "        </textarea>\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <style>\n",
       "        #ipyplot-imgs-container-div-B6odnqQhJLpueWqQgJxVSH {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            margin: 0%;\n",
       "            overflow: auto;\n",
       "            position: relative;\n",
       "            overflow-y: scroll;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-placeholder-div-B6odnqQhJLpueWqQgJxVSH {\n",
       "            width: 200px;\n",
       "            display: inline-block;\n",
       "            margin: 3px;\n",
       "            position: relative;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH {\n",
       "            width: 200px;\n",
       "            background: white;\n",
       "            display: inline-block;\n",
       "            vertical-align: top;\n",
       "            text-align: center;\n",
       "            position: relative;\n",
       "            border: 2px solid #ddd;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH span.ipyplot-img-close {\n",
       "            display: none;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH span {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            position: absolute;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH img {\n",
       "            width: 200px;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH span.ipyplot-img-close:hover {\n",
       "            cursor: zoom-out;\n",
       "        }\n",
       "        div.ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH span.ipyplot-img-expand:hover {\n",
       "            cursor: zoom-in;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH]:target {\n",
       "            transform: scale(2.5);\n",
       "            transform-origin: left top;\n",
       "            z-index: 5000;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "            position: absolute;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH]:target span.ipyplot-img-close {\n",
       "            display: block;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH]:target span.ipyplot-img-expand {\n",
       "            display: none;\n",
       "        }\n",
       "        </style>\n",
       "    <div id=\"ipyplot-imgs-container-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "    <div class=\"ipyplot-placeholder-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "        <div id=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-h5xYr8Tr6neZyTxpAzcA5a\" class=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: man standing at desert   <br/>\n",
       "                     Distance: 10.47\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/eUSfRwB_qY8.jpg</h4><img src=\"../data/raw/images/eUSfRwB_qY8.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-h5xYr8Tr6neZyTxpAzcA5a\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "        <div id=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-8TW59DANECPvGQ3Cn579We\" class=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: man jumping on desert   <br/>\n",
       "                     Distance: 10.47\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/rCPE_BjGaBo.jpg</h4><img src=\"../data/raw/images/rCPE_BjGaBo.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-8TW59DANECPvGQ3Cn579We\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "        <div id=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-abBQWLHWmazNFWkypgEbWR\" class=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: man standing while facing desert during daytime   <br/>\n",
       "                     Distance: 8.70\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/BEwrVP6o0yQ.jpg</h4><img src=\"../data/raw/images/BEwrVP6o0yQ.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-abBQWLHWmazNFWkypgEbWR\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "        <div id=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-is4CZ4eecFBhGtPLWFcfr2\" class=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: man and woman walking on desert during daytime   <br/>\n",
       "                     Distance: 8.23\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/MasnAA5KqDY.jpg</h4><img src=\"../data/raw/images/MasnAA5KqDY.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-is4CZ4eecFBhGtPLWFcfr2\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "        <div id=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-8ve3Z34ZjULLCP8GetzxTY\" class=\"ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: man in black shirt standing on desert lang   <br/>\n",
       "                     Distance: 8.23\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/L3HrRyVyINo.jpg</h4><img src=\"../data/raw/images/L3HrRyVyINo.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-B6odnqQhJLpueWqQgJxVSH-8ve3Z34ZjULLCP8GetzxTY\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "_shards": {
        "failed": 0,
        "skipped": 0,
        "successful": 1,
        "total": 1
       },
       "hits": {
        "hits": [
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:man in 4744) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 582
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 3.727481
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 4
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.56048656
                }
               ],
               "value": 4.5962467
              }
             ],
             "value": 4.5962467
            },
            {
             "description": "weight(description_final:desert in 4744) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 207
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 4.759679
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 4
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.56048656
                }
               ],
               "value": 5.869019
              }
             ],
             "value": 5.869019
            }
           ],
           "value": 10.465265
          },
          "_id": "eUSfRwB_qY8",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 10.465265,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "man standing at desert",
           "photo_image_url": "https://images.unsplash.com/photo-1551286663-102d2d654aa0"
          }
         },
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:man in 5410) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 582
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 3.727481
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 4
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.56048656
                }
               ],
               "value": 4.5962467
              }
             ],
             "value": 4.5962467
            },
            {
             "description": "weight(description_final:desert in 5410) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 207
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 4.759679
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 4
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.56048656
                }
               ],
               "value": 5.869019
              }
             ],
             "value": 5.869019
            }
           ],
           "value": 10.465265
          },
          "_id": "rCPE_BjGaBo",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 10.465265,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "man jumping on desert",
           "photo_image_url": "https://images.unsplash.com/photo-1570747985092-67396e190644"
          }
         },
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:man in 3959) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 582
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 3.727481
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 7
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.4657039
                }
               ],
               "value": 3.818986
              }
             ],
             "value": 3.818986
            },
            {
             "description": "weight(description_final:desert in 3959) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 207
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 4.759679
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 7
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.4657039
                }
               ],
               "value": 4.8765225
              }
             ],
             "value": 4.8765225
            }
           ],
           "value": 8.695509
          },
          "_id": "BEwrVP6o0yQ",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 8.695509,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "man standing while facing desert during daytime",
           "photo_image_url": "https://images.unsplash.com/photo-1547038651-456412e990d7"
          }
         },
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:man in 4556) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 582
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 3.727481
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 8
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.44085336
                }
               ],
               "value": 3.6151996
              }
             ],
             "value": 3.6151996
            },
            {
             "description": "weight(description_final:desert in 4556) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 207
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 4.759679
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 8
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.44085336
                }
               ],
               "value": 4.616305
              }
             ],
             "value": 4.616305
            }
           ],
           "value": 8.231504
          },
          "_id": "MasnAA5KqDY",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 8.231504,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "man and woman walking on desert during daytime",
           "photo_image_url": "https://images.unsplash.com/photo-1580238459404-f1b956ebc8ca"
          }
         },
         {
          "_explanation": {
           "description": "sum of:",
           "details": [
            {
             "description": "weight(description_final:man in 4022) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 582
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 3.727481
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 8
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.44085336
                }
               ],
               "value": 3.6151996
              }
             ],
             "value": 3.6151996
            },
            {
             "description": "weight(description_final:desert in 4022) [PerFieldSimilarity], result of:",
             "details": [
              {
               "description": "score(freq=1.0), computed as boost * idf * tf from:",
               "details": [
                {
                 "description": "boost",
                 "details": [],
                 "value": 2.2
                },
                {
                 "description": "idf, computed as log(1 + (N - n + 0.5) / (n + 0.5)) from:",
                 "details": [
                  {
                   "description": "n, number of documents containing term",
                   "details": [],
                   "value": 207
                  },
                  {
                   "description": "N, total number of documents with field",
                   "details": [],
                   "value": 24216
                  }
                 ],
                 "value": 4.759679
                },
                {
                 "description": "tf, computed as freq / (freq + k1 * (1 - b + b * dl / avgdl)) from:",
                 "details": [
                  {
                   "description": "freq, occurrences of term within document",
                   "details": [],
                   "value": 1
                  },
                  {
                   "description": "k1, term saturation parameter",
                   "details": [],
                   "value": 1.2
                  },
                  {
                   "description": "b, length normalization parameter",
                   "details": [],
                   "value": 0.75
                  },
                  {
                   "description": "dl, length of field",
                   "details": [],
                   "value": 8
                  },
                  {
                   "description": "avgdl, average length of field",
                   "details": [],
                   "value": 7.4354973
                  }
                 ],
                 "value": 0.44085336
                }
               ],
               "value": 4.616305
              }
             ],
             "value": 4.616305
            }
           ],
           "value": 8.231504
          },
          "_id": "L3HrRyVyINo",
          "_index": "unsplash",
          "_node": "m-t7ipRASpSgHsL3a6irdQ",
          "_score": 8.231504,
          "_shard": "[unsplash][0]",
          "_source": {
           "description_final": "man in black shirt standing on desert lang",
           "photo_image_url": "https://images.unsplash.com/photo-1547005353-9dfaf6fde952"
          }
         }
        ],
        "max_score": 10.465265,
        "total": {
         "relation": "eq",
         "value": 782
        }
       },
       "timed_out": false,
       "took": 3
      },
      "text/plain": [
       "<IPython.core.display.JSON object>"
      ]
     },
     "execution_count": 138,
     "metadata": {
      "application/json": {
       "expanded": false,
       "root": "root"
      }
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_results( \"image of a man in a desert\", analyzer=\"english\", explain=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3bbacdc-4dc3-437b-8a72-d7d2f6ce4cf5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "0dbc785c-d6c3-4073-a14b-b61c500d9212",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T19:31:32.351794Z",
     "iopub.status.busy": "2023-04-25T19:31:32.351012Z",
     "iopub.status.idle": "2023-04-25T19:31:32.377903Z",
     "shell.execute_reply": "2023-04-25T19:31:32.377203Z",
     "shell.execute_reply.started": "2023-04-25T19:31:32.351762Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h3>Query: light at the end of the tunnel </h3>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'tokens'</span>: <span style=\"font-weight: bold\">[</span>\n",
       "        <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'light'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'at'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'the'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'end'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">13</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'of'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">17</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">19</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'the'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">23</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'position'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">}</span><span style=\"color: #000000; text-decoration-color: #000000\">,</span>\n",
       "<span style=\"color: #000000; text-decoration-color: #000000\">        </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'token'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'tunnel'</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'start_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">24</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'end_offset'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">30</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008000; text-decoration-color: #008000\">'type'</span><span style=\"color: #000000; text-decoration-color: #000000\">: </span><span style=\"color: #008000; text-decoration-color: #008000\">'&lt;ALPHANUM&gt;'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'position'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span><span style=\"font-weight: bold\">}</span>\n",
       "    <span style=\"font-weight: bold\">]</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'tokens'\u001b[0m: \u001b[1m[\u001b[0m\n",
       "        \u001b[1m{\u001b[0m\u001b[32m'token'\u001b[0m: \u001b[32m'light'\u001b[0m, \u001b[32m'start_offset'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'end_offset'\u001b[0m: \u001b[1;36m5\u001b[0m, \u001b[32m'type'\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m<\u001b[0m\u001b[32mALPHANUM\u001b[0m\u001b[32m>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'at'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m6\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m8\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'the'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m9\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m12\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'end'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m13\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m16\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'of'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m17\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m19\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'the'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m20\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m23\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM>'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'position'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n",
       "\u001b[39m        \u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'token'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'tunnel'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'start_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m24\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'end_offset'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m30\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'type'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'<ALPHANUM\u001b[0m\u001b[32m>\u001b[0m\u001b[32m'\u001b[0m, \u001b[32m'position'\u001b[0m: \u001b[1;36m6\u001b[0m\u001b[1m}\u001b[0m\n",
       "    \u001b[1m]\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <style>\n",
       "        #ipyplot-html-viewer-toggle-kV3FgEAfLz5qWQQuhKgTAZ {\n",
       "            position: absolute;\n",
       "            top: -9999px;\n",
       "            left: -9999px;\n",
       "            visibility: hidden;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-label-kV3FgEAfLz5qWQQuhKgTAZ { \n",
       "            position: relative;\n",
       "            display: inline-block;\n",
       "            cursor: pointer;\n",
       "            color: blue;\n",
       "            text-decoration: underline;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-textarea-kV3FgEAfLz5qWQQuhKgTAZ {\n",
       "            background: lightgrey;\n",
       "            width: 100%;\n",
       "            height: 0px;\n",
       "            display: none;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-toggle-kV3FgEAfLz5qWQQuhKgTAZ:checked ~ #ipyplot-html-viewer-textarea-kV3FgEAfLz5qWQQuhKgTAZ {\n",
       "            height: 200px;\n",
       "            display: block;\n",
       "        }\n",
       "\n",
       "        #ipyplot-html-viewer-toggle-kV3FgEAfLz5qWQQuhKgTAZ:checked + #ipyplot-html-viewer-label-kV3FgEAfLz5qWQQuhKgTAZ:after {\n",
       "            content: \"hide html\";\n",
       "            position: absolute;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "            right: 0;\n",
       "            bottom: 0;\n",
       "            background: white;\n",
       "            cursor: pointer;\n",
       "            color: blue;\n",
       "            text-decoration: underline;\n",
       "        }\n",
       "    </style>\n",
       "    <div>\n",
       "        <input type=\"checkbox\" id=\"ipyplot-html-viewer-toggle-kV3FgEAfLz5qWQQuhKgTAZ\">\n",
       "        <label id=\"ipyplot-html-viewer-label-kV3FgEAfLz5qWQQuhKgTAZ\" for=\"ipyplot-html-viewer-toggle-kV3FgEAfLz5qWQQuhKgTAZ\">show html</label>\n",
       "        <textarea id=\"ipyplot-html-viewer-textarea-kV3FgEAfLz5qWQQuhKgTAZ\" readonly>\n",
       "            \n",
       "        <style>\n",
       "        #ipyplot-imgs-container-div-HWchwqNRuM4ivQZBpeewJ8 {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            margin: 0%;\n",
       "            overflow: auto;\n",
       "            position: relative;\n",
       "            overflow-y: scroll;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-placeholder-div-HWchwqNRuM4ivQZBpeewJ8 {\n",
       "            width: 200px;\n",
       "            display: inline-block;\n",
       "            margin: 3px;\n",
       "            position: relative;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8 {\n",
       "            width: 200px;\n",
       "            background: white;\n",
       "            display: inline-block;\n",
       "            vertical-align: top;\n",
       "            text-align: center;\n",
       "            position: relative;\n",
       "            border: 2px solid #ddd;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8 span.ipyplot-img-close {\n",
       "            display: none;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8 span {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            position: absolute;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8 img {\n",
       "            width: 200px;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8 span.ipyplot-img-close:hover {\n",
       "            cursor: zoom-out;\n",
       "        }\n",
       "        div.ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8 span.ipyplot-img-expand:hover {\n",
       "            cursor: zoom-in;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8]:target {\n",
       "            transform: scale(2.5);\n",
       "            transform-origin: left top;\n",
       "            z-index: 5000;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "            position: absolute;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8]:target span.ipyplot-img-close {\n",
       "            display: block;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8]:target span.ipyplot-img-expand {\n",
       "            display: none;\n",
       "        }\n",
       "        </style>\n",
       "    <div id=\"ipyplot-imgs-container-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "    <div class=\"ipyplot-placeholder-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "        <div id=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-8uAsUqNxmdCz5ZiYtoPxER\" class=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: At the end of the tunnel   <br/>\n",
       "                     Distance: 25.83\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/bNCXoisclCo.jpg</h4><img src=\"../data/raw/images/bNCXoisclCo.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-8uAsUqNxmdCz5ZiYtoPxER\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "        <div id=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-nZLeTAYjEknEnn8AnsoZv5\" class=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: Beach at the end of a day   <br/>\n",
       "                     Distance: 15.37\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/rj1KGW8CtAc.jpg</h4><img src=\"../data/raw/images/rj1KGW8CtAc.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-nZLeTAYjEknEnn8AnsoZv5\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "        <div id=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-LXjYdTnpnDb38k6GSXSR5U\" class=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: Peace at the end of a sunny day...   <br/>\n",
       "                     Distance: 14.55\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/kCcyt7bYP3w.jpg</h4><img src=\"../data/raw/images/kCcyt7bYP3w.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-LXjYdTnpnDb38k6GSXSR5U\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "        <div id=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-jisTzntocV7HruDu5skX6c\" class=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: Tunnel View under the night sky.   <br/>\n",
       "                     Distance: 12.44\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/f3ytCVDbSGU.jpg</h4><img src=\"../data/raw/images/f3ytCVDbSGU.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-jisTzntocV7HruDu5skX6c\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "        <div id=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-5q978tmVDZS2maoyr2pTpz\" class=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: The end is always good   <br/>\n",
       "                     Distance: 12.13\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/CNMZqMmSjbc.jpg</h4><img src=\"../data/raw/images/CNMZqMmSjbc.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-5q978tmVDZS2maoyr2pTpz\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    </div>\n",
       "        </textarea>\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <style>\n",
       "        #ipyplot-imgs-container-div-HWchwqNRuM4ivQZBpeewJ8 {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            margin: 0%;\n",
       "            overflow: auto;\n",
       "            position: relative;\n",
       "            overflow-y: scroll;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-placeholder-div-HWchwqNRuM4ivQZBpeewJ8 {\n",
       "            width: 200px;\n",
       "            display: inline-block;\n",
       "            margin: 3px;\n",
       "            position: relative;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8 {\n",
       "            width: 200px;\n",
       "            background: white;\n",
       "            display: inline-block;\n",
       "            vertical-align: top;\n",
       "            text-align: center;\n",
       "            position: relative;\n",
       "            border: 2px solid #ddd;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8 span.ipyplot-img-close {\n",
       "            display: none;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8 span {\n",
       "            width: 100%;\n",
       "            height: 100%;\n",
       "            position: absolute;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8 img {\n",
       "            width: 200px;\n",
       "        }\n",
       "\n",
       "        div.ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8 span.ipyplot-img-close:hover {\n",
       "            cursor: zoom-out;\n",
       "        }\n",
       "        div.ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8 span.ipyplot-img-expand:hover {\n",
       "            cursor: zoom-in;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8]:target {\n",
       "            transform: scale(2.5);\n",
       "            transform-origin: left top;\n",
       "            z-index: 5000;\n",
       "            top: 0;\n",
       "            left: 0;\n",
       "            position: absolute;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8]:target span.ipyplot-img-close {\n",
       "            display: block;\n",
       "        }\n",
       "\n",
       "        div[id^=ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8]:target span.ipyplot-img-expand {\n",
       "            display: none;\n",
       "        }\n",
       "        </style>\n",
       "    <div id=\"ipyplot-imgs-container-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "    <div class=\"ipyplot-placeholder-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "        <div id=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-8uAsUqNxmdCz5ZiYtoPxER\" class=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: At the end of the tunnel   <br/>\n",
       "                     Distance: 25.83\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/bNCXoisclCo.jpg</h4><img src=\"../data/raw/images/bNCXoisclCo.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-8uAsUqNxmdCz5ZiYtoPxER\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "        <div id=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-nZLeTAYjEknEnn8AnsoZv5\" class=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: Beach at the end of a day   <br/>\n",
       "                     Distance: 15.37\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/rj1KGW8CtAc.jpg</h4><img src=\"../data/raw/images/rj1KGW8CtAc.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-nZLeTAYjEknEnn8AnsoZv5\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "        <div id=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-LXjYdTnpnDb38k6GSXSR5U\" class=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: Peace at the end of a sunny day...   <br/>\n",
       "                     Distance: 14.55\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/kCcyt7bYP3w.jpg</h4><img src=\"../data/raw/images/kCcyt7bYP3w.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-LXjYdTnpnDb38k6GSXSR5U\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "        <div id=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-jisTzntocV7HruDu5skX6c\" class=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: Tunnel View under the night sky.   <br/>\n",
       "                     Distance: 12.44\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/f3ytCVDbSGU.jpg</h4><img src=\"../data/raw/images/f3ytCVDbSGU.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-jisTzntocV7HruDu5skX6c\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <div class=\"ipyplot-placeholder-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "        <div id=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-5q978tmVDZS2maoyr2pTpz\" class=\"ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8\">\n",
       "            <h4 style=\"font-size: 12px; word-wrap: break-word;\">\n",
       "                     Photo title: The end is always good   <br/>\n",
       "                     Distance: 12.13\n",
       "            \n",
       "                     </h4>\n",
       "            <h4 style=\"font-size: 9px; padding-left: 10px; padding-right: 10px; width: 95%; word-wrap: break-word; white-space: normal;\">../data/raw/images/CNMZqMmSjbc.jpg</h4><img src=\"../data/raw/images/CNMZqMmSjbc.jpg\"/>\n",
       "            <a href=\"#!\">\n",
       "                <span class=\"ipyplot-img-close\"/>\n",
       "            </a>\n",
       "            <a href=\"#ipyplot-content-div-HWchwqNRuM4ivQZBpeewJ8-5q978tmVDZS2maoyr2pTpz\">\n",
       "                <span class=\"ipyplot-img-expand\"/>\n",
       "            </a>\n",
       "        </div>\n",
       "    </div>\n",
       "    </div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "find_results( \"light at the end of the tunnel\", analyzer=\"standard\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9de0b531-7895-41fe-86e9-bd78022fdc2f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ca361fe-c0d8-445a-af59-db1e2c8322f5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40bf9523-5dec-42fa-bb16-d16214b2617d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e996b1a-7777-49a8-91f3-e3ee0c1c0646",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "3db4aa35-7304-48c5-a7f5-0e612b1f2adc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T22:10:28.585613Z",
     "iopub.status.busy": "2023-04-25T22:10:28.585228Z",
     "iopub.status.idle": "2023-04-25T22:10:28.590890Z",
     "shell.execute_reply": "2023-04-25T22:10:28.590042Z",
     "shell.execute_reply.started": "2023-04-25T22:10:28.585582Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sample_queries = [\n",
    "\n",
    "\"person on top of mountain\"\n",
    ", \"picture of a man in a desert\"\n",
    ", \"person in a desert\"    \n",
    "\n",
    ", \"the boy and girl on a beach\"\n",
    ", \"children in beach\"    \n",
    "\n",
    ", \"Two dogs playing in the snow\"\n",
    "\n",
    ", \"light at the end of the tunnel\"\n",
    ", \"seven wonders of the world\"\n",
    "    \n",
    "    \n",
    ", \"water droplets on a leaf\"\n",
    "    \n",
    ", \"ripley's aquarium of canada, toronto, canada\"\n",
    ", \"the butterfly atrium at hershey gardens\"\n",
    "    \n",
    ", \"salar de uyuni uyuni bolivia\"\n",
    ", \"沙漠青蛙 沙漠青蛙\" #(desert frog)\n",
    ", \"por do sol no mar\"\n",
    ", \"conhece te a ti mesmo\" #\t ( Greek for know thyself)\n",
    "\n",
    "\n",
    ", \"there is no planet b\"\n",
    "\n",
    "\n",
    ", \"nova scotia duck tolling retriever\"\n",
    "    \n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8859627d-19a1-4f94-af81-50b0ed8aa310",
   "metadata": {},
   "source": [
    "Questions to Ask:\n",
    "\n",
    "How does the model do for clear "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9a852a7-8b18-44eb-84c2-07ed25eac24e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc5dfe77-63f2-465b-bb9f-c925f456ce03",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "44a70fe7-36af-4e69-8486-ccc347d3b3bf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T22:10:30.486273Z",
     "iopub.status.busy": "2023-04-25T22:10:30.485467Z",
     "iopub.status.idle": "2023-04-25T22:10:30.491405Z",
     "shell.execute_reply": "2023-04-25T22:10:30.490536Z",
     "shell.execute_reply.started": "2023-04-25T22:10:30.486237Z"
    },
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n\\n\"ripley\\'s aquarium of canada, toronto, canada\" (not there)\\nsalar de uyuni uyuni bolivia (not there)\\n\\n沙漠青蛙 沙漠青蛙 desert frog (not tokenizing correclty) \\nconhece te a ti mesmo (not understandin latin)\\n\\nplanet B (because not tokenized)\\n\\nseven wonders of the world: not understanding intent\\n'"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "\n",
    "\"ripley's aquarium of canada, toronto, canada\" (not there)\n",
    "salar de uyuni uyuni bolivia (not there)\n",
    "\n",
    "沙漠青蛙 沙漠青蛙 desert frog (not tokenizing correclty) \n",
    "conhece te a ti mesmo (not understandin latin)\n",
    "\n",
    "planet B (because not tokenized)\n",
    "\n",
    "seven wonders of the world: not understanding intent\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "2617eddc-84ef-4e57-ba98-fb0232777a1f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T23:14:43.167361Z",
     "iopub.status.busy": "2023-04-25T23:14:43.167012Z",
     "iopub.status.idle": "2023-04-25T23:14:43.219331Z",
     "shell.execute_reply": "2023-04-25T23:14:43.218621Z",
     "shell.execute_reply.started": "2023-04-25T23:14:43.167333Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "52a69e74e0ff42acac14952302f154b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(Dropdown(description='query', options=('person on top of mountain', 'picture of a man in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "@interact\n",
    "def interact_find_results(query=sample_queries, analyzer=[\"english\",\"stop\",\"standard\"], explain=[True,False], k =10):\n",
    "    find_results( query, analyzer=analyzer,explain=explain, k =k)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9340883e-7e08-4e51-9107-bf996ad49498",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "3c14c723-4a32-42d1-b092-26f498c24334",
   "metadata": {},
   "source": [
    "1) Do the results of `person on top of mountain` differ between the tokenization?\n",
    "\n",
    "2) How do the results of `picture of a man in a desert` and `person in a desert` compare ? \n",
    "\n",
    "3) How do `the boy and girl on a beach` and `children in beach` compare ?\n",
    "\n",
    "4) Are the results of `seven wonders of the world`relevnat ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "527ccb47-044b-46b3-9052-7a11e960f5e1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0263d116-79b4-4768-a759-96f1a2c60d6f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a27d93c7-243a-44e4-902e-3e95aebbaa12",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2fcbe756-6f51-4f1b-b7d0-582ea2618744",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e47d1fe7-4522-46ae-ad76-6cdf804cc404",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94884694-42e8-4339-a28f-872cb9b185c5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "294dcac6-9836-468c-8458-ab35f66fe1cb",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26f817fd-bfd5-41a3-a679-b53f2e6321e7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "dd2646c1-0e3c-4dac-b0db-cd5ed05b34b5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-25T22:20:37.753359Z",
     "iopub.status.busy": "2023-04-25T22:20:37.753011Z",
     "iopub.status.idle": "2023-04-25T22:20:39.368885Z",
     "shell.execute_reply": "2023-04-25T22:20:39.367802Z",
     "shell.execute_reply.started": "2023-04-25T22:20:37.753332Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.97 ms ± 65.3 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "search_query = \"Two dogs playing in the snow\"\n",
    "k =5 \n",
    "top_items = fetch_results(client,query=query,num_hits=k)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb46caee-08eb-462b-a510-7770fcf953d2",
   "metadata": {},
   "source": [
    "We are able to search 25k docs in ~3 ms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ecb66bb-df77-4c07-94dd-4522b291d3d8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04322b4f-b08e-47a3-9cc8-64db1103cc30",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d58ee0b-c255-42ce-b11a-5ce5586b6a18",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa9aac9b-3731-4e99-8249-3cb33ac9e929",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5523cadb-1a4c-411d-8de6-0c5b23e62257",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee09e948-364b-43c4-956d-9cf8e8cf1519",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f53f0fa-841e-4eb1-ad78-c4041d8db469",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "af4232ac-3420-4783-a61b-a63d68c0f392",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Other"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7190203-e9b8-442d-8591-a88b5344f46b",
   "metadata": {},
   "source": [
    "other topics to be covered if there is time"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e08ec2e6-da80-4350-9a9e-6f675fafcf97",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Built in tokenizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6da5298a-94d0-448b-a740-66a3f342f2c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def elastic_tokenize(tokenizer,  text, url = ELASTIC_FULL_URL+\"/_analyze\"):\n",
    "    r =requests.post(url, \n",
    "              json =\n",
    "                    {\n",
    "                      \"tokenizer\": tokenizer ,\n",
    "                      \"text\": text\n",
    "                    }\n",
    "    \n",
    "    \n",
    "        )\n",
    "\n",
    "    rich.print (r.json() )\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89b6db9b-005e-4374-b5de-5e662b8d5ec5",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = \"<p> ELASTICSEARCH is built on top of the open-source <b>Apache Lucene</b>. </p>\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fad38f7-7370-461d-86c0-57cf32de8eac",
   "metadata": {},
   "source": [
    "whitespace tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1495b666-4e9f-46d8-b15d-bceadd7f0009",
   "metadata": {},
   "outputs": [],
   "source": [
    "elastic_tokenize (tokenizer= \"whitespace\",  text= sentence)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "313fa801-15e1-4c13-b06b-62722f6c78c3",
   "metadata": {},
   "source": [
    "standard tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bab0571a-6e1e-42e3-be14-db54855d1b08",
   "metadata": {},
   "outputs": [],
   "source": [
    "elastic_tokenize (tokenizer= \"standard\",  text= sentence)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d31a8e8e-46e0-4b4a-8cb8-9b6bf2513d3b",
   "metadata": {},
   "source": [
    "ngram tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad01a29f-5e1b-44a4-b024-e7909576ae01",
   "metadata": {},
   "outputs": [],
   "source": [
    "elastic_tokenize (tokenizer= \"ngram\",  text= \"Quick\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c88fef79-661d-4e6b-a923-55730513df72",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "a9a9ee23-bce2-4b56-be22-9ec7d7ccd342",
   "metadata": {},
   "source": [
    "note that by default, the default schema for text content stored content as full text and keywords.      \n",
    "It is ignored as keyword, if the length is greater than 256 tokens\n",
    "\n",
    "[ignore_above reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/ignore-above.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4de6ad08-3c8d-4406-9475-3d8c89caec3a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bc60f8a-d78f-4e50-8407-fd307975a0c3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e00f9947-e165-4a03-961e-5eb11f09a7c3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "211e25a5-13fc-4cb2-a25f-0245d4c0d71b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb22afbe-9d48-4f2c-b42b-0721f21ba945",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3038c8d6-a1db-45f4-a3f8-a6b40b75ffb0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ead5140f-9d0c-4076-ab00-82963f3c5164",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60d8b74f-1a64-4713-b1da-599019d83b80",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee31471c-300d-41ac-b84f-2dd53eb99e29",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc42cc6e-7e38-4965-a05a-ce8a29b951c2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38c3cd61-96c0-41a3-af9e-b610187d7ef5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d069160-8191-4cbd-9c08-5c3d40692fd3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b04eb975-1e46-45c8-adfc-23454d5eac19",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "165e920d-742b-48ef-8110-dcadd6e09504",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "7066871e-4dae-4220-8727-d5397f353464",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Distributed tf-idf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5957c0b-2497-4d52-aa3f-06641746decb",
   "metadata": {},
   "source": [
    "we are running an elastic search cluster with three shards.\n",
    "\n",
    "ES has two ways to compute the distributed term frequencies\n",
    "\n",
    "\n",
    "\n",
    "`query_then_fetch`     \n",
    "(Default) Distributed term frequencies are calculated locally for each shard running the search.    \n",
    "\n",
    "We recommend this option for faster searches with potentially less accurate scoring.\n",
    "\n",
    "`dfs_query_then_fetch`    \n",
    "Distributed term frequencies are calculated globally, using information gathered from all shards running the search.   \n",
    "While this option increases the accuracy of scoring, it adds a round-trip to each shard, which can result in slower searches.\n",
    "\n",
    "taken from ES [docs](https://www.elastic.co/guide/en/elasticsearch/reference/8.4/search-search.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93dd724d-de7a-463a-a6e2-64cb981584d6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "19827c91-66be-4f6d-9842-ee1555614732",
   "metadata": {},
   "source": [
    "searching with the default mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ada31079-047e-4d1c-a4ca-30f59e142a1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "resp = client.search(\n",
    "    query = {\n",
    "            \"multi_match\": {\n",
    "                \"query\": query,\n",
    "                 \"fields\": [\"description_final\"],\n",
    "                            }\n",
    "            }\n",
    "    , size=2\n",
    "    #, explain=True\n",
    "    , source = [\"description_final\"]\n",
    "    , search_type = \"query_then_fetch\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "308aaab1-228d-4c74-a811-745d12f77f06",
   "metadata": {},
   "outputs": [],
   "source": [
    "JSON (resp.body , expanded=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b03a90d7-5b7b-4010-8e6a-86bd353aec51",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8ef264d1-136e-49ac-9f1c-691af5c5a0b8",
   "metadata": {},
   "source": [
    "searching with the global dfs mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eaf23bd9-8079-438d-be3f-318e5f1b19ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "resp = client.search(\n",
    "    query = {\n",
    "            \"multi_match\": {\n",
    "                \"query\": query,\n",
    "                 \"fields\": [\"description_final\"],\n",
    "                            }\n",
    "            }\n",
    "    , size=2\n",
    "    #, explain=True\n",
    "    , source = [\"description_final\"]\n",
    "    , search_type = \"dfs_query_then_fetch\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0043720f-72d6-43cb-be68-1ad15afdbfd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "JSON (resp.body , expanded=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b76d354-c48c-49aa-a260-b0fb2df1ff92",
   "metadata": {},
   "source": [
    "score difference between the different search types\n",
    "\n",
    "before: 14.775831, 13.5637\n",
    "    \n",
    "after:  14.706409, 13.708656"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "712fbc24-bee4-4ab8-ab29-3373aa561b5e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca8797fa-ed9b-4595-b513-b4595cb7db71",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e00ac71-e029-4a3a-bd0e-b3a3a9782fde",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22895c00-51e0-4c67-bfa5-a756cc30c939",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4ceb031-74a9-4028-9ac3-4d84d11d2308",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "daa729bd-30b6-4ec5-8116-b7d69cc5a95c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77adec2d-dce6-4201-b239-295251e1eb3b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17cd58c2-c23d-40fa-9dd7-544fecbf52bd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a800f1e1-9c52-4373-8c45-7bce21cf4f96",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5eaca044-83e7-4432-ac37-e12cfd7f8dd4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "890a8140-e0da-4bc6-b137-22994006ba0d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee539a4f-bef3-4878-9683-38439f223511",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ea7ddc8-5960-4ef1-a5ad-4f0cf8341322",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09a8cbc7-837c-42be-b7b4-5900710e0779",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c205b1d1-8531-4cb8-98f8-1ef698eef590",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd4f17e4-dd10-4622-9d35-c85b9798ecf6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0dcf9ab5-132a-4ec2-bee1-03a70c71db3e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10ad187a-fc44-4ee8-8157-485ac0421547",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d443c6bd-72d6-401f-bec6-b62015a8786c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc151777-7d58-4da6-b70f-9c5bbe00d156",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba1b523e-13b9-40db-9e68-2e078e4c772f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd1446d1-258b-4db2-92f9-0683651b2909",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69ed9542-1ae0-4703-a0a9-fcbdcf558fb5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "workshop",
   "name": "pytorch-gpu.1-13.m107",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/pytorch-gpu.1-13:m107"
  },
  "kernelspec": {
   "display_name": "workshop",
   "language": "python",
   "name": "workshop"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "toc-autonumbering": false,
  "toc-showtags": false
 },
 "nbformat": 4,
 "nbformat_minor": 5
}