Assestment of how many pages to scrape Fediushkin.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2a0dd69f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5f2f3419",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_csv(r'C:/Users/Armit/Python/Scripts/Project/games_list_categories.csv', sep = '\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f1c13394",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.read_csv(r'C:/Users/Armit/Python/Scripts/Project/games_list_categories200_600.csv', sep = '\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fad9cdc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df3 = pd.read_csv(r'C:/Users/Armit/Python/Scripts/Project/games_list_categories600_1000.csv', sep = '\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "845eadca",
   "metadata": {},
   "outputs": [],
   "source": [
    "df4 = pd.read_csv(r'C:/Users/Armit/Python/Scripts/Project/games_list_categories1000_1400_.csv', sep = '\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "da58ad46",
   "metadata": {},
   "outputs": [],
   "source": [
    "df5 = pd.read_csv(r'C:/Users/Armit/Python/Scripts/Project/games_list_categories1000_1460_.csv', sep = '\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "1ee56a11",
   "metadata": {},
   "outputs": [],
   "source": [
    "df6 = pd.read_csv(r'C:/Users/Armit/Python/Scripts/Project/games_list_categories1460_1590_.csv', sep = '\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6635374c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df7= pd.read_csv(r'C:/Users/Armit/Python/Scripts/Project/games_list_categories1590_2152_.csv', sep = '\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8c44fd27",
   "metadata": {},
   "outputs": [],
   "source": [
    "df8 = pd.read_csv(r'C:/Users/Armit/Python/Scripts/Project/games_list_categories2152_2695_.csv', sep = '\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "441d25a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df9 = pd.read_csv(r'C:/Users/Armit/Python/Scripts/Project/games_list_categories_additional.csv', sep = '\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "eb70cc5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df9 = df9.set_index('index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "3c3f019c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = df1[0:200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "19c6733c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2= df2[200:600]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "271fb832",
   "metadata": {},
   "outputs": [],
   "source": [
    "# missed 600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "2a2cf8d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df3 = df3[600:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e6aed5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df4 = df4[1000:1400]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "154caa51",
   "metadata": {},
   "outputs": [],
   "source": [
    "df5 = df5[1000:1460]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "f728d006",
   "metadata": {},
   "outputs": [],
   "source": [
    "df6 = df6[1460:1590]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "a170319c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "30375080",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "400"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84568292",
   "metadata": {},
   "outputs": [],
   "source": [
    "df3 = df3[1460:1590]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "2b98a78e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "400"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "5c124e21",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "460"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "3957d48a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "174"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "71b50965",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "562"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "dd6d0ff3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "543"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66519fda",
   "metadata": {},
   "outputs": [],
   "source": [
    "df7 = df7[600:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "3238d025",
   "metadata": {},
   "outputs": [],
   "source": [
    "df9 = df9.drop(['index'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "3842df45",
   "metadata": {},
   "outputs": [],
   "source": [
    "df9 = df9[['name', 'id', 'url', 'full_url', 'overview_url','categories']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "aca23611",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>full_url</th>\n",
       "      <th>overview_url</th>\n",
       "      <th>categories</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>600</th>\n",
       "      <td>Star Trek: Fleet Captains</td>\n",
       "      <td>79127</td>\n",
       "      <td>/boardgame/79127/star-trek-fleet-captains</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/79127/star...</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/79127/star...</td>\n",
       "      <td>['Fighting', 'Miniatures', 'Movies / TV / Radi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1299</th>\n",
       "      <td>Cubitos</td>\n",
       "      <td>298069</td>\n",
       "      <td>/boardgame/298069/cubitos</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/298069/cub...</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/298069/cub...</td>\n",
       "      <td>['Dice', 'Racing']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1305</th>\n",
       "      <td>Cubitos</td>\n",
       "      <td>298069</td>\n",
       "      <td>/boardgame/298069/cubitos</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/298069/cub...</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/298069/cub...</td>\n",
       "      <td>['Dice', 'Racing']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1416</th>\n",
       "      <td>Project: ELITE</td>\n",
       "      <td>171726</td>\n",
       "      <td>/boardgame/171726/project-elite</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/171726/pro...</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/171726/pro...</td>\n",
       "      <td>['Fighting', 'Miniatures', 'Real-time', 'Scien...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1417</th>\n",
       "      <td>Babel</td>\n",
       "      <td>986</td>\n",
       "      <td>/boardgame/986/babel</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/986/babel/...</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/986/babel</td>\n",
       "      <td>['Ancient', 'Card Game', 'City Building']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2147</th>\n",
       "      <td>Viva Topo!</td>\n",
       "      <td>8195</td>\n",
       "      <td>/boardgame/8195/viva-topo</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/8195/viva-...</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/8195/viva-...</td>\n",
       "      <td>['Animals', \"Children's Game\", 'Dice']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2148</th>\n",
       "      <td>GoodCritters</td>\n",
       "      <td>255615</td>\n",
       "      <td>/boardgame/255615/goodcritters</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/255615/goo...</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/255615/goo...</td>\n",
       "      <td>['Animals', 'Bluffing', 'Negotiation', 'Party ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2149</th>\n",
       "      <td>Limes</td>\n",
       "      <td>153623</td>\n",
       "      <td>/boardgame/153623/limes</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/153623/lim...</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/153623/limes</td>\n",
       "      <td>['Card Game', 'City Building', 'Puzzle']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2150</th>\n",
       "      <td>Space Explorers</td>\n",
       "      <td>235817</td>\n",
       "      <td>/boardgame/235817/space-explorers</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/235817/spa...</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/235817/spa...</td>\n",
       "      <td>['Card Game', 'Economic', 'Space Exploration']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2151</th>\n",
       "      <td>Koryŏ</td>\n",
       "      <td>140535</td>\n",
       "      <td>/boardgame/140535/kory</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/140535/kor...</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/140535/kory</td>\n",
       "      <td>['Card Game']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>63 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            name      id  \\\n",
       "index                                      \n",
       "600    Star Trek: Fleet Captains   79127   \n",
       "1299                     Cubitos  298069   \n",
       "1305                     Cubitos  298069   \n",
       "1416              Project: ELITE  171726   \n",
       "1417                       Babel     986   \n",
       "...                          ...     ...   \n",
       "2147                  Viva Topo!    8195   \n",
       "2148                GoodCritters  255615   \n",
       "2149                       Limes  153623   \n",
       "2150             Space Explorers  235817   \n",
       "2151                       Koryŏ  140535   \n",
       "\n",
       "                                             url  \\\n",
       "index                                              \n",
       "600    /boardgame/79127/star-trek-fleet-captains   \n",
       "1299                   /boardgame/298069/cubitos   \n",
       "1305                   /boardgame/298069/cubitos   \n",
       "1416             /boardgame/171726/project-elite   \n",
       "1417                        /boardgame/986/babel   \n",
       "...                                          ...   \n",
       "2147                   /boardgame/8195/viva-topo   \n",
       "2148              /boardgame/255615/goodcritters   \n",
       "2149                     /boardgame/153623/limes   \n",
       "2150           /boardgame/235817/space-explorers   \n",
       "2151                      /boardgame/140535/kory   \n",
       "\n",
       "                                                full_url  \\\n",
       "index                                                      \n",
       "600    https://boardgamegeek.com/boardgame/79127/star...   \n",
       "1299   https://boardgamegeek.com/boardgame/298069/cub...   \n",
       "1305   https://boardgamegeek.com/boardgame/298069/cub...   \n",
       "1416   https://boardgamegeek.com/boardgame/171726/pro...   \n",
       "1417   https://boardgamegeek.com/boardgame/986/babel/...   \n",
       "...                                                  ...   \n",
       "2147   https://boardgamegeek.com/boardgame/8195/viva-...   \n",
       "2148   https://boardgamegeek.com/boardgame/255615/goo...   \n",
       "2149   https://boardgamegeek.com/boardgame/153623/lim...   \n",
       "2150   https://boardgamegeek.com/boardgame/235817/spa...   \n",
       "2151   https://boardgamegeek.com/boardgame/140535/kor...   \n",
       "\n",
       "                                            overview_url  \\\n",
       "index                                                      \n",
       "600    https://boardgamegeek.com/boardgame/79127/star...   \n",
       "1299   https://boardgamegeek.com/boardgame/298069/cub...   \n",
       "1305   https://boardgamegeek.com/boardgame/298069/cub...   \n",
       "1416   https://boardgamegeek.com/boardgame/171726/pro...   \n",
       "1417       https://boardgamegeek.com/boardgame/986/babel   \n",
       "...                                                  ...   \n",
       "2147   https://boardgamegeek.com/boardgame/8195/viva-...   \n",
       "2148   https://boardgamegeek.com/boardgame/255615/goo...   \n",
       "2149    https://boardgamegeek.com/boardgame/153623/limes   \n",
       "2150   https://boardgamegeek.com/boardgame/235817/spa...   \n",
       "2151     https://boardgamegeek.com/boardgame/140535/kory   \n",
       "\n",
       "                                              categories  \n",
       "index                                                     \n",
       "600    ['Fighting', 'Miniatures', 'Movies / TV / Radi...  \n",
       "1299                                  ['Dice', 'Racing']  \n",
       "1305                                  ['Dice', 'Racing']  \n",
       "1416   ['Fighting', 'Miniatures', 'Real-time', 'Scien...  \n",
       "1417           ['Ancient', 'Card Game', 'City Building']  \n",
       "...                                                  ...  \n",
       "2147              ['Animals', \"Children's Game\", 'Dice']  \n",
       "2148   ['Animals', 'Bluffing', 'Negotiation', 'Party ...  \n",
       "2149            ['Card Game', 'City Building', 'Puzzle']  \n",
       "2150      ['Card Game', 'Economic', 'Space Exploration']  \n",
       "2151                                       ['Card Game']  \n",
       "\n",
       "[63 rows x 6 columns]"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "0c8047e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "frames = [df1, df2, df3, df5, df6, df7, df8,df9]\n",
    "\n",
    "result = pd.concat(frames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "cd574f83",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = result.drop(['url','full_url','overview_url'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "5afa1c02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>id</th>\n",
       "      <th>categories</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2148</th>\n",
       "      <td>GoodCritters</td>\n",
       "      <td>255615</td>\n",
       "      <td>['Animals', 'Bluffing', 'Negotiation', 'Party ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              name      id                                         categories\n",
       "2148  GoodCritters  255615  ['Animals', 'Bluffing', 'Negotiation', 'Party ..."
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result[result['id'] ==255615]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "88473623",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>id</th>\n",
       "      <th>categories</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1305</th>\n",
       "      <td>Cubitos</td>\n",
       "      <td>298069</td>\n",
       "      <td>['Dice', 'Racing']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Project: ELITE</td>\n",
       "      <td>171726</td>\n",
       "      <td>['Fighting', 'Miniatures', 'Real-time', 'Scien...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Babel</td>\n",
       "      <td>986</td>\n",
       "      <td>['Ancient', 'Card Game', 'City Building']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Historia</td>\n",
       "      <td>157096</td>\n",
       "      <td>['Civilization']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Hare &amp; Tortoise</td>\n",
       "      <td>361</td>\n",
       "      <td>['Animals', 'Racing']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1457</th>\n",
       "      <td>Axis &amp; Allies</td>\n",
       "      <td>98</td>\n",
       "      <td>['Economic', 'Wargame', 'World War II']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1458</th>\n",
       "      <td>Founding Fathers</td>\n",
       "      <td>37358</td>\n",
       "      <td>['Political']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1459</th>\n",
       "      <td>Gingerbread House</td>\n",
       "      <td>258444</td>\n",
       "      <td>['Fantasy', 'Humor', 'Mythology', 'Novel-based...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2097</th>\n",
       "      <td>Dual Powers: Revolution 1917</td>\n",
       "      <td>241987</td>\n",
       "      <td>['Deduction', 'Political']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2100</th>\n",
       "      <td>Dual Powers: Revolution 1917</td>\n",
       "      <td>241987</td>\n",
       "      <td>['Deduction', 'Political']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>93 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              name      id  \\\n",
       "1305                       Cubitos  298069   \n",
       "0                   Project: ELITE  171726   \n",
       "1                            Babel     986   \n",
       "2                         Historia  157096   \n",
       "3                  Hare & Tortoise     361   \n",
       "...                            ...     ...   \n",
       "1457                 Axis & Allies      98   \n",
       "1458              Founding Fathers   37358   \n",
       "1459             Gingerbread House  258444   \n",
       "2097  Dual Powers: Revolution 1917  241987   \n",
       "2100  Dual Powers: Revolution 1917  241987   \n",
       "\n",
       "                                             categories  \n",
       "1305                                 ['Dice', 'Racing']  \n",
       "0     ['Fighting', 'Miniatures', 'Real-time', 'Scien...  \n",
       "1             ['Ancient', 'Card Game', 'City Building']  \n",
       "2                                      ['Civilization']  \n",
       "3                                 ['Animals', 'Racing']  \n",
       "...                                                 ...  \n",
       "1457            ['Economic', 'Wargame', 'World War II']  \n",
       "1458                                      ['Political']  \n",
       "1459  ['Fantasy', 'Humor', 'Mythology', 'Novel-based...  \n",
       "2097                         ['Deduction', 'Political']  \n",
       "2100                         ['Deduction', 'Political']  \n",
       "\n",
       "[93 rows x 3 columns]"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result[result.duplicated() == True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "fe7c731e",
   "metadata": {},
   "outputs": [],
   "source": [
    "result.drop_duplicates(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "9c672bf0",
   "metadata": {},
   "outputs": [],
   "source": [
    "result.drop_duplicates('id', keep='last', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "41ce26ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "games_list = pd.read_csv(r'C:/Users/Armit/Python/Scripts/Project/games_100_pages.csv', sep = '\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "c8f99d28",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(games_list,result[['id','categories']],on='id', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "b0bbbdc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[0:2695]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "802d3ecc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>name</th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>full_url</th>\n",
       "      <th>categories</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>734</th>\n",
       "      <td>734</td>\n",
       "      <td>Pandemic: Rising Tide</td>\n",
       "      <td>234671</td>\n",
       "      <td>/boardgame/234671/pandemic-rising-tide</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/234671/pan...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>738</th>\n",
       "      <td>738</td>\n",
       "      <td>Hawaii</td>\n",
       "      <td>106217</td>\n",
       "      <td>/boardgame/106217/hawaii</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/106217/haw...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>754</th>\n",
       "      <td>754</td>\n",
       "      <td>Lanterns: The Harvest Festival</td>\n",
       "      <td>160851</td>\n",
       "      <td>/boardgame/160851/lanterns-harvest-festival</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/160851/lan...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>995</td>\n",
       "      <td>Reavers of Midgard</td>\n",
       "      <td>242684</td>\n",
       "      <td>/boardgame/242684/reavers-midgard</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/242684/rea...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1074</th>\n",
       "      <td>1074</td>\n",
       "      <td>Royals</td>\n",
       "      <td>165986</td>\n",
       "      <td>/boardgame/165986/royals</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/165986/roy...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1409</th>\n",
       "      <td>1409</td>\n",
       "      <td>Expedition to Newdale</td>\n",
       "      <td>272682</td>\n",
       "      <td>/boardgame/272682/expedition-newdale</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/272682/exp...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1426</th>\n",
       "      <td>1426</td>\n",
       "      <td>Mangrovia</td>\n",
       "      <td>165948</td>\n",
       "      <td>/boardgame/165948/mangrovia</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/165948/man...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1713</th>\n",
       "      <td>1713</td>\n",
       "      <td>Keltis</td>\n",
       "      <td>34585</td>\n",
       "      <td>/boardgame/34585/keltis</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/34585/kelt...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1735</th>\n",
       "      <td>1735</td>\n",
       "      <td>Jórvík</td>\n",
       "      <td>193739</td>\n",
       "      <td>/boardgame/193739/jorvik</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/193739/jor...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1739</th>\n",
       "      <td>1739</td>\n",
       "      <td>Machi Koro Legacy</td>\n",
       "      <td>259081</td>\n",
       "      <td>/boardgame/259081/machi-koro-legacy</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/259081/mac...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1892</th>\n",
       "      <td>1892</td>\n",
       "      <td>Piece o' Cake</td>\n",
       "      <td>37371</td>\n",
       "      <td>/boardgame/37371/piece-o-cake</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/37371/piec...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1970</th>\n",
       "      <td>1970</td>\n",
       "      <td>Kingdomino Duel</td>\n",
       "      <td>281960</td>\n",
       "      <td>/boardgame/281960/kingdomino-duel</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/281960/kin...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2257</th>\n",
       "      <td>2257</td>\n",
       "      <td>Victorian Masterminds</td>\n",
       "      <td>189453</td>\n",
       "      <td>/boardgame/189453/victorian-masterminds</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/189453/vic...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2492</th>\n",
       "      <td>2492</td>\n",
       "      <td>Adventure Games: Monochrome Inc.</td>\n",
       "      <td>267814</td>\n",
       "      <td>/boardgame/267814/adventure-games-monochrome-inc</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/267814/adv...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2493</th>\n",
       "      <td>2493</td>\n",
       "      <td>Rival Restaurants</td>\n",
       "      <td>262114</td>\n",
       "      <td>/boardgame/262114/rival-restaurants</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/262114/riv...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2665</th>\n",
       "      <td>2665</td>\n",
       "      <td>The Great Dinosaur Rush</td>\n",
       "      <td>179071</td>\n",
       "      <td>/boardgame/179071/great-dinosaur-rush</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/179071/gre...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0                              name      id  \\\n",
       "734          734             Pandemic: Rising Tide  234671   \n",
       "738          738                            Hawaii  106217   \n",
       "754          754    Lanterns: The Harvest Festival  160851   \n",
       "995          995                Reavers of Midgard  242684   \n",
       "1074        1074                            Royals  165986   \n",
       "1409        1409             Expedition to Newdale  272682   \n",
       "1426        1426                         Mangrovia  165948   \n",
       "1713        1713                            Keltis   34585   \n",
       "1735        1735                            Jórvík  193739   \n",
       "1739        1739                 Machi Koro Legacy  259081   \n",
       "1892        1892                     Piece o' Cake   37371   \n",
       "1970        1970                   Kingdomino Duel  281960   \n",
       "2257        2257             Victorian Masterminds  189453   \n",
       "2492        2492  Adventure Games: Monochrome Inc.  267814   \n",
       "2493        2493                 Rival Restaurants  262114   \n",
       "2665        2665           The Great Dinosaur Rush  179071   \n",
       "\n",
       "                                                   url  \\\n",
       "734             /boardgame/234671/pandemic-rising-tide   \n",
       "738                           /boardgame/106217/hawaii   \n",
       "754        /boardgame/160851/lanterns-harvest-festival   \n",
       "995                  /boardgame/242684/reavers-midgard   \n",
       "1074                          /boardgame/165986/royals   \n",
       "1409              /boardgame/272682/expedition-newdale   \n",
       "1426                       /boardgame/165948/mangrovia   \n",
       "1713                           /boardgame/34585/keltis   \n",
       "1735                          /boardgame/193739/jorvik   \n",
       "1739               /boardgame/259081/machi-koro-legacy   \n",
       "1892                     /boardgame/37371/piece-o-cake   \n",
       "1970                 /boardgame/281960/kingdomino-duel   \n",
       "2257           /boardgame/189453/victorian-masterminds   \n",
       "2492  /boardgame/267814/adventure-games-monochrome-inc   \n",
       "2493               /boardgame/262114/rival-restaurants   \n",
       "2665             /boardgame/179071/great-dinosaur-rush   \n",
       "\n",
       "                                               full_url categories  \n",
       "734   https://boardgamegeek.com/boardgame/234671/pan...        NaN  \n",
       "738   https://boardgamegeek.com/boardgame/106217/haw...        NaN  \n",
       "754   https://boardgamegeek.com/boardgame/160851/lan...        NaN  \n",
       "995   https://boardgamegeek.com/boardgame/242684/rea...        NaN  \n",
       "1074  https://boardgamegeek.com/boardgame/165986/roy...        NaN  \n",
       "1409  https://boardgamegeek.com/boardgame/272682/exp...        NaN  \n",
       "1426  https://boardgamegeek.com/boardgame/165948/man...        NaN  \n",
       "1713  https://boardgamegeek.com/boardgame/34585/kelt...        NaN  \n",
       "1735  https://boardgamegeek.com/boardgame/193739/jor...        NaN  \n",
       "1739  https://boardgamegeek.com/boardgame/259081/mac...        NaN  \n",
       "1892  https://boardgamegeek.com/boardgame/37371/piec...        NaN  \n",
       "1970  https://boardgamegeek.com/boardgame/281960/kin...        NaN  \n",
       "2257  https://boardgamegeek.com/boardgame/189453/vic...        NaN  \n",
       "2492  https://boardgamegeek.com/boardgame/267814/adv...        NaN  \n",
       "2493  https://boardgamegeek.com/boardgame/262114/riv...        NaN  \n",
       "2665  https://boardgamegeek.com/boardgame/179071/gre...        NaN  "
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['categories'].isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "7c68d0d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>name</th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>full_url</th>\n",
       "      <th>categories</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>734</th>\n",
       "      <td>734</td>\n",
       "      <td>Pandemic: Rising Tide</td>\n",
       "      <td>234671</td>\n",
       "      <td>/boardgame/234671/pandemic-rising-tide</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/234671/pan...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0                   name      id  \\\n",
       "734         734  Pandemic: Rising Tide  234671   \n",
       "\n",
       "                                        url  \\\n",
       "734  /boardgame/234671/pandemic-rising-tide   \n",
       "\n",
       "                                              full_url categories  \n",
       "734  https://boardgamegeek.com/boardgame/234671/pan...        NaN  "
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['id'] == 234671]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "2af692d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://boardgamegeek.com/boardgame/281960/kingdomino-duel/ratings'"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['categories'].isna()]['full_url'][1970]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "38ce0782",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop('Unnamed: 0', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "504f7084",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(r'./df_categories_.csv', sep='\\t', encoding='utf-8', header='true',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "a7559713",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>full_url</th>\n",
       "      <th>categories</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Gloomhaven</td>\n",
       "      <td>174430</td>\n",
       "      <td>/boardgame/174430/gloomhaven</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/174430/glo...</td>\n",
       "      <td>['Adventure', 'Exploration', 'Fantasy', 'Fight...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Pandemic Legacy: Season 1</td>\n",
       "      <td>161936</td>\n",
       "      <td>/boardgame/161936/pandemic-legacy-season-1</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/161936/pan...</td>\n",
       "      <td>['Environmental', 'Medical']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Brass: Birmingham</td>\n",
       "      <td>224517</td>\n",
       "      <td>/boardgame/224517/brass-birmingham</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/224517/bra...</td>\n",
       "      <td>['Economic', 'Industry / Manufacturing', 'Trai...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        name      id  \\\n",
       "0                 Gloomhaven  174430   \n",
       "1  Pandemic Legacy: Season 1  161936   \n",
       "2          Brass: Birmingham  224517   \n",
       "\n",
       "                                          url  \\\n",
       "0                /boardgame/174430/gloomhaven   \n",
       "1  /boardgame/161936/pandemic-legacy-season-1   \n",
       "2          /boardgame/224517/brass-birmingham   \n",
       "\n",
       "                                            full_url  \\\n",
       "0  https://boardgamegeek.com/boardgame/174430/glo...   \n",
       "1  https://boardgamegeek.com/boardgame/161936/pan...   \n",
       "2  https://boardgamegeek.com/boardgame/224517/bra...   \n",
       "\n",
       "                                          categories  \n",
       "0  ['Adventure', 'Exploration', 'Fantasy', 'Fight...  \n",
       "1                       ['Environmental', 'Medical']  \n",
       "2  ['Economic', 'Industry / Manufacturing', 'Trai...  "
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "07e9291c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_cattegories = pd.read_csv(r'C:/Users/Armit/Python/Scripts/Project/df_categories_.csv', sep = '\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "8d0c42c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2695"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_cattegories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "0b8b27e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pages = pd.read_csv(r'C:/Users/Armit/Python/Scripts/Project/games_list_rating_1000.csv', sep = '\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "bfd7cb6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pages2 = pd.read_csv(r'C:/Users/Armit/Python/Scripts/Project/games_list_rating_1000_2000.csv', sep = '\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "429b1441",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pages3 = pd.read_csv(r'C:/Users/Armit/Python/Scripts/Project/games_list_rating_2000_3000.csv', sep = '\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "adec1ba2",
   "metadata": {},
   "outputs": [],
   "source": [
    "frames = [df_pages, df_pages2, df_pages3]\n",
    "\n",
    "result_pages = pd.concat(frames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "24298a68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>pages</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>174430</td>\n",
       "      <td>900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>161936</td>\n",
       "      <td>873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>224517</td>\n",
       "      <td>430</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  pages\n",
       "0  174430    900\n",
       "1  161936    873\n",
       "2  224517    430"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_pages.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "d20420ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_pages = result_pages.drop(['name','full_url','url'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "26213dd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final = pd.merge(df_cattegories,result_pages[['id','pages']],on='id', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "id": "c1c0b73d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.to_csv(r'./games_final.csv', sep='\\t', encoding='utf-8', header='true',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "id": "4fa9e398",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2699"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_final)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acfb890f",
   "metadata": {},
   "source": [
    "### Full Time to scrape 2500+ games"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "id": "5199f8c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['full_time'] = df_final['pages'] * 13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "id": "160ad11d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['full_time_minutes'] = df_final['full_time']/60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "f99a282b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "days to scrape ratings of 2699 games:  43.58881944444439\n"
     ]
    }
   ],
   "source": [
    "print('days to scrape ratings of 2699 games: ', sum(df_final['full_time_minutes'])/60/24)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ac49cf6",
   "metadata": {},
   "source": [
    "# Time to scrape with smarter approach"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "697e0f07",
   "metadata": {},
   "source": [
    "Scrape approaches:\n",
    "1. If we take 200 recent pages from each game \n",
    "\n",
    "2. If we take 50 recent pages from each game (convinient)\n",
    "\n",
    "3. If we take 50 random  pages from each game "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "b785dd1c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>full_url</th>\n",
       "      <th>categories</th>\n",
       "      <th>pages</th>\n",
       "      <th>full_time</th>\n",
       "      <th>full_time_minutes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2338</th>\n",
       "      <td>Take 5!</td>\n",
       "      <td>153</td>\n",
       "      <td>/boardgame/153/take-5</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/153/take-5...</td>\n",
       "      <td>['Card Game']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1709</th>\n",
       "      <td>Seikatsu</td>\n",
       "      <td>216428</td>\n",
       "      <td>/boardgame/216428/seikatsu</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/216428/sei...</td>\n",
       "      <td>['Abstract Strategy', 'Animals', 'Puzzle']</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2570</th>\n",
       "      <td>Unmatched Game System</td>\n",
       "      <td>295564</td>\n",
       "      <td>/boardgame/295564/unmatched-game-system</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/295564/unm...</td>\n",
       "      <td>['Fighting', 'Miniatures']</td>\n",
       "      <td>7</td>\n",
       "      <td>91</td>\n",
       "      <td>1.516667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2557</th>\n",
       "      <td>1880: China</td>\n",
       "      <td>69601</td>\n",
       "      <td>/boardgame/69601/1880-china</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/69601/1880...</td>\n",
       "      <td>['Economic', 'Trains']</td>\n",
       "      <td>8</td>\n",
       "      <td>104</td>\n",
       "      <td>1.733333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2631</th>\n",
       "      <td>Sherlock Holmes Consulting Detective: The Bake...</td>\n",
       "      <td>296345</td>\n",
       "      <td>/boardgame/296345/sherlock-holmes-consulting-d...</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/296345/she...</td>\n",
       "      <td>['Deduction', 'Murder/Mystery', 'Novel-based']</td>\n",
       "      <td>8</td>\n",
       "      <td>104</td>\n",
       "      <td>1.733333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   name      id  \\\n",
       "2338                                            Take 5!     153   \n",
       "1709                                           Seikatsu  216428   \n",
       "2570                              Unmatched Game System  295564   \n",
       "2557                                        1880: China   69601   \n",
       "2631  Sherlock Holmes Consulting Detective: The Bake...  296345   \n",
       "\n",
       "                                                    url  \\\n",
       "2338                              /boardgame/153/take-5   \n",
       "1709                         /boardgame/216428/seikatsu   \n",
       "2570            /boardgame/295564/unmatched-game-system   \n",
       "2557                        /boardgame/69601/1880-china   \n",
       "2631  /boardgame/296345/sherlock-holmes-consulting-d...   \n",
       "\n",
       "                                               full_url  \\\n",
       "2338  https://boardgamegeek.com/boardgame/153/take-5...   \n",
       "1709  https://boardgamegeek.com/boardgame/216428/sei...   \n",
       "2570  https://boardgamegeek.com/boardgame/295564/unm...   \n",
       "2557  https://boardgamegeek.com/boardgame/69601/1880...   \n",
       "2631  https://boardgamegeek.com/boardgame/296345/she...   \n",
       "\n",
       "                                          categories  pages  full_time  \\\n",
       "2338                                   ['Card Game']      0          0   \n",
       "1709      ['Abstract Strategy', 'Animals', 'Puzzle']      0          0   \n",
       "2570                      ['Fighting', 'Miniatures']      7         91   \n",
       "2557                          ['Economic', 'Trains']      8        104   \n",
       "2631  ['Deduction', 'Murder/Mystery', 'Novel-based']      8        104   \n",
       "\n",
       "      full_time_minutes  \n",
       "2338           0.000000  \n",
       "1709           0.000000  \n",
       "2570           1.516667  \n",
       "2557           1.733333  \n",
       "2631           1.733333  "
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final.sort_values(by = 'pages',ascending = True).head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "65e73aa1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>full_url</th>\n",
       "      <th>categories</th>\n",
       "      <th>pages</th>\n",
       "      <th>full_time</th>\n",
       "      <th>full_time_minutes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Gloomhaven</td>\n",
       "      <td>174430</td>\n",
       "      <td>/boardgame/174430/gloomhaven</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/174430/glo...</td>\n",
       "      <td>['Adventure', 'Exploration', 'Fantasy', 'Fight...</td>\n",
       "      <td>900</td>\n",
       "      <td>11700</td>\n",
       "      <td>195.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Pandemic Legacy: Season 1</td>\n",
       "      <td>161936</td>\n",
       "      <td>/boardgame/161936/pandemic-legacy-season-1</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/161936/pan...</td>\n",
       "      <td>['Environmental', 'Medical']</td>\n",
       "      <td>873</td>\n",
       "      <td>11349</td>\n",
       "      <td>189.150000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Brass: Birmingham</td>\n",
       "      <td>224517</td>\n",
       "      <td>/boardgame/224517/brass-birmingham</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/224517/bra...</td>\n",
       "      <td>['Economic', 'Industry / Manufacturing', 'Trai...</td>\n",
       "      <td>430</td>\n",
       "      <td>5590</td>\n",
       "      <td>93.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Terraforming Mars</td>\n",
       "      <td>167791</td>\n",
       "      <td>/boardgame/167791/terraforming-mars</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/167791/ter...</td>\n",
       "      <td>['Economic', 'Environmental', 'Industry / Manu...</td>\n",
       "      <td>1372</td>\n",
       "      <td>17836</td>\n",
       "      <td>297.266667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Twilight Imperium: Fourth Edition</td>\n",
       "      <td>233078</td>\n",
       "      <td>/boardgame/233078/twilight-imperium-fourth-edi...</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/233078/twi...</td>\n",
       "      <td>['Civilization', 'Economic', 'Exploration', 'N...</td>\n",
       "      <td>289</td>\n",
       "      <td>3757</td>\n",
       "      <td>62.616667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Gloomhaven: Jaws of the Lion</td>\n",
       "      <td>291457</td>\n",
       "      <td>/boardgame/291457/gloomhaven-jaws-lion</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/291457/glo...</td>\n",
       "      <td>['Adventure', 'Exploration', 'Fantasy', 'Fight...</td>\n",
       "      <td>212</td>\n",
       "      <td>2756</td>\n",
       "      <td>45.933333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Through the Ages: A New Story of Civilization</td>\n",
       "      <td>182028</td>\n",
       "      <td>/boardgame/182028/through-ages-new-story-civil...</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/182028/thr...</td>\n",
       "      <td>['Card Game', 'Civilization', 'Economic']</td>\n",
       "      <td>489</td>\n",
       "      <td>6357</td>\n",
       "      <td>105.950000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Gaia Project</td>\n",
       "      <td>220308</td>\n",
       "      <td>/boardgame/220308/gaia-project</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/220308/gai...</td>\n",
       "      <td>['Economic', 'Science Fiction', 'Space Explora...</td>\n",
       "      <td>351</td>\n",
       "      <td>4563</td>\n",
       "      <td>76.050000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Star Wars: Rebellion</td>\n",
       "      <td>187645</td>\n",
       "      <td>/boardgame/187645/star-wars-rebellion</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/187645/sta...</td>\n",
       "      <td>['Civil War', 'Fighting', 'Miniatures', 'Movie...</td>\n",
       "      <td>489</td>\n",
       "      <td>6357</td>\n",
       "      <td>105.950000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Twilight Struggle</td>\n",
       "      <td>12333</td>\n",
       "      <td>/boardgame/12333/twilight-struggle</td>\n",
       "      <td>https://boardgamegeek.com/boardgame/12333/twil...</td>\n",
       "      <td>['Modern Warfare', 'Political', 'Wargame']</td>\n",
       "      <td>885</td>\n",
       "      <td>11505</td>\n",
       "      <td>191.750000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            name      id  \\\n",
       "0                                     Gloomhaven  174430   \n",
       "1                      Pandemic Legacy: Season 1  161936   \n",
       "2                              Brass: Birmingham  224517   \n",
       "3                              Terraforming Mars  167791   \n",
       "4              Twilight Imperium: Fourth Edition  233078   \n",
       "5                   Gloomhaven: Jaws of the Lion  291457   \n",
       "6  Through the Ages: A New Story of Civilization  182028   \n",
       "7                                   Gaia Project  220308   \n",
       "8                           Star Wars: Rebellion  187645   \n",
       "9                              Twilight Struggle   12333   \n",
       "\n",
       "                                                 url  \\\n",
       "0                       /boardgame/174430/gloomhaven   \n",
       "1         /boardgame/161936/pandemic-legacy-season-1   \n",
       "2                 /boardgame/224517/brass-birmingham   \n",
       "3                /boardgame/167791/terraforming-mars   \n",
       "4  /boardgame/233078/twilight-imperium-fourth-edi...   \n",
       "5             /boardgame/291457/gloomhaven-jaws-lion   \n",
       "6  /boardgame/182028/through-ages-new-story-civil...   \n",
       "7                     /boardgame/220308/gaia-project   \n",
       "8              /boardgame/187645/star-wars-rebellion   \n",
       "9                 /boardgame/12333/twilight-struggle   \n",
       "\n",
       "                                            full_url  \\\n",
       "0  https://boardgamegeek.com/boardgame/174430/glo...   \n",
       "1  https://boardgamegeek.com/boardgame/161936/pan...   \n",
       "2  https://boardgamegeek.com/boardgame/224517/bra...   \n",
       "3  https://boardgamegeek.com/boardgame/167791/ter...   \n",
       "4  https://boardgamegeek.com/boardgame/233078/twi...   \n",
       "5  https://boardgamegeek.com/boardgame/291457/glo...   \n",
       "6  https://boardgamegeek.com/boardgame/182028/thr...   \n",
       "7  https://boardgamegeek.com/boardgame/220308/gai...   \n",
       "8  https://boardgamegeek.com/boardgame/187645/sta...   \n",
       "9  https://boardgamegeek.com/boardgame/12333/twil...   \n",
       "\n",
       "                                          categories  pages  full_time  \\\n",
       "0  ['Adventure', 'Exploration', 'Fantasy', 'Fight...    900      11700   \n",
       "1                       ['Environmental', 'Medical']    873      11349   \n",
       "2  ['Economic', 'Industry / Manufacturing', 'Trai...    430       5590   \n",
       "3  ['Economic', 'Environmental', 'Industry / Manu...   1372      17836   \n",
       "4  ['Civilization', 'Economic', 'Exploration', 'N...    289       3757   \n",
       "5  ['Adventure', 'Exploration', 'Fantasy', 'Fight...    212       2756   \n",
       "6          ['Card Game', 'Civilization', 'Economic']    489       6357   \n",
       "7  ['Economic', 'Science Fiction', 'Space Explora...    351       4563   \n",
       "8  ['Civil War', 'Fighting', 'Miniatures', 'Movie...    489       6357   \n",
       "9         ['Modern Warfare', 'Political', 'Wargame']    885      11505   \n",
       "\n",
       "   full_time_minutes  \n",
       "0         195.000000  \n",
       "1         189.150000  \n",
       "2          93.166667  \n",
       "3         297.266667  \n",
       "4          62.616667  \n",
       "5          45.933333  \n",
       "6         105.950000  \n",
       "7          76.050000  \n",
       "8         105.950000  \n",
       "9         191.750000  "
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "id": "d920fe39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "338"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_final[df_final['pages'] > 199])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "id": "c1050749",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of feedback:  3380000\n"
     ]
    }
   ],
   "source": [
    "print('number of feedback: ', 200*338*50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "id": "f05b311b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "days to scrape 200 pages of 338 games: 4.068518518518519\n"
     ]
    }
   ],
   "source": [
    "print('days to scrape 200 pages of 338 games:',200*13*338/60/60/60)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "id": "47395838",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "702"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_final[df_final['pages'] > 100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "id": "629f2e16",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "days to scrape 50 pages of 1307 games: 4.225\n"
     ]
    }
   ],
   "source": [
    "print('days to scrape 50 pages of 1307 games:',100*13*702/60/60/60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "id": "968139d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1307"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_final[df_final['pages'] > 50])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "id": "57f8a94a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "days to scrape 50 pages of 1307 games: 3.933101851851852\n"
     ]
    }
   ],
   "source": [
    "print('days to scrape 50 pages of 1307 games:',50*13*1307/60/60/60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23f7793e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}