examples/pqdm_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "290e7a5a-179e-428d-a02b-3091dd92e581",
   "metadata": {},
   "source": [
    "# This notebook is to demonstate two shortcomings of PQDM:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84788f26-ce04-47aa-8736-10ee95565fe2",
   "metadata": {},
   "source": [
    "## 1. The need to set the process start method on MacOS to fork"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3f2aa83b-e645-48fc-8db4-e2c797a7b9bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import multiprocessing as mp\n",
    "import platform\n",
    "if 'darwin' in platform.system().lower():\n",
    "    mp.set_start_method('fork')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9519d9e2-c307-4f45-a828-a62979772b67",
   "metadata": {},
   "source": [
    "## 2. The lack of memory-efficient lazy iteration even with `bounded` set to True\n",
    "Although setting `bounded` to True limits the number of item in the input queue, the full result set is still stored in the output array."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2247be89-a0d3-4165-9a48-cad3cb479b56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "333fe6b1cdc944ef9fb14a38220ee788",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "QUEUEING TASKS | : 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a07f867d55bc4c55bda09823e078412b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PROCESSING TASKS | :   0%|          | 0/20 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "079e3b3a376a49f69ad977add3aa06d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "COLLECTING RESULTS | :   0%|          | 0/20 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[0,\n",
       " 1,\n",
       " 4,\n",
       " 9,\n",
       " 16,\n",
       " 25,\n",
       " 36,\n",
       " 49,\n",
       " 64,\n",
       " 81,\n",
       " 100,\n",
       " 121,\n",
       " 144,\n",
       " 169,\n",
       " 196,\n",
       " 225,\n",
       " 256,\n",
       " 289,\n",
       " 324,\n",
       " 361]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from time import sleep\n",
    "from pqdm.processes import pqdm\n",
    "\n",
    "def artificially_slow_square(a):\n",
    "    b = a\n",
    "    for repeat in range(200_000):\n",
    "        b = b * a\n",
    "        \n",
    "    return a*a\n",
    "\n",
    "input_arr = list(range(20))\n",
    "\n",
    "def input_arr_generator():\n",
    "    for e in input_arr:\n",
    "        yield e\n",
    "\n",
    "n_jobs = 4 \n",
    "result = pqdm(input_arr_generator(), artificially_slow_square, n_jobs, bounded=True)\n",
    "\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33102f0f-046d-4652-8bf6-45c453151d74",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}