examples/mtasklite_pqdm_spacy_tokenization_demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "713e6a76-9063-4860-a4bd-cbdaba8e718c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Install Spacy the first time you run this notebook\n",
    "!pip install -U pip setuptools wheel\n",
    "!pip install -U spacy\n",
    "!python -m spacy download en_core_web_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "aace78b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from mtasklite import delayed_init\n",
    "\n",
    "SPACY_POS = 'tagger'\n",
    "SPACY_NER = 'ner'\n",
    "SPACY_PARSER = 'parser'\n",
    "\n",
    "SPACY_MODEL = 'en_core_web_sm'\n",
    "\n",
    "@delayed_init\n",
    "class SpacyTextParser:\n",
    "    def __init__(self, model_name, disable_modules=[SPACY_NER, SPACY_PARSER]):\n",
    "        # It's important to load spacy here\n",
    "        import spacy\n",
    "        self._nlp = spacy.load(model_name, disable=disable_modules)\n",
    "\n",
    "    def __call__(self, text):\n",
    "        return [e.text for e in self._nlp(text)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "327f1a6b-9ff5-4bed-977b-5b64c7965791",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['This', 'is', 'a', 'simple', 'text', '!']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parser = SpacyTextParser(SPACY_MODEL)\n",
    "parser('This is a simple text!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "baf65c3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89e9b3882cbf4c5da75ba52f4532f2c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      "8it [00:00, 10.48it/s]\u001b[A\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from mtasklite.processes import pqdm\n",
    "\n",
    "input_arr = [\n",
    "    'Accelerate is a library that enables the same PyTorch code to be run across any distributed configuration by adding just four lines of code!',\n",
    "    'In short, training and inference at scale made simple, efficient and adaptable.',\n",
    "    'Built on torch_xla and torch.distributed, Accelerate takes care of the heavy lifting, so you don’t have to write any custom code to adapt to these platforms.',\n",
    "    'Convert existing codebases to utilize DeepSpeed, perform fully sharded data parallelism, and have automatic support for mixed-precision training!',\n",
    "    'Welcome to the Accelerate tutorials!',\n",
    "    'These introductory guides will help catch you up to speed on working with Accelerate.',\n",
    "    'You’ll learn how to modify your code to have it work with the API seamlessly, how to launch your script properly, and more!',\n",
    "    'These tutorials assume some basic knowledge of Python and familiarity with the PyTorch framework.'\n",
    "]\n",
    "\n",
    "# Exactly 4 workers initialized with a given model name\n",
    "N_JOBS=4\n",
    "\n",
    "with pqdm(input_arr, N_JOBS * [SpacyTextParser(SPACY_MODEL)]) as pbar:\n",
    "    result = list(tqdm(pbar))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3ae39a56-7d72-48cd-8c13-09f3af9d7a6d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['Accelerate', 'is', 'a', 'library', 'that', 'enables', 'the', 'same', 'PyTorch', 'code', 'to', 'be', 'run', 'across', 'any', 'distributed', 'configuration', 'by', 'adding', 'just', 'four', 'lines', 'of', 'code', '!'], ['In', 'short', ',', 'training', 'and', 'inference', 'at', 'scale', 'made', 'simple', ',', 'efficient', 'and', 'adaptable', '.'], ['Built', 'on', 'torch_xla', 'and', 'torch.distributed', ',', 'Accelerate', 'takes', 'care', 'of', 'the', 'heavy', 'lifting', ',', 'so', 'you', 'do', 'n’t', 'have', 'to', 'write', 'any', 'custom', 'code', 'to', 'adapt', 'to', 'these', 'platforms', '.'], ['Convert', 'existing', 'codebases', 'to', 'utilize', 'DeepSpeed', ',', 'perform', 'fully', 'sharded', 'data', 'parallelism', ',', 'and', 'have', 'automatic', 'support', 'for', 'mixed', '-', 'precision', 'training', '!'], ['Welcome', 'to', 'the', 'Accelerate', 'tutorials', '!'], ['These', 'introductory', 'guides', 'will', 'help', 'catch', 'you', 'up', 'to', 'speed', 'on', 'working', 'with', 'Accelerate', '.'], ['You', '’ll', 'learn', 'how', 'to', 'modify', 'your', 'code', 'to', 'have', 'it', 'work', 'with', 'the', 'API', 'seamlessly', ',', 'how', 'to', 'launch', 'your', 'script', 'properly', ',', 'and', 'more', '!'], ['These', 'tutorials', 'assume', 'some', 'basic', 'knowledge', 'of', 'Python', 'and', 'familiarity', 'with', 'the', 'PyTorch', 'framework', '.']]\n"
     ]
    }
   ],
   "source": [
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f86b8c9f-a5e0-4106-95b9-ff17b2df9a0b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}