00 - preparing_dependency.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==== loading data ====\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<KeysViewHDF5 ['sents', 'synts']>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import h5py, os\n",
    "print(\"==== loading data ====\")\n",
    "nmt_train = h5py.File(os.path.join('./data/train_data.h5'), 'r')\n",
    "nmt_train.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(b'you were trying to take something back .',\n",
       " b'(ROOT (S (NP (PRP you)) (VP (VBD were) (VP (VBG trying) (S (VP (TO to) (VP (VB take) (NP (NN something)) (PRT (RB back))))))) (. .)))')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nmt_train['sents'][1], nmt_train['synts'][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<KeysViewHDF5 ['sents', 'synts']>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nmt_val = h5py.File(os.path.join('./data/valid_data.h5'), 'r')\n",
    "nmt_val.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(45377426, 12800)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(nmt_train['sents']), len(nmt_val['sents'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(b\"ah , yes , i 'm kang-mo ... - do n't , dad !\",\n",
       " b\"(ROOT (S (S (ADVP (RB ah)) (, ,) (INTJ (UH yes)) (, ,) (FW i) (VP (VBP 'm) (ADJP (JJ kang-mo)))) (: ...) (S (: -) (VP (VBP do) (RB n't) (, ,) (NP (NN dad)))) (. !)))\")"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nmt_train['sents'][0],nmt_train['synts'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_dep = ['ACL', 'ACOMP', 'ADVCL', 'ADVMOD', 'AGENT', 'AMOD', 'APPOS', 'ATTR', 'AUX', 'AUXPASS', 'CASE','CC', 'CCOMP', 'COMPOUND', 'CONJ', 'CSUBJ', 'CSUBJPASS', 'DATIVE','DEP','DET', 'DOBJ', 'EXPL', 'INTJ', 'MARK', 'META','NEG', 'NOUNMOD', 'NPMOD', 'NSUBJ', 'NSUBJPASS', 'NUMMOD', 'OPRD', 'PARATAXIS', 'PCOMP', 'POBJ', 'POSS', 'PRECONJ', 'PREDET', 'PREP', 'PRT', 'PUNCT', 'QUANTMOD', 'RELCL', 'ROOT', 'XCOMP']\n",
    "deprecated = ['COMPLM', 'INFMOD', 'PARTMOD', 'HMOD', 'HYPH', 'IOBJ', 'NUM', 'NUMBER', 'NMOD','NN', 'NPADVMOD', 'POSSESSIVE', 'RCMOD']\n",
    "\n",
    "dependency_tags = new_dep + deprecated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open(\"./data/dictionary.pkl\", \"rb\") as file:\n",
    "    dictionary = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(31414, 31414+len(dependency_tags)):\n",
    "    # print(i,dependency_tags[i-31414])\n",
    "    dictionary.idx2word[i] = dependency_tags[i-31414].upper()\n",
    "    a = dependency_tags[i-31414].upper()\n",
    "    dictionary.word2idx[a] = i "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4827, 'RELCL', 31457)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dictionary.word2idx['root'], dictionary.idx2word[31456], dictionary.word2idx['ROOT']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_paren(tok):\n",
    "    return tok == \")\" or tok == \"(\"\n",
    "\n",
    "def getleaf(tree):\n",
    "    nonleaves = ''\n",
    "    for w in str(tree).replace('\\n', '').split():\n",
    "        w = w.replace('(', '( ').replace(')', ' )')\n",
    "        nonleaves += w + ' '\n",
    "    \n",
    "    leaves = []\n",
    "    arr = nonleaves.split()\n",
    "    for n, i in enumerate(arr):\n",
    "        if n + 1 < len(arr):\n",
    "            tok1 = arr[n]\n",
    "            tok2 = arr[n + 1]\n",
    "            if not is_paren(tok1) and not is_paren(tok2):\n",
    "                leaves.append(arr[n])\n",
    "\n",
    "    return leaves\n",
    "\n",
    "def deleaf(tree):\n",
    "    nonleaves = ''\n",
    "    for w in str(tree).replace('\\n', '').split():\n",
    "        w = w.replace('(', '( ').replace(')', ' )')\n",
    "        nonleaves += w + ' '\n",
    "\n",
    "    arr = nonleaves.split()\n",
    "    for n, i in enumerate(arr):\n",
    "        if n + 1 < len(arr):\n",
    "            tok1 = arr[n]\n",
    "            tok2 = arr[n + 1]\n",
    "            if not is_paren(tok1) and not is_paren(tok2):\n",
    "                arr[n + 1] = \"\"\n",
    "\n",
    "    nonleaves = \" \".join(arr)\n",
    "    return nonleaves.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Tokenizer BPE\n",
    "from subwordnmt.apply_bpe import BPE, read_vocabulary\n",
    "import codecs\n",
    "import numpy as np\n",
    "\n",
    "# load bpe codes\n",
    "bpe_codes = codecs.open('./data/bpe.codes', encoding='utf-8')\n",
    "bpe_vocab = codecs.open('./data/vocab.txt', encoding='utf-8')\n",
    "bpe_vocab = read_vocabulary(bpe_vocab, 50)\n",
    "bpe = BPE(bpe_codes, '@@', bpe_vocab, None)\n",
    "\n",
    "def bpe_tokenizer(sent_, target = False):\n",
    " # bpe segment and convert to tensor\n",
    "    sent_ = bpe.segment(sent_).split()\n",
    "    sent_ = [dictionary.word2idx[w] if w in dictionary.word2idx else dictionary.word2idx[\"<unk>\"] for w in sent_]\n",
    "    if target:\n",
    "        sent_ = [dictionary.word2idx[\"<sos>\"]] + sent_ + [dictionary.word2idx[\"<eos>\"]]\n",
    "    return sent_\n",
    "\n",
    "#syntax to syntatic tokenzier\n",
    "from nltk import ParentedTree\n",
    "def parser_tokenizer(synt_):\n",
    "    synt_  = ParentedTree.fromstring(synt_)\n",
    "    synt_ = deleaf(synt_)\n",
    "    synt_ = [dictionary.word2idx[f\"<{w}>\"] for w in synt_ if f\"<{w}>\" in dictionary.word2idx]\n",
    "    synt_ = [dictionary.word2idx[\"<sos>\"]] + synt_ + [dictionary.word2idx[\"<eos>\"]]\n",
    "    return synt_\n",
    "\n",
    "import pickle\n",
    "with open('synt_vocab.pkl', 'rb') as f:\n",
    "    synt_vocab = pickle.load(f)\n",
    "\n",
    "def bow(synt_):\n",
    "    synt_bow = np.ones(74)\n",
    "    synt_ = ['<sos>'] + deleaf(synt_) + ['<eos>']\n",
    "    for tag in synt_:\n",
    "        if tag != '<sos>' and tag != '<eos>':\n",
    "            synt_bow[synt_vocab[tag]-3] += 1\n",
    "    synt_bow /= synt_bow.sum()\n",
    "    return synt_bow \n",
    "\n",
    "import spacy\n",
    "nlp = spacy.load('en_core_web_md')\n",
    "def dependency_parser(sent_):\n",
    "    dep_ = [token.dep_.upper() for token in nlp(sent_)]\n",
    "    dep_ = [dictionary.word2idx[w] for w in dep_]\n",
    "    return dep_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dependency_parser(nmt_train['sents'][1].decode()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "def prepare_paraphrase_dataset(sent1, synt2, sent2):\n",
    "    lists_ = list()\n",
    "    for sen1, syn2, sen2 in tqdm(zip(sent1,synt2,sent2)):\n",
    "        sent_ = bpe_tokenizer(sen1.decode())\n",
    "        syn_  = parser_tokenizer(syn2.decode())\n",
    "        trg_  = bpe_tokenizer(sen2.decode(), target = True)\n",
    "        dep_  = dependency_parser(sen2.decode()) \n",
    "        # bow_  = bow(syn2.decode())\n",
    "        lists_.append((sent_, syn_, trg_, dep_))\n",
    "    return lists_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1000000it [1:59:49, 139.10it/s]\n"
     ]
    }
   ],
   "source": [
    "num = 1000000\n",
    "nmt_trainset = prepare_paraphrase_dataset(nmt_train['sents'][:num], nmt_train['synts'][:num], nmt_train['sents'][:num])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./data/nmt_trainset_dep.pkl', 'wb') as f:\n",
    "    pickle.dump(nmt_trainset, f, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5000it [00:36, 135.65it/s]\n"
     ]
    }
   ],
   "source": [
    "num = 5000\n",
    "nmt_validset = prepare_paraphrase_dataset(nmt_val['sents'][:num], nmt_val['synts'][:num], nmt_val['sents'][:num])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./data/nmt_validset_dep.pkl', 'wb') as f:\n",
    "    pickle.dump(nmt_validset, f, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}