GDSC_recommendation_model_Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Name        : Halloween Search - The Efficient Reference Recommendation Engine - Training the model part\n",
    "Description : Extract and store the data and finally exports all the data in a pickle file\n",
    "Purpose     : Created for the Capgemini Global Data Science Challenge IV\n",
    "Author:     : Arka Prava Bandyopadhyay\n",
    "Team        : Team_Halloween\n",
    "Date:       : 21/03/2018\n",
    "Version:    : 3.33\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "##########################################################################\n",
    "###########       Convert ppt files to xml file  #########################\n",
    "##########################################################################\n",
    "import os\n",
    "# ***IMPORTANT*** Please put the correct path below where the ppts are saved ***IMPORTANT***\n",
    "path=\"PLEASE GIVE PATH\"\n",
    "\n",
    "content= '''#!/bin/bash\n",
    "\n",
    "path='''+str('\"'+path+'\"')+'''\n",
    "libreoffice --headless --invisible --convert-to pptx \"$path\"/*.ppt --outdir \"$path\"\n",
    "FILES=\"$path\"/*.pptx\n",
    "cd \"$path\"\n",
    "mkdir -p data_txt\n",
    "for f in $FILES\n",
    "do\n",
    "base_name=$(basename \"$f\")\n",
    "filename=\"${base_name%.*}\"\n",
    "unzip -qc \"$f\" ppt/slides/slide*.xml   | grep -oP '(?<=\\<a:t\\>).*?(?=\\</a:t\\>)' > \"$path\"/data_txt/\"$filename\".txt\n",
    "done\n",
    "'''\n",
    "\n",
    "\n",
    "with open(str(path+'/to_text.sh'),'w') as out:\n",
    "    out.write(content)\n",
    "    out.close()\n",
    "rc = os.system(str('chmod 777 '+path+'/to_text.sh;bash '+path+'/to_text.sh;'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'path' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-fa8dd9f60ec7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     47\u001b[0m         \u001b[0mtext\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mre\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msub\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"([a-z])([A-Z])\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m\"\\g<1> \\g<2>\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     48\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mtext\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 49\u001b[1;33m \u001b[0mmypath\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mglob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mglob\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m'/data_txt/'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m\"*.txt\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     50\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'file_name'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'Client_name'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'Technology Name'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m\"Full_text\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     51\u001b[0m \u001b[0mdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'path' is not defined"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import pandas as pd\n",
    "import os\n",
    "import glob\n",
    "import html\n",
    "def extractText(mypath, df):\n",
    "        list_files=mypath\n",
    "        for f in list_files:\n",
    "            text_file = open(f, encoding=\"utf8\")\n",
    "            text_from_txt = text_file.read()\n",
    "            text_file.close()\n",
    "            #put text in lowercase\n",
    "            text_lower = str(text_from_txt).lower()\n",
    "\n",
    "            #Extract paragraphs from text \n",
    "            client_name = f.split('data_txt/')[1].split('.')[0].split('_')[0]\n",
    "            file_name=f.split('data_txt/')[1].split('.')[0]\n",
    "            technology_name= re.split('\\d_',file_name)[-1]\n",
    "            full_text=text_from_txt\n",
    "            vars=[file_name,client_name,technology_name,full_text]\n",
    "            df.loc[len(df.index)] = vars\n",
    "        return df\n",
    "\n",
    "    \n",
    "def regexp(word1, word2, text):\n",
    "    found =\"\"\n",
    "    m=re.search(word1+\"(.*)\"+word2, text, re.DOTALL)\n",
    "    if m:\n",
    "        found = m.group(1)\n",
    "        #print(found)\n",
    "    else:\n",
    "        m=re.search(word1+\"(.*)\", text, re.DOTALL)\n",
    "        if m:\n",
    "            found = m.group(1)\n",
    "        #else:\n",
    "            #print('not found')\n",
    "    return found\n",
    "\n",
    "\n",
    "def CleanClientName(text):\n",
    "        if '_' in list(text):\n",
    "            text=text.split('_')\n",
    "            text= ' '.join(text)\n",
    "        if '&' in list(text):\n",
    "            text=text.split('&')\n",
    "            text= ' '.join(text)\n",
    "        text = re.sub(\"([a-z])([A-Z])\",\"\\g<1> \\g<2>\",text)\n",
    "        return text\n",
    "mypath=glob.glob(os.path.join(str(path+'/data_txt/'),\"*.txt\"))          \n",
    "columns=['file_name','Client_name', 'Technology Name',\"Full_text\"]\n",
    "df = pd.DataFrame(columns=columns)   \n",
    "df1=extractText(mypath, df)\n",
    "\n",
    "df1['Client_name']=df1['Client_name'].apply(CleanClientName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Country Name List\n",
    "country_name = ['Aruba', 'Afghanistan', 'Angola', 'Anguilla', 'Åland Islands', 'Albania', 'Andorra', 'United Arab Emirates', 'Argentina', 'Armenia', 'American Samoa', 'Antarctica', 'French Southern Territories', 'Antigua and Barbuda', 'Australia', 'Austria', 'Azerbaijan', 'Burundi', 'Belgium', 'Benin', 'Bonaire, Sint Eustatius and Saba', 'Burkina Faso', 'Bangladesh', 'Bulgaria', 'Bahrain', 'Bahamas', 'Bosnia and Herzegovina', 'Saint Barthélemy', 'Belarus', 'Belize', 'Bermuda', 'Bolivia, Plurinational State of', 'Brazil', 'Barbados', 'Brunei Darussalam', 'Bhutan', 'Bouvet Island', 'Botswana', 'Central African Republic', 'Canada', 'Cocos (Keeling) Islands', 'Switzerland', 'Chile', 'China', \"Côte d'Ivoire\", 'Cameroon', 'Congo, The Democratic Republic of the', 'Congo', 'Cook Islands', 'Colombia', 'Comoros', 'Cabo Verde', 'Costa Rica', 'Cuba', 'Curaçao', 'Christmas Island', 'Cayman Islands', 'Cyprus', 'Czechia', 'Germany', 'Djibouti', 'Dominica', 'Denmark', 'Dominican Republic', 'Algeria', 'Ecuador', 'Egypt', 'Eritrea', 'Western Sahara', 'Spain', 'Estonia', 'Ethiopia', 'Finland', 'Fiji', 'Falkland Islands (Malvinas)', 'France', 'Faroe Islands', 'Micronesia, Federated States of', 'Gabon', 'United Kingdom', 'Georgia', 'Guernsey', 'Ghana', 'Gibraltar', 'Guinea', 'Guadeloupe', 'Gambia', 'Guinea-Bissau', 'Equatorial Guinea', 'Greece', 'Grenada', 'Greenland', 'Guatemala', 'French Guiana', 'Guam', 'Guyana', 'Hong Kong', 'Heard Island and McDonald Islands', 'Honduras', 'Croatia', 'Haiti', 'Hungary', 'Indonesia', 'Isle of Man', 'India', 'British Indian Ocean Territory', 'Ireland', 'Iran, Islamic Republic of', 'Iraq', 'Iceland', 'Israel', 'Italy', 'Jamaica', 'Jersey', 'Jordan', 'Japan', 'Kazakhstan', 'Kenya', 'Kyrgyzstan', 'Cambodia', 'Kiribati', 'Saint Kitts and Nevis', 'Korea, Republic of', 'Kuwait', \"Lao People's Democratic Republic\", 'Lebanon', 'Liberia', 'Libya', 'Saint Lucia', 'Liechtenstein', 'Sri Lanka', 'Lesotho', 'Lithuania', 'Luxembourg', 'Latvia', 'Macao', 'Saint Martin (French part)', 'Morocco', 'Monaco', 'Moldova, Republic of', 'Madagascar', 'Maldives', 'Mexico', 'Marshall Islands', 'Macedonia, Republic of', 'Mali', 'Malta', 'Myanmar', 'Montenegro', 'Mongolia', 'Northern Mariana Islands', 'Mozambique', 'Mauritania', 'Montserrat', 'Martinique', 'Mauritius', 'Malawi', 'Malaysia', 'Mayotte', 'Namibia', 'New Caledonia', 'Niger', 'Norfolk Island', 'Nigeria', 'Nicaragua', 'Niue', 'Netherlands', 'Norway', 'Nepal', 'Nauru', 'New Zealand', 'Oman', 'Pakistan', 'Panama', 'Pitcairn', 'Peru', 'Philippines', 'Palau', 'Papua New Guinea', 'Poland', 'Puerto Rico', \"Korea, Democratic People's Republic of\", 'Portugal', 'Paraguay', 'Palestine, State of', 'French Polynesia', 'Qatar', 'Réunion', 'Romania', 'Russian Federation', 'Rwanda', 'Saudi Arabia', 'Sudan', 'Senegal', 'Singapore', 'South Georgia and the South Sandwich Islands', 'Saint Helena, Ascension and Tristan da Cunha', 'Svalbard and Jan Mayen', 'Solomon Islands', 'Sierra Leone', 'El Salvador', 'San Marino', 'Somalia', 'Saint Pierre and Miquelon', 'Serbia', 'South Sudan', 'Sao Tome and Principe', 'Suriname', 'Slovakia', 'Slovenia', 'Sweden', 'Swaziland', 'Sint Maarten (Dutch part)', 'Seychelles', 'Syrian Arab Republic', 'Turks and Caicos Islands', 'Chad', 'Togo', 'Thailand', 'Tajikistan', 'Tokelau', 'Turkmenistan', 'Timor-Leste', 'Tonga', 'Trinidad and Tobago', 'Tunisia', 'Turkey', 'Tuvalu', 'Taiwan, Province of China', 'Tanzania, United Republic of', 'Uganda', 'Ukraine', 'United States Minor Outlying Islands', 'Uruguay', 'United States', 'Uzbekistan', 'Holy See (Vatican City State)', 'Saint Vincent and the Grenadines', 'Venezuela, Bolivarian Republic of', 'Virgin Islands, British', 'Virgin Islands, U.S.', 'Viet Nam', 'Vanuatu', 'Wallis and Futuna', 'Samoa', 'Yemen', 'South Africa', 'Zambia', 'Zimbabwe', 'UK','US']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################################\n",
    "############        Scraping The Country   ##########################\n",
    "#####################################################################\n",
    "\n",
    "\n",
    "df1['country'] = ''\n",
    "short_cn = ['UK','US']\n",
    "import pycountry\n",
    "for c,i in enumerate(df1['Full_text']):\n",
    "    i=regexp(r'Country[&a-zA-Z ]*:','Deal',html.unescape(i).replace('\\n',' '))\n",
    "    for country in pycountry.countries:\n",
    "        if country.name in i:\n",
    "            if df1.loc[c,('country')] == '':\n",
    "                df1.loc[c,('country')]=country.name\n",
    "            else:\n",
    "                df1.loc[c,('country')] = df1.loc[c,('country')] + ',' + country.name\n",
    "    for j in short_cn:\n",
    "            if j in i:\n",
    "                df1.loc[c,('country')]=j\n",
    "                \n",
    "for c,i in enumerate(df1['country']):\n",
    "    if i== 'United Kingdom':\n",
    "        df1.loc[c,('country')]= 'UK'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################################################\n",
    "########### Taking the entire file name with the suffix(.pptx or ppt) ##############\n",
    "####################################################################################\n",
    "\n",
    "df1['file_name_actual']=''\n",
    "mypath=glob.glob(os.path.join(str(path+'/data_txt/'),\"*\"))\n",
    "#mypath_ppt=glob.glob(os.path.join('C:\\\\Users\\\\abandyop\\\\DS\\\\Presentations\\\\',\"*\"))\n",
    "string_file_name=''\n",
    "for i in range(len(mypath_ppt)):\n",
    "    string_file_name+=mypath_ppt[i].split('\\\\')[-1]+' | '\n",
    "# Just extracting the ppt or pptx from the file name and writing it as actual file name\n",
    "for i in range(len(df1)):\n",
    "    pos=string_file_name.find(df1.loc[i,('file_name')]+'.') + len(df1.loc[i,('file_name')]+'.')\n",
    "    df1.loc[i,('file_name_actual')] = df1.loc[i,('file_name')]+'.'+ string_file_name[pos:pos+4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "###################################################################################\n",
    "###########  Extracting the year and month for each document ######################\n",
    "###################################################################################\n",
    "\n",
    "df1['year']=''\n",
    "df1['month']=''\n",
    "for count,i in enumerate(mypath):\n",
    "    try:\n",
    "        k=re.findall(r'\\d{4,8}',i.split('txt\\\\')[1].split('.')[0])[0]\n",
    "        if (len(k)==6) or (len(k)==8):\n",
    "            df1.loc[count,('year')] = k[0:4]\n",
    "            df1.loc[count,('month')] =k[4:6]\n",
    "        if len(k)==4:\n",
    "            if k[0:2]=='20':\n",
    "                df1.loc[count,('year')] = k[0:4]\n",
    "                df1.loc[count,('month')] ='' \n",
    "            else:\n",
    "                df1.loc[count,('year')] = str('20'+k[0:2])\n",
    "                df1.loc[count,('month')] =k[2:4]           \n",
    "    except:\n",
    "        df1.loc[count,('year')] = ''\n",
    "        df1.loc[count,('month')] =''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "###################################################################################################################\n",
    "########################                  The Sector Dictionary          ##########################################\n",
    "###################################################################################################################\n",
    "\n",
    "sector_dict={'Automotive': ['Automotive','Audi','BMW','Borg','automobile','auto','automobiles'] ,\n",
    "'Electronics & High Tech': ['HTECH' ,'High','Tech','ISOLA','Isola','Electronics'],\n",
    "'Government & Public Sector': ['Public', 'Sector','PS','Railways','Maharashtra','Lufthansa','HMRC','Corning',\n",
    "                               'Amtrak','Rail','Municipal','Anser','Government'],\n",
    "'Consumer Products & Retail': ['CPRDT','Consumer', 'Products', 'Retail','CPRDT','CPRD','Retails','Bumble','Adidas','Mc',\n",
    "                               \"Lowe's\",'Ahold','Unilever','Cargill','Beiersdorf','FMCGGiant','Loves','Nestle','Burberry'],\n",
    "'Healthcare & Life Sciences':['Bio', 'Pharma' , 'Health', 'Life','Pharmaceutical','Pharma','Pharmacitical','Hospital',\n",
    "                              'USPharma','Astellas','Healthcare'],\n",
    "'Financial Services':['Financial','FS'],\n",
    "'Insurance':['Insurance','Allianz','MAAF'],\n",
    "'Banking & Capital Markets':['Bank','Banking','Deutsche'],\n",
    "'Manufacturing & Industrial Products': ['Manufacturing','Ferro','Siemens','Boeing','Meggitt','MALS'],\n",
    "'Natural Resources':['Energy', 'Utilities','Gas','Net','Electric'],\n",
    "'Media & Entertainment': ['Media & Entertainment' ,'Media','Movie','Studio','Netflix','Entertainment'],\n",
    "'Telecoms':['TME','Telecom','Telco','Telefonica','T-Mobile','Cable','Telecoms'],\n",
    "'End User Computing': ['Hydro','SUEZ','GDF']\n",
    "          }\n",
    "\n",
    "           \n",
    "           \n",
    "#'Transportation & Distribution':'Others':\n",
    "\n",
    "###################################################################################################################\n",
    "########################            Decoding Sector for each Row         ##########################################\n",
    "###################################################################################################################\n",
    "df1['Sector'] = ''\n",
    "\n",
    "for k,i in enumerate(df1['Full_text']):\n",
    "    i=html.unescape(i)\n",
    "    for key,value in sector_dict.items():\n",
    "        for item in value:\n",
    "            item=item.lower()\n",
    "            if item in regexp('Sector:','',i).lower().split()[0:4]:\n",
    "                df1.loc[k,['Sector']] = key\n",
    "            elif item in regexp('Client Industry:','',i).lower().split()[0:4]:\n",
    "                df1.loc[k,['Sector']] = key\n",
    "            elif item in regexp('Industry\\n:','',i).lower().split()[0:4]:\n",
    "                df1.loc[k,['Sector']] = key            \n",
    "            elif item in df1['Client_name'][k].lower().split():\n",
    "                df1.loc[k,['Sector']] = key\n",
    "                \n",
    "###################################################################################################################\n",
    "####################    Changing Financial Services to banking and Insurance   ####################################\n",
    "###################################################################################################################                \n",
    "\n",
    "values=df1[df1['Sector']=='Financial Services'].index.values        \n",
    "for count in values:\n",
    "    i=df1.loc[count,'Full_text']\n",
    "    if i.lower().find('bank')!=-1:\n",
    "        df1.loc[count,('Sector')] = 'Banking & Capital Markets'\n",
    "    elif i.lower().find('insurance')!=-1:\n",
    "        df1.loc[count,('Sector')] = 'Insurance'\n",
    "    elif i.lower().find('card')!=-1:\n",
    "        df1.loc[count,('Sector')] = 'Banking & Capital Markets'\n",
    "        \n",
    "# Correcting wrong data input by ppt makers\n",
    "df1.loc[df1['file_name']=='RBS_201706_HRAnalitics','Sector']='Banking & Capital Markets'\n",
    "df1.loc[df1['file_name']=='BorgWarner_201308_SupplyChain','country']='Germany'\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "#############################################################################################\n",
    "####################            Technologies Used      ######################################\n",
    "#############################################################################################\n",
    "df1['Tech']=''\n",
    "for count,i in enumerate(df1['Full_text']):\n",
    "    i=html.unescape(i.replace('\\n', ' ').replace('\\t',' '))\n",
    "    m=' '.join(regexp('Technologies:','FTES deployed',i).split(' ')[0:20])\n",
    "    if(m==''):\n",
    "        m=' '.join(regexp('Technologies :','FTES deployed',i).split(' ')[0:20])\n",
    "    if(m==''):\n",
    "        m=' '.join(regexp('Package:','Alliance',i).split(' ')[0:20])\n",
    "    m=m.split('Project Contacts')[0]\n",
    "    m=m.split('FTEs deployed')[0]\n",
    "    m=m.split('Alliance Partners')[0]  \n",
    "    m=m.replace('<<Details of technologies>>','')\n",
    "    m=m.replace('<<Technologies>>','')\n",
    "    m=m.replace('<<technology>>','')\n",
    "    df1.loc[count,['Tech']]=m\n",
    "    #print(m)\n",
    "    #print(' '.join(regexp('Technologies:','FTES deployed',i).split(' ')[0:20]))\n",
    "    #print('-'*50)\n",
    "\n",
    "#TechCorpus\n",
    "\n",
    "TechCorpus=''\n",
    "for i in df1['Tech']:\n",
    "    TechCorpus+=i\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "import codecs\n",
    "import string\n",
    "import subprocess \n",
    "import nltk\n",
    "from nltk.stem.snowball import EnglishStemmer\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from string import digits\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "###########################################\n",
    "##### Define text cleaning function #######\n",
    "###########################################\n",
    "def text_cleaning(text, escape_list=[], stop=[]):\n",
    "    l=[]\n",
    "    \"\"\"\n",
    "    Text cleaning function:\n",
    "        Input: \n",
    "            -text: a string variable, the text to be cleaned\n",
    "            -escape_list : words not to transform by the cleaning process (only lowcase transformation is needed)  \n",
    "            -stop : custom stopwords\n",
    "        Output:\n",
    "            -text cleaned and stemmed           \n",
    "    \"\"\"\n",
    "    \n",
    "    \n",
    "    \"\"\" Get stop word list from package\"\"\"\n",
    "    StopWords = list(set(stopwords.words('english')))\n",
    "    custom_stop = StopWords + stop\n",
    "    \n",
    "    \"\"\" Step 1: Parse html entities\"\"\"\n",
    "    text = html.unescape(text)\n",
    "    text=text.replace('\\n',' ').replace('\\t',' ').replace('’','')\n",
    "    \n",
    "    \n",
    "    \n",
    "    \"\"\" Step 2: Decode special caracters\"\"\"\n",
    "    text = text.encode('utf8').decode('unicode_escape')\n",
    "    \n",
    " \n",
    "    \"\"\" Step 3: Tokenise text: spliting text elements with the TreeBankWordTokenizer method\"\"\"\n",
    "    tokenizer = TreebankWordTokenizer()\n",
    "    tokenz=[','.join(tokenizer.tokenize(mot)) if mot  not in escape_list else mot  for mot in text.split()  ]\n",
    "    \n",
    "    \n",
    "    \"\"\" Step 4: Drop punctuations \"\"\"\n",
    "    tokenz=[re.sub(r'[^\\w\\s]',' ',mot) if mot  not in escape_list else mot  for mot in tokenz  ]\n",
    "    tokenz = ' '.join(tokenz).split()\n",
    "       \n",
    "    \"\"\" Step 5.1: Remove stop words \"\"\"\n",
    "    tokenz=([token for token in tokenz if token not in custom_stop])\n",
    "    \n",
    "    \n",
    "    \"\"\" Step 5.2: Delete digits from text \"\"\"\n",
    "    tokenz=([token for token in tokenz if (  (token.isdigit())==False)  ])  \n",
    "\n",
    "    \"\"\" Step 5.3: Remove digits from tokens\"\"\"\n",
    "    remove_digits = str.maketrans('', '', digits)\n",
    "    tokenz=[token.translate(remove_digits)  if token not in  escape_list else token for token in tokenz   ]\n",
    "    \n",
    "    \"\"\" Step 6.1: Lowcase the text\"\"\"\n",
    "    tokenz=([token.lower() for token in tokenz])\n",
    "    \n",
    "    \"\"\" Step 6.2: Lemmatize the text \n",
    "     \n",
    "'''tokenz=[WordNetLemmatizer().lemmatize(token) if token not in escape_list else token for token in tokenz ]'''\"\"\"\n",
    "    \"\"\" Step 6.2: Stem the text \"\"\"\n",
    "    tokenz=[EnglishStemmer().stem(token) if token not in escape_list else token for token in tokenz ]\n",
    "\n",
    "    \"\"\" Step 6.3: Drop words with one caratcter and proceed last check for stop words after Stemming\"\"\"\n",
    "    tokenz=[token for token in tokenz if (token not in  custom_stop and len(token)>1) ]\n",
    "\n",
    "    return ' '.join(tokenz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\abandyop\\AppData\\Local\\Continuum\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:39: DeprecationWarning: invalid escape sequence '\\ '\n"
     ]
    }
   ],
   "source": [
    "# list of words not to transform by the cleaning process\n",
    "escape_list = []\n",
    "\n",
    "# Custom stopwords to remove\n",
    "stop = [\"client\", \"capgemini\", \"copyright\", \"understand\", \"right\", \"reserved\", \"project\"]\n",
    "\n",
    "df1['Full_text']=df1['Full_text'].apply(text_cleaning,args=(escape_list,stop))\n",
    "###################################################################\n",
    "####### Cleaning the need or purpose or technology name ###########\n",
    "###################################################################\n",
    "def TechnologyNameClean(text):\n",
    "    if 'iotanalytics' in text.lower().split():\n",
    "        text = text.replace('iotanalytics','iot analytics')\n",
    "    if 'saphana' in text.lower().split():\n",
    "        text = text.replace('saphana','sap hana')\n",
    "        \n",
    "    if '_' in list(text):\n",
    "        text=text.split('_')\n",
    "        text= ' '.join(text)\n",
    "    if '&' in list(text):\n",
    "        text=text.split('&')\n",
    "        text= ' '.join(text)\n",
    "    text = re.sub(r\"([a-z])([A-Z])\",\"\\g<1> \\g<2>\",text)\n",
    "    return text\n",
    "        \n",
    "\n",
    "\n",
    "df1['Technology Name']= df1['Technology Name'].apply(TechnologyNameClean)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Counting Text Length for each ppt\n",
    "df1['text_length']=''\n",
    "df1['text_length']=df1['Full_text'].apply(lambda x:len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################################################\n",
    "##############      Exporting the Model   ####################\n",
    "##############################################################\n",
    "\n",
    "df1.drop(df1[df1['file_name']== 'Data Science Challenge IV Kickoff_v3'].index,inplace=True)\n",
    "# Same file so deleting\n",
    "df1.drop(df1[df1['file_name']== 'Beiersdorf_201305_SAP_BW2'].index,inplace=True)\n",
    "df1.drop(df1[df1['file_name']== 'Beiersdorf_201306_SAP_BW2'].index,inplace=True)\n",
    "df1.drop(df1[df1['file_name']== 'Beiersdorf_201312_InventoryManagement2'].index,inplace=True)\n",
    "df1.drop(df1[df1['file_name']== 'SiemensBT_201609_InnovativeAnalyticsBasedServicesUsingIoT.pptx'].index,inplace=True)\n",
    "\n",
    "\n",
    "df1.reset_index(drop=True,inplace=True)\n",
    "import pickle\n",
    "df1.to_pickle(\"trained_df.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}