GenerateKLPFile.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf8')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>topic</th>\n",
       "      <th>text::text::S</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>595524450503815168</td>\n",
       "      <td>-Prendere i libri in copisteria-Fare la spesa-Spararmi in bocca-Farmi la doccia</td>\n",
       "      <td>TWITA</td>\n",
       "      <td>-Prendere::-prendere::V i::i::RD libri::libro::S in::in::E copisteria-Fare::copisteria-fare::V la::la::RD spesa-Spararmi::spesa-spararmi::S in::in::E bocca-Farmi::bocca-farmi::S la::la::RD doccia::doccia::S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>578468106504433665</td>\n",
       "      <td>...comunque con una crociera Costa se non ti ammazza Schettino prima ti spara il terrorista dopo...</td>\n",
       "      <td>HSC</td>\n",
       "      <td>.::.::FF comunque::comunque::B con::con::E una::una::RI crociera::crociera::S Costa::costa::SP se::se::PC non::non::BN ti::ti::PC ammazza::ammazza::V Schettino::schettino::SP prima::prima::B ti::ti::PC spara::sparare::V il::il::RD terrorista::terrorista::S dopo::dopo::B .::.::FS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>577791521174466560</td>\n",
       "      <td>“@TweetComici: Ogni ragazza: \\\"non sono una ragazza gelosa.\\\"*3 minuti dopo*\\\"CHI CAZZO È QUELLA PUTTANA?\\\"”</td>\n",
       "      <td>TWITA</td>\n",
       "      <td>“::“::FB @TweetComici::__user::USR __COLON__::__COLON__::FC Ogni::ogni::SP ragazza::ragazzo::S __COLON__::__COLON__::FC \\::\\::FB \"::\"::FB non::non::BN sono::essere::V una::una::RI ragazza::ragazzo::S gelosa::geloso::A .::.::FF \\::\\::FB \"::\"::FB *::*::FB 3::3::N minuti::minuto::S dopo*\\::dopo*\\::A \"::\"::FB CHI::chi::PR cazzo::cazzo::A È::essere::V quella::quello::PD puttana::puttana::A ?::?::FS...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   id  \\\n",
       "0  595524450503815168   \n",
       "1  578468106504433665   \n",
       "2  577791521174466560   \n",
       "\n",
       "                                                                                                           text  \\\n",
       "0                               -Prendere i libri in copisteria-Fare la spesa-Spararmi in bocca-Farmi la doccia   \n",
       "1           ...comunque con una crociera Costa se non ti ammazza Schettino prima ti spara il terrorista dopo...   \n",
       "2  “@TweetComici: Ogni ragazza: \\\"non sono una ragazza gelosa.\\\"*3 minuti dopo*\\\"CHI CAZZO È QUELLA PUTTANA?\\\"”   \n",
       "\n",
       "   topic  \\\n",
       "0  TWITA   \n",
       "1    HSC   \n",
       "2  TWITA   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                    text::text::S   \n",
       "0                                                                                                                                                                                                  -Prendere::-prendere::V i::i::RD libri::libro::S in::in::E copisteria-Fare::copisteria-fare::V la::la::RD spesa-Spararmi::spesa-spararmi::S in::in::E bocca-Farmi::bocca-farmi::S la::la::RD doccia::doccia::S   \n",
       "1                                                                                                                         .::.::FF comunque::comunque::B con::con::E una::una::RI crociera::crociera::S Costa::costa::SP se::se::PC non::non::BN ti::ti::PC ammazza::ammazza::V Schettino::schettino::SP prima::prima::B ti::ti::PC spara::sparare::V il::il::RD terrorista::terrorista::S dopo::dopo::B .::.::FS   \n",
       "2  “::“::FB @TweetComici::__user::USR __COLON__::__COLON__::FC Ogni::ogni::SP ragazza::ragazzo::S __COLON__::__COLON__::FC \\::\\::FB \"::\"::FB non::non::BN sono::essere::V una::una::RI ragazza::ragazzo::S gelosa::geloso::A .::.::FF \\::\\::FB \"::\"::FB *::*::FB 3::3::N minuti::minuto::S dopo*\\::dopo*\\::A \"::\"::FB CHI::chi::PR cazzo::cazzo::A È::essere::V quella::quello::PD puttana::puttana::A ?::?::FS...  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = True\n",
    "\n",
    "if(test):\n",
    "    NOME_FILE = \"test_dataset.klp\"\n",
    "    path = \"../test_ironita2018_revnlt_processed.tsv\"\n",
    "else:\n",
    "    NOME_FILE = \"train_dataset.klp\"\n",
    "    path = \"../training_ironita2018_renlt_processed.tsv\"\n",
    "    #path = \"../training_ironita2018.tsv\"\n",
    "\n",
    "df = pd.read_csv(path, sep='\\t')\n",
    "df.head(3)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploratory Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "if(not test):\n",
    "    pd.options.display.max_colwidth = 100\n",
    "    df[(df.irony==0) & (df.sarcasm ==1)].head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "if(not test):   \n",
    "    pd.options.display.max_colwidth = 400\n",
    "    df[(df.irony==1) | (df.sarcasm ==1)][['text','irony','sarcasm']].sample(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "if(not test):\n",
    "    df[(df.irony==1) | (df.sarcasm ==1)].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generazione file KLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generaArraydataset(df=df, target=\"irony\", lower = True):\n",
    "    arrayDataset = []\n",
    "    arrayTarget = []\n",
    "    for index, row in df.iterrows():\n",
    "        if(lower):\n",
    "            arrayDataset.append(row[\"text\"].lower())\n",
    "        else:\n",
    "            arrayDataset.append(row[\"text\"])\n",
    "        try:\n",
    "            if(row[target] == 0):\n",
    "                arrayTarget.append(-1)\n",
    "            else:\n",
    "                arrayTarget.append(row[target])\n",
    "        except:\n",
    "            arrayTarget.append(-1)\n",
    "    return [arrayDataset, arrayTarget]\n",
    "\n",
    "array = generaArraydataset()\n",
    "X_train = array[0]\n",
    "y_train_irony = array[1]\n",
    "y_train_sarcasm = generaArraydataset(target=\"sarcasm\")[1]\n",
    "\n",
    "'''\n",
    "from nltk import word_tokenize\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "tknzr = TweetTokenizer(strip_handles=True)\n",
    "\n",
    "#token = nltk.word_tokenize(X_train[2])\n",
    "print(tknzr.tokenize(X_train[2]))\n",
    "'''\n",
    "print(y_train_irony[2])\n",
    "print(y_train_sarcasm[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import word_tokenize\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "import hashlib\n",
    "\n",
    "def creaBoW(dataset, target, name = \"dataset.klp\"):\n",
    "    f = open(name,\"w+\")\n",
    "    for index, riga in enumerate(dataset):\n",
    "        string = str(target[index]) + \" |BV:bow| \"\n",
    "        tknzr = TweetTokenizer(strip_handles=True)\n",
    "        tweettoknz = tknzr.tokenize(dataset[index])\n",
    "        for word in tweettoknz:\n",
    "            string = string + \"_\" + hashlib.md5(''.join(word)).hexdigest() + \":1.0 \"\n",
    "        f.write(string + \"|EV| |BS:comment| \"+ dataset[index] +\" |ES| |BS:IDTweet| \"+ str(df.iloc[index]['id']) +\" |ES| \\n\")    \n",
    "    f.close()\n",
    "\n",
    "    \n",
    "def creaBoW2(df=df, name = \"dataset.klp\", test=False):\n",
    "    f = open(name,\"w+\")\n",
    "    for index, row in df.iterrows():\n",
    "        if(test):\n",
    "            try:\n",
    "                if(row[\"irony\"] == 1):\n",
    "                    ironia = \"Irony\"\n",
    "            except:\n",
    "                ironia = \"NOTIrony\"\n",
    "            try:    \n",
    "                if(row[\"sarcasm\"] == 1):\n",
    "                    sarcasmo = \"Sarcasmo\"\n",
    "            except:\n",
    "                sarcasmo = \"NOTSarcasmo\"\n",
    "        else:\n",
    "            if(row[\"irony\"] == 1):\n",
    "                ironia = \"Irony\"\n",
    "            else:\n",
    "                ironia = \"NOTIrony\"\n",
    "            if(row[\"sarcasm\"] == 1):\n",
    "                sarcasmo = \"Sarcasmo\"\n",
    "            else:\n",
    "                sarcasmo = \"NOTSarcasmo\"\n",
    "        string = ironia + \" \" + sarcasmo + \" |BV:bow| \"\n",
    "        tknzr = TweetTokenizer(strip_handles=True)\n",
    "        tweettoknz = tknzr.tokenize(row[\"text\"])\n",
    "        for word in tweettoknz:\n",
    "            string = string + \"_\" + hashlib.md5(''.join(word)).hexdigest() + \":1.0 \"\n",
    "        #f.write(string + \"|EV| |BS:comment| \"+ row[\"text\"] +\" |ES| |BS:IDTweet| \"+ str(row[\"id\"]) +\" |ES| \\n\")\n",
    "        f.write(string + \"|EV| |BS:IDTweet| \"+ str(row[\"id\"]) +\" |ES| \\n\") \n",
    "    f.close()\n",
    "    \n",
    "\n",
    "#creaBoW(X_train, y_train_irony, NOME_FILE)\n",
    "creaBoW2(name = NOME_FILE, test=test)\n",
    "#NOME_FILE = \"dataset_sarcasmo.klp\"\n",
    "#creaBoW(X_train, y_train_sarcasm, NOME_FILE)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df = pd.read_csv(\"../training_ironita2018_renlt_processed.tsv\", sep='\\t')\n",
    "#df.head(3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generaArrayTokenizzatoLemmi(df=df):\n",
    "    arrayLemmi = []\n",
    "    for index, row in df.iterrows():\n",
    "        arrayLemmi.append(row[\"text::text::S \"])\n",
    "    arrayTokenizzatoLemmi = []\n",
    "    #per ogni elemento del dataset\n",
    "    for text in arrayLemmi:\n",
    "        arrayParoleLemm = text.split()\n",
    "        arrayFrase = []\n",
    "        #per ogni token text::text::S\n",
    "        for parola in arrayParoleLemm:\n",
    "            arrayParola = parola.split(\"::\")\n",
    "            #divido ogni singolo token e prendo il lemma\n",
    "            arrayFrase.append(arrayParola[1])\n",
    "        arrayTokenizzatoLemmi.append(arrayFrase)\n",
    "    return arrayTokenizzatoLemmi\n",
    "\n",
    "def generaArrayTokenizzatoSurface(df=df, listPOS = ['A','B','S','V','HTG'], lower = True):\n",
    "    arrayLemmi = []\n",
    "    for index, row in df.iterrows():\n",
    "        arrayLemmi.append(row[\"text::text::S \"])\n",
    "    arrayTokenizzatoLemmi = []\n",
    "    for text in arrayLemmi:\n",
    "        arrayParoleLemm = text.split()\n",
    "        arrayFrase = []\n",
    "        for parola in arrayParoleLemm:\n",
    "            arrayParola = parola.split(\"::\")\n",
    "            if((arrayParola[2] in listPOS) and listPOS != []):\n",
    "                if(lower):\n",
    "                    arrayFrase.append(arrayParola[0].lower())\n",
    "                else:\n",
    "                    arrayFrase.append(arrayParola[0])\n",
    "            if(listPOS == []):\n",
    "                if(lower):\n",
    "                    arrayFrase.append(arrayParola[0].lower())\n",
    "                else:\n",
    "                    arrayFrase.append(arrayParola[0])\n",
    "        arrayTokenizzatoLemmi.append(arrayFrase)\n",
    "    return arrayTokenizzatoLemmi\n",
    "    \n",
    "prova = generaArrayTokenizzatoLemmi()\n",
    "print prova[2]\n",
    "prova2 = generaArrayTokenizzatoSurface()\n",
    "print prova2[2]\n",
    "arrayCompleto = generaArrayTokenizzatoSurface(listPOS = [])\n",
    "print arrayCompleto[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.util import ngrams\n",
    "\n",
    "def generaArrayNgrams(dataset, ngram=2):\n",
    "    arrayBigrammi = []\n",
    "    for item in dataset:\n",
    "        arrayBigrammi.append(list(ngrams(item,ngram)))\n",
    "    return arrayBigrammi\n",
    "\n",
    "\n",
    "arrayBigrammiLemmi = generaArrayNgrams(prova)\n",
    "arrayBigrammiSurface = generaArrayNgrams(arrayCompleto)\n",
    "\n",
    "\n",
    "arrayBigrammiChar = generaArrayNgrams(X_train)\n",
    "array3grammiChar = generaArrayNgrams(X_train, 3)\n",
    "array4grammiChar = generaArrayNgrams(X_train, 4)\n",
    "array5grammiChar = generaArrayNgrams(X_train, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy import asarray\n",
    "\n",
    "def importaWordSpace(pathWS = \"../ws\"):\n",
    "    # load the whole embedding into memory\n",
    "    embeddings_index = dict()\n",
    "    f = open(pathWS)\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        #print(word)\n",
    "        string = ''.join(values[3:]).split(',')\n",
    "        for x in string:\n",
    "            x = float(x)\n",
    "        #print(string)\n",
    "        coefs = asarray(string, dtype='float32')\n",
    "        embeddings_index[word] = coefs\n",
    "    f.close()\n",
    "    print('Loaded %s word vectors.' % len(embeddings_index))\n",
    "    return embeddings_index\n",
    "\n",
    "def importaWordSpace2(pathWS = '../DPL-IT_lrec2016.txt'):\n",
    "    # load the whole embedding into memory\n",
    "    embeddings_index = dict()\n",
    "    f = open(pathWS)\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        #print(word)\n",
    "        string = ''.join(values[1:]).split(',')\n",
    "        for x in string:\n",
    "            x = float(x)\n",
    "        #print(string)\n",
    "        coefs = asarray(string, dtype='float32')\n",
    "        embeddings_index[word] = coefs\n",
    "    f.close()\n",
    "    print('Loaded %s word vectors.' % len(embeddings_index))\n",
    "    return embeddings_index\n",
    "\n",
    "def importaWordSpace3(pathWS = '../emoji2vec.txt'):\n",
    "    # load the whole embedding into memory\n",
    "    embeddings_index = dict()\n",
    "    f = open(pathWS)\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        coefs = asarray(values[1:], dtype='float32')\n",
    "        embeddings_index[word] = coefs\n",
    "    f.close()\n",
    "    print('Loaded %s word vectors.' % len(embeddings_index))\n",
    "    return embeddings_index\n",
    "\n",
    "def importaWordSpace4(pathWS = '../sentix'):\n",
    "    # load the whole embedding into memory\n",
    "    embeddings_index = dict()\n",
    "    f = open(pathWS)\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        coefs = asarray(values[3:], dtype='float32')\n",
    "        embeddings_index[word] = coefs\n",
    "    f.close()\n",
    "    print('Loaded %s word vectors.' % len(embeddings_index))\n",
    "    return embeddings_index\n",
    "\n",
    "def generaCombWordSpace(dataset = prova2):\n",
    "    embeddings_index = importaWordSpace()\n",
    "    combDataset = []\n",
    "    for tweet in dataset:\n",
    "        comb = np.zeros(250)\n",
    "        norm = 0.0\n",
    "        for parola in tweet:\n",
    "            if(parola in embeddings_index):\n",
    "                comb = comb + embeddings_index[parola]\n",
    "                norm += 1\n",
    "        if(norm != 0):\n",
    "            combDataset.append(comb/norm)\n",
    "        else:\n",
    "            combDataset.append(comb)\n",
    "    return combDataset\n",
    "\n",
    "def generaCombDPL2(df=df):\n",
    "    arrayLemmi = []\n",
    "    embeddings_index = importaWordSpace2()\n",
    "    for index, row in df.iterrows():\n",
    "        arrayLemmi.append(row[\"text::text::S \"])\n",
    "    arrayTokenizzatoLemmi = []\n",
    "    for text in arrayLemmi:\n",
    "        arraySmooth = [0.3333 , 0.3333 , 0.3333]\n",
    "        arNomi = np.zeros(3)\n",
    "        cnomi = 0\n",
    "        arVerb = np.zeros(3)\n",
    "        cverb = 0\n",
    "        arAdj = np.zeros(3)\n",
    "        cadj = 0\n",
    "        arAdv = np.zeros(3)\n",
    "        cadv = 0\n",
    "        arTOT = np.zeros(3)\n",
    "        ctot = 0\n",
    "        arrayParoleLemm = text.split()\n",
    "        arrayFrase = []\n",
    "        for parola in arrayParoleLemm:\n",
    "            arrayParola = parola.split(\"::\")\n",
    "            if(arrayParola[2] == 'A'):\n",
    "                try:\n",
    "                    arAdj += embeddings_index[arrayParola[0].lower()]\n",
    "                    arTOT += embeddings_index[arrayParola[0].lower()]\n",
    "                    cadj += 1\n",
    "                    ctot += 1\n",
    "                except:\n",
    "                    arAdj += arraySmooth\n",
    "                    arTOT += arraySmooth\n",
    "                    cadj += 1\n",
    "                    ctot += 1\n",
    "            if(arrayParola[2] == 'B'):\n",
    "                try:\n",
    "                    arAdv += embeddings_index[arrayParola[0].lower()]\n",
    "                    arTOT += embeddings_index[arrayParola[0].lower()]\n",
    "                    cadv += 1\n",
    "                    ctot += 1\n",
    "                except:\n",
    "                    arAdv += arraySmooth\n",
    "                    arTOT += arraySmooth\n",
    "                    cadv += 1\n",
    "                    ctot += 1\n",
    "            if(arrayParola[2] == 'S'):\n",
    "                try:\n",
    "                    arNomi += embeddings_index[arrayParola[0].lower()]\n",
    "                    arTOT += embeddings_index[arrayParola[0].lower()]\n",
    "                    cnomi += 1\n",
    "                    ctot += 1\n",
    "                except:\n",
    "                    arNomi += arraySmooth\n",
    "                    arTOT += arraySmooth\n",
    "                    cnomi += 1\n",
    "                    ctot += 1\n",
    "            if(arrayParola[2] == 'V'):\n",
    "                try:\n",
    "                    arVerb += embeddings_index[arrayParola[0].lower()]\n",
    "                    arTOT += embeddings_index[arrayParola[0].lower()]\n",
    "                    cverb += 1\n",
    "                    ctot += 1\n",
    "                except:\n",
    "                    arVerb += arraySmooth\n",
    "                    arTOT += arraySmooth\n",
    "                    cverb += 1\n",
    "                    ctot += 1\n",
    "            else:\n",
    "                try:\n",
    "                    arTOT += embeddings_index[arrayParola[0].lower()]\n",
    "                    ctot += 1\n",
    "                except:\n",
    "                    arTOT += arraySmooth\n",
    "                    ctot += 1\n",
    "        if(cnomi == 0):\n",
    "            cnomi = 1\n",
    "        if(cverb == 0):\n",
    "            cverb = 1\n",
    "        if(cadj == 0):\n",
    "            cadj = 1\n",
    "        if(cadv == 0):\n",
    "            cadv = 1\n",
    "        if(ctot == 0):\n",
    "            ctot = 1\n",
    "        arrayFrase = np.concatenate((arNomi/cnomi, arVerb/cverb, arAdj/cadj, arAdv/cadv, arTOT/ctot), axis=None)            \n",
    "        arrayTokenizzatoLemmi.append(arrayFrase)\n",
    "    return arrayTokenizzatoLemmi\n",
    "\n",
    "\n",
    "def generaCombDPL(dataset = generaArrayTokenizzatoSurface(listPOS = ['A','S','V'])):\n",
    "    embeddings_index = importaWordSpace2()\n",
    "    combDataset = []\n",
    "    for tweet in dataset:\n",
    "        comb = [0.0]\n",
    "        for parola in tweet:\n",
    "            if(parola in embeddings_index):\n",
    "                comb = np.concatenate((comb, embeddings_index[parola]), axis=None)\n",
    "        combDataset.append(comb)\n",
    "    return combDataset\n",
    "\n",
    "def appendBowDPL( nomerappresentazione, dataset = generaArrayTokenizzatoSurface(listPOS = ['A','S','V']), file_name = \"dataset.klp\"):\n",
    "    f = open(file_name, 'r')\n",
    "    embeddings_index = importaWordSpace2()\n",
    "    file_lines = []\n",
    "    for index, x in enumerate(f.readlines()):\n",
    "        string = \" |BV:\"+ nomerappresentazione + \"| \"\n",
    "        for word in dataset[index]:\n",
    "            try:\n",
    "                vecPol = embeddings_index[word]\n",
    "            except:\n",
    "                vecPol = [0, 0, 0]\n",
    "            pal = hashlib.md5(''.join(word)).hexdigest()\n",
    "            string = string + \"_\" + pal +\"_pos:\"+ str(vecPol[0]) + \" \" + pal +\"_neg:\"+ str(vecPol[1]) + \" \" + pal +\"_net:\"+ str(vecPol[2]) + \" \"\n",
    "        string = string + \"|EV| \"\n",
    "        file_lines.append(''.join([x.strip(), string, '\\n']))\n",
    "\n",
    "    with open(file_name, 'w') as f:\n",
    "        f.writelines(file_lines)\n",
    "\n",
    "\n",
    "def generaCombSentix(dataset = prova2):\n",
    "    embeddings_index = importaWordSpace4()\n",
    "    combDataset = []\n",
    "    for tweet in dataset:\n",
    "        comb = np.zeros(4)\n",
    "        norm = 0.0\n",
    "        for parola in tweet:\n",
    "            if(parola in embeddings_index):\n",
    "                try:\n",
    "                    comb = comb + embeddings_index[parola]\n",
    "                except:\n",
    "                    print parola\n",
    "                    print embeddings_index[parola]\n",
    "                norm += 1\n",
    "        if(norm == 0):\n",
    "            norm = 1\n",
    "        combDataset.append(comb/norm)\n",
    "    return combDataset\n",
    "\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import word_tokenize\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "import numpy as np\n",
    "\n",
    "def generaEmoji(dataset = X_train):\n",
    "    embeddings_index = importaWordSpace3()\n",
    "    combDataset = []\n",
    "    for tweet in dataset:\n",
    "        tknzr = TweetTokenizer(strip_handles=True)\n",
    "        tweettoknz = tknzr.tokenize(tweet)\n",
    "        comb = np.zeros(300)\n",
    "        for parola in tweettoknz:\n",
    "            if(parola in embeddings_index):\n",
    "                print parola\n",
    "                comb = comb + embeddings_index[parola]\n",
    "        combDataset.append(comb)\n",
    "    return combDataset\n",
    "\n",
    "def feturesPunteggiatura(dataset = X_train):\n",
    "    combDataset = []\n",
    "    for tweet in dataset:\n",
    "        charset = [0.0,0.0,0.0,0.0,0.0,0.0]\n",
    "        numUpperCase = 0.0\n",
    "        numchar = 0.0\n",
    "        for char in tweet:\n",
    "            numchar += 1\n",
    "            if(char.istitle()):\n",
    "                numUpperCase += 1\n",
    "            if(char == \"!\"):\n",
    "                charset[0] += 1\n",
    "            if(char == \"?\"):\n",
    "                charset[1] += 1\n",
    "            if(char == \".\"):\n",
    "                charset[2] += 1    \n",
    "            if(char == \",\"):\n",
    "                charset[3] += 1\n",
    "            if(char == \";\"):\n",
    "                charset[4] += 1\n",
    "            if(char == \":\"):\n",
    "                charset[5] += 1\n",
    "        charset.append(numUpperCase/numchar)\n",
    "        combDataset.append(charset)\n",
    "    return combDataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.spatial import distance\n",
    "\n",
    "#Combinazione Lineare delle parole del Word Space moltiplicate per la distanza dal centroide\n",
    "#Per ogni Elemento del dataset ritorna la tupla (Bow, Var, Mean)\n",
    "def generaIronySpecificBow(dataset, arrayCentroide = generaCombWordSpace(dataset=prova)):\n",
    "    embeddings_index = importaWordSpace()\n",
    "    combDataset = []\n",
    "    for index, tweet in enumerate(dataset):\n",
    "        comb = np.zeros(250)\n",
    "        arrayDistanze = []\n",
    "        for parola in tweet:\n",
    "            if(parola in embeddings_index):\n",
    "                comb = comb + embeddings_index[parola] * distance.cosine(embeddings_index[parola], arrayCentroide[index])\n",
    "                arrayDistanze.append(distance.euclidean(embeddings_index[parola], arrayCentroide[index]))\n",
    "        x = asarray(arrayDistanze)\n",
    "        aList = (comb, np.var(x), np.mean(x))\n",
    "        combDataset.append(tuple(aList))\n",
    "    return combDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.spatial import distance\n",
    "\n",
    "def calcolaDistanzaSmooth(dataset=arrayCompleto, embeddings_index = importaWordSpace(), arrayCentroide = generaCombWordSpace(dataset=prova)):\n",
    "    arrayDistanzeDataset = []\n",
    "    for index, tweet in enumerate(dataset):\n",
    "        arrayDistanzetweet = []\n",
    "        for word in tweet:\n",
    "            if(word in embeddings_index):\n",
    "                distanza = 1 - distance.cosine(arrayCentroide[index], embeddings_index[word])\n",
    "                arrayDistanzetweet.append(distanza)\n",
    "        arrayDistanzeDataset.append(np.mean(arrayDistanzetweet))\n",
    "    return np.mean(arrayDistanzeDataset)\n",
    "\n",
    "#calcolaDistanzaSmooth()\n",
    "#Output: 0.4273651809245167"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.spatial import distance\n",
    "                \n",
    "\n",
    "def appendBowIronySpecific(dataset, nomerappresentazione, file_name = \"dataset.klp\", embeddings_index = importaWordSpace(), arrayCentroide = generaCombWordSpace(dataset=prova)):\n",
    "    f = open(file_name, 'r')\n",
    "    smooth = 0.4273651809245167\n",
    "    file_lines = []\n",
    "    combDataset = []\n",
    "    missingWord = 0.0\n",
    "    totalWord = 0.0\n",
    "    for index, x in enumerate(f.readlines()):\n",
    "        string = \" |BV:\"+ nomerappresentazione + \"| \"\n",
    "        arrayDistanze = []\n",
    "        flag = 0\n",
    "        distanzaMax = 0\n",
    "        distanzaMin = 4\n",
    "        for word in dataset[index]:\n",
    "            #calcola distanza tra word e centroide[index]\n",
    "            if(word in embeddings_index):\n",
    "                distanza = 1 - distance.cosine(arrayCentroide[index], embeddings_index[word])\n",
    "                if(distanza > distanzaMax):\n",
    "                    distanzaMax = distanza\n",
    "                if(distanza < distanzaMin):\n",
    "                    distanzaMin = distanza\n",
    "                arrayDistanze.append(1 - distance.cosine(arrayCentroide[index], embeddings_index[word]))\n",
    "                string = string + \"_\" + hashlib.md5(''.join(word)).hexdigest() + \":\"+ str(distanza)+\" \"\n",
    "                flag = 1\n",
    "            else:\n",
    "                distanza = smooth\n",
    "                missingWord += 1\n",
    "            totalWord += 1\n",
    "                \n",
    "        string = string + \"|EV| \"\n",
    "        '''\n",
    "        if(flag == 1):\n",
    "            file_lines.append(''.join([x.strip(), string, '\\n']))\n",
    "        else:\n",
    "            file_lines.append(''.join([x.strip(), \"\", '\\n']))\n",
    "        '''\n",
    "        file_lines.append(''.join([x.strip(), string, '\\n']))\n",
    "        x = asarray(arrayDistanze)\n",
    "        if(np.isnan(np.var(x)) or np.isnan(np.mean(x))):\n",
    "            aList = [-2,-2,-2,-2]\n",
    "        else:\n",
    "            aList = [np.var(x), np.mean(x), distanzaMin, distanzaMax]\n",
    "        combDataset.append(aList)\n",
    "\n",
    "    with open(file_name, 'w') as f:\n",
    "        f.writelines(file_lines)\n",
    "    print \"Parole mancanti: \" + str(missingWord/totalWord)\n",
    "    return combDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "import hashlib\n",
    "\n",
    "\n",
    "def appendBow(dataset, nomerappresentazione, file_name = \"dataset.klp\", tupla = True):\n",
    "    f = open(file_name, 'r')\n",
    "    file_lines = []\n",
    "    for index, x in enumerate(f.readlines()):\n",
    "        string = \" |BV:\"+ nomerappresentazione + \"| \"\n",
    "        for word in dataset[index]:\n",
    "            if(tupla):\n",
    "                string = string + \"_\" + hashlib.md5(''.join(word)).hexdigest() + \":1.0 \"\n",
    "            else:\n",
    "                string = string + \"_\" + word + \":1.0 \"\n",
    "        string = string + \"|EV| \"\n",
    "        file_lines.append(''.join([x.strip(), string, '\\n']))\n",
    "\n",
    "    with open(file_name, 'w') as f:\n",
    "        f.writelines(file_lines) \n",
    "        \n",
    "        \n",
    "\n",
    "\n",
    "def appendDenseRapr(dataset, nomerappresentazione, file_name = \"dataset.klp\"):\n",
    "    f = open(file_name, 'r')\n",
    "    file_lines = []\n",
    "    for index, x in enumerate(f.readlines()):\n",
    "        string = \" |BDV:\"+ nomerappresentazione + \"| \"\n",
    "        for num in dataset[index]:\n",
    "            string = string + str(num) + \" \"\n",
    "        string = string + \"|EDV| \"\n",
    "        file_lines.append(''.join([x.strip(), string, '\\n']))\n",
    "        \n",
    "    with open(file_name, 'w') as f:\n",
    "        f.writelines(file_lines)\n",
    "\n",
    "\n",
    "def appendDenseRaprTuple(dataset, nomerappresentazione, idTupla, file_name = \"dataset.klp\"):\n",
    "    f = open(file_name, 'r')\n",
    "    file_lines = []\n",
    "    for index, x in enumerate(f.readlines()):\n",
    "        string = \" |BDV:\"+ nomerappresentazione + \"| \"\n",
    "        if( idTupla < 1):\n",
    "            for num in dataset[index][idTupla]:\n",
    "                string = string + str(num) + \" \"\n",
    "        else:\n",
    "            string = string + str(dataset[index][idTupla]) + \" \"\n",
    "        string = string + \"|EDV| \"\n",
    "        file_lines.append(''.join([x.strip(), string, '\\n']))\n",
    "        \n",
    "    with open(file_name, 'w') as f:\n",
    "        f.writelines(file_lines)\n",
    "\n",
    "        \n",
    "def padding(combDPL):\n",
    "    massima = 0\n",
    "    nuovoarray = []\n",
    "    for array in combDPL:\n",
    "        if len(array) > massima:\n",
    "            massima = len(array)\n",
    "    for array in combDPL:\n",
    "        arrayzeri = np.zeros(massima - len(array))\n",
    "        array2 = np.concatenate((array, arrayzeri), axis=None)\n",
    "        nuovoarray.append(array2)\n",
    "    return nuovoarray"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Append delle rappresentazioni KLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "combWS = generaCombWordSpace()\n",
    "appendDenseRapr(combWS, \"WSSurface\",NOME_FILE)\n",
    "\n",
    "\n",
    "\n",
    "comDPL = generaCombDPL2()\n",
    "appendDenseRapr(comDPL, \"combDPL\",NOME_FILE)\n",
    "\n",
    "comSentix = generaCombSentix()\n",
    "appendDenseRapr(comSentix, \"combSentix\",NOME_FILE)\n",
    "\n",
    "combEmoji = generaEmoji()\n",
    "appendDenseRapr(combEmoji, \"combEmoji\",NOME_FILE)\n",
    "\n",
    "featuresPuntegg = feturesPunteggiatura()\n",
    "appendDenseRapr(featuresPuntegg, \"featPunt\",NOME_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "meanVarArray = appendBowIronySpecific(prova, \"bowIronySpecific\", NOME_FILE)\n",
    "appendDenseRapr(meanVarArray, \"VarMean\",NOME_FILE)\n",
    "\n",
    "IronySpecificA = generaArrayTokenizzatoSurface(listPOS = ['A'])\n",
    "meanVarArrayA = appendBowIronySpecific(IronySpecificA, \"bowIronySpecificA\", NOME_FILE)\n",
    "appendDenseRapr(meanVarArrayA, \"VarMeanA\",NOME_FILE)\n",
    "\n",
    "IronySpecificS = generaArrayTokenizzatoSurface(listPOS = ['S'])\n",
    "meanVarArrayS = appendBowIronySpecific(IronySpecificS, \"bowIronySpecificS\", NOME_FILE)\n",
    "appendDenseRapr(meanVarArrayS, \"VarMeanS\",NOME_FILE)\n",
    "\n",
    "IronySpecificV = generaArrayTokenizzatoSurface(listPOS = ['V'])\n",
    "meanVarArrayV = appendBowIronySpecific(IronySpecificV, \"bowIronySpecificV\", NOME_FILE)\n",
    "appendDenseRapr(meanVarArrayV, \"VarMeanV\",NOME_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "irony_specific_bow = generaIronySpecificBow(arrayCompleto)\n",
    "appendDenseRaprTuple(irony_specific_bow, \"IronySpecificWS\", 0 ,NOME_FILE)\n",
    "#appendDenseRaprTuple(irony_specific_bow, \"Var\", 1 ,NOME_FILE)\n",
    "#appendDenseRaprTuple(irony_specific_bow, \"Mean\", 2 ,NOME_FILE)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "appendBow(prova, \"bowLemmi\", NOME_FILE, tupla = False)\n",
    "appendBowDPL(nomerappresentazione = \"bowDPL\", file_name = NOME_FILE)\n",
    "appendBow(arrayBigrammiLemmi, \"bowBigramLemmi\", NOME_FILE)\n",
    "appendBow(arrayBigrammiSurface, \"bowBigramSurface\", NOME_FILE)\n",
    "\n",
    "\n",
    "\n",
    "appendBow(arrayBigrammiChar, \"bow2gramChar\", NOME_FILE)\n",
    "appendBow(array3grammiChar, \"bow3gramChar\", NOME_FILE)\n",
    "appendBow(array4grammiChar, \"bow4gramChar\", NOME_FILE)\n",
    "appendBow(array5grammiChar, \"bow5gramChar\", NOME_FILE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Irony Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "#Tipo 0 Surface\n",
    "#Tipo 1 Lemma\n",
    "def arrayIrony(path=\"../irony_corpus_sentences_processed.txt\", ok=False, tipo=0, lower = True, listPOS = ['A','S','V']):\n",
    "    arrayLemmi = open(path, \"r\").readlines()\n",
    "    arrayTokenizzatoLemmi = []\n",
    "    for text in arrayLemmi:\n",
    "        arrayParoleLemm = text.split()\n",
    "        arrayFrase = []\n",
    "        for parola in arrayParoleLemm:\n",
    "            arrayParola = parola.split(\"::\")\n",
    "            if((arrayParola[2] in listPOS) or ok):\n",
    "                if(lower):\n",
    "                    arrayFrase.append(arrayParola[tipo].lower())\n",
    "                else:\n",
    "                    arrayFrase.append(arrayParola[tipo])\n",
    "        arrayTokenizzatoLemmi.append(arrayFrase)\n",
    "    return arrayTokenizzatoLemmi\n",
    "\n",
    "def merge_two_dicts(x, y):\n",
    "    z = x.copy()   # start with x's keys and values\n",
    "    z.update(y)    # modifies z with y's keys and values & returns None\n",
    "    return z\n",
    "\n",
    "def generaDictNgram(arr, numNgram = 0):\n",
    "    dic = dict()\n",
    "    for line in tqdm(arr):\n",
    "        if numNgram != 0:\n",
    "            dic = merge_two_dicts(FreqDist(ngrams(line, numNgram)), dic)\n",
    "        else:\n",
    "            dic = merge_two_dicts(FreqDist(line), dic)\n",
    "    return dic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Genero dizionari con frequenza su file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### NB: il codice qui sotto è da eseguire solo una volta (la prima, cambiando la tipologia di cella da markdown a code) per la generazione su file del dizionario delle frequenze delle parole nell' Irony Corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "from nltk import ngrams, FreqDist\n",
    "import pickle\n",
    "\n",
    "arr = arrayIrony(ok=True)\n",
    "arr2 = arrayIrony(tipo=1)\n",
    "\n",
    "all_counts = dict()\n",
    "\n",
    "for size in 0, 2, 3:\n",
    "    if(size == 0):\n",
    "        all_counts[size] = generaDictNgram(arr2, size)\n",
    "    else:\n",
    "        all_counts[size] = generaDictNgram(arr, size)\n",
    "        \n",
    "with open('unigrammi.pickle', 'wb') as handle:\n",
    "    pickle.dump(all_counts[0], handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "with open('bigrammi.pickle', 'wb') as handle:\n",
    "    pickle.dump(all_counts[2], handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "with open('trigrammi.pickle', 'wb') as handle:\n",
    "    pickle.dump(all_counts[3], handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('unigrammi.pickle', 'rb') as handle:\n",
    "    unigramDict = pickle.load(handle)\n",
    "\n",
    "with open('bigrammi.pickle', 'rb') as handle:\n",
    "    bigramDict = pickle.load(handle)\n",
    "\n",
    "with open('trigrammi.pickle', 'rb') as handle:\n",
    "    trigramDict = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def appendBowIronyCorpus(dataset, nomerappresentazione, dizionarioFrequenze, file_name = \"dataset.klp\"):\n",
    "    f = open(file_name, 'r')\n",
    "    file_lines = []\n",
    "    for index, x in enumerate(f.readlines()):\n",
    "        string = \" |BV:\"+ nomerappresentazione + \"| \"\n",
    "        for word in dataset[index]:\n",
    "            if(word in dizionarioFrequenze):\n",
    "                if(dizionarioFrequenze[word] >= 1):\n",
    "                    string = string + \"_\" + hashlib.md5(''.join(word)).hexdigest() + \":\" + str(np.log(dizionarioFrequenze[word] + 1))  + \" \"\n",
    "        string = string + \"|EV| \"\n",
    "        file_lines.append(''.join([x.strip(), string, '\\n']))\n",
    "    with open(file_name, 'w') as f:\n",
    "        f.writelines(file_lines)\n",
    "        \n",
    "def appendBowBinaryIronyCorpus(dataset, nomerappresentazione, dizionarioFrequenze, file_name = \"dataset.klp\"):\n",
    "    f = open(file_name, 'r')\n",
    "    file_lines = []\n",
    "    for index, x in enumerate(f.readlines()):\n",
    "        string = \" |BV:\"+ nomerappresentazione + \"| \"\n",
    "        for word in dataset[index]:\n",
    "            if(word in dizionarioFrequenze):\n",
    "                if(dizionarioFrequenze[word] >= 1):\n",
    "                    string = string + \"_\" + hashlib.md5(''.join(word)).hexdigest() + \":1.0 \"\n",
    "        string = string + \"|EV| \"\n",
    "        file_lines.append(''.join([x.strip(), string, '\\n']))\n",
    "    with open(file_name, 'w') as f:\n",
    "        f.writelines(file_lines)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "arrayTrigrammiSurface = generaArrayNgrams(arrayCompleto, 3)\n",
    "\n",
    "appendBowIronyCorpus(prova, \"bowIC\", unigramDict, NOME_FILE)\n",
    "appendBowIronyCorpus(arrayBigrammiSurface, \"bowIC2gramSurface\",bigramDict, NOME_FILE)\n",
    "appendBowIronyCorpus(arrayTrigrammiSurface, \"bowIC3gramSurface\",trigramDict, NOME_FILE)\n",
    "\n",
    "appendBowBinaryIronyCorpus(prova, \"bowICBIN\", unigramDict, NOME_FILE)\n",
    "appendBowBinaryIronyCorpus(arrayBigrammiSurface, \"bowICBIN2gramSurface\",bigramDict, NOME_FILE)\n",
    "appendBowBinaryIronyCorpus(arrayTrigrammiSurface, \"bowICBIN3gramSurface\",trigramDict, NOME_FILE)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generaArrayDenso(dataset, dizionarioFrequenze):\n",
    "    combDataset = []\n",
    "    for index, tweet in enumerate(dataset):\n",
    "        comb = np.zeros(4)\n",
    "        intnumparl1 = 0.0\n",
    "        intnumparl0 = 0.0\n",
    "        massimo = 0.0\n",
    "        minimo = 0.0\n",
    "        for word in tweet:\n",
    "            if(word in dizionarioFrequenze):\n",
    "                intnumparl1 += 1\n",
    "                comb[0] += dizionarioFrequenze[word]\n",
    "                if(dizionarioFrequenze[word] > massimo):\n",
    "                    massimo = dizionarioFrequenze[word]\n",
    "                if(dizionarioFrequenze[word] < minimo or intnumparl0 == 0 ):\n",
    "                    minimo = dizionarioFrequenze[word]\n",
    "            intnumparl0 += 1\n",
    "        if(intnumparl0 == 0):\n",
    "            intnumparl0 = 1\n",
    "        comb[0] = comb[0] / intnumparl0\n",
    "        comb[1] = intnumparl1 / intnumparl0\n",
    "        comb[2] = massimo\n",
    "        comb[3] = minimo\n",
    "        combDataset.append(comb)\n",
    "    return combDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "densoUnigram = generaArrayDenso(prova, unigramDict)\n",
    "densoBigram = generaArrayDenso(arrayBigrammiSurface, bigramDict)\n",
    "densoTrigram = generaArrayDenso(arrayTrigrammiSurface, trigramDict)\n",
    "\n",
    "\n",
    "appendDenseRapr(densoUnigram, \"densUnigram\",NOME_FILE)\n",
    "appendDenseRapr(densoBigram, \"densBigram\",NOME_FILE)\n",
    "appendDenseRapr(densoTrigram, \"densTrigram\",NOME_FILE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Controllo del file prima dell'invio "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "def controllaFileOutput(nomeoutput = \"sanitycheck.tsv\", pathTest = \"../test_ironita2018_revnlt_processed.tsv\", pathOutput = \"../testOutput.tsv\"):\n",
    "    f = open(nomeoutput,\"w+\")\n",
    "    test = pd.read_csv(pathTest, sep='\\t')\n",
    "    out = pd.read_csv(pathOutput, sep='\\t')\n",
    "    for x, line in test.iterrows():\n",
    "        lineaOut = out.iloc[x]\n",
    "        if(line[\"id\"] == lineaOut[\"id\"]):\n",
    "            f.write(str(line[\"id\"])+\"\\t\"+str(lineaOut[\"irony\"])+\"\\t\"+str(lineaOut[\"sarcasm\"])+\"\\t\"+line[\"text\"]+\"\\n\")\n",
    "        else:\n",
    "            raise ValueError('Gli id '+ str(line[\"id\"]) + \" e \" + str(lineaOut[\"id\"]) + \" non sono uguali \\n Linea: \" + str(x))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "controllaFileOutput()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Softmax normalizzazione Kernel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.50396867, 0.49603133])"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "def softmax(x):\n",
    "    \"\"\"Compute softmax values for each sets of scores in x.\"\"\"\n",
    "    e_x = np.exp(x - np.max(x))\n",
    "    return e_x / e_x.sum()\n",
    "\n",
    "scores = [0.6985084, 0.6826334]\n",
    "softmax(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "! open ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}