sentiment_analysis_ces.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentence-level analysis on a single book: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T18:39:34.173316Z",
     "start_time": "2018-06-03T18:39:32.882301Z"
    }
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "import re\n",
    "import pandas as pd\n",
    "\n",
    "import numpy as np\n",
    "from scipy.stats import mode\n",
    "# for cosine distance calculation:\n",
    "from scipy import spatial\n",
    "\n",
    "import sklearn.feature_extraction.text as text\n",
    "from sklearn import decomposition\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from textblob import TextBlob\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from nltk.tokenize import wordpunct_tokenize\n",
    "\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T19:08:53.186320Z",
     "start_time": "2018-06-03T19:08:52.601358Z"
    }
   },
   "outputs": [],
   "source": [
    "#unpickle\n",
    "\n",
    "pickloader=open( \"lemmed_books.pkl\", \"rb\" )\n",
    "clean_books = pickle.load(pickloader)\n",
    "pickloader.close()\n",
    "\n",
    "pickloader=open(\"book_titles.pkl\",\"rb\")\n",
    "titles = pickle.load(pickloader)\n",
    "pickloader.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T18:55:49.503001Z",
     "start_time": "2018-06-03T18:55:49.497007Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"the time machine , by h . g . well [ 1898 ] i the time traveller ( for so it will be convenient to speak of him ) wa expounding a recondite matter to u . his grey eye shone and twinkled , and his usually pale face wa flushed and animated . the fire burned brightly , and the soft radiance of the incandescent light in the lily of silver caught the bubble that flashed and passed in our glass . our chair , being his patent , embraced and caressed u rather than submitted to be sat upon , and there wa that luxurious after - dinner atmosphere when thought roams gracefully free of the trammel of precision . and he put it to u in this way -- marking the point with a lean forefinger -- a we sat and lazily admired his earnestness over this new paradox ( a we thought it ) and his fecundity . ' you must follow me carefully . i shall have to controvert one or two idea that are almost universally accepted . the geometry , for instance , they taught you at school is founded on a misconception .' ' is \""
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_books[5][0:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T20:45:40.856012Z",
     "start_time": "2018-06-03T20:45:40.852090Z"
    }
   },
   "outputs": [],
   "source": [
    "pickler = open(\"novel_names_dash.pkl\",\"wb\")\n",
    "pickle.dump(list(titles), pickler)\n",
    "pickler.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_books = [book.lower() for book in clean_books]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T19:02:27.030110Z",
     "start_time": "2018-06-03T18:58:17.408858Z"
    }
   },
   "outputs": [],
   "source": [
    "#I did this already; this is redundant code in case you're starting from preprocessing\n",
    "\n",
    "# Lemming: gentler than stemming \n",
    "lemma=nltk.stem.WordNetLemmatizer()\n",
    "\n",
    "lemma_books = []\n",
    "for book in clean_books:\n",
    "    lemma_book = [lemma.lemmatize(word) for word in wordpunct_tokenize(book)]\n",
    "    lemma_book = (' ').join(lemma_book)\n",
    "    lemma_books.append(lemma_book)\n",
    "\n",
    "lemma_books[0][0:100]\n",
    "\n",
    "clean_books = lemma_books\n",
    "\n",
    "#contractions\n",
    "import re\n",
    "cList = {\n",
    "  \"ain't\": \"am not\",\n",
    "  \"aren't\": \"are not\",\n",
    "  \"can't\": \"cannot\",\n",
    "  \"can't've\": \"cannot have\",\n",
    "  \"'cause\": \"because\",\n",
    "  \"could've\": \"could have\",\n",
    "  \"couldn't\": \"could not\",\n",
    "  \"couldn't've\": \"could not have\",\n",
    "  \"didn't\": \"did not\",\n",
    "  \"doesn't\": \"does not\",\n",
    "  \"don't\": \"do not\",\n",
    "  \"hadn't\": \"had not\",\n",
    "  \"hadn't've\": \"had not have\",\n",
    "  \"hasn't\": \"has not\",\n",
    "  \"haven't\": \"have not\",\n",
    "  \"he'd\": \"he would\",\n",
    "  \"he'd've\": \"he would have\",\n",
    "  \"he'll\": \"he will\",\n",
    "  \"he'll've\": \"he will have\",\n",
    "  \"he's\": \"he is\",\n",
    "  \"how'd\": \"how did\",\n",
    "  \"how'd'y\": \"how do you\",\n",
    "  \"how'll\": \"how will\",\n",
    "  \"how's\": \"how is\",\n",
    "  \"i'd\": \"i would\",\n",
    "  \"i'd've\": \"i would have\",\n",
    "  \"i'll\": \"i will\",\n",
    "  \"i'll've\": \"i will have\",\n",
    "  \"i'm\": \"i am\",\n",
    "  \"i've\": \"i have\",\n",
    "  \"isn't\": \"is not\",\n",
    "  \"it'd\": \"it had\",\n",
    "  \"it'd've\": \"it would have\",\n",
    "  \"it'll\": \"it will\",\n",
    "  \"it'll've\": \"it will have\",\n",
    "  \"it's\": \"it is\",\n",
    "  \"let's\": \"let us\",\n",
    "  \"ma'am\": \"madam\",\n",
    "  \"mayn't\": \"may not\",\n",
    "  \"might've\": \"might have\",\n",
    "  \"mightn't\": \"might not\",\n",
    "  \"mightn't've\": \"might not have\",\n",
    "  \"must've\": \"must have\",\n",
    "  \"mustn't\": \"must not\",\n",
    "  \"mustn't've\": \"must not have\",\n",
    "  \"needn't\": \"need not\",\n",
    "  \"needn't've\": \"need not have\",\n",
    "  \"o'clock\": \"of the clock\",\n",
    "  \"oughtn't\": \"ought not\",\n",
    "  \"oughtn't've\": \"ought not have\",\n",
    "  \"shan't\": \"shall not\",\n",
    "  \"sha'n't\": \"shall not\",\n",
    "  \"shan't've\": \"shall not have\",\n",
    "  \"she'd\": \"she would\",\n",
    "  \"she'd've\": \"she would have\",\n",
    "  \"she'll\": \"she will\",\n",
    "  \"she'll've\": \"she will have\",\n",
    "  \"she's\": \"she is\",\n",
    "  \"should've\": \"should have\",\n",
    "  \"shouldn't\": \"should not\",\n",
    "  \"shouldn't've\": \"should not have\",\n",
    "  \"so've\": \"so have\",\n",
    "  \"so's\": \"so is\",\n",
    "  \"that'd\": \"that would\",\n",
    "  \"that'd've\": \"that would have\",\n",
    "  \"that's\": \"that is\",\n",
    "  \"there'd\": \"there had\",\n",
    "  \"there'd've\": \"there would have\",\n",
    "  \"there's\": \"there is\",\n",
    "  \"they'd\": \"they would\",\n",
    "  \"they'd've\": \"they would have\",\n",
    "  \"they'll\": \"they will\",\n",
    "  \"they'll've\": \"they will have\",\n",
    "  \"they're\": \"they are\",\n",
    "  \"they've\": \"they have\",\n",
    "  \"to've\": \"to have\",\n",
    "  \"wasn't\": \"was not\",\n",
    "  \"we'd\": \"we had\",\n",
    "  \"we'd've\": \"we would have\",\n",
    "  \"we'll\": \"we will\",\n",
    "  \"we'll've\": \"we will have\",\n",
    "  \"we're\": \"we are\",\n",
    "  \"we've\": \"we have\",\n",
    "  \"weren't\": \"were not\",\n",
    "  \"what'll\": \"what will\",\n",
    "  \"what'll've\": \"what will have\",\n",
    "  \"what're\": \"what are\",\n",
    "  \"what's\": \"what is\",\n",
    "  \"what've\": \"what have\",\n",
    "  \"when's\": \"when is\",\n",
    "  \"when've\": \"when have\",\n",
    "  \"where'd\": \"where did\",\n",
    "  \"where's\": \"where is\",\n",
    "  \"where've\": \"where have\",\n",
    "  \"who'll\": \"who will\",\n",
    "  \"who'll've\": \"who will have\",\n",
    "  \"who's\": \"who is\",\n",
    "  \"who've\": \"who have\",\n",
    "  \"why's\": \"why is\",\n",
    "  \"why've\": \"why have\",\n",
    "  \"will've\": \"will have\",\n",
    "  \"won't\": \"will not\",\n",
    "  \"won't've\": \"will not have\",\n",
    "  \"would've\": \"would have\",\n",
    "  \"wouldn't\": \"would not\",\n",
    "  \"wouldn't've\": \"would not have\",\n",
    "  \"y'all\": \"you all\",\n",
    "  \"y'alls\": \"you alls\",\n",
    "  \"y'all'd\": \"you all would\",\n",
    "  \"y'all'd've\": \"you all would have\",\n",
    "  \"y'all're\": \"you all are\",\n",
    "  \"y'all've\": \"you all have\",\n",
    "  \"you'd\": \"you had\",\n",
    "  \"you'd've\": \"you would have\",\n",
    "  \"you'll\": \"you will\",\n",
    "  \"you'll've\": \"you will have\",\n",
    "  \"you're\": \"you are\",\n",
    "  \"you've\": \"you have\"\n",
    "}\n",
    "\n",
    "c_re = re.compile('(%s)' % '|'.join(cList.keys()))\n",
    "\n",
    "def expandContractions(text, c_re=c_re):\n",
    "    def replace(match):\n",
    "        return cList[match.group(0)]\n",
    "    return c_re.sub(replace, text)\n",
    "\n",
    "expandContractions(clean_books[0].lower())\n",
    "\n",
    "clean_books_expanded = [expandContractions(book.lower()) for book in clean_books]\n",
    "\n",
    "clean_books = clean_books_expanded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T19:03:51.437331Z",
     "start_time": "2018-06-03T19:02:27.031967Z"
    }
   },
   "outputs": [],
   "source": [
    "#tokenize as sentences for sentiment analysis\n",
    "\n",
    "clean_book_sentences = []\n",
    "for book in clean_books:\n",
    "    sentences = sent_tokenize(book)\n",
    "    clean_book_sentences.append(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T19:03:53.342174Z",
     "start_time": "2018-06-03T19:03:51.439024Z"
    }
   },
   "outputs": [],
   "source": [
    "pickler = open(\"clean_book_sentences.pkl\",\"wb\")\n",
    "pickle.dump(clean_book_sentences, pickler)\n",
    "pickler.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function for creating an average sentiment polarity for each bin:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_book_sentences[0][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentiments = [TextBlob(sentence).sentiment.polarity for sentence in clean_book_sentences[novel_index]]\n",
    "sentiments[:5] #etc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T19:04:12.092704Z",
     "start_time": "2018-06-03T19:04:12.089634Z"
    }
   },
   "outputs": [],
   "source": [
    "#you might want to play with this depending on article/book/text format\n",
    "number_of_buckets = 5 \n",
    "\n",
    "#Novel has Exposition, Rising Action, Climax, Falling Action, Resolution, hence 5\n",
    "\n",
    "def get_sentiment(novel_index):\n",
    "\n",
    "    #make list of sentiment by sentences\n",
    "    sentiments = [TextBlob(sentence).sentiment.polarity for sentence in clean_book_sentences[novel_index]]\n",
    "    \n",
    "    #bin list into buckets\n",
    "    sentiments_binned = list(np.array_split(sentiments, number_of_buckets))\n",
    "\n",
    "    #take the mean of each bucket for graphing\n",
    "    sentiments_binned = [np.mean(bin) for bin in sentiments_binned]\n",
    "    \n",
    "    return(sentiments_binned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#now chose a book:\n",
    "novel_index=0\n",
    "print('Novel name:', novel_names[novel_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-03T19:04:14.958864Z",
     "start_time": "2018-06-03T19:04:14.154611Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "sentiments_binned = get_sentiment(novel_index)\n",
    "\n",
    "y_pos = np.arange(len(sentiments_binned))\n",
    "plt.figure(figsize=(10,5))\n",
    "plt.bar(y_pos, height=sentiments_binned);\n",
    "plt.ylabel(\"Sentiment polarity\")\n",
    "plt.xlabel(\"Book narrative -->\")\n",
    "plt.title(f'Sentiment trajectory for {novel_names[novel_index]}');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Source for some of this code:\n",
    "- https://de.dariah.eu/\n",
    "- https://de.dariah.eu/tatom/topic_model_python.html\n",
    "- https://de.dariah.eu/tatom/topic_model_visualization.html#topic-model-visualization\n",
    "- https://github.com/Jack-Etheredge/Project-guten-bag-of-words"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}