Papers_Preproc.ipynb

{"cells":[{"cell_type":"markdown","source":["# Text mining and search exam notebook: Papers Preprocessing\n","Authors: Gaetano Chiriaco, Riccardo Porcedda, Gianmarco Russo"],"metadata":{"id":"jeOXPhoFPn0p"}},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"FtUWo8-V2OHZ","executionInfo":{"status":"ok","timestamp":1674315083461,"user_tz":-60,"elapsed":49330,"user":{"displayName":"Gaetano Chiriaco","userId":"10967778284345158401"}},"outputId":"d65f182e-d0ae-4995-ef4c-56410d5d82e7"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"code","execution_count":null,"metadata":{"id":"e2LVz6wZQiIY"},"outputs":[],"source":["import pandas as pd\n","# Import papers\n","papers = pd.read_csv(\"drive/MyDrive/TM&S/data/papers.csv\")"]},{"cell_type":"markdown","source":["#Extract Abstract and Main Text"],"metadata":{"id":"26w_V51a0fe9"}},{"cell_type":"code","source":["papers.head(3)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":240},"id":"LQ8ooXRPbGxW","executionInfo":{"status":"ok","timestamp":1674315089092,"user_tz":-60,"elapsed":36,"user":{"displayName":"Gaetano Chiriaco","userId":"10967778284345158401"}},"outputId":"a568b9f5-edb7-413a-de3b-abf8baf4bb4f"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["    id  year                                              title event_type  \\\n","0    1  1987  Self-Organization of Associative Database and ...        NaN   \n","1   10  1987  A Mean Field Theory of Layer IV of Visual Cort...        NaN   \n","2  100  1988  Storing Covariance by the Associative Long-Ter...        NaN   \n","\n","                                            pdf_name          abstract  \\\n","0  1-self-organization-of-associative-database-an...  Abstract Missing   \n","1  10-a-mean-field-theory-of-layer-iv-of-visual-c...  Abstract Missing   \n","2  100-storing-covariance-by-the-associative-long...  Abstract Missing   \n","\n","                                          paper_text  \n","0  767\\n\\nSELF-ORGANIZATION OF ASSOCIATIVE DATABA...  \n","1  683\\n\\nA MEAN FIELD THEORY OF LAYER IV OF VISU...  \n","2  394\\n\\nSTORING COVARIANCE BY THE ASSOCIATIVE\\n...  "],"text/html":["\n","  <div id=\"df-f8df1238-7ced-4aa6-b732-27eb280935cc\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>year</th>\n","      <th>title</th>\n","      <th>event_type</th>\n","      <th>pdf_name</th>\n","      <th>abstract</th>\n","      <th>paper_text</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>1987</td>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>NaN</td>\n","      <td>1-self-organization-of-associative-database-an...</td>\n","      <td>Abstract Missing</td>\n","      <td>767\\n\\nSELF-ORGANIZATION OF ASSOCIATIVE DATABA...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>10</td>\n","      <td>1987</td>\n","      <td>A Mean Field Theory of Layer IV of Visual Cort...</td>\n","      <td>NaN</td>\n","      <td>10-a-mean-field-theory-of-layer-iv-of-visual-c...</td>\n","      <td>Abstract Missing</td>\n","      <td>683\\n\\nA MEAN FIELD THEORY OF LAYER IV OF VISU...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>100</td>\n","      <td>1988</td>\n","      <td>Storing Covariance by the Associative Long-Ter...</td>\n","      <td>NaN</td>\n","      <td>100-storing-covariance-by-the-associative-long...</td>\n","      <td>Abstract Missing</td>\n","      <td>394\\n\\nSTORING COVARIANCE BY THE ASSOCIATIVE\\n...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f8df1238-7ced-4aa6-b732-27eb280935cc')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-f8df1238-7ced-4aa6-b732-27eb280935cc button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-f8df1238-7ced-4aa6-b732-27eb280935cc');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":3}]},{"cell_type":"code","source":["papers.paper_text[0]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":174},"id":"Uzy2F33v9OEK","executionInfo":{"status":"ok","timestamp":1674315110119,"user_tz":-60,"elapsed":306,"user":{"displayName":"Gaetano Chiriaco","userId":"10967778284345158401"}},"outputId":"414da8b0-ab3a-487e-9d33-e05f2ebb0284"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'767\\n\\nSELF-ORGANIZATION OF ASSOCIATIVE DATABASE\\nAND ITS APPLICATIONS\\nHisashi Suzuki and Suguru Arimoto\\nOsaka University, Toyonaka, Osaka 560, Japan\\nABSTRACT\\nAn efficient method of self-organizing associative databases is proposed together with\\napplications to robot eyesight systems. The proposed databases can associate any input\\nwith some output. In the first half part of discussion, an algorithm of self-organization is\\nproposed. From an aspect of hardware, it produces a new style of neural network. In the\\nlatter half part, an applicability to handwritten letter recognition and that to an autonomous\\nmobile robot system are demonstrated.\\n\\nINTRODUCTION\\nLet a mapping f : X -+ Y be given. Here, X is a finite or infinite set, and Y is another\\nfinite or infinite set. A learning machine observes any set of pairs (x, y) sampled randomly\\nfrom X x Y. (X x Y means the Cartesian product of X and Y.) And, it computes some\\nestimate j : X -+ Y of f to make small, the estimation error in some measure.\\nUsually we say that: the faster the decrease of estimation error with increase of the number of samples, the better the learning machine. However, such expression on performance\\nis incomplete. Since, it lacks consideration on the candidates of J of j assumed preliminarily. Then, how should we find out good learning machines? To clarify this conception,\\nlet us discuss for a while on some types of learning machines. And, let us advance the\\nunderstanding of the self-organization of associative database .\\n. Parameter Type\\nAn ordinary type of learning machine assumes an equation relating x\\'s and y\\'s with\\nparameters being indefinite, namely, a structure of f. It is equivalent to define implicitly a\\nset F of candidates of\\n(F is some subset of mappings from X to Y.) And, it computes\\nvalues of the parameters based on the observed samples. We call such type a parameter\\ntype.\\nFor a learning machine defined well, if F 3 f, j approaches f as the number of samples\\nincreases. In the alternative case, however, some estimation error remains eternally. Thus,\\na problem of designing a learning machine returns to find out a proper structure of f in this\\nsense.\\nOn the other hand, the assumed structure of f is demanded to be as compact as possible\\nto achieve a fast learning. In other words, the number of parameters should be small. Since,\\nif the parameters are few, some j can be uniquely determined even though the observed\\nsamples are few. However, this demand of being proper contradicts to that of being compact.\\nConsequently, in the parameter type, the better the compactness of the assumed structure\\nthat is proper, the better the learning machine. This is the most elementary conception\\nwhen we design learning machines .\\n\\n1.\\n\\n. Universality and Ordinary Neural Networks\\nNow suppose that a sufficient knowledge on f is given though J itself is unknown. In\\nthis case, it is comparatively easy to find out proper and compact structures of J. In the\\nalternative case, however, it is sometimes difficult. A possible solution is to give up the\\ncompactness and assume an almighty structure that can cover various 1\\'s. A combination\\nof some orthogonal bases of the infinite dimension is such a structure. Neural networks 1 ,2\\nare its approximations obtained by truncating finitely the dimension for implementation.\\n\\n? American Institute of Physics 1988\\n\\n\\x0c768\\nA main topic in designing neural networks is to establish such desirable structures of 1.\\nThis work includes developing practical procedures that compute values of coefficients from\\nthe observed samples. Such discussions are :flourishing since 1980 while many efficient methods have been proposed. Recently, even hardware units computing coefficients in parallel\\nfor speed-up are sold, e.g., ANZA, Mark III, Odyssey and E-1.\\nNevertheless, in neural networks, there always exists a danger of some error remaining\\neternally in estimating /. Precisely speaking, suppose that a combination of the bases of a\\nfinite number can define a structure of 1 essentially. In other words, suppose that F 3 /, or\\n1 is located near F. In such case, the estimation error is none or negligible. However, if 1\\nis distant from F, the estimation error never becomes negligible. Indeed, many researches\\nreport that the following situation appears when 1 is too complex. Once the estimation\\nerror converges to some value (> 0) as the number of samples increases, it decreases hardly\\neven though the dimension is heighten. This property sometimes is a considerable defect of\\nneural networks .\\n. Recursi ve Type\\nThe recursive type is founded on another methodology of learning that should be as\\nfollows. At the initial stage of no sample, the set Fa (instead of notation F) of candidates\\nof I equals to the set of all mappings from X to Y. After observing the first sample\\n(Xl, Yl) E X x Y, Fa is reduced to Fi so that I(xt) = Yl for any I E F. After observing\\nthe second sample (X2\\' Y2) E X x Y, Fl is further reduced to F2 so that i(xt) = Yl and\\nI(X2) = Y2 for any I E F. Thus, the candidate set F becomes gradually small as observation\\nof samples proceeds. The after observing i-samples, which we write\\nis one of the most\\nlikelihood estimation of 1 selected in fi;. Hence, contrarily to the parameter type, the\\nrecursive type guarantees surely that j approaches to 1 as the number of samples increases.\\nThe recursive type, if observes a sample (x\" yd, rewrites values 1,-l(X),S to I,(x)\\'s for\\nsome x\\'s correlated to the sample. Hence, this type has an architecture composed of a rule\\nfor rewriting and a free memory space. Such architecture forms naturally a kind of database\\nthat builds up management systems of data in a self-organizing way. However, this database\\ndiffers from ordinary ones in the following sense. It does not only record the samples already\\nobserved, but computes some estimation of l(x) for any x E X. We call such database an\\nassociative database.\\nThe first subject in constructing associative databases is how we establish the rule for\\nrewri ting. For this purpose, we adap t a measure called the dissimilari ty. Here, a dissimilari ty\\nmeans a mapping d : X x X -+ {reals > O} such that for any (x, x) E X x X, d(x, x) > 0\\nwhenever l(x) t /(x). However, it is not necessarily defined with a single formula. It is\\ndefinable with, for example, a collection of rules written in forms of \"if? .. then?? .. \"\\nThe dissimilarity d defines a structure of 1 locally in X x Y. Hence, even though\\nthe knowledge on f is imperfect, we can re:flect it on d in some heuristic way. Hence,\\ncontrarily to neural networks, it is possible to accelerate the speed of learning by establishing\\nd well. Especially, we can easily find out simple d\\'s for those l\\'s which process analogically\\ninformation like a human. (See the applications in this paper.) And, for such /\\'s, the\\nrecursive type shows strongly its effectiveness.\\nWe denote a sequence of observed samples by (Xl, Yd, (X2\\' Y2),???. One of the simplest\\nconstructions of associative databases after observing i-samples (i = 1,2,.,,) is as follows.\\n\\ni\\n\\ni\"\\n\\nI,\\n\\nAlgorithm 1. At the initial stage, let So be the empty set. For every i =\\n1,2\" .. , let i,-l(x) for any x E X equal some y* such that (x*,y*) E S,-l and\\n\\nd(x, x*) =\\n\\nmin\\n(%,y)ES.-t\\n\\nd(x, x) .\\n\\nFurthermore, add (x\" y,) to S;-l to produce Sa, i.e., S, = S,_l U {(x\"\\n\\n(1)\\n\\ny,n.\\n\\n\\x0c769\\n\\nAnother version improved to economize the memory is as follows.\\n\\nAlgorithm 2, At the initial stage, let So be composed of an arbitrary element\\nin X x Y. For every i = 1,2\"\", let ii-lex) for any x E X equal some y. such\\nthat (x?, y.) E Si-l and\\nd(x, x?) =\\n\\nmin\\n\\nd(x, x) .\\n\\n(i,i)ES.-l\\n\\nFurthermore, if ii-l(Xi) # Yi then let Si = Si-l, or add (Xi, Yi) to Si-l to\\nproduce Si, i.e., Si = Si-l U {(Xi, Yi)}\\'\\nIn either construction, ii approaches to f as i increases. However, the computation time\\ngrows proportionally to the size of Si. The second subject in constructing associative\\ndatabases is what addressing rule we should employ to economize the computation time. In\\nthe subsequent chapters, a construction of associative database for this purpose is proposed.\\nIt manages data in a form of binary tree.\\n\\nSELF-ORGANIZATION OF ASSOCIATIVE DATABASE\\nGiven a sample sequence (Xl, Yl), (X2\\' Y2), .. \" the algorithm for constructing associative\\ndatabase is as follows.\\n\\nAlgorithm 3,\\'\\n\\nStep I(Initialization): Let (x[root], y[root]) = (Xl, Yd. Here, x[.] and y[.] are\\nvariables assigned for respective nodes to memorize data.. Furthermore, let t = 1.\\nStep 2: Increase t by 1, and put x, in. After reset a pointer n to the root, repeat\\nthe following until n arrives at some terminal node, i.e., leaf.\\nNotations nand\\nd(xt, x[n)), let n\\n\\nn mean the descendant nodes of n.\\n=n. Otherwise, let n =n.\\n\\nIf d(x\" r[n)) ~\\n\\nStep 3: Display yIn] as the related information. Next, put y, in. If yIn] = y\" back\\nto step 2. Otherwise, first establish new descendant nodes n and n. Secondly,\\nlet\\n\\n(x[n], yIn))\\n(x[n], yIn))\\n\\n(x[n], yIn)),\\n(Xt, y,).\\n\\n(2)\\n(3)\\n\\nFinally, back to step 2. Here, the loop of step 2-3 can be stopped at any time\\nand also can be continued.\\nNow, suppose that gate elements, namely, artificial \"synapses\" that play the role of branching by d are prepared. Then, we obtain a new style of neural network with gate elements\\nbeing randomly connected by this algorithm.\\n\\nLETTER RECOGNITION\\nRecen tly, the vertical slitting method for recognizing typographic English letters3 , the\\nelastic matching method for recognizing hand written discrete English letters4 , the global\\ntraining and fuzzy logic search method for recognizing Chinese characters written in square\\nstyleS, etc. are published. The self-organization of associative database realizes the recognition of handwritten continuous English letters.\\n\\n\\x0c770\\n\\n9 /wn\"\\n\\nNOV\\n\\n~ ~ ~ -xk :La.t\\n\\n~~ ~ ~~~\\n\\ndw1lo\\'\\n\\n~~~~~of~~\\n\\n~~~ 4,-?~~4Fig. 1. Source document.\\n2~~---------------\\'\\n\\nlOO~---------------\\'\\n\\nH\\n\\no\\n\\no\\nFig. 2. Windowing.\\n\\n1000\\n\\n2000\\n\\n3000\\n\\n4000\\n\\nNumber of samples\\n\\no\\n\\n1000\\n\\n2000\\n\\n3000\\n\\n4000\\n\\nNUAlber of sampl es\\n\\nFig. 3. An experiment result.\\n\\nAn image scanner takes a document image (Fig. 1). The letter recognizer uses a parallelogram window that at least can cover the maximal letter (Fig. 2), and processes the\\nsequence of letters while shifting the window. That is, the recognizer scans a word in a\\nslant direction. And, it places the window so that its left vicinity may be on the first black\\npoint detected. Then, the window catches a letter and some part of the succeeding letter.\\nIf recognition of the head letter is performed, its end position, namely, the boundary line\\nbetween two letters becomes known. Hence, by starting the scanning from this boundary\\nand repeating the above operations, the recognizer accomplishes recursively the task. Thus\\nthe major problem comes to identifying the head letter in the window.\\nConsidering it, we define the following.\\n? Regard window images as x\\'s, and define X accordingly.\\n? For a (x, x) E X x X, denote by B a black point in the left area from the boundary on\\nwindow image X. Project each B onto window image x. Then, measure the Euclidean\\ndistance 6 between fj and a black point B on x being the closest to B. Let d(x, x) be\\nthe summation of 6\\'s for all black points B\\'s on x divided by the number of B\\'s.\\n? Regard couples of the \"reading\" and the position of boundary as y\\'s, and define Y\\naccordingly.\\nAn operator teaches the recognizer in interaction the relation between window image and\\nreading& boundary with algorithm 3. Precisely, if the recalled reading is incorrect, the\\noperator teaches a correct reading via the console. Moreover, if the boundary position is\\nincorrect, he teaches a correct position via the mouse.\\nFig. 1 shows partially a document image used in this experiment. Fig. 3 shows the\\nchange of the number of nodes and that of the recognition rate defined as the relative\\nfrequency of correct answers in the past 1000 trials. Speciiications of the window are height\\n= 20dot, width = 10dot, and slant angular = 68deg. In this example, the levels of tree\\nwere distributed in 6-19 at time 4000 and the recognition rate converged to about 74%.\\nExperimentally, the recognition rate converges to about 60-85% in most cases, and to 95% at\\na rare case. However, it does not attain 100% since, e.g., \"c\" and \"e\" are not distinguishable\\nbecause of excessive lluctuation in writing. If the consistency of the x, y-relation is not\\nassured like this, the number of nodes increases endlessly (d. Fig. 3). Hence, it is clever to\\nstop the learning when the recognition rate attains some upper limit. To improve further\\nthe recognition rate, we must consider the spelling of words. It is one of future subjects.\\n\\n\\x0c771\\n\\nOBSTACLE AVOIDING MOVEMENT\\nVarious systems of camera type autonomous mobile robot are reported flourishingly6-1O.\\nThe system made up by the authors (Fig. 4) also belongs to this category. Now, in mathematical methodologies, we solve usually the problem of obstacle avoiding movement as\\na cost minimization problem under some cost criterion established artificially. Contrarily,\\nthe self-organization of associative database reproduces faithfully the cost criterion of an\\noperator. Therefore, motion of the robot after learning becomes very natural.\\nNow, the length, width and height of the robot are all about O.7m, and the weight is\\nabout 30kg. The visual angle of camera is about 55deg. The robot has the following three\\nfactors of motion. It turns less than ?30deg, advances less than 1m, and controls speed less\\nthan 3km/h. The experiment was done on the passageway of wid th 2.5m inside a building\\nwhich the authors\\' laboratories exist in (Fig. 5). Because of an experimental intention, we\\narrange boxes, smoking stands, gas cylinders, stools, handcarts, etc. on the passage way at\\nrandom. We let the robot take an image through the camera, recall a similar image, and\\ntrace the route preliminarily recorded on it. For this purpose, we define the following.\\n? Let the camera face 28deg downward to take an image, and process it through a low\\npass filter. Scanning vertically the filtered image from the bottom to the top, search\\nthe first point C where the luminance changes excessively. Then, su bstitu te all points\\nfrom the bottom to C for white, and all points from C to the top for black (Fig. 6).\\n(If no obstacle exists just in front of the robot, the white area shows the \\'\\'free\\'\\' area\\nwhere the robot can move around.) Regard binary 32 x 32dot images processed thus\\nas x\\'s, and define X accordingly.\\n? For every (x, x) E X x X, let d(x, x) be the number of black points on the exclusive-or\\nimage between x and X.\\n? Regard as y\\'s the images obtained by drawing routes on images x\\'s, and define Y\\naccordingly.\\nThe robot superimposes, on the current camera image x, the route recalled for x, and\\ninquires the operator instructions. The operator judges subjectively whether the suggested\\nroute is appropriate or not. In the negative answer, he draws a desirable route on x with the\\nmouse to teach a new y to the robot. This opera.tion defines implicitly a sample sequence\\nof (x, y) reflecting the cost criterion of the operator.\\n\\n.::l\" !\\n-\\n\\nIibUBe\\n\\n_. -\\n\\n22\\n\\n11\\n\\nRoan\\n\\n12\\n\\n{-\\n\\n13\\n\\nStationary uni t\\n\\nFig. 4. Configuration of\\nautonomous mobile robot system.\\n\\n~\\n\\nI\\n\\n,\\n\\n23\\n\\n24\\n\\nNorth\\n14\\n\\nrmbi Ie unit (robot)\\n\\n-\\n\\nRoan\\n\\ny\\n\\nt\\n\\nFig. 5. Experimental\\nenvironment.\\n\\n\\x0c772\\n\\nWall\\n\\nCamera image\\n\\nPreprocessing\\n\\nA\\n\\n::: !fa\\n\\n?\\n\\nPreprocessing\\n\\n0\\n\\nO\\n\\nCourse\\nsuggest ion\\n\\n??\\n\\n..\\n\\nSearch\\n\\nA\\n\\nFig. 6. Processing for\\nobstacle avoiding movement.\\n\\nx\\n\\nFig. 1. Processing for\\nposition identification.\\nWe define the satisfaction rate by the relative frequency of acceptable suggestions of\\nroute in the past 100 trials. In a typical experiment, the change of satisfaction rate showed\\na similar tendency to Fig. 3, and it attains about 95% around time 800. Here, notice that\\nthe rest 5% does not mean directly the percentage of collision. (In practice, we prevent the\\ncollision by adopting some supplementary measure.) At time 800, the number of nodes was\\n145, and the levels of tree were distributed in 6-17.\\nThe proposed method reflects delicately various characters of operator. For example, a\\nrobot trained by an operator 0 moves slowly with enough space against obstacles while one\\ntrained by another operator 0\\' brushes quickly against obstacles. This fact gives us a hint\\non a method of printing \"characters\" into machines.\\nPOSITION IDENTIFICATION\\nThe robot can identify its position by recalling a similar landscape with the position data\\nto a camera image. For this purpose, in principle, it suffices to regard camera images and\\nposition data as x\\'s and y\\'s, respectively. However, the memory capacity is finite in actual\\ncompu ters. Hence, we cannot but compress the camera images at a slight loss of information.\\nSuch compression is admittable as long as the precision of position identification is in an\\nacceptable area. Thus, the major problem comes to find out some suitable compression\\nmethod.\\nIn the experimental environment (Fig. 5), juts are on the passageway at intervals of\\n3.6m, and each section between adjacent juts has at most one door. The robot identifies\\nroughly from a surrounding landscape which section itself places in. And, it uses temporarily\\na triangular surveying technique if an exact measure is necessary. To realize the former task,\\nwe define the following .\\n? Turn the camera to take a panorama image of 360deg. Scanning horizontally the\\ncenter line, substitute the points where the luminance excessively changes for black\\nand the other points for white (Fig. 1). Regard binary 360dot line images processed\\nthus as x\\'s, and define X accordingly.\\n? For every (x, x) E X x X, project each black point A on x onto x. And, measure the\\nEuclidean distance 6 between A and a black point A on x being the closest to A. Let\\nthe summation of 6 be S. Similarly, calculate S by exchanging the roles of x and X.\\nDenoting the numbers of A\\'s and A\\'s respectively by nand n, define\\n\\n\\x0c773\\n\\nd(x, x) =\\n\\n~(~\\n+ ~).\\n2 n\\nn\\n\\n(4)\\n\\n? Regard positive integers labeled on sections as y\\'s (cf. Fig. 5), and define Y accordingly.\\nIn the learning mode, the robot checks exactly its position with a counter that is reset periodically by the operator. The robot runs arbitrarily on the passageways within 18m area\\nand learns the relation between landscapes and position data. (Position identification beyond 18m area is achieved by crossing plural databases one another.) This task is automatic\\nexcepting the periodic reset of counter, namely, it is a kind of learning without teacher.\\nWe define the identification rate by the relative frequency of correct recalls of position\\ndata in the past 100 trials. In a typical example, it converged to about 83% around time\\n400. At time 400, the number of levels was 202, and the levels oftree were distributed in 522. Since the identification failures of 17% can be rejected by considering the trajectory, no\\npro blem arises in practical use. In order to improve the identification rate, the compression\\nratio of camera images must be loosened. Such possibility depends on improvement of the\\nhardware in the future.\\nFig. 8 shows an example of actual motion of the robot based on the database for obstacle\\navoiding movement and that for position identification. This example corresponds to a case\\nof moving from 14 to 23 in Fig. 5. Here, the time interval per frame is about 40sec.\\n\\n,~. .~ (\\n;~\"i..\\n~\\n\\n\"\\n\\n\"\\n\\n.\\n\\n..I\\n\\nI\\n\\n?\\n?\\n\\n\"\\n\\nI\\'\\n.\\n\\'.1\\nt\\n\\n;\\n\\ni\\n\\n-:\\n, . . , \\'II\\n\\nFig. 8. Actual motion of the robot.\\n\\n\\x0c774\\n\\nCONCLUSION\\nA method of self-organizing associative databases was proposed with the application to\\nrobot eyesight systems. The machine decomposes a global structure unknown into a set of\\nlocal structures known and learns universally any input-output response. This framework\\nof problem implies a wide application area other than the examples shown in this paper.\\nA defect of the algorithm 3 of self-organization is that the tree is balanced well only\\nfor a subclass of structures of f. A subject imposed us is to widen the class. A probable\\nsolution is to abolish the addressing rule depending directly on values of d and, instead, to\\nestablish another rule depending on the distribution function of values of d. It is now under\\ninvestigation.\\n\\nREFERENCES\\n1. Hopfield, J. J. and D. W. Tank, \"Computing with Neural Circuit: A Model/\\'\\n\\nScience 233 (1986), pp. 625-633.\\n2. Rumelhart, D. E. et al., \"Learning Representations by Back-Propagating Errors,\" Nature 323 (1986), pp. 533-536.\\n\\n3. Hull, J. J., \"Hypothesis Generation in a Computational Model for Visual Word\\nRecognition,\" IEEE Expert, Fall (1986), pp. 63-70.\\n4. Kurtzberg, J. M., \"Feature Analysis for Symbol Recognition by Elastic Matching,\" IBM J. Res. Develop. 31-1 (1987), pp. 91-95.\\n\\n5. Wang, Q. R. and C. Y. Suen, \"Large Tree Classifier with Heuristic Search and\\nGlobal Training,\" IEEE Trans. Pattern. Anal. & Mach. Intell. PAMI 9-1\\n(1987) pp. 91-102.\\n6. Brooks, R. A. et al, \"Self Calibration of Motion and Stereo Vision for Mobile\\nRobots,\" 4th Int. Symp. of Robotics Research (1987), pp. 267-276.\\n7. Goto, Y. and A. Stentz, \"The CMU System for Mobile Robot Navigation,\" 1987\\nIEEE Int. Conf. on Robotics & Automation (1987), pp. 99-105.\\n8. Madarasz, R. et al., \"The Design of an Autonomous Vehicle for the Disabled,\"\\nIEEE Jour. of Robotics & Automation RA 2-3 (1986), pp. 117-125.\\n9. Triendl, E. and D. J. Kriegman, \"Stereo Vision and Navigation within Buildings,\" 1987 IEEE Int. Conf. on Robotics & Automation (1987), pp. 1725-1730.\\n10. Turk, M. A. et al., \"Video Road-Following for the Autonomous Land Vehicle,\"\\n1987 IEEE Int. Conf. on Robotics & Automation (1987), pp. 273-279.\\n\\n\\x0c'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":5}]},{"cell_type":"code","source":["len(papers)"],"metadata":{"id":"DBGImWyvO8bE","executionInfo":{"status":"ok","timestamp":1674315091809,"user_tz":-60,"elapsed":7,"user":{"displayName":"Gaetano Chiriaco","userId":"10967778284345158401"}},"outputId":"10f8c6fe-9696-492f-d7cc-e3e9dc8f800b","colab":{"base_uri":"https://localhost:8080/"}},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["7241"]},"metadata":{},"execution_count":4}]},{"cell_type":"markdown","source":["## Where are the words \"abstract\", \"introduction\" and \"references\" located?"],"metadata":{"id":"MCKKmSIva2K0"}},{"cell_type":"code","source":["import re\n","import numpy as np\n","\n","def get_word_position(word: str, text: str, take=\"first\") -> float:\n","\n","  valid_take={\"first\",\"last\"}\n","  if take not in valid_take:\n","    raise ValueError(\"'take' must be one of %r.\" % valid_take)\n","\n","  #remove linebreaks\n","  text = text.replace(\"\\n\",\" \")\n","\n","  try:\n","    #If both upper and capitalize are present, consider the first/last one (indicated by take)\n","    a = re.search(r\"\\b%s\\b\" % word.upper(), text).start()/len(text)\n","    b = re.search(r\"\\b%s\\b\" % word.capitalize(), text).start()/len(text)\n","\n","    if take==\"first\":\n","      return min([a,b])\n","    elif take==\"last\":\n","      return max([a,b])\n","\n","  except AttributeError:\n","    try:\n","      return re.search(r\"\\b%s\\b\" % word.upper(), text).start()/len(text)\n","    except AttributeError:\n","      try:\n","        return re.search(r\"\\b%s\\b\" % word.capitalize(), text).start()/len(text)\n","      except AttributeError:\n","        return -1"],"metadata":{"id":"1IDcH9ptRrmy"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import matplotlib.pyplot as plt\n","\n","# Relative position of the term \"abstract\"\n","hist1 = plt.hist(papers[\"paper_text\"].apply(lambda x: get_word_position(\"abstract\",x)),bins=500,range=(0,0.2))\n","plt.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":54},"id":"nzhrD1t6TgXH","executionInfo":{"status":"ok","timestamp":1670426785051,"user_tz":-60,"elapsed":18491,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"61ef09c9-7096-4f91-ae7c-28a5a1992675"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["<Figure size 432x288 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXcAAAD5CAYAAADcDXXiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAATfklEQVR4nO3df7DldX3f8edLfqw2JgHClVl30Yt2rQOxWdIr0qZpFWJFOsmSxhJsotTSWZPgTJxkOlliO6adMmOmTUicNjhkUJY0ilsTx52AbQmSOs4UdMF15UcIlx8Ou1lhI0o0NDTgu3+cz+rhcn+fe87e/dznY+bM+Z7P9/P9ft/3e768znc/3+85pKqQJPXlRce6AEnS2jPcJalDhrskdchwl6QOGe6S1CHDXZI6dOJSHZK8GPgssKn1/0RVvT/JDcA/Bp5qXf9lVe1PEuC3gYuBp1v73Ytt4/TTT6/p6elV/xGStBHdddddf1FVU/PNWzLcgWeAC6rqW0lOAj6X5NNt3r+pqk/M6f9WYFt7vAG4tj0vaHp6mn379i2jFEnSUUm+stC8JYdlauBb7eVJ7bHYN592ADe25e4ATkmyeSUFS5JGs6wx9yQnJNkPPAHcWlV3tllXJzmQ5Jokm1rbFuCxocUPtjZJ0oQsK9yr6rmq2g5sBc5L8oPAVcBrgdcDpwG/spINJ9mZZF+SfUeOHFlh2ZKkxazobpmq+gZwO3BRVR1uQy/PAB8BzmvdDgFnDi22tbXNXdd1VTVTVTNTU/NeD5AkrdKS4Z5kKskpbfolwJuBPz06jt7ujrkEuKctshd4ZwbOB56qqsNjqV6SNK/l3C2zGdid5AQGHwZ7quqPknwmyRQQYD/wc63/LQxug5xlcCvku9a+bEnSYpYM96o6AJw7T/sFC/Qv4MrRS5MkrZbfUJWkDhnuktShDRfu07tuPtYlSNLYbbhwl6SNwHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1KENFe7+roykjWJDhbskbRSGuyR1aEOGu8Mzknq3IcNdknpnuEtShwx3SeqQ4S5JHVoy3JO8OMnnk3wpyb1J/n1rPyvJnUlmk3w8ycmtfVN7PdvmT4/3T5AkzbWcM/dngAuq6oeA7cBFSc4Hfh24pqr+NvB14IrW/wrg6639mtZPkjRBS4Z7DXyrvTypPQq4APhEa98NXNKmd7TXtPkXJsmaVSxJWtKyxtyTnJBkP/AEcCvwEPCNqnq2dTkIbGnTW4DHANr8p4AfWMuiJUmLW1a4V9VzVbUd2AqcB7x21A0n2ZlkX5J9R44cGXV1kqQhK7pbpqq+AdwO/H3glCQntllbgUNt+hBwJkCb//3A1+ZZ13VVNVNVM1NTU6ssX5I0n+XcLTOV5JQ2/RLgzcD9DEL+ba3b5cCn2vTe9po2/zNVVWtZtCRpcScu3YXNwO4kJzD4MNhTVX+U5D7gpiT/EfgicH3rfz3we0lmgSeBy8ZQtyRpEUuGe1UdAM6dp/1hBuPvc9v/Gvjna1KdJGlV/IaqJHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUMbJtynd918rEuQpInZMOE+HwNfUq82bLgb7JJ6tmHDXZJ6ZrhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHlgz3JGcmuT3JfUnuTfKLrf3XkhxKsr89Lh5a5qoks0keSPKWcf4BkqQXOnEZfZ4Ffrmq7k7yvcBdSW5t866pqv883DnJ2cBlwDnAy4E/TvKaqnpuLQuXJC1syTP3qjpcVXe36W8C9wNbFllkB3BTVT1TVY8As8B5a1GsJGl5VjTmnmQaOBe4szW9J8mBJB9Ocmpr2wI8NrTYQRb/MJAkrbFlh3uSlwJ/ALy3qv4SuBZ4NbAdOAz8xko2nGRnkn1J9h05cmQli0qSlrCscE9yEoNg//2q+kOAqnq8qp6rqm8Dv8t3h14OAWcOLb61tT1PVV1XVTNVNTM1NTXK3yBJmmM5d8sEuB64v6p+c6h981C3nwTuadN7gcuSbEpyFrAN+PzalSxJWspy7pb5EeAdwJeT7G9tvwq8Pcl2oIBHgXcDVNW9SfYA9zG40+ZK75SRpMlaMtyr6nNA5pl1yyLLXA1cPUJdkqQR+A1VSeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUoc2RLhP77p5VfMk6Xi1IcJ9KQa8pN4Y7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocM98bfl5HUkyXDPcmZSW5Pcl+Se5P8Yms/LcmtSR5sz6e29iT5YJLZJAeS/PC4/whJ0vMt58z9WeCXq+ps4HzgyiRnA7uA26pqG3Bbew3wVmBbe+wErl3zqiVJi1oy3KvqcFXd3aa/CdwPbAF2ALtbt93AJW16B3BjDdwBnJJk85pXLkla0IrG3JNMA+cCdwJnVNXhNuurwBltegvw2NBiB1ubJGlClh3uSV4K/AHw3qr6y+F5VVVArWTDSXYm2Zdk35EjR1ayqCRpCcsK9yQnMQj236+qP2zNjx8dbmnPT7T2Q8CZQ4tvbW3PU1XXVdVMVc1MTU2ttn5J0jyWc7dMgOuB+6vqN4dm7QUub9OXA58aan9nu2vmfOCpoeEbSdIEnLiMPj8CvAP4cpL9re1XgQ8Ae5JcAXwFuLTNuwW4GJgFngbetaYVS5KWtGS4V9XngCww+8J5+hdw5Yh1SZJG4DdUJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktSh7sN9etfNx7oESZq47sNdkjYiw12SOmS4S1KHDPchjs9L6oXhLkkdMtwlqUOGuyR1yHCXpA4tGe5JPpzkiST3DLX9WpJDSfa3x8VD865KMpvkgSRvGVfhkqSFLefM/Qbgonnar6mq7e1xC0CSs4HLgHPaMr+T5IS1KnYSvGNGUg+WDPeq+izw5DLXtwO4qaqeqapHgFngvBHqkyStwihj7u9JcqAN25za2rYAjw31OdjaJEkTtNpwvxZ4NbAdOAz8xkpXkGRnkn1J9h05cmSVZUiS5rOqcK+qx6vquar6NvC7fHfo5RBw5lDXra1tvnVcV1UzVTUzNTW1mjIkSQtYVbgn2Tz08ieBo3fS7AUuS7IpyVnANuDzo5UoSVqpE5fqkORjwBuB05McBN4PvDHJdqCAR4F3A1TVvUn2APcBzwJXVtVz4yldkrSQJcO9qt4+T/P1i/S/Grh6lKIkSaPxG6qS1CHDfR5+kUnS8c5wl6QOdR3unoFL2qi6DndJ2qgMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcN9Af4WvKTjmeG+CANe0vHKcJekDi0Z7kk+nOSJJPcMtZ2W5NYkD7bnU1t7knwwyWySA0l+eJzFS5Lmt5wz9xuAi+a07QJuq6ptwG3tNcBbgW3tsRO4dm3KlCStxJLhXlWfBZ6c07wD2N2mdwOXDLXfWAN3AKck2bxWxUqSlme1Y+5nVNXhNv1V4Iw2vQV4bKjfwdYmSZqgkS+oVlUBtdLlkuxMsi/JviNHjoxahiRpyGrD/fGjwy3t+YnWfgg4c6jf1tb2AlV1XVXNVNXM1NTUKsuQJM1nteG+F7i8TV8OfGqo/Z3trpnzgaeGhm8kSRNy4lIdknwMeCNwepKDwPuBDwB7klwBfAW4tHW/BbgYmAWeBt41hpolSUtYMtyr6u0LzLpwnr4FXDlqUZKk0fgNVUnqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwz3JUzvuvlYlyBJK2a4L4MBL+l4Y7hLUocM92Xy7F3S8cRwl6QOnTjKwkkeBb4JPAc8W1UzSU4DPg5MA48Cl1bV10crU5K0Emtx5v6mqtpeVTPt9S7gtqraBtzWXkuSJmgcwzI7gN1tejdwyRi2cUw47i7peDFquBfwv5LclWRnazujqg636a8CZ4y4jVUxiCVtZCONuQP/sKoOJXkZcGuSPx2eWVWVpOZbsH0Y7AR4xSteMWIZkqRhI525V9Wh9vwE8EngPODxJJsB2vMTCyx7XVXNVNXM1NTUKGVIkuZYdbgn+Z4k33t0GvgnwD3AXuDy1u1y4FOjFilJWplRhmXOAD6Z5Oh6PlpV/yPJF4A9Sa4AvgJcOnqZkqSVWHW4V9XDwA/N0/414MJRipIkjcZvqEpShwz3FfIWS0nHA8NdkjrUZbh7di1po+sy3CVpozPcJalDhvsqOfQjaT0z3CWpQ4a7JHXIcF8Fh2QkrXeGuyR1aNTfc9/whs/iH/3APz2GlUjSd3nmPgKHZyStV4a7JHXIcJekDhnua+joMI3DNZKONcNdkjpkuEtShwz3NebQjKT1wHCXpA51F+6eMUtSh+G+niz1QeMHkaRxGVu4J7koyQNJZpPsGtd21rvpXTcb4pImbiy/LZPkBOC/Am8GDgJfSLK3qu4bx/aOWu8hut7rk9SPcZ25nwfMVtXDVfX/gJuAHWPaFrD+g3Oh+hZrX+9/k6T1a1zhvgV4bOj1wda25noIwKNBPt/fsti84T5rWcti6z1e9vdy6lztB+jxsg+0saWq1n6lyduAi6rqX7fX7wDeUFXvGeqzE9jZXv4d4IFVbu504C9GKHdc1mtdsH5rs66Vsa6V6bGuV1bV1HwzxvV77oeAM4deb21t31FV1wHXjbqhJPuqambU9ay19VoXrN/arGtlrGtlNlpd4xqW+QKwLclZSU4GLgP2jmlbkqQ5xnLmXlXPJnkP8D+BE4APV9W949iWJOmFxva/2auqW4BbxrX+ISMP7YzJeq0L1m9t1rUy1rUyG6qusVxQlSQdW/78gCR1aN2F+1I/W5BkU5KPt/l3JpkemndVa38gyVuWu85x1pXkzUnuSvLl9nzB0DJ/0ta5vz1eNsG6ppP836Ftf2homb/X6p1N8sEkmWBdPzNU0/4k306yvc2bxP76R0nuTvJsu6V3eN7lSR5sj8uH2iexv+atK8n2JP8nyb1JDiT56aF5NyR5ZGh/bZ9UXW3ec0Pb3jvUflZ7z2fbMXDypOpK8qY5x9dfJ7mkzZvE/vqlJPe19+q2JK8cmre2x1dVrZsHg4uvDwGvAk4GvgScPafPLwAfatOXAR9v02e3/puAs9p6TljOOsdc17nAy9v0DwKHhpb5E2DmGO2vaeCeBdb7eeB8IMCngbdOqq45fV4HPDTh/TUN/F3gRuBtQ+2nAQ+351Pb9KkT3F8L1fUaYFubfjlwGDilvb5huO8k91eb960F1rsHuKxNfwj4+UnWNec9fRL4WxPcX28a2t7P893/Htf8+FpvZ+7L+dmCHcDuNv0J4ML2SbYDuKmqnqmqR4DZtr61+CmEVddVVV+sqj9v7fcCL0myaYXbX/O6Flphks3A91XVHTU4sm4ELjlGdb29LbtWlqyrqh6tqgPAt+cs+xbg1qp6sqq+DtwKXDSp/bVQXVX1Z1X1YJv+c+AJYN4vtazCKPtrXu09voDBew6DY2Bi+2uOtwGfrqqnV7j9Ueq6fWh7dzD4DhCM4fhab+G+nJ8t+E6fqnoWeAr4gUWWXYufQhilrmE/BdxdVc8MtX2k/RPw363in/Oj1nVWki8m+d9JfnSo/8El1jnuuo76aeBjc9rGvb9Wuuyk9teSkpzH4IzxoaHmq9sQwDWrOKkYta4XJ9mX5I6jQx8M3uNvtPd8Netci7qOuowXHl+T3F9XMDgTX2zZVR9f6y3cu5XkHODXgXcPNf9MVb0O+NH2eMcESzoMvKKqzgV+Cfhoku+b4PYXleQNwNNVdc9Q87HcX+taO8P7PeBdVXX0bPUq4LXA6xn8c/9XJlzWK2vwzct/AfxWkldPePsLavvrdQy+i3PUxPZXkp8FZoD/NK5trLdwX/JnC4b7JDkR+H7ga4ssu5x1jrMukmwFPgm8s6q+c1ZVVYfa8zeBjzL4Z91E6mrDV19r27+Lwdnea1r/rUPLT3x/NS84q5rQ/lrpspPaXwtqH8o3A++rqjuOtlfV4Rp4BvgIk91fw+/Xwwyul5zL4D0+pb3nK17nWtTVXAp8sqr+ZqjeieyvJD8GvA/4iaF/xa/98bXaiwfjeDD4UtXDDC6IHr0gcc6cPlfy/Atxe9r0OTz/gurDDC5wLLnOMdd1Suv/z+ZZ5+lt+iQGY5A/N8G6poAT2vSr2gFzWs1/AefiSdXVXr+o1fOqSe+vob438MILqo8wuNh1apue2P5apK6TgduA987Td3N7DvBbwAcmWNepwKY2fTrwIO3iIvDfef4F1V+YVF1D7XcAb5r0/mLwAfcQ7SL4OI+vZRc+qQdwMfBnbQe8r7X9BwafcgAvbgfHbPujhwPgfW25Bxi6ojzfOidVF/Bvgb8C9g89XgZ8D3AXcIDBhdbfpoXthOr6qbbd/cDdwI8PrXMGuKet87/Qvuw2wffxjcAdc9Y3qf31egbjmn/F4Czz3qFl/1Wrd5bB8Mck99e8dQE/C/zNnONre5v3GeDLrbb/Brx0gnX9g7btL7XnK4bW+ar2ns+2Y2DThN/HaQYnDy+as85J7K8/Bh4feq/2juv48huqktSh9TbmLklaA4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkd+v+X6zPYonKUPAAAAABJRU5ErkJggg==\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":["# Compute the quantile\n","abs_quant = np.quantile(papers[\"paper_text\"].apply(lambda x: get_word_position(\"abstract\",x)),0.95)\n","abs_quant"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"NwsvNotpZYie","executionInfo":{"status":"ok","timestamp":1670426795359,"user_tz":-60,"elapsed":10316,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"365d7ff1-5ed6-4b30-e59d-71648b64f039"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.020143429656190676"]},"metadata":{},"execution_count":7}]},{"cell_type":"code","source":["# Relative position of the term \"introduction\"\n","hist2 = plt.hist(papers[\"paper_text\"].apply(lambda x: get_word_position(\"introduction\",x)),bins=500,range=(0,1))\n","plt.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":54},"id":"9Bz6z2oqVqPc","executionInfo":{"status":"ok","timestamp":1670426806968,"user_tz":-60,"elapsed":11614,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"0999309b-d306-46a1-ebac-1b86a0ca6f09"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["<Figure size 432x288 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAN+UlEQVR4nO3df4xlZ13H8feHLgWN2Jbu2DS7KwOhqBuM0GxqCYliC6ZU021iISUCq1ndAMVgMNE1/OHPP+APqZIQdLUNC0HaisZuaI3B/kgjscWpLYW2QYZa7K6FHUq7aghI5esf96kO6+zeuzv3R+9z36/kZp7znGfO+T5zZz5z5txzz6SqkCT15TmzLkCSNH6GuyR1yHCXpA4Z7pLUIcNdkjq0ZdYFAGzdurWWl5dnXYYkzZV77733a1W1tNG6Z0W4Ly8vs7KyMusyJGmuJPnyidZ5WkaSOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjq0UOG+vP+WWZcgSVOxUOEuSYvCcJekDhnuktQhw12SOrRw4e6LqpIWwcKFuyQtgoUJd4/YJS2ShQl3SVokhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR1ayHD3DU2SejdyuCc5I8l9ST7Zll+c5J4kq0luTHJm639eW15t65cnU7ok6URO5cj9XcDD65bfB1xbVS8FngT2tv69wJOt/9o2TpI0RSOFe5LtwM8Af9aWA1wCfKINOQhc2dq72zJt/aVtvCRpSkY9cv9D4NeB77Tlc4GnqurptnwY2Nba24DHANr6Y228JGlKhoZ7kp8FjlbVvePccZJ9SVaSrKytrY1z05K08EY5cn81cEWSR4EbGJyO+SPg7CRb2pjtwJHWPgLsAGjrzwKeOH6jVXWgqnZV1a6lpaVNTUKS9N2GhntV/WZVba+qZeBq4Paq+nngDuCqNmwPcHNrH2rLtPW3V1WNtWpJ0klt5jr33wDenWSVwTn161r/dcC5rf/dwP7NlShJOlVbhg/5P1V1J3Bnaz8CXLTBmG8CbxhDbZKk07SQ71CVpN4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDi1EuC/vv2XWJUjSVC1EuEvSojHcJalDhrskdchwl6QOGe6S1CHDXZI6tLDh7uWRknq2sOEuST0z3CWpQwsd7p6akdSrhQ53SeqV4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0aGu5Jnp/kM0k+m+TBJL/T+l+c5J4kq0luTHJm639eW15t65cnOwVJ0vFGOXL/FnBJVf0Y8ArgsiQXA+8Drq2qlwJPAnvb+L3Ak63/2jZOkjRFQ8O9Bv6zLT63PQq4BPhE6z8IXNnau9sybf2lSTK2iiVJQ410zj3JGUnuB44CnwK+BDxVVU+3IYeBba29DXgMoK0/Bpy7wTb3JVlJsrK2tra5WUiSvstI4V5V/11VrwC2AxcBP7zZHVfVgaraVVW7lpaWNrs5SdI6p3S1TFU9BdwBvAo4O8mWtmo7cKS1jwA7ANr6s4AnxlKtJGkko1wts5Tk7Nb+HuB1wMMMQv6qNmwPcHNrH2rLtPW3V1WNs2hJ0sltGT6E84GDSc5g8Mvgpqr6ZJKHgBuS/D5wH3BdG38d8NEkq8DXgasnULck6SSGhntVPQC8coP+Rxicfz++/5vAG8ZSnSTptPgOVUnqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqUPfhvrz/llmXIElT1324D2P4S+rRwoe7JPXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUoaHhnmRHkjuSPJTkwSTvav0vTPKpJF9sH89p/UnygSSrSR5IcuGkJyFJ+m6jHLk/DfxaVe0ELgauSbIT2A/cVlUXALe1ZYDXAxe0xz7gQ2OvWpJ0UkPDvaoer6p/au3/AB4GtgG7gYNt2EHgytbeDXykBu4Gzk5y/tgrlySd0Cmdc0+yDLwSuAc4r6oeb6u+ApzX2tuAx9Z92uHWd/y29iVZSbKytrZ2imVLkk5m5HBP8n3AXwK/WlX/vn5dVRVQp7LjqjpQVbuqatfS0tKpfKokaYiRwj3JcxkE+8eq6q9a91efOd3SPh5t/UeAHes+fXvrkyRNyShXywS4Dni4qt6/btUhYE9r7wFuXtf/1nbVzMXAsXWnbyRJUzDKkfurgbcAlyS5vz0uB94LvC7JF4HXtmWAW4FHgFXgT4F3jL/s8fJf7UnqzZZhA6rq74GcYPWlG4wv4JpN1iVJ2gTfoSpJHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGe+P/UZXUE8NdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOG+jveXkdQLw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1aGi4J7k+ydEkn1/X98Ikn0ryxfbxnNafJB9IsprkgSQXTrJ4SdLGRjly/zBw2XF9+4HbquoC4La2DPB64IL22Ad8aDxlSpJOxdBwr6q7gK8f170bONjaB4Er1/V/pAbuBs5Ocv64ipUkjeZ0z7mfV1WPt/ZXgPNaexvw2Lpxh1vf/5NkX5KVJCtra2unWYYkaSObfkG1qgqo0/i8A1W1q6p2LS0tbbYMSdI6pxvuX33mdEv7eLT1HwF2rBu3vfXNhPeKkbSoTjfcDwF7WnsPcPO6/re2q2YuBo6tO30jSZqSLcMGJPk48Bpga5LDwG8B7wVuSrIX+DLwxjb8VuByYBX4BvCLE6hZkjTE0HCvqjedYNWlG4wt4JrNFiVJ2hzfoSpJHTLcJalDhrskdchwl6QOGe6S1CHDfQO++UnSvDPcJalDhvtxPGqX1APDXZI6ZLhLUocMd0nqULfh7rlzSYus23CXpEVmuEtShwx3SeqQ4S5JHTLcJalDhvsJeLWNpHlmuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe4n4eWQkuaV4S5JHTLcJalDhrskdchwH8Lz7pLmkeEuSR0y3CWpQ4a7JHXIcJekDhnuI/BFVUnzxnCXpA4Z7iPy6F3SPOky3CcVxAa8pHnRZbhPmiEv6dluIuGe5LIkX0iymmT/JPZxIpMO3me2v7z/FkNe0rPW2MM9yRnAB4HXAzuBNyXZOe79PJsY9JKGmXZOTOLI/SJgtaoeqar/Am4Adk9gP8Dsj6TX7/NE+19f47D1pzuHefjlMskaR932ycYN28asv8az3v+sjGveoz6/z/wcbvQ42fbWjzlRe6Oxk5KqGu8Gk6uAy6rql9ryW4Afr6p3HjduH7CvLf4Q8IXT3OVW4Gun+bnzyjkvBue8GDYz5xdV1dJGK7acfj2bU1UHgAOb3U6SlaraNYaS5oZzXgzOeTFMas6TOC1zBNixbnl765MkTckkwv0fgQuSvDjJmcDVwKEJ7EeSdAJjPy1TVU8neSfwt8AZwPVV9eC497POpk/tzCHnvBic82KYyJzH/oKqJGn2fIeqJHXIcJekDs1NuA+7pUGS5yW5sa2/J8ny9KscrxHm/O4kDyV5IMltSV40izrHadRbVyT5uSSVZO4vmxtlzkne2J7rB5P8+bRrHLcRvrd/MMkdSe5r39+Xz6LOcUlyfZKjST5/gvVJ8oH29XggyYWb3mlVPesfDF6Y/RLwEuBM4LPAzuPGvAP449a+Grhx1nVPYc4/BXxva799Eebcxr0AuAu4G9g167qn8DxfANwHnNOWf2DWdU9hzgeAt7f2TuDRWde9yTn/BHAh8PkTrL8c+BsgwMXAPZvd57wcuY9yS4PdwMHW/gRwaZJMscZxGzrnqrqjqr7RFu9m8J6CeTbqrSt+D3gf8M1pFjcho8z5l4EPVtWTAFV1dMo1jtsocy7g+1v7LODfpljf2FXVXcDXTzJkN/CRGrgbODvJ+ZvZ57yE+zbgsXXLh1vfhmOq6mngGHDuVKqbjFHmvN5eBr/559nQObc/V3dUVS83WhnleX4Z8LIkn05yd5LLplbdZIwy598G3pzkMHAr8CvTKW1mTvXnfaiZ3X5A45PkzcAu4CdnXcskJXkO8H7gF2ZcyrRtYXBq5jUM/jq7K8mPVtVTM61qst4EfLiq/iDJq4CPJnl5VX1n1oXNi3k5ch/llgb/OybJFgZ/yj0xleomY6TbOCR5LfAe4Iqq+taUapuUYXN+AfBy4M4kjzI4N3lozl9UHeV5PgwcqqpvV9W/AP/MIOzn1Shz3gvcBFBV/wA8n8ENtno19tu2zEu4j3JLg0PAnta+Cri92isVc2ronJO8EvgTBsE+7+dhYcicq+pYVW2tquWqWmbwOsMVVbUym3LHYpTv7b9mcNROkq0MTtM8Ms0ix2yUOf8rcClAkh9hEO5rU61yug4Bb21XzVwMHKuqxze1xVm/inwKrzZfzuCI5UvAe1rf7zL44YbBk/8XwCrwGeAls655CnP+O+CrwP3tcWjWNU96zseNvZM5v1pmxOc5DE5HPQR8Drh61jVPYc47gU8zuJLmfuCnZ13zJuf7ceBx4NsM/hLbC7wNeNu65/iD7evxuXF8X3v7AUnq0LyclpEknQLDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXofwC3dvBSoRClvgAAAABJRU5ErkJggg==\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":["# Compute the quantile\n","intro_quant = np.quantile(papers[\"paper_text\"].apply(lambda x: get_word_position(\"introduction\",x)),0.95)\n","intro_quant"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"siX2IUe6aH1a","executionInfo":{"status":"ok","timestamp":1670426817439,"user_tz":-60,"elapsed":10478,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"6ae454cb-b89c-4616-8311-c4f97cf10cc7"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.0713353115727003"]},"metadata":{},"execution_count":9}]},{"cell_type":"code","source":["# Relative position of the term \"references\"\n","hist3 = plt.hist(papers[\"paper_text\"].apply(lambda x: get_word_position(\"references\",x,take=\"last\")),bins=500,range=(0,1))\n","plt.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":54},"id":"cO0gLP8ZaSal","executionInfo":{"status":"ok","timestamp":1670426834014,"user_tz":-60,"elapsed":16586,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"3d2463e8-af00-4841-98fa-0434cbd973b0"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["<Figure size 432x288 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAQnElEQVR4nO3df4xlZX3H8fdHttrUYkF3JIQfjhgwRduudkJtWi2KtYgNaGsopCpa6oqVpq1NmrUm1diYWCuamFrsGjdgowhK0U3AVkpVUiPqIHRdUBRwqbtdd0ewaKqlAt/+MWfb6zjL3Jn7a+4z71dyM+c855x7vs/cmc+c+9xzzqSqkCS15VGTLkCSNHyGuyQ1yHCXpAYZ7pLUIMNdkhq0adIFAGzevLlmZ2cnXYYkTZWbb77521U1s9yydRHus7OzzM/PT7oMSZoqSe453DKHZSSpQYa7JDXIcJekBhnuktSgFcM9yY4kB5Ps7mm7Msmt3WNPklu79tkkP+hZ9t5RFi9JWl4/Z8tcBvwN8IFDDVX1O4emk1wC3N+z/l1VtWVYBUqSVm/FcK+qG5PMLrcsSYBzgecNtyxJ0iAGHXN/NnCgqr7e0/bkJLck+UySZw/4/JKkNRj0IqbzgSt65vcDJ1bVvUl+EfhYkqdV1XeXbphkK7AV4MQTTxywDElSrzUfuSfZBPwWcOWhtqp6oKru7aZvBu4CTllu+6raXlVzVTU3M7Ps1bOSpDUaZFjm+cBXq2rvoYYkM0mO6KZPAk4G7h6sREnSavVzKuQVwOeApybZm+TCbtF5/OiQDMBzgF3dqZEfBS6qqvuGWbAkDcvstmsnXcLI9HO2zPmHaX/lMm1XA1cPXpYkaRBeoSpJDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLmnDafk/MB1iuEtSgwx3SWqQ4S5JDTLcJalBK4Z7kh1JDibZ3dP25iT7ktzaPc7qWfaGJHcmuSPJb4yqcEnS4fVz5H4ZcOYy7e+qqi3d4zqAJKcC5wFP67b52yRHDKtYSVJ/Vgz3qroRuK/P5zsH+HBVPVBV3wDuBE4boD5J0hoMMuZ+cZJd3bDN0V3bccA3e9bZ27X9mCRbk8wnmV9YWBigDEnSUmsN90uBpwBbgP3AJat9gqraXlVzVTU3MzOzxjIkabhaucBpTeFeVQeq6qGqehh4H/8/9LIPOKFn1eO7NknSGK0p3JMc2zP7EuDQmTQ7gfOSPCbJk4GTgS8MVqIkabU2rbRCkiuA04HNSfYCbwJOT7IFKGAP8BqAqrotyVXA7cCDwOuq6qHRlC5JOpwVw72qzl+m+f2PsP5bgbcOUpQkjdvstmvZ87YXTbqMofEKVUkbSisfmK7EcJekBhnuktQgw12SGmS4S9qQWh97N9wlqUGGuyQt0cJRveEuacNrIcyXMtwlqUGGuyQ1yHCXtGG0OPxyOIa7JB3GNP8xMNwlqTPNYb6U4S5JDTLcJalBhrukDa2loZhehrskNchwl6QGGe6S1CDDXZJWMLvt2qkbmzfcJalBK4Z7kh1JDibZ3dP210m+mmRXkmuSHNW1zyb5QZJbu8d7R1m8JGl5/Ry5XwacuaTteuDpVfXzwNeAN/Qsu6uqtnSPi4ZTpiRpNVYM96q6EbhvSdsnq+rBbvYm4PgR1CZJWqNhjLn/HvCJnvknJ7klyWeSPPtwGyXZmmQ+yfzCwsIQypCk4Zu2D1IPGSjck7wReBD4YNe0Hzixqp4BvB74UJLHLbdtVW2vqrmqmpuZmRmkDEnSEmsO9ySvBH4T+N2qKoCqeqCq7u2mbwbuAk4ZQp2SpFVYU7gnORP4M+Dsqvp+T/tMkiO66ZOAk4G7h1GoJKl/m1ZaIckVwOnA5iR7gTexeHbMY4DrkwDc1J0Z8xzgLUl+CDwMXFRV9y37xJI0RtM6dr5WK4Z7VZ2/TPP7D7Pu1cDVgxYlSRqMV6hKUoMMd0nNW8uQzLQP4xjuktQgw12SGmS4S1KDDHdJTZv2sfO1MtwlqU/T9IfCcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlaRWm5XRIw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAb1Fe5JdiQ5mGR3T9vjk1yf5Ovd16O79iR5d5I7k+xK8sxRFS9JWl6/R+6XAWcuadsG3FBVJwM3dPMALwRO7h5bgUsHL1OStBp9hXtV3Qjct6T5HODybvpy4MU97R+oRTcBRyU5dhjFSpL6M8iY+zFVtb+b/hZwTDd9HPDNnvX2dm0/IsnWJPNJ5hcWFgYoQ5LGaxpuQTCUD1SrqoBa5Tbbq2ququZmZmaGUYYkqTNIuB84NNzSfT3Yte8DTuhZ7/iuTZI0JoOE+07ggm76AuDjPe2v6M6aeRZwf8/wjSRpDDb1s1KSK4DTgc1J9gJvAt4GXJXkQuAe4Nxu9euAs4A7ge8DrxpyzZKkFfQV7lV1/mEWnbHMugW8bpCiJEmD8QpVSWqQ4S5JDTLcJalBhrukZk3DxUajYrhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdktZoPd8vvq9/kL2cJE8FruxpOgn4C+Ao4NXAQtf+51V13ZorlKR1aD0HOwwQ7lV1B7AFIMkRwD7gGuBVwLuq6h1DqVCStGrDGpY5A7irqu4Z0vNJkgYwrHA/D7iiZ/7iJLuS7Ehy9JD2IUnq08DhnuTRwNnAR7qmS4GnsDhksx+45DDbbU0yn2R+YWFhuVUkSWs0jCP3FwJfqqoDAFV1oKoeqqqHgfcBpy23UVVtr6q5qpqbmZkZQhmSpEOGEe7n0zMkk+TYnmUvAXYPYR+StCrr/WyWUVvz2TIASR4L/Drwmp7mtyfZAhSwZ8kySdIYDBTuVfVfwBOWtL18oIokSQPzClVJapDhLkkNMtwlqUGGuyQ1yHCX1IzZbddu+FMgDzHcJalBhrskNchwl6QGGe6S1CDDXVJz/FDVcJekgazXPySGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJWkI1tv57oa7JDXIcJekBm0a9AmS7AG+BzwEPFhVc0keD1wJzAJ7gHOr6juD7kuS1J9hHbk/t6q2VNVcN78NuKGqTgZu6OYlSWMyqmGZc4DLu+nLgRePaD+SNHHr7cNUGE64F/DJJDcn2dq1HVNV+7vpbwHHLN0oydYk80nmFxYWhlCGJOmQgcfcgV+tqn1Jnghcn+SrvQurqpLU0o2qajuwHWBubu7HlkuS1m7gI/eq2td9PQhcA5wGHEhyLED39eCg+5Ek9W+gcE/y2CRHHpoGXgDsBnYCF3SrXQB8fJD9SJJWZ9BhmWOAa5Iceq4PVdU/JvkicFWSC4F7gHMH3I8kaRUGCvequhv4hWXa7wXOGOS5Jakfh85U2fO2F024kvXFK1QlqUGGu6QmrMdzzSfJcJekBhnuktQgw13S1HIo5vAMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJGpLZbdeumwurDHdJapDhLkkNMtwlqUGGuyQN2XoYdzfcJalBhrukqbMejozXO8Ndkhq05nBPckKSTyW5PcltSf6oa39zkn1Jbu0eZw2vXElSPzYNsO2DwJ9W1ZeSHAncnOT6btm7quodg5cnSVqLNYd7Ve0H9nfT30vyFeC4YRUmSctxvL0/QxlzTzILPAP4fNd0cZJdSXYkOfow22xNMp9kfmFhYRhlSNpADPlHNnC4J/lp4Grgj6vqu8ClwFOALSwe2V+y3HZVtb2q5qpqbmZmZtAyJEk9Bgr3JD/BYrB/sKr+AaCqDlTVQ1X1MPA+4LTBy5QkrcYgZ8sEeD/wlap6Z0/7sT2rvQTYvfbyJElrMciR+68ALweet+S0x7cn+XKSXcBzgT8ZRqGSNjbH2FdnkLNl/hXIMouuW3s5kqRh8ApVSRqBSb/TMNwlrWu9ITnpwJwmhrukdc9QXz3DXZIaZLhLWpc8Wh+M4S5JDTLcJa0rHrEPh+EuSQ0y3CWpQYa7JDXIcJe0bjn+vnaGu6R1odUgn1S/DHdJGpFJ/sEy3CVNzNLwOzTf6lH8OBnukoZmuVA2qCfzPTDcJWlMxhnyhrskNchwlzQSqzlKdehm+Ax3SWNxuA9PNRqGu7SEoTM6s9uu9fs7Joa7pFU53L+9e6R/h+dR+/j7PLJwT3JmkjuS3Jlk26j2I2n1VgqacY6Xb8SgH4eRhHuSI4D3AC8ETgXOT3LqKPYlSfpxozpyPw24s6rurqr/AT4MnDOifQ3Nej6C6Le2UfWhn7fVjzSe2nvl4UrrDGI1b/8P1bLao9jV9mESV2EuHSJZzevVT9/6/R6s59+pSRnX9yRVNfwnTV4KnFlVv9/Nvxz4paq6uGedrcDWbvapwB0D7HIz8O0Btp82G62/YJ83Cvu8Ok+qqpnlFmxaez2DqartwPZhPFeS+aqaG8ZzTYON1l+wzxuFfR6eUQ3L7ANO6Jk/vmuTJI3BqML9i8DJSZ6c5NHAecDOEe1LkrTESIZlqurBJBcD/wQcAeyoqttGsa/OUIZ3pshG6y/Y543CPg/JSD5QlSRNlleoSlKDDHdJatDUhPtKtzNI8pgkV3bLP59kdvxVDlcffX59ktuT7EpyQ5InTaLOYer3thVJfjtJJZn60+b66XOSc7vX+rYkHxp3jcPWx8/2iUk+leSW7uf7rEnUOSxJdiQ5mGT3YZYnybu778euJM8ceKdVte4fLH4oexdwEvBo4N+AU5es8wfAe7vp84ArJ133GPr8XOCnuunXboQ+d+sdCdwI3ATMTbruMbzOJwO3AEd380+cdN1j6PN24LXd9KnAnknXPWCfnwM8E9h9mOVnAZ8AAjwL+Pyg+5yWI/d+bmdwDnB5N/1R4IwkGWONw7Zin6vqU1X1/W72JhavJ5hm/d624i+BvwL+e5zFjUg/fX418J6q+g5AVR0cc43D1k+fC3hcN/0zwH+Msb6hq6obgfseYZVzgA/UopuAo5IcO8g+pyXcjwO+2TO/t2tbdp2qehC4H3jCWKobjX763OtCFv/yT7MV+9y9XT2hqlq5aUk/r/MpwClJPpvkpiRnjq260einz28GXpZkL3Ad8IfjKW1iVvv7vqKJ3X5Aw5PkZcAc8GuTrmWUkjwKeCfwygmXMm6bWByaOZ3Fd2c3Jvm5qvrPiVY1WucDl1XVJUl+Gfj7JE+vqocnXdi0mJYj935uZ/B/6yTZxOJbuXvHUt1o9HULhyTPB94InF1VD4yptlFZqc9HAk8HPp1kD4tjkzun/EPVfl7nvcDOqvphVX0D+BqLYT+t+unzhcBVAFX1OeAnWbzBVquGfsuWaQn3fm5nsBO4oJt+KfAv1X1SMaVW7HOSZwB/x2KwT/s4LKzQ56q6v6o2V9VsVc2y+DnD2VU1P5lyh6Kfn+2PsXjUTpLNLA7T3D3OIoesnz7/O3AGQJKfZTHcF8Za5XjtBF7RnTXzLOD+qto/0DNO+lPkVXzafBaLRyx3AW/s2t7C4i83LL74HwHuBL4AnDTpmsfQ538GDgC3do+dk6551H1esu6nmfKzZfp8ncPicNTtwJeB8yZd8xj6fCrwWRbPpLkVeMGkax6wv1cA+4EfsvhO7ELgIuCintf4Pd3348vD+Ln29gOS1KBpGZaRJK2C4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIa9L+DvAbPyht0BAAAAABJRU5ErkJggg==\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":["# Compute the quantile\n","ref_quant = np.quantile(papers[\"paper_text\"].apply(lambda x: get_word_position(\"references\",x)),0.05)\n","ref_quant"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"UB8KFWjOaZeE","executionInfo":{"status":"ok","timestamp":1670426849788,"user_tz":-60,"elapsed":15788,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"9b6dd0b6-c6dc-44b0-c1fd-7a802cdb056c"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.8056631324995238"]},"metadata":{},"execution_count":11}]},{"cell_type":"markdown","source":["## Use the quantiles"],"metadata":{"id":"SJEaFYZAbVd5"}},{"cell_type":"code","source":["# Get first/last term occurence and its relative position\n","def get_word_and_position(word: str, text: str, take=\"first\"):\n","\n","  valid_take={\"first\",\"last\"}\n","  if take not in valid_take:\n","    raise ValueError(\"'take' must be one of %r.\" % valid_take)\n","\n","  #remove linebreaks\n","  text = text.replace(\"\\n\",\" \")\n","\n","  try:\n","    #If both upper and capitalize are present, consider the first/last one (indicated by take)\n","    a = re.search(r\"\\b%s\\b\" % word.upper(), text)\n","    b = re.search(r\"\\b%s\\b\" % word.capitalize(), text)\n","    \n","    if take==\"first\":\n","      if a.start()/len(text) < b.start()/len(text):\n","        return a.group(), a.start()/len(text)\n","      else:\n","        return b.group(), b.start()/len(text)\n","\n","    elif take==\"last\":\n","      if a.start()/len(text) > b.start()/len(text):\n","        return a.group(), a.start()/len(text)\n","      else:\n","        return b.group(), b.start()/len(text)\n","\n","  except AttributeError:\n","    try:\n","      a = re.search(r\"\\b%s\\b\" % word.upper(), text)\n","      return a.group(), a.start()/len(text)\n","    except AttributeError:\n","      try:\n","        a = re.search(r\"\\b%s\\b\" % word.capitalize(), text)\n","        return a.group(), a.start()/len(text)\n","      except AttributeError:\n","        return \"no match\", -1"],"metadata":{"id":"XFaxve-aHQ_c"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["#TEST\n","\n","get_word_and_position(\"introduction\",\"This is an introduction where we use the words Introduction and INTRODUCTION\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cY5zorxVLvhb","executionInfo":{"status":"ok","timestamp":1670426849791,"user_tz":-60,"elapsed":37,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"dd2f09dd-92ed-447e-a96d-e410eeee21fd"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["('Introduction', 0.618421052631579)"]},"metadata":{},"execution_count":13}]},{"cell_type":"code","source":["get_word_and_position(\"introduction\",\"This is an introduction where we use the words INTRODUCTION and INTRODUCTION\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"VKZLYsilnlps","executionInfo":{"status":"ok","timestamp":1670426849793,"user_tz":-60,"elapsed":33,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"ce929e51-e23f-40ba-97e6-1a9df4f11f68"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["('INTRODUCTION', 0.618421052631579)"]},"metadata":{},"execution_count":14}]},{"cell_type":"code","source":["get_word_and_position(\"references\",\"Should take not the first REFERENCES, but the last References\",take=\"last\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"pJmDqyrXM00d","executionInfo":{"status":"ok","timestamp":1670426849795,"user_tz":-60,"elapsed":30,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"3fa1ef93-bb74-45d7-fb04-ab7805dae427"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["('References', 0.8360655737704918)"]},"metadata":{},"execution_count":15}]},{"cell_type":"code","execution_count":null,"metadata":{"id":"x_cwajsJ4xx5"},"outputs":[],"source":["# Extract main text and abstract using keywords and relative chosen quantiles\n","def clean_extract(text: str, abs: str, intro: str, ref: str) -> str:\n","\n","  #remove linebreaks\n","  text = text.replace(\"\\n\",\" \")\n","\n","  #get keywords and their position, if present\n","  intro, intro_pos = get_word_and_position(intro,text)\n","  abs, abs_pos = get_word_and_position(abs,text)\n","  ref, ref_pos = get_word_and_position(ref,text,take=\"last\")\n","\n","  #Check intro position\n","  if intro==\"no match\" or intro_pos > intro_quant:\n","    ab = \"Abstract Missing\"\n","    main = \"no match\"\n","    return ab, main\n","\n","  #If position is ok, continue\n","  else:\n","    #Check abstract position\n","    if abs!=\"no match\" and abs_pos < abs_quant:\n","      #extract abstract\n","      try:\n","        ab = re.search('%s(.*?)%s' % (abs, intro), text).group(1)\n","      except AttributeError:\n","        print(intro+\" found before \"+abs)\n","        ab = \"Abstract Missing\"\n","        main = \"no match\"\n","        return ab, main\n","    else:\n","      ab = \"Abstract Missing\"\n","\n","    #If references position is ok, main text is between intro and ref\n","    if ref!=\"no match\" and ref_pos > ref_quant:\n","      main = re.search('%s(.*)%s' % (intro, ref), text).group(1)\n","    #otherwise main text is everything after intro\n","    else:\n","      main = re.search('%s(.*)' % intro, text).group(1)\n","      print(\"References not found\")\n","\n","    return ab, main"]},{"cell_type":"code","source":["papers[\"Abstract\"]=papers[\"abstract\"]\n","papers[\"Main\"]=papers[\"paper_text\"]\n","\n","for i in range(len(papers)):\n","  papers[\"Abstract\"][i], papers[\"Main\"][i] = clean_extract(papers[\"paper_text\"][i], \"abstract\", \"introduction\", \"references\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"TuhMDqhsZ6v0","executionInfo":{"status":"ok","timestamp":1670426905207,"user_tz":-60,"elapsed":55436,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"b8807043-783f-4e99-93a1-d5e1edb3886c"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["<ipython-input-17-5ecf13c3305d>:5: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  papers[\"Abstract\"][i], papers[\"Main\"][i] = clean_extract(papers[\"paper_text\"][i], \"abstract\", \"introduction\", \"references\")\n"]},{"output_type":"stream","name":"stdout","text":["References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","INTRODUCTION found before Abstract\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n","References not found\n"]}]},{"cell_type":"code","source":["# Number of papers where extraction of main text failed\n","len(papers[\"Main\"][papers[\"Main\"]==\"no match\"])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"x7qoHq4EiXGC","executionInfo":{"status":"ok","timestamp":1670426905211,"user_tz":-60,"elapsed":45,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"35ddb801-d2b7-448a-c0e1-acacd0b44f94"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["866"]},"metadata":{},"execution_count":19}]},{"cell_type":"code","source":["# Where Abstract was missing in the original column, insert the extracted abstract if present\n","papers[\"abstract\"][papers[\"abstract\"]==\"Abstract Missing\"]=papers[\"Abstract\"]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"t2HBE7otsSn-","executionInfo":{"status":"ok","timestamp":1670426905213,"user_tz":-60,"elapsed":39,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"8d01e844-9a72-4004-d5d3-7cdf4ff69c72"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["<ipython-input-21-45066019a5d5>:1: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  papers[\"abstract\"][papers[\"abstract\"]==\"Abstract Missing\"]=papers[\"Abstract\"]\n"]}]},{"cell_type":"code","source":["# Number of papers where extraction of abstract failed\n","len(papers[\"abstract\"][papers[\"abstract\"]==\"Abstract Missing\"])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Xqb7WgK6s5kY","executionInfo":{"status":"ok","timestamp":1670426905215,"user_tz":-60,"elapsed":39,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"7ab7ed00-e158-4e99-b43c-a23f54e96ed8"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["910"]},"metadata":{},"execution_count":22}]},{"cell_type":"code","source":["# Drop not useful columns\n","papers.drop([\"id\",\"year\",\"paper_text\",\"event_type\",\"pdf_name\",\"Abstract\"], axis=1, inplace=True)"],"metadata":{"id":"VXydQY6Rtewt"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Drop papers with main text or abstract missing\n","papers = papers[(papers[\"Main\"]!=\"no match\") & (papers[\"abstract\"]!=\"Abstract Missing\")]\n","len(papers)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"WL1rm-yctt26","executionInfo":{"status":"ok","timestamp":1670426905218,"user_tz":-60,"elapsed":35,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"b160e78d-c2a6-41bf-cd71-bc3fc1d5bb9a"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["6194"]},"metadata":{},"execution_count":25}]},{"cell_type":"code","source":["papers.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":652},"id":"NUmgtEV3vEnf","executionInfo":{"status":"ok","timestamp":1670426905222,"user_tz":-60,"elapsed":38,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"5dd9728c-835b-435f-972e-0c1293cb4483"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                                title  \\\n","0   Self-Organization of Associative Database and ...   \n","3   Bayesian Query Construction for Neural Network...   \n","4   Neural Network Ensembles, Cross Validation, an...   \n","9   Pulsestream Synapses with Non-Volatile Analogu...   \n","10                 Learning to Play the Game of Chess   \n","\n","                                             abstract  \\\n","0    An efficient method of self-organizing associ...   \n","3    If data collection is costly, there is much t...   \n","4    Learning of continuous valued functions using...   \n","9    This paper presents results from the first us...   \n","10   This paper presents NeuroChess, a program whi...   \n","\n","                                                 Main  \n","0    Let a mapping f : X -+ Y be given. Here, X is...  \n","3     In this paper we consider the situation wher...  \n","4     It is well known that a combination of many ...  \n","9     Fusion of the nuclei of hydrogen provides th...  \n","10   Throughout the last decades, the game of ches...  "],"text/html":["\n","  <div id=\"df-a4c7c96f-6784-4459-9e6f-319cb7986f13\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>abstract</th>\n","      <th>Main</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Bayesian Query Construction for Neural Network...</td>\n","      <td>If data collection is costly, there is much t...</td>\n","      <td>In this paper we consider the situation wher...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Neural Network Ensembles, Cross Validation, an...</td>\n","      <td>Learning of continuous valued functions using...</td>\n","      <td>It is well known that a combination of many ...</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>Pulsestream Synapses with Non-Volatile Analogu...</td>\n","      <td>This paper presents results from the first us...</td>\n","      <td>Fusion of the nuclei of hydrogen provides th...</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>Learning to Play the Game of Chess</td>\n","      <td>This paper presents NeuroChess, a program whi...</td>\n","      <td>Throughout the last decades, the game of ches...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a4c7c96f-6784-4459-9e6f-319cb7986f13')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-a4c7c96f-6784-4459-9e6f-319cb7986f13 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-a4c7c96f-6784-4459-9e6f-319cb7986f13');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":26}]},{"cell_type":"code","source":["papers.to_csv(\"drive/MyDrive/TM&S/data/abs_and_main.csv\", index=False)"],"metadata":{"id":"lwxp8W3oyaeH"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["#Preprocess"],"metadata":{"id":"Tk8pqGJ20lKw"}},{"cell_type":"code","source":["import nltk\n","from nltk.tokenize import sent_tokenize\n","nltk.download('punkt')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"AOle3ZXiH2aS","executionInfo":{"status":"ok","timestamp":1670435102468,"user_tz":-60,"elapsed":1099,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"9f6608d9-60ae-40cf-a598-2b57ef8e76c3"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["[nltk_data] Downloading package punkt to /root/nltk_data...\n","[nltk_data]   Package punkt is already up-to-date!\n"]},{"output_type":"execute_result","data":{"text/plain":["True"]},"metadata":{},"execution_count":2}]},{"cell_type":"code","source":["import string\n","import re\n","from nltk.stem.snowball import SnowballStemmer\n","from nltk.tokenize import WordPunctTokenizer\n","from nltk.stem import WordNetLemmatizer\n","from nltk.corpus import stopwords\n","nltk.download('stopwords')\n","nltk.download('wordnet')\n","nltk.download('omw-1.4')\n","\n","\n","def remove_punctuation(text):\n","    # String punctuation: !\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~\n","    punctuationfree = \"\".join([i for i in text if i not in string.punctuation])\n","    return punctuationfree\n","\n","def character_repetition(text):\n","    # Pattern matching for all case alphabets\n","    # \\1   It refers to the first capturing group.\n","    # {1,} It means we are matching for repetition that occurs more than one time.\n","    # r’\\1\\1' → It limits all the repetition to two characters.\n","    Pattern_alpha = re.compile(r\"([A-Za-z])\\1{1,}\", re.DOTALL)\n","    \n","    Formatted_text = Pattern_alpha.sub(r\"\\1\\1\", text) \n","    return Formatted_text\n","\n","def remove_spaces(text):\n","    return text.replace('\\\\n', ' ').replace('\\n', ' ').replace('\\t',' ').replace('  ', ' ').replace('\\n\\n',' ')\n","\n","\n","def remove_stopwords(tokenized_text):\n","  stop_words = stopwords.words('english')\n","  tokenized_text_without_stopwords = []\n","  for token in tokenized_text:\n","    if token.lower() not in stop_words and len(token)>2:\n","      tokenized_text_without_stopwords.append(token)\n","  return tokenized_text_without_stopwords\n","\n","\n","def lemmatization(tokenized_text):\n","  wnl = WordNetLemmatizer()\n","  n_words = len(tokenized_text)\n","  for idx in range(n_words):\n","    tokenized_text[idx] = wnl.lemmatize(tokenized_text[idx])\n","  return tokenized_text\n","\n","\n","def stemming(tokenized_text):\n","    stemmer = SnowballStemmer(\"english\")\n","    n_words = len(tokenized_text)\n","    for idx in range(n_words):\n","      tokenized_text[idx] = stemmer.stem(tokenized_text[idx])\n","    return tokenized_text\n","    \n","\n","def preprocess(text,stopword = True, lemma = True,stem=False):\n","  tokenized_text =  WordPunctTokenizer().tokenize(text)\n","  if stopword:\n","    tokenized_text = remove_stopwords(tokenized_text)\n","  \n","  if lemma:\n","    tokenized_text = lemmatization(tokenized_text)\n","  \n","  if stem:\n","    tokenized_text = stemming(tokenized_text)\n","\n","  return tokenized_text"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cxvY5zTmNYfz","executionInfo":{"status":"ok","timestamp":1670435102469,"user_tz":-60,"elapsed":8,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"7104e302-c86c-485b-bb30-321e4338e6a5"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["[nltk_data] Downloading package stopwords to /root/nltk_data...\n","[nltk_data]   Package stopwords is already up-to-date!\n","[nltk_data] Downloading package wordnet to /root/nltk_data...\n","[nltk_data]   Package wordnet is already up-to-date!\n","[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n","[nltk_data]   Package omw-1.4 is already up-to-date!\n"]}]},{"cell_type":"code","source":["df = pd.read_csv(\"drive/MyDrive/TM&S/data/abs_and_main.csv\")"],"metadata":{"id":"1hn7JypTDa2o"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"ZSt2J-oTDkho","executionInfo":{"status":"ok","timestamp":1670426915530,"user_tz":-60,"elapsed":38,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"39a988d1-513f-48fc-fe49-d91292372427"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Bayesian Query Construction for Neural Network...   \n","2  Neural Network Ensembles, Cross Validation, an...   \n","3  Pulsestream Synapses with Non-Volatile Analogu...   \n","4                 Learning to Play the Game of Chess   \n","\n","                                            abstract  \\\n","0   An efficient method of self-organizing associ...   \n","1   If data collection is costly, there is much t...   \n","2   Learning of continuous valued functions using...   \n","3   This paper presents results from the first us...   \n","4   This paper presents NeuroChess, a program whi...   \n","\n","                                                Main  \n","0   Let a mapping f : X -+ Y be given. Here, X is...  \n","1    In this paper we consider the situation wher...  \n","2    It is well known that a combination of many ...  \n","3    Fusion of the nuclei of hydrogen provides th...  \n","4   Throughout the last decades, the game of ches...  "],"text/html":["\n","  <div id=\"df-dfb467b9-0d87-4075-b3f8-bf6a9c848c80\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>abstract</th>\n","      <th>Main</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Bayesian Query Construction for Neural Network...</td>\n","      <td>If data collection is costly, there is much t...</td>\n","      <td>In this paper we consider the situation wher...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Neural Network Ensembles, Cross Validation, an...</td>\n","      <td>Learning of continuous valued functions using...</td>\n","      <td>It is well known that a combination of many ...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Pulsestream Synapses with Non-Volatile Analogu...</td>\n","      <td>This paper presents results from the first us...</td>\n","      <td>Fusion of the nuclei of hydrogen provides th...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Learning to Play the Game of Chess</td>\n","      <td>This paper presents NeuroChess, a program whi...</td>\n","      <td>Throughout the last decades, the game of ches...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-dfb467b9-0d87-4075-b3f8-bf6a9c848c80')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-dfb467b9-0d87-4075-b3f8-bf6a9c848c80 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-dfb467b9-0d87-4075-b3f8-bf6a9c848c80');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":31}]},{"cell_type":"markdown","source":["## Abstract"],"metadata":{"id":"x7yyhVmUvjOL"}},{"cell_type":"code","source":["abs_sent=df[[\"title\",\"abstract\"]]\n","abs_sent.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"LDaZC3P8vlIl","executionInfo":{"status":"ok","timestamp":1670426915533,"user_tz":-60,"elapsed":33,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"56b350d3-5281-4509-e381-281d7a3e79cf"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Bayesian Query Construction for Neural Network...   \n","2  Neural Network Ensembles, Cross Validation, an...   \n","3  Pulsestream Synapses with Non-Volatile Analogu...   \n","4                 Learning to Play the Game of Chess   \n","\n","                                            abstract  \n","0   An efficient method of self-organizing associ...  \n","1   If data collection is costly, there is much t...  \n","2   Learning of continuous valued functions using...  \n","3   This paper presents results from the first us...  \n","4   This paper presents NeuroChess, a program whi...  "],"text/html":["\n","  <div id=\"df-5bfa6907-4c2f-48ff-a05c-a1d3f6f0840c\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>abstract</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Bayesian Query Construction for Neural Network...</td>\n","      <td>If data collection is costly, there is much t...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Neural Network Ensembles, Cross Validation, an...</td>\n","      <td>Learning of continuous valued functions using...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Pulsestream Synapses with Non-Volatile Analogu...</td>\n","      <td>This paper presents results from the first us...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Learning to Play the Game of Chess</td>\n","      <td>This paper presents NeuroChess, a program whi...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5bfa6907-4c2f-48ff-a05c-a1d3f6f0840c')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-5bfa6907-4c2f-48ff-a05c-a1d3f6f0840c button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-5bfa6907-4c2f-48ff-a05c-a1d3f6f0840c');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":32}]},{"cell_type":"code","source":["#Subset abstracts into sentences \n","abs_sent[\"sentences\"] = abs_sent[\"abstract\"].apply(lambda x: sent_tokenize(x))"],"metadata":{"id":"lM0SzPfsv4bF"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["abs_sent = abs_sent.explode(column=[\"sentences\"],ignore_index=True)\n","abs_sent"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":424},"id":"y0XFt_9SwCA8","executionInfo":{"status":"ok","timestamp":1670426916056,"user_tz":-60,"elapsed":17,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"37a4e7ee-227a-48b8-990c-412cdb9ac293"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                                   title  \\\n","0      Self-Organization of Associative Database and ...   \n","1      Self-Organization of Associative Database and ...   \n","2      Self-Organization of Associative Database and ...   \n","3      Self-Organization of Associative Database and ...   \n","4      Self-Organization of Associative Database and ...   \n","...                                                  ...   \n","39475  Correlation and Interpolation Networks for Rea...   \n","39476  Correlation and Interpolation Networks for Rea...   \n","39477  Correlation and Interpolation Networks for Rea...   \n","39478  Correlation and Interpolation Networks for Rea...   \n","39479  Correlation and Interpolation Networks for Rea...   \n","\n","                                                abstract  \\\n","0       An efficient method of self-organizing associ...   \n","1       An efficient method of self-organizing associ...   \n","2       An efficient method of self-organizing associ...   \n","3       An efficient method of self-organizing associ...   \n","4       An efficient method of self-organizing associ...   \n","...                                                  ...   \n","39475   We describe a framework for real-time trackin...   \n","39476   We describe a framework for real-time trackin...   \n","39477   We describe a framework for real-time trackin...   \n","39478   We describe a framework for real-time trackin...   \n","39479   We describe a framework for real-time trackin...   \n","\n","                                               sentences  \n","0       An efficient method of self-organizing associ...  \n","1      The proposed databases can associate any input...  \n","2      In the first half part of discussion, an algor...  \n","3      From an aspect of hardware, it produces a new ...  \n","4      In the latter half part, an applicability to h...  \n","...                                                  ...  \n","39475  A distributed view-based representation is use...  \n","39476  The ensemble response of the set of view corre...  \n","39477  Activation levels of the motor state correspon...  \n","39478  By integrating fast and robust 2-D processing ...  \n","39479                                                  1  \n","\n","[39480 rows x 3 columns]"],"text/html":["\n","  <div id=\"df-4a151f46-4d23-4d60-b520-fbbfc5743130\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>abstract</th>\n","      <th>sentences</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>The proposed databases can associate any input...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>In the first half part of discussion, an algor...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>From an aspect of hardware, it produces a new ...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>In the latter half part, an applicability to h...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>39475</th>\n","      <td>Correlation and Interpolation Networks for Rea...</td>\n","      <td>We describe a framework for real-time trackin...</td>\n","      <td>A distributed view-based representation is use...</td>\n","    </tr>\n","    <tr>\n","      <th>39476</th>\n","      <td>Correlation and Interpolation Networks for Rea...</td>\n","      <td>We describe a framework for real-time trackin...</td>\n","      <td>The ensemble response of the set of view corre...</td>\n","    </tr>\n","    <tr>\n","      <th>39477</th>\n","      <td>Correlation and Interpolation Networks for Rea...</td>\n","      <td>We describe a framework for real-time trackin...</td>\n","      <td>Activation levels of the motor state correspon...</td>\n","    </tr>\n","    <tr>\n","      <th>39478</th>\n","      <td>Correlation and Interpolation Networks for Rea...</td>\n","      <td>We describe a framework for real-time trackin...</td>\n","      <td>By integrating fast and robust 2-D processing ...</td>\n","    </tr>\n","    <tr>\n","      <th>39479</th>\n","      <td>Correlation and Interpolation Networks for Rea...</td>\n","      <td>We describe a framework for real-time trackin...</td>\n","      <td>1</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>39480 rows × 3 columns</p>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4a151f46-4d23-4d60-b520-fbbfc5743130')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-4a151f46-4d23-4d60-b520-fbbfc5743130 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-4a151f46-4d23-4d60-b520-fbbfc5743130');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":34}]},{"cell_type":"code","source":["# Preprocess sentences\n","abs_sent[\"clean_sent\"] = abs_sent[\"sentences\"].apply(lambda x: remove_punctuation(x))\n","abs_sent[\"clean_sent\"] = abs_sent[\"clean_sent\"].apply(lambda x: x.lower())\n","abs_sent[\"clean_sent\"] = abs_sent[\"clean_sent\"].apply(lambda x: re.sub(r'\\d+', '', x))\n","abs_sent[\"clean_sent\"] = abs_sent[\"clean_sent\"].apply(lambda x: character_repetition(x))\n","abs_sent[\"clean_sent\"] = abs_sent[\"clean_sent\"].apply(lambda x: remove_spaces(x))\n","abs_sent[\"clean_sent\"] = abs_sent[\"clean_sent\"].apply(lambda x: preprocess(x))"],"metadata":{"id":"ey-y2dfUwKkf"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["abs_sent.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":337},"id":"pwxR2h1bwX0h","executionInfo":{"status":"ok","timestamp":1670426929379,"user_tz":-60,"elapsed":57,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"0a600df7-cb59-489b-e771-740720ed3798"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Self-Organization of Associative Database and ...   \n","2  Self-Organization of Associative Database and ...   \n","3  Self-Organization of Associative Database and ...   \n","4  Self-Organization of Associative Database and ...   \n","\n","                                            abstract  \\\n","0   An efficient method of self-organizing associ...   \n","1   An efficient method of self-organizing associ...   \n","2   An efficient method of self-organizing associ...   \n","3   An efficient method of self-organizing associ...   \n","4   An efficient method of self-organizing associ...   \n","\n","                                           sentences  \\\n","0   An efficient method of self-organizing associ...   \n","1  The proposed databases can associate any input...   \n","2  In the first half part of discussion, an algor...   \n","3  From an aspect of hardware, it produces a new ...   \n","4  In the latter half part, an applicability to h...   \n","\n","                                          clean_sent  \n","0  [efficient, method, selforganizing, associativ...  \n","1     [proposed, database, associate, input, output]  \n","2  [first, half, part, discussion, algorithm, sel...  \n","3  [aspect, hardware, produce, new, style, neural...  \n","4  [latter, half, part, applicability, handwritte...  "],"text/html":["\n","  <div id=\"df-b9646343-f175-479e-adf9-590a7a1829d3\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>abstract</th>\n","      <th>sentences</th>\n","      <th>clean_sent</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>[efficient, method, selforganizing, associativ...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>The proposed databases can associate any input...</td>\n","      <td>[proposed, database, associate, input, output]</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>In the first half part of discussion, an algor...</td>\n","      <td>[first, half, part, discussion, algorithm, sel...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>From an aspect of hardware, it produces a new ...</td>\n","      <td>[aspect, hardware, produce, new, style, neural...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>In the latter half part, an applicability to h...</td>\n","      <td>[latter, half, part, applicability, handwritte...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b9646343-f175-479e-adf9-590a7a1829d3')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-b9646343-f175-479e-adf9-590a7a1829d3 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-b9646343-f175-479e-adf9-590a7a1829d3');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":36}]},{"cell_type":"code","source":["# Compute sentence length\n","abs_sent[\"sent_len\"] = abs_sent[\"clean_sent\"].apply(lambda x: len(x))\n","abs_sent.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":337},"id":"aocnwLTjwWuD","executionInfo":{"status":"ok","timestamp":1670426929381,"user_tz":-60,"elapsed":54,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"902a20ff-7277-4d2c-8313-aab1f4b3136f"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Self-Organization of Associative Database and ...   \n","2  Self-Organization of Associative Database and ...   \n","3  Self-Organization of Associative Database and ...   \n","4  Self-Organization of Associative Database and ...   \n","\n","                                            abstract  \\\n","0   An efficient method of self-organizing associ...   \n","1   An efficient method of self-organizing associ...   \n","2   An efficient method of self-organizing associ...   \n","3   An efficient method of self-organizing associ...   \n","4   An efficient method of self-organizing associ...   \n","\n","                                           sentences  \\\n","0   An efficient method of self-organizing associ...   \n","1  The proposed databases can associate any input...   \n","2  In the first half part of discussion, an algor...   \n","3  From an aspect of hardware, it produces a new ...   \n","4  In the latter half part, an applicability to h...   \n","\n","                                          clean_sent  sent_len  \n","0  [efficient, method, selforganizing, associativ...        11  \n","1     [proposed, database, associate, input, output]         5  \n","2  [first, half, part, discussion, algorithm, sel...         7  \n","3  [aspect, hardware, produce, new, style, neural...         7  \n","4  [latter, half, part, applicability, handwritte...        12  "],"text/html":["\n","  <div id=\"df-ca77ae7c-a46a-4930-a04c-b1d8606c431e\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>abstract</th>\n","      <th>sentences</th>\n","      <th>clean_sent</th>\n","      <th>sent_len</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>[efficient, method, selforganizing, associativ...</td>\n","      <td>11</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>The proposed databases can associate any input...</td>\n","      <td>[proposed, database, associate, input, output]</td>\n","      <td>5</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>In the first half part of discussion, an algor...</td>\n","      <td>[first, half, part, discussion, algorithm, sel...</td>\n","      <td>7</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>From an aspect of hardware, it produces a new ...</td>\n","      <td>[aspect, hardware, produce, new, style, neural...</td>\n","      <td>7</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>In the latter half part, an applicability to h...</td>\n","      <td>[latter, half, part, applicability, handwritte...</td>\n","      <td>12</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ca77ae7c-a46a-4930-a04c-b1d8606c431e')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-ca77ae7c-a46a-4930-a04c-b1d8606c431e button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-ca77ae7c-a46a-4930-a04c-b1d8606c431e');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":37}]},{"cell_type":"code","source":["import matplotlib.pyplot as plt\n","import numpy as np\n","\n","# Show distribution of sentences length in the abstracts\n","hist_len = plt.hist(abs_sent[\"sent_len\"],bins=np.max(abs_sent[\"sent_len\"]))\n","plt.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":265},"id":"tEHhxH3gxD01","executionInfo":{"status":"ok","timestamp":1670426929384,"user_tz":-60,"elapsed":52,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"5c214454-3266-49c0-d5b1-845107b32874"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["<Figure size 432x288 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAARlElEQVR4nO3df6zddX3H8efLKv6OgNw1ta1rp52mLrGYBjGaxcmEgmbFxDnIpo3B1D8g08VlQf/BHyPBRGWaKFmVzro4kfgjNI7IOmRx/iFQhCEFCXcI0qbQqyC6meHq3vvjfOrOrvf23t5fp/d+no/k5n6/7+/3nPM533zv63zu53zO96SqkCT14WmjboAkaekY+pLUEUNfkjpi6EtSRwx9SerI00fdgOM544wzasOGDaNuhiQtK3fcccePq2psqm0ndehv2LCB/fv3j7oZkrSsJHl4um0O70hSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkdO6k/krmQbLv/HXy8/dNWbRtgSST2xpy9JHTH0JakjDu+cBIaHesDhHkmLZ8aefpJnJbktyb8lOZDkQ62+McmtScaTfDnJKa3+zLY+3rZvGLqv97f6/UnOW6wnJUma2myGd54C3lBVrwS2ANuSnA18FLi6ql4KPAFc0va/BHii1a9u+5FkM3AR8ApgG/CZJKsW8slIko5vxtCvgf9oq89oPwW8AfhKq+8BLmzL29s6bfs5SdLq11XVU1X1Q2AcOGtBnoUkaVZm9UZuklVJ7gKOAPuAfwd+WlVH2y4HgbVteS3wCEDb/iTwwuH6FLeRJC2BWYV+Vf2qqrYA6xj0zl++WA1KsjPJ/iT7JyYmFuthJKlLJzRls6p+CtwCvAY4Ncmx2T/rgENt+RCwHqBtfwHwk+H6FLcZfoxdVbW1qraOjU35FY+SpDmazeydsSSntuVnA28E7mMQ/m9tu+0AbmjLe9s6bfu3qqpa/aI2u2cjsAm4baGeiCRpZrOZp78G2NNm2jwNuL6qvpHkXuC6JH8N3Alc2/a/Fvj7JOPA4wxm7FBVB5JcD9wLHAUurapfLezTkSQdz4yhX1V3A2dOUX+QKWbfVNV/AX88zX1dCVx54s2UJC0EL8MgSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOuL19E9CfpWipMVi6C+RyV+UIkmj4PCOJHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xE/knuQmf5LXyzJImg97+pLUEUNfkjpi6EtSRwx9SeqIoS9JHXH2ziLyGvqSTjb29CWpIzOGfpL1SW5Jcm+SA0ne0+ofTHIoyV3t54Kh27w/yXiS+5OcN1Tf1mrjSS5fnKckSZrObIZ3jgLvq6rvJXk+cEeSfW3b1VX1seGdk2wGLgJeAbwI+Ockv9s2fxp4I3AQuD3J3qq6dyGeiCRpZjOGflUdBg635Z8nuQ9Ye5ybbAeuq6qngB8mGQfOatvGq+pBgCTXtX0NfUlaIic0pp9kA3AmcGsrXZbk7iS7k5zWamuBR4ZudrDVpqtPfoydSfYn2T8xMXEizZMkzWDWoZ/kecBXgfdW1c+Aa4CXAFsY/Cfw8YVoUFXtqqqtVbV1bGxsIe5SktTMaspmkmcwCPwvVtXXAKrqsaHtnwW+0VYPAeuHbr6u1ThOXZK0BGYzeyfAtcB9VfWJofqaod3eAtzTlvcCFyV5ZpKNwCbgNuB2YFOSjUlOYfBm796FeRqSpNmYTU//tcDbge8nuavVPgBcnGQLUMBDwLsBqupAkusZvEF7FLi0qn4FkOQy4CZgFbC7qg4s4HORJM1gNrN3vgNkik03Huc2VwJXTlG/8Xi3kyQtLj+RK0kdMfQlqSOGviR1xKtsLjPDV+70+3IlnSh7+pLUkRXd07dXLEn/nz19SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpIyv62jtLbfhaP5J0MrKnL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjoyY+gnWZ/kliT3JjmQ5D2tfnqSfUkeaL9Pa/Uk+VSS8SR3J3nV0H3taPs/kGTH4j0tSdJUZtPTPwq8r6o2A2cDlybZDFwO3FxVm4Cb2zrA+cCm9rMTuAYGLxLAFcCrgbOAK469UEiSlsaMoV9Vh6vqe23558B9wFpgO7Cn7bYHuLAtbwe+UAPfBU5NsgY4D9hXVY9X1RPAPmDbgj4bSdJxndCYfpINwJnArcDqqjrcNj0KrG7La4FHhm52sNWmq09+jJ1J9ifZPzExcSLNkyTNYNaXYUjyPOCrwHur6mdJfr2tqipJLUSDqmoXsAtg69atC3KfK9Xkyz48dNWbRtQSScvFrHr6SZ7BIPC/WFVfa+XH2rAN7feRVj8ErB+6+bpWm64uSVois5m9E+Ba4L6q+sTQpr3AsRk4O4AbhurvaLN4zgaebMNANwHnJjmtvYF7bqtJkpbIbIZ3Xgu8Hfh+krta7QPAVcD1SS4BHgbe1rbdCFwAjAO/AN4JUFWPJ/kIcHvb78NV9fiCPAtJ0qzMGPpV9R0g02w+Z4r9C7h0mvvaDew+kQZKkhaOn8iVpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkdm/SUqOvkNf6mKX6giaSr29CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6ojz9OdheF68JC0H9vQlqSOGviR1xNCXpI7MGPpJdic5kuSeodoHkxxKclf7uWBo2/uTjCe5P8l5Q/VtrTae5PKFfyqSpJnMpqf/eWDbFPWrq2pL+7kRIMlm4CLgFe02n0myKskq4NPA+cBm4OK2ryRpCc04e6eqvp1kwyzvbztwXVU9BfwwyThwVts2XlUPAiS5ru177wm3WLMyeWaRV92UBPMb078syd1t+Oe0VlsLPDK0z8FWm67+G5LsTLI/yf6JiYl5NE+SNNlcQ/8a4CXAFuAw8PGFalBV7aqqrVW1dWxsbKHuVpLEHD+cVVWPHVtO8lngG231ELB+aNd1rcZx6pKkJTKnnn6SNUOrbwGOzezZC1yU5JlJNgKbgNuA24FNSTYmOYXBm717595sSdJczNjTT/Il4PXAGUkOAlcAr0+yBSjgIeDdAFV1IMn1DN6gPQpcWlW/avdzGXATsArYXVUHFvzZSJKOazazdy6eonztcfa/ErhyivqNwI0n1DpJ0oLyE7mS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdmdOXqGj5Gf7OXL8vV+qXPX1J6oihL0kdcXjnBA0Pk0jScmNPX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0JekjswY+kl2JzmS5J6h2ulJ9iV5oP0+rdWT5FNJxpPcneRVQ7fZ0fZ/IMmOxXk6kqTjmU1P//PAtkm1y4Gbq2oTcHNbBzgf2NR+dgLXwOBFArgCeDVwFnDFsRcKSdLSmTH0q+rbwOOTytuBPW15D3DhUP0LNfBd4NQka4DzgH1V9XhVPQHs4zdfSCRJi2yuY/qrq+pwW34UWN2W1wKPDO13sNWmq/+GJDuT7E+yf2JiYo7NkyRNZd5v5FZVAbUAbTl2f7uqamtVbR0bG1uou5UkMfcLrj2WZE1VHW7DN0da/RCwfmi/da12CHj9pPq/zPGxNU+TLxrn9fWlfsy1p78XODYDZwdww1D9HW0Wz9nAk20Y6Cbg3CSntTdwz201SdISmrGnn+RLDHrpZyQ5yGAWzlXA9UkuAR4G3tZ2vxG4ABgHfgG8E6CqHk/yEeD2tt+Hq2rym8OSpEU2Y+hX1cXTbDpnin0LuHSa+9kN7D6h1kmSFpSfyJWkjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1ZK7X3tEK4rV4pH7Y05ekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSPO05/B5DnskrSc2dOXpI7Y09dvGP7vxk/nSiuLPX1J6oihL0kdMfQlqSOGviR1xNCXpI7MK/STPJTk+0nuSrK/1U5Psi/JA+33aa2eJJ9KMp7k7iSvWognIEmavYXo6f9BVW2pqq1t/XLg5qraBNzc1gHOBza1n53ANQvw2JKkE7AYwzvbgT1teQ9w4VD9CzXwXeDUJGsW4fElSdOY74ezCvinJAX8bVXtAlZX1eG2/VFgdVteCzwydNuDrXYYnbT8KkVpZZlv6L+uqg4l+S1gX5IfDG+sqmovCLOWZCeD4R9e/OIXz7N5kqRh8xreqapD7fcR4OvAWcBjx4Zt2u8jbfdDwPqhm69rtcn3uauqtlbV1rGxsfk0T5I0yZxDP8lzkzz/2DJwLnAPsBfY0XbbAdzQlvcC72izeM4GnhwaBpIkLYH5DO+sBr6e5Nj9/ENVfTPJ7cD1SS4BHgbe1va/EbgAGAd+AbxzHo+9qLycsqSVas6hX1UPAq+cov4T4Jwp6gVcOtfHkyTNn5/IlaSOGPqS1BG/REUnxC9YkZY3e/qS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI07Z1Jx52WVp+bGnL0kdsaePF1iT1A97+pLUEUNfkjri8I4WjNflkU5+9vQlqSP29LUonM4pnZy6CX1n6EiSwzuS1JVuevoaLd/klU4O9vQlqSP29LXkfJNXGh17+pLUEXv6GjnH+6WlY+jrpOLQj7S4ljz0k2wDPgmsAj5XVVctdRu0fMz18xW+WEhTW9LQT7IK+DTwRuAgcHuSvVV171K2Qyuf/zFIU1vqnv5ZwHhVPQiQ5DpgO2Doa1Etxieyh19I5vMic7z3NGb7focvcpqtVNXSPVjyVmBbVb2rrb8deHVVXTa0z05gZ1t9GXD/PB7yDODH87j9SuVxmZ7HZnoem+mdbMfmt6tqbKoNJ90buVW1C9i1EPeVZH9VbV2I+1pJPC7T89hMz2MzveV0bJZ6nv4hYP3Q+rpWkyQtgaUO/duBTUk2JjkFuAjYu8RtkKRuLenwTlUdTXIZcBODKZu7q+rAIj7kggwTrUAel+l5bKbnsZnesjk2S/pGriRptLz2jiR1xNCXpI6syNBPsi3J/UnGk1w+6vaMUpL1SW5Jcm+SA0ne0+qnJ9mX5IH2+7RRt3UUkqxKcmeSb7T1jUlubefOl9uEg+4kOTXJV5L8IMl9SV7jOTOQ5C/a39I9Sb6U5FnL6bxZcaE/dKmH84HNwMVJNo+2VSN1FHhfVW0GzgYubcfjcuDmqtoE3NzWe/Qe4L6h9Y8CV1fVS4EngEtG0qrR+yTwzap6OfBKBseo+3MmyVrgz4GtVfV7DCakXMQyOm9WXOgzdKmHqvolcOxSD12qqsNV9b22/HMGf7xrGRyTPW23PcCFo2nh6CRZB7wJ+FxbD/AG4Cttl16PywuA3weuBaiqX1bVT/GcOebpwLOTPB14DnCYZXTerMTQXws8MrR+sNW6l2QDcCZwK7C6qg63TY8Cq0fUrFH6G+CvgP9p6y8EflpVR9t6r+fORmAC+Ls29PW5JM/Fc4aqOgR8DPgRg7B/EriDZXTerMTQ1xSSPA/4KvDeqvrZ8LYazNvtau5ukjcDR6rqjlG35ST0dOBVwDVVdSbwn0wayunxnAFo72NsZ/DC+CLgucC2kTbqBK3E0PdSD5MkeQaDwP9iVX2tlR9LsqZtXwMcGVX7RuS1wB8leYjBEOAbGIxjn9r+bYd+z52DwMGqurWtf4XBi0Dv5wzAHwI/rKqJqvpv4GsMzqVlc96sxND3Ug9D2jj1tcB9VfWJoU17gR1teQdww1K3bZSq6v1Vta6qNjA4R75VVX8K3AK8te3W3XEBqKpHgUeSvKyVzmFw+fOuz5nmR8DZSZ7T/raOHZtlc96syE/kJrmAwXjtsUs9XDniJo1MktcB/wp8n/8bu/4Ag3H964EXAw8Db6uqx0fSyBFL8nrgL6vqzUl+h0HP/3TgTuDPquqpUbZvFJJsYfAG9ynAg8A7GXQSuz9nknwI+BMGM+PuBN7FYAx/WZw3KzL0JUlTW4nDO5KkaRj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSP/C/q7ANN082hrAAAAAElFTkSuQmCC\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":["#We will use this value to filter out sentences in the main text which are not relevant\n","max_len = np.max(abs_sent[\"sent_len\"])\n","max_len"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"TYIo0h1T2Z2L","executionInfo":{"status":"ok","timestamp":1670426929385,"user_tz":-60,"elapsed":48,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"06f31058-a281-4f75-fa36-ba0c53779b7a"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["86"]},"metadata":{},"execution_count":39}]},{"cell_type":"code","source":["#Filter out sentences with less than 3 tokens\n","abs_sent = abs_sent[abs_sent[\"sent_len\"]>2]"],"metadata":{"id":"sk1UAlQpyMow"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["#Get relative position of every sentence in its belonging document\n","abs_sent[\"sent_pos\"] = (abs_sent.groupby(\"title\",sort=False).cumcount() + 1).astype(float)\n","abs_sent[\"total_length\"] = 0.\n","abs_sent = abs_sent.join(abs_sent.groupby(\"title\",sort=False)[\"total_length\"].size(), on=\"title\", rsuffix='_r')\n","\n","abs_sent[\"sent_pos\"] = abs_sent[\"sent_pos\"] / abs_sent[\"total_length_r\"]\n","\n","abs_sent.drop([\"total_length\",\"total_length_r\"],axis=1,inplace=True)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Ll0ssTFJ8tz5","executionInfo":{"status":"ok","timestamp":1670426929388,"user_tz":-60,"elapsed":48,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"07fac5e5-cf57-4fd8-e0c3-9a15b07ab23b"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["<ipython-input-41-af72f580c685>:2: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  abs_sent[\"sent_pos\"] = (abs_sent.groupby(\"title\",sort=False).cumcount() + 1).astype(float)\n","<ipython-input-41-af72f580c685>:3: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  abs_sent[\"total_length\"] = 0.\n"]}]},{"cell_type":"code","source":["# Drop the abstract column (we already have all sentences exploded)\n","abs_sent.drop([\"abstract\"],axis=1,inplace=True)\n","abs_sent.columns"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"JOUlZI4t9AGj","executionInfo":{"status":"ok","timestamp":1670426929389,"user_tz":-60,"elapsed":46,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"17c0a31c-4385-4e3d-e926-dc3e701ce02a"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Index(['title', 'sentences', 'clean_sent', 'sent_len', 'sent_pos'], dtype='object')"]},"metadata":{},"execution_count":42}]},{"cell_type":"code","source":["abs_sent.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":337},"id":"Rq1qUTsR4uyf","executionInfo":{"status":"ok","timestamp":1670426929390,"user_tz":-60,"elapsed":44,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"0fd730a5-b725-4288-bb5b-778f1e10ddd1"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Self-Organization of Associative Database and ...   \n","2  Self-Organization of Associative Database and ...   \n","3  Self-Organization of Associative Database and ...   \n","4  Self-Organization of Associative Database and ...   \n","\n","                                           sentences  \\\n","0   An efficient method of self-organizing associ...   \n","1  The proposed databases can associate any input...   \n","2  In the first half part of discussion, an algor...   \n","3  From an aspect of hardware, it produces a new ...   \n","4  In the latter half part, an applicability to h...   \n","\n","                                          clean_sent  sent_len  sent_pos  \n","0  [efficient, method, selforganizing, associativ...        11       0.2  \n","1     [proposed, database, associate, input, output]         5       0.4  \n","2  [first, half, part, discussion, algorithm, sel...         7       0.6  \n","3  [aspect, hardware, produce, new, style, neural...         7       0.8  \n","4  [latter, half, part, applicability, handwritte...        12       1.0  "],"text/html":["\n","  <div id=\"df-34515cce-2b07-4b07-8ff3-ada03650bfe5\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>sentences</th>\n","      <th>clean_sent</th>\n","      <th>sent_len</th>\n","      <th>sent_pos</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>[efficient, method, selforganizing, associativ...</td>\n","      <td>11</td>\n","      <td>0.2</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>The proposed databases can associate any input...</td>\n","      <td>[proposed, database, associate, input, output]</td>\n","      <td>5</td>\n","      <td>0.4</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>In the first half part of discussion, an algor...</td>\n","      <td>[first, half, part, discussion, algorithm, sel...</td>\n","      <td>7</td>\n","      <td>0.6</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>From an aspect of hardware, it produces a new ...</td>\n","      <td>[aspect, hardware, produce, new, style, neural...</td>\n","      <td>7</td>\n","      <td>0.8</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>In the latter half part, an applicability to h...</td>\n","      <td>[latter, half, part, applicability, handwritte...</td>\n","      <td>12</td>\n","      <td>1.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-34515cce-2b07-4b07-8ff3-ada03650bfe5')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-34515cce-2b07-4b07-8ff3-ada03650bfe5 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-34515cce-2b07-4b07-8ff3-ada03650bfe5');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":43}]},{"cell_type":"code","source":["abs_sent.to_csv(\"drive/MyDrive/TM&S/data/abs_features.csv\", index=False)"],"metadata":{"id":"N_3usjrWw4Yz"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Main text"],"metadata":{"id":"EmJnAxgLvYtt"}},{"cell_type":"code","source":["main_sent=df[[\"title\",\"Main\"]]\n","main_sent.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"NPqzY3Z1EPZu","executionInfo":{"status":"ok","timestamp":1670426931253,"user_tz":-60,"elapsed":17,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"416d4fbd-4792-4661-f5ac-133c14c0072d"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Bayesian Query Construction for Neural Network...   \n","2  Neural Network Ensembles, Cross Validation, an...   \n","3  Pulsestream Synapses with Non-Volatile Analogu...   \n","4                 Learning to Play the Game of Chess   \n","\n","                                                Main  \n","0   Let a mapping f : X -+ Y be given. Here, X is...  \n","1    In this paper we consider the situation wher...  \n","2    It is well known that a combination of many ...  \n","3    Fusion of the nuclei of hydrogen provides th...  \n","4   Throughout the last decades, the game of ches...  "],"text/html":["\n","  <div id=\"df-a545a8e9-64e4-490d-bbaf-23cb1360cd87\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>Main</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Bayesian Query Construction for Neural Network...</td>\n","      <td>In this paper we consider the situation wher...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Neural Network Ensembles, Cross Validation, an...</td>\n","      <td>It is well known that a combination of many ...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Pulsestream Synapses with Non-Volatile Analogu...</td>\n","      <td>Fusion of the nuclei of hydrogen provides th...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Learning to Play the Game of Chess</td>\n","      <td>Throughout the last decades, the game of ches...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a545a8e9-64e4-490d-bbaf-23cb1360cd87')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-a545a8e9-64e4-490d-bbaf-23cb1360cd87 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-a545a8e9-64e4-490d-bbaf-23cb1360cd87');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":45}]},{"cell_type":"code","source":["# Subset main text into sentences\n","main_sent[\"sentences\"] = main_sent[\"Main\"].apply(lambda x: sent_tokenize(x))"],"metadata":{"id":"noFdCu7UvqVS"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["main_sent = main_sent.explode(column=[\"sentences\"],ignore_index=True)\n","main_sent"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":424},"id":"43vKzR96I-qP","executionInfo":{"status":"ok","timestamp":1670426978315,"user_tz":-60,"elapsed":26,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"3cb73bec-e312-4f5d-9a53-95778cce8d53"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                                     title  \\\n","0        Self-Organization of Associative Database and ...   \n","1        Self-Organization of Associative Database and ...   \n","2        Self-Organization of Associative Database and ...   \n","3        Self-Organization of Associative Database and ...   \n","4        Self-Organization of Associative Database and ...   \n","...                                                    ...   \n","1947687  Correlation and Interpolation Networks for Rea...   \n","1947688  Correlation and Interpolation Networks for Rea...   \n","1947689  Correlation and Interpolation Networks for Rea...   \n","1947690  Correlation and Interpolation Networks for Rea...   \n","1947691  Correlation and Interpolation Networks for Rea...   \n","\n","                                                      Main  \\\n","0         Let a mapping f : X -+ Y be given. Here, X is...   \n","1         Let a mapping f : X -+ Y be given. Here, X is...   \n","2         Let a mapping f : X -+ Y be given. Here, X is...   \n","3         Let a mapping f : X -+ Y be given. Here, X is...   \n","4         Let a mapping f : X -+ Y be given. Here, X is...   \n","...                                                    ...   \n","1947687   An important task for natural and artificial ...   \n","1947688   An important task for natural and artificial ...   \n","1947689   An important task for natural and artificial ...   \n","1947690   An important task for natural and artificial ...   \n","1947691   An important task for natural and artificial ...   \n","\n","                                                 sentences  \n","0                       Let a mapping f : X -+ Y be given.  \n","1        Here, X is a finite or infinite set, and Y is ...  \n","2        A learning machine observes any set of pairs (...  \n","3          (X x Y means the Cartesian product of X and Y.)  \n","4        And, it computes some estimate j : X -+ Y of f...  \n","...                                                    ...  \n","1947687  A distributed view-based representation is com...  \n","1947688  3-D constraints on facial shape are achieved t...  \n","1947689  With this framework we have been able to achei...  \n","1947690  A modeled face mimics the expression of a user...  \n","1947691  This system has promise as a new approach in t...  \n","\n","[1947692 rows x 3 columns]"],"text/html":["\n","  <div id=\"df-80c10223-9b49-43fd-a8a2-4d4f63f06418\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>Main</th>\n","      <th>sentences</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","      <td>Let a mapping f : X -+ Y be given.</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","      <td>Here, X is a finite or infinite set, and Y is ...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","      <td>A learning machine observes any set of pairs (...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","      <td>(X x Y means the Cartesian product of X and Y.)</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","      <td>And, it computes some estimate j : X -+ Y of f...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>1947687</th>\n","      <td>Correlation and Interpolation Networks for Rea...</td>\n","      <td>An important task for natural and artificial ...</td>\n","      <td>A distributed view-based representation is com...</td>\n","    </tr>\n","    <tr>\n","      <th>1947688</th>\n","      <td>Correlation and Interpolation Networks for Rea...</td>\n","      <td>An important task for natural and artificial ...</td>\n","      <td>3-D constraints on facial shape are achieved t...</td>\n","    </tr>\n","    <tr>\n","      <th>1947689</th>\n","      <td>Correlation and Interpolation Networks for Rea...</td>\n","      <td>An important task for natural and artificial ...</td>\n","      <td>With this framework we have been able to achei...</td>\n","    </tr>\n","    <tr>\n","      <th>1947690</th>\n","      <td>Correlation and Interpolation Networks for Rea...</td>\n","      <td>An important task for natural and artificial ...</td>\n","      <td>A modeled face mimics the expression of a user...</td>\n","    </tr>\n","    <tr>\n","      <th>1947691</th>\n","      <td>Correlation and Interpolation Networks for Rea...</td>\n","      <td>An important task for natural and artificial ...</td>\n","      <td>This system has promise as a new approach in t...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1947692 rows × 3 columns</p>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-80c10223-9b49-43fd-a8a2-4d4f63f06418')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-80c10223-9b49-43fd-a8a2-4d4f63f06418 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-80c10223-9b49-43fd-a8a2-4d4f63f06418');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":47}]},{"cell_type":"code","source":["# Preprocess (same as for abstracts)\n","main_sent[\"clean_sent\"] = main_sent[\"sentences\"].apply(lambda x: remove_punctuation(x))\n","main_sent[\"clean_sent\"] = main_sent[\"clean_sent\"].apply(lambda x: x.lower())\n","main_sent[\"clean_sent\"] = main_sent[\"clean_sent\"].apply(lambda x: re.sub(r'\\d+', '', x))\n","main_sent[\"clean_sent\"] = main_sent[\"clean_sent\"].apply(lambda x: character_repetition(x))\n","main_sent[\"clean_sent\"] = main_sent[\"clean_sent\"].apply(lambda x: remove_spaces(x))\n","main_sent[\"clean_sent\"] = main_sent[\"clean_sent\"].apply(lambda x: preprocess(x))"],"metadata":{"id":"oXgPiye3N--4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["main_sent.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":337},"id":"d54KzcyARySo","executionInfo":{"status":"ok","timestamp":1670427383798,"user_tz":-60,"elapsed":9,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"a168cd37-98c9-4f81-ca87-94ecd2598d54"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Self-Organization of Associative Database and ...   \n","2  Self-Organization of Associative Database and ...   \n","3  Self-Organization of Associative Database and ...   \n","4  Self-Organization of Associative Database and ...   \n","\n","                                                Main  \\\n","0   Let a mapping f : X -+ Y be given. Here, X is...   \n","1   Let a mapping f : X -+ Y be given. Here, X is...   \n","2   Let a mapping f : X -+ Y be given. Here, X is...   \n","3   Let a mapping f : X -+ Y be given. Here, X is...   \n","4   Let a mapping f : X -+ Y be given. Here, X is...   \n","\n","                                           sentences  \\\n","0                 Let a mapping f : X -+ Y be given.   \n","1  Here, X is a finite or infinite set, and Y is ...   \n","2  A learning machine observes any set of pairs (...   \n","3    (X x Y means the Cartesian product of X and Y.)   \n","4  And, it computes some estimate j : X -+ Y of f...   \n","\n","                                          clean_sent  \n","0                              [let, mapping, given]  \n","1  [finite, infinite, set, another, finite, infin...  \n","2  [learning, machine, observes, set, pair, sampl...  \n","3                         [mean, cartesian, product]  \n","4  [computes, estimate, make, small, estimation, ...  "],"text/html":["\n","  <div id=\"df-e4526df1-60b3-4bf5-824d-458d9d637ed9\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>Main</th>\n","      <th>sentences</th>\n","      <th>clean_sent</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","      <td>Let a mapping f : X -+ Y be given.</td>\n","      <td>[let, mapping, given]</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","      <td>Here, X is a finite or infinite set, and Y is ...</td>\n","      <td>[finite, infinite, set, another, finite, infin...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","      <td>A learning machine observes any set of pairs (...</td>\n","      <td>[learning, machine, observes, set, pair, sampl...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","      <td>(X x Y means the Cartesian product of X and Y.)</td>\n","      <td>[mean, cartesian, product]</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","      <td>And, it computes some estimate j : X -+ Y of f...</td>\n","      <td>[computes, estimate, make, small, estimation, ...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e4526df1-60b3-4bf5-824d-458d9d637ed9')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-e4526df1-60b3-4bf5-824d-458d9d637ed9 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-e4526df1-60b3-4bf5-824d-458d9d637ed9');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":49}]},{"cell_type":"code","source":["# Compute sentences length\n","main_sent[\"sent_len\"] = main_sent[\"clean_sent\"].apply(lambda x: len(x))\n","main_sent.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":337},"id":"PLmSRtApR9Zv","executionInfo":{"status":"ok","timestamp":1670427384403,"user_tz":-60,"elapsed":611,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"30192f13-9b05-46c2-ffe2-c7210c66d2b1"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Self-Organization of Associative Database and ...   \n","2  Self-Organization of Associative Database and ...   \n","3  Self-Organization of Associative Database and ...   \n","4  Self-Organization of Associative Database and ...   \n","\n","                                                Main  \\\n","0   Let a mapping f : X -+ Y be given. Here, X is...   \n","1   Let a mapping f : X -+ Y be given. Here, X is...   \n","2   Let a mapping f : X -+ Y be given. Here, X is...   \n","3   Let a mapping f : X -+ Y be given. Here, X is...   \n","4   Let a mapping f : X -+ Y be given. Here, X is...   \n","\n","                                           sentences  \\\n","0                 Let a mapping f : X -+ Y be given.   \n","1  Here, X is a finite or infinite set, and Y is ...   \n","2  A learning machine observes any set of pairs (...   \n","3    (X x Y means the Cartesian product of X and Y.)   \n","4  And, it computes some estimate j : X -+ Y of f...   \n","\n","                                          clean_sent  sent_len  \n","0                              [let, mapping, given]         3  \n","1  [finite, infinite, set, another, finite, infin...         7  \n","2  [learning, machine, observes, set, pair, sampl...         7  \n","3                         [mean, cartesian, product]         3  \n","4  [computes, estimate, make, small, estimation, ...         7  "],"text/html":["\n","  <div id=\"df-9f8a4fc4-f319-411b-8285-f91708db03e0\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>Main</th>\n","      <th>sentences</th>\n","      <th>clean_sent</th>\n","      <th>sent_len</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","      <td>Let a mapping f : X -+ Y be given.</td>\n","      <td>[let, mapping, given]</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","      <td>Here, X is a finite or infinite set, and Y is ...</td>\n","      <td>[finite, infinite, set, another, finite, infin...</td>\n","      <td>7</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","      <td>A learning machine observes any set of pairs (...</td>\n","      <td>[learning, machine, observes, set, pair, sampl...</td>\n","      <td>7</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","      <td>(X x Y means the Cartesian product of X and Y.)</td>\n","      <td>[mean, cartesian, product]</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","      <td>And, it computes some estimate j : X -+ Y of f...</td>\n","      <td>[computes, estimate, make, small, estimation, ...</td>\n","      <td>7</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9f8a4fc4-f319-411b-8285-f91708db03e0')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-9f8a4fc4-f319-411b-8285-f91708db03e0 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-9f8a4fc4-f319-411b-8285-f91708db03e0');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":50}]},{"cell_type":"code","source":["# As for the asbtracts, drop the main text (we already have all the sentences)\n","main_sent.drop([\"Main\"],axis=1,inplace=True)\n","main_sent.columns"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"zrpd3S7fWO7A","executionInfo":{"status":"ok","timestamp":1670427384403,"user_tz":-60,"elapsed":6,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"55242743-cc5a-49b4-98c8-9875ca644c5b"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Index(['title', 'sentences', 'clean_sent', 'sent_len'], dtype='object')"]},"metadata":{},"execution_count":51}]},{"cell_type":"code","source":["#Filter out sentences with less than 3 tokens and more than max_len tokens\n","main_sent = main_sent[ (main_sent[\"sent_len\"]>2) & (main_sent[\"sent_len\"]<=max_len) ]"],"metadata":{"id":"lpX_2U3F3N6d"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import matplotlib.pyplot as plt\n","import numpy as np\n","\n","# Show distribution of sentences length in main texts\n","hist_len = plt.hist(main_sent[\"sent_len\"],bins=max_len, range=(0,max_len))\n","plt.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":265},"id":"NUgPwXvDsy-W","executionInfo":{"status":"ok","timestamp":1670427384936,"user_tz":-60,"elapsed":537,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"3c350f20-7db9-461f-af88-2ceb5bb53e6c"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["<Figure size 432x288 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYkAAAD4CAYAAAAZ1BptAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAARxUlEQVR4nO3df4xdZZ3H8fdnW1HAKCANwZbddmOjqSSu2GANG2PAhYLG8oeyEHdpCGv/ENcfceMW/yGrsoHEiLBREgLVYoxIkIRGUNIAZnf/ABnEiICECYq0KTJafrgaxep3/7hP8TrMU+jcdu5M5/1Kbuac73nOPc/cnM6nz/l1U1VIkjSTvxp3ByRJ85chIUnqMiQkSV2GhCSpy5CQJHUtHXcHDrRjjz22Vq5cOe5uSNKCct999/2yqpZNrx9yIbFy5UomJibG3Q1JWlCSPD5T3cNNkqQuQ0KS1GVISJK6DAlJUpchIUnqMiQkSV2GhCSpy5CQJHUZEpKkrkPujuu5snLzrS9M/+yy94yxJ5J08DiSkCR1GRKSpC5DQpLU9ZIhkWRLkqeS/HiodkyS7UkebT+PbvUkuSrJZJIfJTlpaJ2Nrf2jSTYO1d+W5IG2zlVJsq9tSJLmzssZSXwVWD+tthm4o6pWA3e0eYAzgdXttQm4GgZ/8IFLgLcDJwOXDP3Rvxr40NB6619iG5KkOfKSIVFV/w3snlbeAGxt01uBs4fq19fA3cBRSY4HzgC2V9Xuqnoa2A6sb8teU1V3V1UB1097r5m2IUmaI7M9J3FcVe1q008Cx7Xp5cATQ+12tNq+6jtmqO9rGy+SZFOSiSQTU1NTs/h1JEkzGfnEdRsB1AHoy6y3UVXXVNXaqlq7bNmLvn1PkjRLsw2JX7RDRbSfT7X6TuCEoXYrWm1f9RUz1Pe1DUnSHJltSGwD9l6htBG4Zah+frvKaR3wbDtkdDtwepKj2wnr04Hb27LnkqxrVzWdP+29ZtrGvLNy861/8ZKkQ8VLPpYjyTeAdwHHJtnB4Cqly4Abk1wIPA6c05rfBpwFTAK/BS4AqKrdST4L3Nvafaaq9p4M/zCDK6gOB77TXuxjG5KkOfKSIVFV53UWnTZD2wIu6rzPFmDLDPUJ4MQZ6r+aaRuSpLnjHdeSpC5DQpLUZUhIkroMCUlSl186dBD4hUSSDhWOJCRJXY4kDrLpN9c5spC0kDiSkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSeryEtg55o12khYSRxKSpC5DQpLUZUhIkroMCUlSlyEhSery6qYx8uF/kuY7RxKSpC5DQpLUZUhIkroMCUlSlyEhSeoyJCRJXYaEJKnLkJAkdRkSkqQuQ0KS1OVjOeYRv5BI0nwz0kgiySeSPJjkx0m+keRVSVYluSfJZJJvJjmstX1lm59sy1cOvc/Frf5IkjOG6utbbTLJ5lH6Kknaf7MOiSTLgY8Ca6vqRGAJcC5wOXBFVb0BeBq4sK1yIfB0q1/R2pFkTVvvzcB64MtJliRZAnwJOBNYA5zX2kqS5sio5ySWAocnWQocAewCTgVuasu3Ame36Q1tnrb8tCRp9Ruq6vdV9VNgEji5vSar6rGqeh64obWVJM2RWYdEVe0EPg/8nEE4PAvcBzxTVXtasx3A8ja9HHiirbuntX/dcH3aOr36iyTZlGQiycTU1NRsfyVJ0jSjHG46msH/7FcBrweOZHC4aM5V1TVVtbaq1i5btmwcXZCkQ9Ioh5veDfy0qqaq6g/AzcApwFHt8BPACmBnm94JnADQlr8W+NVwfdo6vbokaY6MEhI/B9YlOaKdWzgNeAi4C3h/a7MRuKVNb2vztOV3VlW1+rnt6qdVwGrg+8C9wOp2tdRhDE5ubxuhv5Kk/TTr+ySq6p4kNwE/APYA9wPXALcCNyT5XKtd11a5DvhakklgN4M/+lTVg0luZBAwe4CLquqPAEk+AtzO4MqpLVX14Gz7K0nafyPdTFdVlwCXTCs/xuDKpOltfwd8oPM+lwKXzlC/DbhtlD5KkmbPx3JIkroMCUlSlyEhSeoyJCRJXT4Fdp4afiIs+FRYSePhSEKS1GVISJK6DAlJUpchIUnqMiQkSV2GhCSpy5CQJHUZEpKkLkNCktRlSEiSugwJSVKXISFJ6jIkJEldPgV2gfCpsJLGwZGEJKnLkJAkdRkSkqQuQ0KS1GVISJK6DAlJUpchIUnqMiQkSV2GhCSpy5CQJHUZEpKkLkNCktQ1UkgkOSrJTUl+kuThJO9IckyS7UkebT+Pbm2T5Kokk0l+lOSkoffZ2No/mmTjUP1tSR5o61yVJKP0V5K0f0YdSVwJfLeq3gS8BXgY2AzcUVWrgTvaPMCZwOr22gRcDZDkGOAS4O3AycAle4OltfnQ0HrrR+zvIWPl5ltfeEnSwTLrkEjyWuCdwHUAVfV8VT0DbAC2tmZbgbPb9Abg+hq4GzgqyfHAGcD2qtpdVU8D24H1bdlrquruqirg+qH3kiTNgVFGEquAKeArSe5Pcm2SI4HjqmpXa/MkcFybXg48MbT+jlbbV33HDPUXSbIpyUSSiampqRF+JUnSsFFCYilwEnB1Vb0V+A1/PrQEQBsB1AjbeFmq6pqqWltVa5ctW3awNydJi8YoIbED2FFV97T5mxiExi/aoSLaz6fa8p3ACUPrr2i1fdVXzFCXJM2RWYdEVT0JPJHkja10GvAQsA3Ye4XSRuCWNr0NOL9d5bQOeLYdlrodOD3J0e2E9enA7W3Zc0nWtauazh96L0nSHBj1O67/Ffh6ksOAx4ALGATPjUkuBB4HzmltbwPOAiaB37a2VNXuJJ8F7m3tPlNVu9v0h4GvAocD32kvSdIcGSkkquqHwNoZFp02Q9sCLuq8zxZgywz1CeDEUfooSZo977iWJHUZEpKkLkNCktRlSEiSugwJSVKXISFJ6jIkJEldo95Mp3lg+uPCf3bZe8bUE0mHGkcSkqQuQ0KS1GVISJK6DAlJUpchIUnqMiQkSV2GhCSpy5CQJHUZEpKkLkNCktRlSEiSugwJSVKXISFJ6jIkJEldPir8EDT86HAfGy5pFI4kJEldhoQkqcuQkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSeoaOSSSLElyf5Jvt/lVSe5JMpnkm0kOa/VXtvnJtnzl0Htc3OqPJDljqL6+1SaTbB61r5Kk/XMgRhIfAx4emr8cuKKq3gA8DVzY6hcCT7f6Fa0dSdYA5wJvBtYDX27BswT4EnAmsAY4r7WVJM2RkUIiyQrgPcC1bT7AqcBNrclW4Ow2vaHN05af1tpvAG6oqt9X1U+BSeDk9pqsqseq6nnghtZWkjRHRh1JfBH4FPCnNv864Jmq2tPmdwDL2/Ry4AmAtvzZ1v6F+rR1evUXSbIpyUSSiampqRF/JUnSXrMOiSTvBZ6qqvsOYH9mpaquqaq1VbV22bJl4+6OJB0yRnkK7CnA+5KcBbwKeA1wJXBUkqVttLAC2Nna7wROAHYkWQq8FvjVUH2v4XV6db1Mw0+EBZ8KK2n/zHokUVUXV9WKqlrJ4MTznVX1QeAu4P2t2Ubglja9rc3Tlt9ZVdXq57arn1YBq4HvA/cCq9vVUoe1bWybbX8lSfvvYHyfxL8DNyT5HHA/cF2rXwd8LckksJvBH32q6sEkNwIPAXuAi6rqjwBJPgLcDiwBtlTVgwehv5KkjgMSElX1PeB7bfoxBlcmTW/zO+ADnfUvBS6doX4bcNuB6KMkaf95x7UkqcuQkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSeo6GDfTaR4bfkyHj+iQ9FIcSUiSugwJSVKXISFJ6jIkJEldhoQkqcuQkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSery2U2L2PBznMBnOUl6MUcSkqQuQ0KS1GVISJK6DAlJUpchIUnqMiQkSV2GhCSpy5CQJHV5M51eMHxznTfWSQJHEpKkfTAkJEldsw6JJCckuSvJQ0keTPKxVj8myfYkj7afR7d6klyVZDLJj5KcNPReG1v7R5NsHKq/LckDbZ2rkmSUX1aStH9GGUnsAT5ZVWuAdcBFSdYAm4E7qmo1cEebBzgTWN1em4CrYRAqwCXA24GTgUv2Bktr86Gh9daP0F9J0n6adUhU1a6q+kGb/jXwMLAc2ABsbc22Ame36Q3A9TVwN3BUkuOBM4DtVbW7qp4GtgPr27LXVNXdVVXA9UPvJUmaAwfknESSlcBbgXuA46pqV1v0JHBcm14OPDG02o5W21d9xwz1mba/KclEkompqamRfhdJ0p+NHBJJXg18C/h4VT03vKyNAGrUbbyUqrqmqtZW1dply5Yd7M1J0qIxUkgkeQWDgPh6Vd3cyr9oh4poP59q9Z3ACUOrr2i1fdVXzFCXJM2RWd9M1640ug54uKq+MLRoG7ARuKz9vGWo/pEkNzA4Sf1sVe1Kcjvwn0Mnq08HLq6q3UmeS7KOwWGs84H/mm1/tX/81jpJMNod16cA/ww8kOSHrfZpBuFwY5ILgceBc9qy24CzgEngt8AFAC0MPgvc29p9pqp2t+kPA18FDge+016SpDky65Coqv8FevctnDZD+wIu6rzXFmDLDPUJ4MTZ9lGSNBrvuJYkdRkSkqQuQ0KS1GVISJK6/D4JvSx+14S0ODmSkCR1GRKSpC5DQpLUZUhIkroMCUlSl1c3ab/58D9p8XAkIUnqMiQkSV2GhCSpy5CQJHV54loj85Ed0qHLkYQkqcuQkCR1ebhJB5T3UEiHFkcSkqQuQ0KS1OXhJh1UHn6SFjZHEpKkLkNCktTl4SbNKW+8kxYWRxKSpC5HEhobT2pL858hoXnDQ1HS/GNIaF5ylCHND4aEFgRHGdJ4GBJacBxlSHNn3odEkvXAlcAS4NqqumzMXdI8Mz00hhkg0mjmdUgkWQJ8CfgHYAdwb5JtVfXQeHumhWJfATLMMJFmNq9DAjgZmKyqxwCS3ABsAAwJHVAvN0zGYXqAeX5Gc2m+h8Ry4Imh+R3A26c3SrIJ2NRm/y/JI7Pc3rHAL2e57qHOz6bvoH42uXx2y+YB95m++fjZ/M1MxfkeEi9LVV0DXDPq+ySZqKq1B6BLhxw/mz4/m5n5ufQtpM9mvj+WYydwwtD8ilaTJM2B+R4S9wKrk6xKchhwLrBtzH2SpEVjXh9uqqo9ST4C3M7gEtgtVfXgQdzkyIesDmF+Nn1+NjPzc+lbMJ9NqmrcfZAkzVPz/XCTJGmMDAlJUpch0SRZn+SRJJNJNo+7P+OS5IQkdyV5KMmDST7W6sck2Z7k0fbz6HH3dVySLElyf5Jvt/lVSe5p+84320UWi06So5LclOQnSR5O8g73m4Ekn2j/nn6c5BtJXrVQ9htDgr94/MeZwBrgvCRrxtursdkDfLKq1gDrgIvaZ7EZuKOqVgN3tPnF6mPAw0PzlwNXVNUbgKeBC8fSq/G7EvhuVb0JeAuDz2jR7zdJlgMfBdZW1YkMLsI5lwWy3xgSAy88/qOqngf2Pv5j0amqXVX1gzb9awb/0Jcz+Dy2tmZbgbPH08PxSrICeA9wbZsPcCpwU2uyKD+bJK8F3glcB1BVz1fVM7jf7LUUODzJUuAIYBcLZL8xJAZmevzH8jH1Zd5IshJ4K3APcFxV7WqLngSOG1O3xu2LwKeAP7X51wHPVNWeNr9Y951VwBTwlXYo7tokR+J+Q1XtBD4P/JxBODwL3McC2W8MCc0oyauBbwEfr6rnhpfV4LrpRXftdJL3Ak9V1X3j7ss8tBQ4Cbi6qt4K/IZph5YW8X5zNIMR1Srg9cCRwPqxdmo/GBIDPv5jSJJXMAiIr1fVza38iyTHt+XHA0+Nq39jdArwviQ/Y3BI8lQGx+GPaocRYPHuOzuAHVV1T5u/iUFouN/Au4GfVtVUVf0BuJnBvrQg9htDYsDHfzTtGPt1wMNV9YWhRduAjW16I3DLXPdt3Krq4qpaUVUrGewjd1bVB4G7gPe3Zov1s3kSeCLJG1vpNAaP9F/0+w2Dw0zrkhzR/n3t/WwWxH7jHddNkrMYHG/e+/iPS8fcpbFI8vfA/wAP8Ofj7p9mcF7iRuCvgceBc6pq91g6OQ8keRfwb1X13iR/y2BkcQxwP/BPVfX7cfZvHJL8HYMT+ocBjwEXMPiP6KLfb5L8B/CPDK4evB/4FwbnIOb9fmNISJK6PNwkSeoyJCRJXYaEJKnLkJAkdRkSkqQuQ0KS1GVISJK6/h/Iy1jCcrNGmQAAAABJRU5ErkJggg==\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":["#Get relative position of every sentence in its belonging document\n","main_sent[\"sent_pos\"] = (main_sent.groupby(\"title\",sort=False).cumcount() + 1).astype(float)\n","main_sent[\"total_length\"] = 0.\n","main_sent = main_sent.join(main_sent.groupby(\"title\",sort=False)[\"total_length\"].size(), on=\"title\", rsuffix='_r')\n","\n","main_sent[\"sent_pos\"] = main_sent[\"sent_pos\"] / main_sent[\"total_length_r\"]\n","\n","main_sent.drop([\"total_length\",\"total_length_r\"],axis=1,inplace=True)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9eZ6E0yODx5b","executionInfo":{"status":"ok","timestamp":1670427386076,"user_tz":-60,"elapsed":1144,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"156252a6-7ec6-49c8-e068-3cf481eaa794"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["<ipython-input-54-e8b0b9bc7f99>:2: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  main_sent[\"sent_pos\"] = (main_sent.groupby(\"title\",sort=False).cumcount() + 1).astype(float)\n","<ipython-input-54-e8b0b9bc7f99>:3: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame.\n","Try using .loc[row_indexer,col_indexer] = value instead\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  main_sent[\"total_length\"] = 0.\n"]}]},{"cell_type":"code","source":["main_sent.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":250},"id":"BgnlMfM68YGA","executionInfo":{"status":"ok","timestamp":1670427386076,"user_tz":-60,"elapsed":5,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"a621101e-dff9-4645-e4ae-bcc22162b14d"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Self-Organization of Associative Database and ...   \n","2  Self-Organization of Associative Database and ...   \n","3  Self-Organization of Associative Database and ...   \n","4  Self-Organization of Associative Database and ...   \n","\n","                                           sentences  \\\n","0                 Let a mapping f : X -+ Y be given.   \n","1  Here, X is a finite or infinite set, and Y is ...   \n","2  A learning machine observes any set of pairs (...   \n","3    (X x Y means the Cartesian product of X and Y.)   \n","4  And, it computes some estimate j : X -+ Y of f...   \n","\n","                                          clean_sent  sent_len  sent_pos  \n","0                              [let, mapping, given]         3  0.005051  \n","1  [finite, infinite, set, another, finite, infin...         7  0.010101  \n","2  [learning, machine, observes, set, pair, sampl...         7  0.015152  \n","3                         [mean, cartesian, product]         3  0.020202  \n","4  [computes, estimate, make, small, estimation, ...         7  0.025253  "],"text/html":["\n","  <div id=\"df-1bd47b4a-d65d-4574-ac25-14d91f224c44\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>sentences</th>\n","      <th>clean_sent</th>\n","      <th>sent_len</th>\n","      <th>sent_pos</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given.</td>\n","      <td>[let, mapping, given]</td>\n","      <td>3</td>\n","      <td>0.005051</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Here, X is a finite or infinite set, and Y is ...</td>\n","      <td>[finite, infinite, set, another, finite, infin...</td>\n","      <td>7</td>\n","      <td>0.010101</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>A learning machine observes any set of pairs (...</td>\n","      <td>[learning, machine, observes, set, pair, sampl...</td>\n","      <td>7</td>\n","      <td>0.015152</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>(X x Y means the Cartesian product of X and Y.)</td>\n","      <td>[mean, cartesian, product]</td>\n","      <td>3</td>\n","      <td>0.020202</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>And, it computes some estimate j : X -+ Y of f...</td>\n","      <td>[computes, estimate, make, small, estimation, ...</td>\n","      <td>7</td>\n","      <td>0.025253</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1bd47b4a-d65d-4574-ac25-14d91f224c44')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-1bd47b4a-d65d-4574-ac25-14d91f224c44 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-1bd47b4a-d65d-4574-ac25-14d91f224c44');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":55}]},{"cell_type":"code","source":["main_sent.to_csv(\"drive/MyDrive/TM&S/data/main_features.csv\", index=False)"],"metadata":{"id":"4f5a6j6EVi5c"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Create dataset for Topic Modeling"],"metadata":{"id":"amL40tNxJn4l"}},{"cell_type":"code","source":["# Sentences, after preprocessing, are put back together in order to have clean documents to feed to Topic Models\n","altrodf = main_sent\n","\n","altrodf= altrodf.groupby(by=\"title\", sort=False)[\"sentences\"].apply(lambda x: ' '.join(x)).reset_index()\n","\n","altrodf.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"ot02f-JNMg0_","executionInfo":{"status":"ok","timestamp":1670427449480,"user_tz":-60,"elapsed":589,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"de950385-ba8f-4f6b-8b26-4eb422938f1c"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Bayesian Query Construction for Neural Network...   \n","2  Neural Network Ensembles, Cross Validation, an...   \n","3  Pulsestream Synapses with Non-Volatile Analogu...   \n","4                 Learning to Play the Game of Chess   \n","\n","                                           sentences  \n","0   Let a mapping f : X -+ Y be given. Here, X is...  \n","1    In this paper we consider the situation wher...  \n","2    It is well known that a combination of many ...  \n","3    Fusion of the nuclei of hydrogen provides th...  \n","4   Throughout the last decades, the game of ches...  "],"text/html":["\n","  <div id=\"df-3d16d22b-95b4-4500-82da-acd7c36b64bc\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>sentences</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given. Here, X is...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Bayesian Query Construction for Neural Network...</td>\n","      <td>In this paper we consider the situation wher...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Neural Network Ensembles, Cross Validation, an...</td>\n","      <td>It is well known that a combination of many ...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Pulsestream Synapses with Non-Volatile Analogu...</td>\n","      <td>Fusion of the nuclei of hydrogen provides th...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Learning to Play the Game of Chess</td>\n","      <td>Throughout the last decades, the game of ches...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3d16d22b-95b4-4500-82da-acd7c36b64bc')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-3d16d22b-95b4-4500-82da-acd7c36b64bc button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-3d16d22b-95b4-4500-82da-acd7c36b64bc');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":61}]},{"cell_type":"code","source":["altrodf.to_csv(\"drive/MyDrive/TM&S/data/main_ridotto.csv\", index=False)"],"metadata":{"id":"TC0FLUC1OhyS"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Find bigrams and trigrams"],"metadata":{"id":"E8za2sIbRCNp"}},{"cell_type":"code","source":["import pandas as pd\n","\n","main_sent = pd.read_csv(\"drive/MyDrive/TM&S/data/main_features.csv\")\n","abs_sent = pd.read_csv(\"drive/MyDrive/TM&S/data/abs_features.csv\")"],"metadata":{"id":"tanhTTTkRTET"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from ast import literal_eval\n","\n","main_sent[\"clean_sent\"] = main_sent[\"clean_sent\"].apply(literal_eval)\n","abs_sent[\"clean_sent\"] = abs_sent[\"clean_sent\"].apply(literal_eval)"],"metadata":{"id":"4SPesb2-Alvf"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!pip install --upgrade gensim\n","\n","import gensim\n","from gensim.models.phrases import Phrases, ENGLISH_CONNECTOR_WORDS"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"I6GwiraARfjU","executionInfo":{"status":"ok","timestamp":1670427954568,"user_tz":-60,"elapsed":7425,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"6ff3dbbd-a89d-4844-da53-bc680247bbcd"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Requirement already satisfied: gensim in /usr/local/lib/python3.8/dist-packages (4.2.0)\n","Requirement already satisfied: numpy>=1.17.0 in /usr/local/lib/python3.8/dist-packages (from gensim) (1.21.6)\n","Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.8/dist-packages (from gensim) (1.7.3)\n","Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.8/dist-packages (from gensim) (5.2.1)\n"]}]},{"cell_type":"code","source":["# Load bigrams and trigrams model created in the LDA notebook\n","bigrams_model = Phrases.load(\"drive/MyDrive/TM&S/models/bigrams_model.pkl\")\n","trigrams_model = Phrases.load(\"drive/MyDrive/TM&S/models/trigrams_model.pkl\")"],"metadata":{"id":"PSyKfC6-RU6m"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Substitute bigrams\n","main_sent[\"clean_sent_grams\"] = bigrams_model[main_sent[\"clean_sent\"]]\n","abs_sent[\"clean_sent_grams\"] = bigrams_model[abs_sent[\"clean_sent\"]]"],"metadata":{"id":"DcceVyK3RvzX"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Substitute trigrams\n","main_sent[\"clean_sent_grams\"] = bigrams_model[main_sent[\"clean_sent_grams\"]]\n","abs_sent[\"clean_sent_grams\"] = bigrams_model[abs_sent[\"clean_sent_grams\"]]"],"metadata":{"id":"eXnXkRjuSVp9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["main_sent.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":337},"id":"cjpDBRmPSu7C","executionInfo":{"status":"ok","timestamp":1670424013465,"user_tz":-60,"elapsed":29,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"27e5a047-0b72-4914-cd03-902cc4d43fc4"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Self-Organization of Associative Database and ...   \n","2  Self-Organization of Associative Database and ...   \n","3  Self-Organization of Associative Database and ...   \n","4  Self-Organization of Associative Database and ...   \n","\n","                                           sentences  \\\n","0                 Let a mapping f : X -+ Y be given.   \n","1  Here, X is a finite or infinite set, and Y is ...   \n","2  A learning machine observes any set of pairs (...   \n","3    (X x Y means the Cartesian product of X and Y.)   \n","4  And, it computes some estimate j : X -+ Y of f...   \n","\n","                                          clean_sent  sent_len  sent_pos  \\\n","0                              [let, mapping, given]         3  0.005051   \n","1  [finite, infinite, set, another, finite, infin...         7  0.010101   \n","2  [learning, machine, observes, set, pair, sampl...         7  0.015152   \n","3                         [mean, cartesian, product]         3  0.020202   \n","4  [computes, estimate, make, small, estimation, ...         7  0.025253   \n","\n","                                    clean_sent_grams  \n","0                              [let, mapping, given]  \n","1  [finite_infinite, set, another, finite_infinit...  \n","2  [learning, machine, observes, set, pair_sample...  \n","3                          [mean, cartesian_product]  \n","4  [computes, estimate, make, small, estimation_e...  "],"text/html":["\n","  <div id=\"df-10bd9b2e-6dd7-4415-811f-36a35e46c460\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>sentences</th>\n","      <th>clean_sent</th>\n","      <th>sent_len</th>\n","      <th>sent_pos</th>\n","      <th>clean_sent_grams</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given.</td>\n","      <td>[let, mapping, given]</td>\n","      <td>3</td>\n","      <td>0.005051</td>\n","      <td>[let, mapping, given]</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Here, X is a finite or infinite set, and Y is ...</td>\n","      <td>[finite, infinite, set, another, finite, infin...</td>\n","      <td>7</td>\n","      <td>0.010101</td>\n","      <td>[finite_infinite, set, another, finite_infinit...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>A learning machine observes any set of pairs (...</td>\n","      <td>[learning, machine, observes, set, pair, sampl...</td>\n","      <td>7</td>\n","      <td>0.015152</td>\n","      <td>[learning, machine, observes, set, pair_sample...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>(X x Y means the Cartesian product of X and Y.)</td>\n","      <td>[mean, cartesian, product]</td>\n","      <td>3</td>\n","      <td>0.020202</td>\n","      <td>[mean, cartesian_product]</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>And, it computes some estimate j : X -+ Y of f...</td>\n","      <td>[computes, estimate, make, small, estimation, ...</td>\n","      <td>7</td>\n","      <td>0.025253</td>\n","      <td>[computes, estimate, make, small, estimation_e...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-10bd9b2e-6dd7-4415-811f-36a35e46c460')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-10bd9b2e-6dd7-4415-811f-36a35e46c460 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-10bd9b2e-6dd7-4415-811f-36a35e46c460');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":13}]},{"cell_type":"code","source":["abs_sent.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":337},"id":"M8UGd4dLS2BG","executionInfo":{"status":"ok","timestamp":1670424013467,"user_tz":-60,"elapsed":25,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"69bb3013-b3c0-4523-c218-6a79db41a8dc"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Self-Organization of Associative Database and ...   \n","2  Self-Organization of Associative Database and ...   \n","3  Self-Organization of Associative Database and ...   \n","4  Self-Organization of Associative Database and ...   \n","\n","                                           sentences  \\\n","0   An efficient method of self-organizing associ...   \n","1  The proposed databases can associate any input...   \n","2  In the first half part of discussion, an algor...   \n","3  From an aspect of hardware, it produces a new ...   \n","4  In the latter half part, an applicability to h...   \n","\n","                                          clean_sent  sent_len  sent_pos  \\\n","0  [efficient, method, selforganizing, associativ...        11       0.2   \n","1     [proposed, database, associate, input, output]         5       0.4   \n","2  [first, half, part, discussion, algorithm, sel...         7       0.6   \n","3  [aspect, hardware, produce, new, style, neural...         7       0.8   \n","4  [latter, half, part, applicability, handwritte...        12       1.0   \n","\n","                                    clean_sent_grams  \n","0  [efficient, method, selforganizing, associativ...  \n","1      [proposed, database, associate, input_output]  \n","2  [first_half, part, discussion, algorithm, self...  \n","3  [aspect, hardware, produce_new, style, neural_...  \n","4  [latter, half, part, applicability, handwritte...  "],"text/html":["\n","  <div id=\"df-74be5bc2-67e5-4ef1-ad04-690737701670\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>sentences</th>\n","      <th>clean_sent</th>\n","      <th>sent_len</th>\n","      <th>sent_pos</th>\n","      <th>clean_sent_grams</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>An efficient method of self-organizing associ...</td>\n","      <td>[efficient, method, selforganizing, associativ...</td>\n","      <td>11</td>\n","      <td>0.2</td>\n","      <td>[efficient, method, selforganizing, associativ...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>The proposed databases can associate any input...</td>\n","      <td>[proposed, database, associate, input, output]</td>\n","      <td>5</td>\n","      <td>0.4</td>\n","      <td>[proposed, database, associate, input_output]</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>In the first half part of discussion, an algor...</td>\n","      <td>[first, half, part, discussion, algorithm, sel...</td>\n","      <td>7</td>\n","      <td>0.6</td>\n","      <td>[first_half, part, discussion, algorithm, self...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>From an aspect of hardware, it produces a new ...</td>\n","      <td>[aspect, hardware, produce, new, style, neural...</td>\n","      <td>7</td>\n","      <td>0.8</td>\n","      <td>[aspect, hardware, produce_new, style, neural_...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>In the latter half part, an applicability to h...</td>\n","      <td>[latter, half, part, applicability, handwritte...</td>\n","      <td>12</td>\n","      <td>1.0</td>\n","      <td>[latter, half, part, applicability, handwritte...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-74be5bc2-67e5-4ef1-ad04-690737701670')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-74be5bc2-67e5-4ef1-ad04-690737701670 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-74be5bc2-67e5-4ef1-ad04-690737701670');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":14}]},{"cell_type":"code","source":["main_sent.to_csv(\"drive/MyDrive/TM&S/data/main_grams_features.csv\", index=False)\n","abs_sent.to_csv(\"drive/MyDrive/TM&S/data/abs_grams_features.csv\", index=False)"],"metadata":{"id":"5qId8z1IS-4a"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Topic similarity"],"metadata":{"id":"DjXBO6e_Iuj2"}},{"cell_type":"code","source":["import pandas as pd\n","from ast import literal_eval\n","\n","main_sent = pd.read_csv(\"drive/MyDrive/TM&S/data/main_grams_features.csv\")\n","\n","\n","main_sent[\"clean_sent\"] = main_sent[\"clean_sent\"].apply(literal_eval)\n","main_sent[\"clean_sent_grams\"] = main_sent[\"clean_sent_grams\"].apply(literal_eval)"],"metadata":{"id":"_gHRhYoYro3s"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import gensim\n","\n","# Load LDA model\n","LDA = gensim.models.LdaModel.load(\"/content/drive/MyDrive/TM&S/models/lda_10trigrams\")"],"metadata":{"id":"SJjVkQOe7qTM"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import gensim.corpora as corpora\n","\n","#Import dictionary used for LDA\n","id2word_main =  gensim.utils.SaveLoad.load(\"drive/MyDrive/TM&S/models/main_bow.pkl\")\n","\n","# Term Document Frequency for every sentence\n","corpus_main = [id2word_main.doc2bow(text) for text in main_sent[\"clean_sent_grams\"]]"],"metadata":{"id":"lZsolBJw8ohJ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Get topics probability for every sentence in the corpus\n","topics = LDA.get_document_topics(corpus_main,minimum_probability=0)"],"metadata":{"id":"Wz95r0En8THl"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["topics[1]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"33HmO6jiYD_I","executionInfo":{"status":"ok","timestamp":1670435540959,"user_tz":-60,"elapsed":3,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"fca7e29b-e225-4fdd-914b-4734793ba6f4"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[(0, 0.080829635),\n"," (1, 0.10600426),\n"," (2, 0.081088275),\n"," (3, 0.07327901),\n"," (4, 0.11034008),\n"," (5, 0.09871563),\n"," (6, 0.07748632),\n"," (7, 0.12347073),\n"," (8, 0.09058568),\n"," (9, 0.15820037)]"]},"metadata":{},"execution_count":21}]},{"cell_type":"code","source":["# Insert new data in the dataframe\n","main_sent[\"topics\"] = [list(zip(*topics[i]))[1] for i in range(len(topics))]"],"metadata":{"id":"JTvdYc-laEgI"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# From sentences to clean main texts\n","altrodf = main_sent\n","altrodf= altrodf.groupby(by=\"title\", sort=False)[\"clean_sent_grams\"].aggregate(func=sum).reset_index()\n","\n","# Term Document Frequency for every paper\n","X = [id2word_main.doc2bow(text) for text in altrodf[\"clean_sent_grams\"]]\n","\n","# Get topics probability for every paper\n","doc_topics = LDA.get_document_topics(X,minimum_probability=0)\n","\n","altrodf[\"topics\"] = [list(zip(*doc_topics[i]))[1] for i in range(len(doc_topics))]\n","\n","altrodf.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"wXI6QacBcm1i","executionInfo":{"status":"ok","timestamp":1670439706278,"user_tz":-60,"elapsed":19681,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"2bb07f74-8191-443d-efe3-4c627f98bf7e"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Bayesian Query Construction for Neural Network...   \n","2  Neural Network Ensembles, Cross Validation, an...   \n","3  Pulsestream Synapses with Non-Volatile Analogu...   \n","4                 Learning to Play the Game of Chess   \n","\n","                                    clean_sent_grams  \\\n","0  [let, mapping, given, finite_infinite, set, an...   \n","1  [paper_consider, situation, data_collection, c...   \n","2  [well_known, combination, many_different, pred...   \n","3  [fusion, nucleus, hydrogen, provides, energy, ...   \n","4  [throughout, last_decade, game_chess, major, t...   \n","\n","                                              topics  \n","0  (0.045496676, 0.50505626, 4.1512463e-05, 3.751...  \n","1  (6.318733e-05, 0.29339448, 0.13820426, 5.72847...  \n","2  (0.7627803, 0.0999377, 5.9926042e-05, 5.415482...  \n","3  (3.0070025e-05, 0.92038685, 3.0166242e-05, 0.0...  \n","4  (0.029601334, 5.556049e-05, 4.2501157e-05, 3.8...  "],"text/html":["\n","  <div id=\"df-8eb8eba2-2084-4e9e-822d-372bfaa08995\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>clean_sent_grams</th>\n","      <th>topics</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>[let, mapping, given, finite_infinite, set, an...</td>\n","      <td>(0.045496676, 0.50505626, 4.1512463e-05, 3.751...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Bayesian Query Construction for Neural Network...</td>\n","      <td>[paper_consider, situation, data_collection, c...</td>\n","      <td>(6.318733e-05, 0.29339448, 0.13820426, 5.72847...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Neural Network Ensembles, Cross Validation, an...</td>\n","      <td>[well_known, combination, many_different, pred...</td>\n","      <td>(0.7627803, 0.0999377, 5.9926042e-05, 5.415482...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Pulsestream Synapses with Non-Volatile Analogu...</td>\n","      <td>[fusion, nucleus, hydrogen, provides, energy, ...</td>\n","      <td>(3.0070025e-05, 0.92038685, 3.0166242e-05, 0.0...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Learning to Play the Game of Chess</td>\n","      <td>[throughout, last_decade, game_chess, major, t...</td>\n","      <td>(0.029601334, 5.556049e-05, 4.2501157e-05, 3.8...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8eb8eba2-2084-4e9e-822d-372bfaa08995')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-8eb8eba2-2084-4e9e-822d-372bfaa08995 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-8eb8eba2-2084-4e9e-822d-372bfaa08995');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":63}]},{"cell_type":"code","source":["# Merge the dataframes, so that in every row we have sentence-level and document-level topic representation (easier to compute a distance later)\n","altrodf.rename(columns = {'topics':'doc_topics'}, inplace = True)\n","main_sent = main_sent.merge(altrodf[[\"title\",\"doc_topics\"]], how=\"left\", on=\"title\")"],"metadata":{"id":"swEsc1AgnEOJ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["main_sent.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":458},"id":"OaZzaEJFj_Wv","executionInfo":{"status":"ok","timestamp":1670439707357,"user_tz":-60,"elapsed":15,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"9a8c032d-978f-4266-88b5-f39baae22257"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Self-Organization of Associative Database and ...   \n","2  Self-Organization of Associative Database and ...   \n","3  Self-Organization of Associative Database and ...   \n","4  Self-Organization of Associative Database and ...   \n","\n","                                           sentences  \\\n","0                 Let a mapping f : X -+ Y be given.   \n","1  Here, X is a finite or infinite set, and Y is ...   \n","2  A learning machine observes any set of pairs (...   \n","3    (X x Y means the Cartesian product of X and Y.)   \n","4  And, it computes some estimate j : X -+ Y of f...   \n","\n","                                          clean_sent  sent_len  sent_pos  \\\n","0                              [let, mapping, given]         3  0.005051   \n","1  [finite, infinite, set, another, finite, infin...         7  0.010101   \n","2  [learning, machine, observes, set, pair, sampl...         7  0.015152   \n","3                         [mean, cartesian, product]         3  0.020202   \n","4  [computes, estimate, make, small, estimation, ...         7  0.025253   \n","\n","                                    clean_sent_grams  \\\n","0                              [let, mapping, given]   \n","1  [finite_infinite, set, another, finite_infinit...   \n","2  [learning, machine, observes, set, pair_sample...   \n","3                          [mean, cartesian_product]   \n","4  [computes, estimate, make, small, estimation_e...   \n","\n","                                              topics  \\\n","0  (0.02530865, 0.033191103, 0.02538963, 0.022944...   \n","1  (0.080829635, 0.10600426, 0.081088275, 0.07327...   \n","2  (0.010663864, 0.013985155, 0.010697986, 0.0096...   \n","3  (0.025395706, 0.033305272, 0.025476966, 0.0230...   \n","4  (0.015006904, 0.019680848, 0.015054923, 0.0136...   \n","\n","                                          doc_topics  \n","0  (0.045496676, 0.50505626, 4.1512463e-05, 3.751...  \n","1  (0.045496676, 0.50505626, 4.1512463e-05, 3.751...  \n","2  (0.045496676, 0.50505626, 4.1512463e-05, 3.751...  \n","3  (0.045496676, 0.50505626, 4.1512463e-05, 3.751...  \n","4  (0.045496676, 0.50505626, 4.1512463e-05, 3.751...  "],"text/html":["\n","  <div id=\"df-fd31830f-e57d-4913-a42b-0abbdd3bbba6\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>sentences</th>\n","      <th>clean_sent</th>\n","      <th>sent_len</th>\n","      <th>sent_pos</th>\n","      <th>clean_sent_grams</th>\n","      <th>topics</th>\n","      <th>doc_topics</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given.</td>\n","      <td>[let, mapping, given]</td>\n","      <td>3</td>\n","      <td>0.005051</td>\n","      <td>[let, mapping, given]</td>\n","      <td>(0.02530865, 0.033191103, 0.02538963, 0.022944...</td>\n","      <td>(0.045496676, 0.50505626, 4.1512463e-05, 3.751...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Here, X is a finite or infinite set, and Y is ...</td>\n","      <td>[finite, infinite, set, another, finite, infin...</td>\n","      <td>7</td>\n","      <td>0.010101</td>\n","      <td>[finite_infinite, set, another, finite_infinit...</td>\n","      <td>(0.080829635, 0.10600426, 0.081088275, 0.07327...</td>\n","      <td>(0.045496676, 0.50505626, 4.1512463e-05, 3.751...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>A learning machine observes any set of pairs (...</td>\n","      <td>[learning, machine, observes, set, pair, sampl...</td>\n","      <td>7</td>\n","      <td>0.015152</td>\n","      <td>[learning, machine, observes, set, pair_sample...</td>\n","      <td>(0.010663864, 0.013985155, 0.010697986, 0.0096...</td>\n","      <td>(0.045496676, 0.50505626, 4.1512463e-05, 3.751...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>(X x Y means the Cartesian product of X and Y.)</td>\n","      <td>[mean, cartesian, product]</td>\n","      <td>3</td>\n","      <td>0.020202</td>\n","      <td>[mean, cartesian_product]</td>\n","      <td>(0.025395706, 0.033305272, 0.025476966, 0.0230...</td>\n","      <td>(0.045496676, 0.50505626, 4.1512463e-05, 3.751...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>And, it computes some estimate j : X -+ Y of f...</td>\n","      <td>[computes, estimate, make, small, estimation, ...</td>\n","      <td>7</td>\n","      <td>0.025253</td>\n","      <td>[computes, estimate, make, small, estimation_e...</td>\n","      <td>(0.015006904, 0.019680848, 0.015054923, 0.0136...</td>\n","      <td>(0.045496676, 0.50505626, 4.1512463e-05, 3.751...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fd31830f-e57d-4913-a42b-0abbdd3bbba6')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-fd31830f-e57d-4913-a42b-0abbdd3bbba6 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-fd31830f-e57d-4913-a42b-0abbdd3bbba6');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":65}]},{"cell_type":"code","source":["import numpy as np\n","\n","main_sent[\"topics\"] = main_sent[\"topics\"].apply(lambda x: np.asarray(x))\n","main_sent[\"doc_topics\"] = main_sent[\"doc_topics\"].apply(lambda x: np.asarray(x))"],"metadata":{"id":"TVk7FiJHoKww"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from scipy.spatial import distance\n","\n","# Compute Jensen-Shannon distance between the sentence-level and document-level topic distributions\n","main_sent[\"topics_sim\"] = pd.Series(\n","    distance.jensenshannon(\n","        np.vstack(main_sent[\"topics\"]),\n","        np.vstack(main_sent[\"doc_topics\"]),\n","        axis=1\n","    )\n",")"],"metadata":{"id":"jKdCbJi8uD2m"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["main_sent.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":511},"id":"AHGgH209wBrg","executionInfo":{"status":"ok","timestamp":1670439941664,"user_tz":-60,"elapsed":472,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"55cf4041-9213-47ba-a0f7-eed25f37d801"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Self-Organization of Associative Database and ...   \n","2  Self-Organization of Associative Database and ...   \n","3  Self-Organization of Associative Database and ...   \n","4  Self-Organization of Associative Database and ...   \n","\n","                                           sentences  \\\n","0                 Let a mapping f : X -+ Y be given.   \n","1  Here, X is a finite or infinite set, and Y is ...   \n","2  A learning machine observes any set of pairs (...   \n","3    (X x Y means the Cartesian product of X and Y.)   \n","4  And, it computes some estimate j : X -+ Y of f...   \n","\n","                                          clean_sent  sent_len  sent_pos  \\\n","0                              [let, mapping, given]         3  0.005051   \n","1  [finite, infinite, set, another, finite, infin...         7  0.010101   \n","2  [learning, machine, observes, set, pair, sampl...         7  0.015152   \n","3                         [mean, cartesian, product]         3  0.020202   \n","4  [computes, estimate, make, small, estimation, ...         7  0.025253   \n","\n","                                    clean_sent_grams  \\\n","0                              [let, mapping, given]   \n","1  [finite_infinite, set, another, finite_infinit...   \n","2  [learning, machine, observes, set, pair_sample...   \n","3                          [mean, cartesian_product]   \n","4  [computes, estimate, make, small, estimation_e...   \n","\n","                                              topics  \\\n","0  [0.02530865, 0.033191103, 0.02538963, 0.022944...   \n","1  [0.080829635, 0.10600426, 0.081088275, 0.07327...   \n","2  [0.010663864, 0.013985155, 0.010697986, 0.0096...   \n","3  [0.025395706, 0.033305272, 0.025476966, 0.0230...   \n","4  [0.015006904, 0.019680848, 0.015054923, 0.0136...   \n","\n","                                          doc_topics  topics_sim  \n","0  [0.045496676, 0.50505626, 4.1512463e-05, 3.751...    0.676909  \n","1  [0.045496676, 0.50505626, 4.1512463e-05, 3.751...    0.474625  \n","2  [0.045496676, 0.50505626, 4.1512463e-05, 3.751...    0.616646  \n","3  [0.045496676, 0.50505626, 4.1512463e-05, 3.751...    0.516420  \n","4  [0.045496676, 0.50505626, 4.1512463e-05, 3.751...    0.664286  "],"text/html":["\n","  <div id=\"df-194ad128-ad39-4e7e-bfc1-c7b36da8378f\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>sentences</th>\n","      <th>clean_sent</th>\n","      <th>sent_len</th>\n","      <th>sent_pos</th>\n","      <th>clean_sent_grams</th>\n","      <th>topics</th>\n","      <th>doc_topics</th>\n","      <th>topics_sim</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given.</td>\n","      <td>[let, mapping, given]</td>\n","      <td>3</td>\n","      <td>0.005051</td>\n","      <td>[let, mapping, given]</td>\n","      <td>[0.02530865, 0.033191103, 0.02538963, 0.022944...</td>\n","      <td>[0.045496676, 0.50505626, 4.1512463e-05, 3.751...</td>\n","      <td>0.676909</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Here, X is a finite or infinite set, and Y is ...</td>\n","      <td>[finite, infinite, set, another, finite, infin...</td>\n","      <td>7</td>\n","      <td>0.010101</td>\n","      <td>[finite_infinite, set, another, finite_infinit...</td>\n","      <td>[0.080829635, 0.10600426, 0.081088275, 0.07327...</td>\n","      <td>[0.045496676, 0.50505626, 4.1512463e-05, 3.751...</td>\n","      <td>0.474625</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>A learning machine observes any set of pairs (...</td>\n","      <td>[learning, machine, observes, set, pair, sampl...</td>\n","      <td>7</td>\n","      <td>0.015152</td>\n","      <td>[learning, machine, observes, set, pair_sample...</td>\n","      <td>[0.010663864, 0.013985155, 0.010697986, 0.0096...</td>\n","      <td>[0.045496676, 0.50505626, 4.1512463e-05, 3.751...</td>\n","      <td>0.616646</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>(X x Y means the Cartesian product of X and Y.)</td>\n","      <td>[mean, cartesian, product]</td>\n","      <td>3</td>\n","      <td>0.020202</td>\n","      <td>[mean, cartesian_product]</td>\n","      <td>[0.025395706, 0.033305272, 0.025476966, 0.0230...</td>\n","      <td>[0.045496676, 0.50505626, 4.1512463e-05, 3.751...</td>\n","      <td>0.516420</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>And, it computes some estimate j : X -+ Y of f...</td>\n","      <td>[computes, estimate, make, small, estimation, ...</td>\n","      <td>7</td>\n","      <td>0.025253</td>\n","      <td>[computes, estimate, make, small, estimation_e...</td>\n","      <td>[0.015006904, 0.019680848, 0.015054923, 0.0136...</td>\n","      <td>[0.045496676, 0.50505626, 4.1512463e-05, 3.751...</td>\n","      <td>0.664286</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-194ad128-ad39-4e7e-bfc1-c7b36da8378f')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-194ad128-ad39-4e7e-bfc1-c7b36da8378f button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-194ad128-ad39-4e7e-bfc1-c7b36da8378f');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":71}]},{"cell_type":"code","source":["# Drop not useful columns\n","main_sent.drop([\"topics\",\"doc_topics\"],axis=1,inplace=True)"],"metadata":{"id":"36w7e6P0_N9R"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["main_sent.head(3)"],"metadata":{"id":"KDWUy3J-_gFz","executionInfo":{"status":"ok","timestamp":1670440166261,"user_tz":-60,"elapsed":449,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"58e63d94-bbea-4cfa-aa6a-d3021c10b303","colab":{"base_uri":"https://localhost:8080/","height":239}},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Self-Organization of Associative Database and ...   \n","2  Self-Organization of Associative Database and ...   \n","\n","                                           sentences  \\\n","0                 Let a mapping f : X -+ Y be given.   \n","1  Here, X is a finite or infinite set, and Y is ...   \n","2  A learning machine observes any set of pairs (...   \n","\n","                                          clean_sent  sent_len  sent_pos  \\\n","0                              [let, mapping, given]         3  0.005051   \n","1  [finite, infinite, set, another, finite, infin...         7  0.010101   \n","2  [learning, machine, observes, set, pair, sampl...         7  0.015152   \n","\n","                                    clean_sent_grams  topics_sim  \n","0                              [let, mapping, given]    0.676909  \n","1  [finite_infinite, set, another, finite_infinit...    0.474625  \n","2  [learning, machine, observes, set, pair_sample...    0.616646  "],"text/html":["\n","  <div id=\"df-8273759a-3862-4fc9-af54-4aafb8cda4e9\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>sentences</th>\n","      <th>clean_sent</th>\n","      <th>sent_len</th>\n","      <th>sent_pos</th>\n","      <th>clean_sent_grams</th>\n","      <th>topics_sim</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given.</td>\n","      <td>[let, mapping, given]</td>\n","      <td>3</td>\n","      <td>0.005051</td>\n","      <td>[let, mapping, given]</td>\n","      <td>0.676909</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Here, X is a finite or infinite set, and Y is ...</td>\n","      <td>[finite, infinite, set, another, finite, infin...</td>\n","      <td>7</td>\n","      <td>0.010101</td>\n","      <td>[finite_infinite, set, another, finite_infinit...</td>\n","      <td>0.474625</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>A learning machine observes any set of pairs (...</td>\n","      <td>[learning, machine, observes, set, pair, sampl...</td>\n","      <td>7</td>\n","      <td>0.015152</td>\n","      <td>[learning, machine, observes, set, pair_sample...</td>\n","      <td>0.616646</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8273759a-3862-4fc9-af54-4aafb8cda4e9')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-8273759a-3862-4fc9-af54-4aafb8cda4e9 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-8273759a-3862-4fc9-af54-4aafb8cda4e9');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":74}]},{"cell_type":"code","source":["main_sent.to_csv(\"drive/MyDrive/TM&S/data/main_grams_features.csv\", index=False)"],"metadata":{"id":"htrYjA4SqQCz"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## ROUGE"],"metadata":{"id":"FsoCxsGA-_PQ"}},{"cell_type":"code","source":["import pandas as pd\n","from ast import literal_eval\n","\n","abs_sent = pd.read_csv(\"drive/MyDrive/TM&S/data/abs_features.csv\")\n","\n","abs_sent[\"clean_sent\"] = abs_sent[\"clean_sent\"].apply(literal_eval)"],"metadata":{"id":"mEuKCbCL_BzG"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["#!pip install rouge\n","from rouge import Rouge\n","\n","# Given a reference and a text, compute ROUGE-2 Recall for each sentence\n","# and return also the relative position of the best matching sentence in the reference\n","def my_rouge(hyps,refs):\n","  rouges = []\n","\n","  #hyps and refs are lists of lists\n","  my_hyps = [\" \".join(words) for words in hyps]\n","  my_refs = [\" \".join(words) for words in refs]\n","\n","  scores = Rouge(metrics=[\"rouge-2\"]).get_scores(my_hyps, my_refs, avg=False)\n","\n","  for i in range(len(scores)):\n","    rouges.append(scores[i][\"rouge-2\"].get(\"r\"))\n","\n","  return np.max(rouges), np.argmax(rouges)"],"metadata":{"id":"zZQoBeEuht_u"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from tqdm.auto import tqdm\n","\n","# Create lists of references (sentences from the abstracts)\n","refs_list = main_sent[[\"title\"]].join(abs_sent.groupby(by=\"title\")[\"clean_sent\"].apply(list),on=\"title\").drop([\"title\"],axis=1).squeeze()\n","refs_pos_list = main_sent[[\"title\"]].join(abs_sent.groupby(by=\"title\")[\"sent_pos\"].apply(list),on=\"title\").drop([\"title\"],axis=1).squeeze()\n","\n","# For each sentence in the main text, repeat it for each sentence in the relative reference\n","# (easier to compute ROUGE-2 later with the my_rouge function)\n","hyps_list = []\n","for i in tqdm(range(len(main_sent))):\n","  hyps_list.append([main_sent[\"clean_sent\"][i]]*len(refs_list[i]))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":49,"referenced_widgets":["2116adf179ba4a998cb0ef08675a27f3","4d270fea1c4e471aad6c62192dd814b9","d1719e6e761945d680a42370aaf6c98c","aa345f1512104e2096a346e8eb6f6364","d52f071103ef4d59918abd8017c8a515","ddfb9981d10b4bf08a2a329250da904f","cea6af79ae9e476aa45d5240ed3188f0","bfc6863e348c414da4c4046da8635d36","7e4ebb9ed6084520b78d2b25cd279500","a9716975110a486899cff1e783259e11","48d4199716444554a5d758b68e82c998"]},"id":"ArHEFXJkoZUi","executionInfo":{"status":"ok","timestamp":1670452105398,"user_tz":-60,"elapsed":31175,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"c51f1dbf-2c5d-48d4-c0bc-9c5f59684a04"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["  0%|          | 0/1178438 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2116adf179ba4a998cb0ef08675a27f3"}},"metadata":{}}]},{"cell_type":"code","source":["main_sent[\"ROUGE-2\"] = 0.\n","main_sent[\"summ_pos\"] = 0.\n","\n","for i in tqdm(range(len(main_sent))):\n","\n","  max_rouge, idx = my_rouge(hyps_list[i],refs_list[i])\n","\n","  main_sent[\"ROUGE-2\"][i] = max_rouge\n","  main_sent[\"summ_pos\"][i] = refs_pos_list[i][idx]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":223,"referenced_widgets":["479b89359a65468a8987973d9d7a64d3","d1eb86e2ef4d458a9c44bd4f6f160e88","0c5ab31a7d6a42a88796c3d6ff3d30eb","563267a481734c33b608ddf6c9fc74e0","188f597ba4e34d79b50b2cf11282a36a","a47d825fee0c41678173fd0734899dc4","dd7cae919a94491e964a813b7d53393e","3ace9c3f2f674960b78e903a956da95a","be532305eaab4951b8039d3670d8cc5a","683c3f4e26f546a0aa0e89e905ba238e","a6e0d9add4a1407abff4c3d4714fa629"]},"id":"Fye7dCKcBdq4","executionInfo":{"status":"ok","timestamp":1670453238747,"user_tz":-60,"elapsed":1123540,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"9ade837b-65d3-49d4-9e0f-02ca5e39443e"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["  0%|          | 0/1178438 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"479b89359a65468a8987973d9d7a64d3"}},"metadata":{}},{"output_type":"stream","name":"stderr","text":["<ipython-input-137-c57cbbd46e69>:9: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  main_sent[\"ROUGE-2\"][i] = max_rouge\n","<ipython-input-137-c57cbbd46e69>:10: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  main_sent[\"summ_pos\"][i] = refs_pos_list[i][idx]\n"]}]},{"cell_type":"code","source":["main_sent[\"summ_pos\"][main_sent[\"ROUGE-2\"]==0]=-1"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"i9h86tiTceNo","executionInfo":{"status":"ok","timestamp":1670453239918,"user_tz":-60,"elapsed":8,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"921e0474-931a-43ac-990a-391e5f233a22"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["<ipython-input-138-5926ecbf5d20>:1: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  main_sent[\"summ_pos\"][main_sent[\"ROUGE-2\"]==0]=-1\n"]}]},{"cell_type":"code","source":["main_sent.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":424},"id":"BvSQjuFpusCt","executionInfo":{"status":"ok","timestamp":1670453239918,"user_tz":-60,"elapsed":5,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"744b5a8a-a6fe-4135-a2d0-24792c853386"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Self-Organization of Associative Database and ...   \n","2  Self-Organization of Associative Database and ...   \n","3  Self-Organization of Associative Database and ...   \n","4  Self-Organization of Associative Database and ...   \n","\n","                                           sentences  \\\n","0                 Let a mapping f : X -+ Y be given.   \n","1  Here, X is a finite or infinite set, and Y is ...   \n","2  A learning machine observes any set of pairs (...   \n","3    (X x Y means the Cartesian product of X and Y.)   \n","4  And, it computes some estimate j : X -+ Y of f...   \n","\n","                                          clean_sent  sent_len  sent_pos  \\\n","0                              [let, mapping, given]         3  0.005051   \n","1  [finite, infinite, set, another, finite, infin...         7  0.010101   \n","2  [learning, machine, observes, set, pair, sampl...         7  0.015152   \n","3                         [mean, cartesian, product]         3  0.020202   \n","4  [computes, estimate, make, small, estimation, ...         7  0.025253   \n","\n","                                    clean_sent_grams  topics_sim  ROUGE-2  \\\n","0                              [let, mapping, given]    0.676909      0.0   \n","1  [finite_infinite, set, another, finite_infinit...    0.474625      0.0   \n","2  [learning, machine, observes, set, pair_sample...    0.616646      0.0   \n","3                          [mean, cartesian_product]    0.516420      0.0   \n","4  [computes, estimate, make, small, estimation_e...    0.664286      0.0   \n","\n","   summ_pos  \n","0      -1.0  \n","1      -1.0  \n","2      -1.0  \n","3      -1.0  \n","4      -1.0  "],"text/html":["\n","  <div id=\"df-15b836ed-d4ac-4509-b093-a81ec52f2844\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>sentences</th>\n","      <th>clean_sent</th>\n","      <th>sent_len</th>\n","      <th>sent_pos</th>\n","      <th>clean_sent_grams</th>\n","      <th>topics_sim</th>\n","      <th>ROUGE-2</th>\n","      <th>summ_pos</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given.</td>\n","      <td>[let, mapping, given]</td>\n","      <td>3</td>\n","      <td>0.005051</td>\n","      <td>[let, mapping, given]</td>\n","      <td>0.676909</td>\n","      <td>0.0</td>\n","      <td>-1.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Here, X is a finite or infinite set, and Y is ...</td>\n","      <td>[finite, infinite, set, another, finite, infin...</td>\n","      <td>7</td>\n","      <td>0.010101</td>\n","      <td>[finite_infinite, set, another, finite_infinit...</td>\n","      <td>0.474625</td>\n","      <td>0.0</td>\n","      <td>-1.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>A learning machine observes any set of pairs (...</td>\n","      <td>[learning, machine, observes, set, pair, sampl...</td>\n","      <td>7</td>\n","      <td>0.015152</td>\n","      <td>[learning, machine, observes, set, pair_sample...</td>\n","      <td>0.616646</td>\n","      <td>0.0</td>\n","      <td>-1.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>(X x Y means the Cartesian product of X and Y.)</td>\n","      <td>[mean, cartesian, product]</td>\n","      <td>3</td>\n","      <td>0.020202</td>\n","      <td>[mean, cartesian_product]</td>\n","      <td>0.516420</td>\n","      <td>0.0</td>\n","      <td>-1.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>And, it computes some estimate j : X -+ Y of f...</td>\n","      <td>[computes, estimate, make, small, estimation, ...</td>\n","      <td>7</td>\n","      <td>0.025253</td>\n","      <td>[computes, estimate, make, small, estimation_e...</td>\n","      <td>0.664286</td>\n","      <td>0.0</td>\n","      <td>-1.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-15b836ed-d4ac-4509-b093-a81ec52f2844')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-15b836ed-d4ac-4509-b093-a81ec52f2844 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-15b836ed-d4ac-4509-b093-a81ec52f2844');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":139}]},{"cell_type":"code","source":["main_sent.to_csv(\"drive/MyDrive/TM&S/data/main_grams_features.csv\", index=False)"],"metadata":{"id":"cWYKrSjHPUlV"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## TF-IDF weighting"],"metadata":{"id":"Pro48QsOMpXO"}},{"cell_type":"code","source":["import pandas as pd\n","from ast import literal_eval\n","\n","main_sent = pd.read_csv(\"drive/MyDrive/TM&S/data/main_grams_features.csv\")\n","main_sent[\"clean_sent_grams\"] = main_sent[\"clean_sent_grams\"].apply(literal_eval)\n","main_sent.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":424},"id":"4DPLVj7JMsrC","executionInfo":{"status":"ok","timestamp":1673196975452,"user_tz":-60,"elapsed":40939,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"fd0b1a68-a1d8-4073-9ef4-c69b7ad50437"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Self-Organization of Associative Database and ...   \n","2  Self-Organization of Associative Database and ...   \n","3  Self-Organization of Associative Database and ...   \n","4  Self-Organization of Associative Database and ...   \n","\n","                                           sentences  \\\n","0                 Let a mapping f : X -+ Y be given.   \n","1  Here, X is a finite or infinite set, and Y is ...   \n","2  A learning machine observes any set of pairs (...   \n","3    (X x Y means the Cartesian product of X and Y.)   \n","4  And, it computes some estimate j : X -+ Y of f...   \n","\n","                                          clean_sent  sent_len  sent_pos  \\\n","0                        ['let', 'mapping', 'given']         3  0.005051   \n","1  ['finite', 'infinite', 'set', 'another', 'fini...         7  0.010101   \n","2  ['learning', 'machine', 'observes', 'set', 'pa...         7  0.015152   \n","3                   ['mean', 'cartesian', 'product']         3  0.020202   \n","4  ['computes', 'estimate', 'make', 'small', 'est...         7  0.025253   \n","\n","                                    clean_sent_grams  topics_sim  ROUGE-2  \\\n","0                              [let, mapping, given]    0.676909      0.0   \n","1  [finite_infinite, set, another, finite_infinit...    0.474625      0.0   \n","2  [learning, machine, observes, set, pair_sample...    0.616646      0.0   \n","3                          [mean, cartesian_product]    0.516420      0.0   \n","4  [computes, estimate, make, small, estimation_e...    0.664286      0.0   \n","\n","   summ_pos  \n","0      -1.0  \n","1      -1.0  \n","2      -1.0  \n","3      -1.0  \n","4      -1.0  "],"text/html":["\n","  <div id=\"df-def4a144-08dd-4322-8f1d-aad6471fa615\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>sentences</th>\n","      <th>clean_sent</th>\n","      <th>sent_len</th>\n","      <th>sent_pos</th>\n","      <th>clean_sent_grams</th>\n","      <th>topics_sim</th>\n","      <th>ROUGE-2</th>\n","      <th>summ_pos</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given.</td>\n","      <td>['let', 'mapping', 'given']</td>\n","      <td>3</td>\n","      <td>0.005051</td>\n","      <td>[let, mapping, given]</td>\n","      <td>0.676909</td>\n","      <td>0.0</td>\n","      <td>-1.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Here, X is a finite or infinite set, and Y is ...</td>\n","      <td>['finite', 'infinite', 'set', 'another', 'fini...</td>\n","      <td>7</td>\n","      <td>0.010101</td>\n","      <td>[finite_infinite, set, another, finite_infinit...</td>\n","      <td>0.474625</td>\n","      <td>0.0</td>\n","      <td>-1.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>A learning machine observes any set of pairs (...</td>\n","      <td>['learning', 'machine', 'observes', 'set', 'pa...</td>\n","      <td>7</td>\n","      <td>0.015152</td>\n","      <td>[learning, machine, observes, set, pair_sample...</td>\n","      <td>0.616646</td>\n","      <td>0.0</td>\n","      <td>-1.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>(X x Y means the Cartesian product of X and Y.)</td>\n","      <td>['mean', 'cartesian', 'product']</td>\n","      <td>3</td>\n","      <td>0.020202</td>\n","      <td>[mean, cartesian_product]</td>\n","      <td>0.516420</td>\n","      <td>0.0</td>\n","      <td>-1.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>And, it computes some estimate j : X -+ Y of f...</td>\n","      <td>['computes', 'estimate', 'make', 'small', 'est...</td>\n","      <td>7</td>\n","      <td>0.025253</td>\n","      <td>[computes, estimate, make, small, estimation_e...</td>\n","      <td>0.664286</td>\n","      <td>0.0</td>\n","      <td>-1.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-def4a144-08dd-4322-8f1d-aad6471fa615')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-def4a144-08dd-4322-8f1d-aad6471fa615 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-def4a144-08dd-4322-8f1d-aad6471fa615');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":13}]},{"cell_type":"code","source":["altrodf = main_sent.copy()\n","\n","altrodf= altrodf.groupby(by=\"title\", sort=False)[\"clean_sent_grams\"].aggregate(func=sum).reset_index()\n","altrodf.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"2tmnFBtPOwA_","executionInfo":{"status":"ok","timestamp":1673196978876,"user_tz":-60,"elapsed":3428,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"a3a6270e-6999-46a4-fe70-06886e6d1047"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                               title  \\\n","0  Self-Organization of Associative Database and ...   \n","1  Bayesian Query Construction for Neural Network...   \n","2  Neural Network Ensembles, Cross Validation, an...   \n","3  Pulsestream Synapses with Non-Volatile Analogu...   \n","4                 Learning to Play the Game of Chess   \n","\n","                                    clean_sent_grams  \n","0  [let, mapping, given, finite_infinite, set, an...  \n","1  [paper_consider, situation, data_collection, c...  \n","2  [well_known, combination, many_different, pred...  \n","3  [fusion, nucleus, hydrogen, provides, energy, ...  \n","4  [throughout, last_decade, game_chess, major, t...  "],"text/html":["\n","  <div id=\"df-3f12c90d-af83-4545-884e-015f6609bcbc\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>clean_sent_grams</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>[let, mapping, given, finite_infinite, set, an...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Bayesian Query Construction for Neural Network...</td>\n","      <td>[paper_consider, situation, data_collection, c...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Neural Network Ensembles, Cross Validation, an...</td>\n","      <td>[well_known, combination, many_different, pred...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Pulsestream Synapses with Non-Volatile Analogu...</td>\n","      <td>[fusion, nucleus, hydrogen, provides, energy, ...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Learning to Play the Game of Chess</td>\n","      <td>[throughout, last_decade, game_chess, major, t...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3f12c90d-af83-4545-884e-015f6609bcbc')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-3f12c90d-af83-4545-884e-015f6609bcbc button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-3f12c90d-af83-4545-884e-015f6609bcbc');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":14}]},{"cell_type":"code","source":["# Create corpus for TF-IDF\n","corpus = [\" \".join(words) for words in altrodf[\"clean_sent_grams\"]]"],"metadata":{"id":"Ho6xamjsTE3z"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from sklearn.feature_extraction.text import TfidfVectorizer\n","\n","vectorizer = TfidfVectorizer()\n","X = vectorizer.fit_transform(corpus)"],"metadata":{"id":"ie75o-ZvVjI-"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# DataFrame from sparse matrix (easier to find document-term elements)\n","tf_idf_df = pd.DataFrame.sparse.from_spmatrix(X, index=altrodf.title, columns=vectorizer.get_feature_names_out())"],"metadata":{"id":"8lJ971b9GLXE"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["tf_idf_df[\"let\"][\"Self-Organization of Associative Database and Its Applications\"]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"L3QQEpxZIOr6","executionInfo":{"status":"ok","timestamp":1673210788266,"user_tz":-60,"elapsed":7,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"8c44a997-af06-49f2-b481-b2f6cd497640"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.09587074734413566"]},"metadata":{},"execution_count":51}]},{"cell_type":"code","source":["tf_idf_df[\"mapping\"][\"Self-Organization of Associative Database and Its Applications\"]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"WaqhUtbeNGKm","executionInfo":{"status":"ok","timestamp":1673211965968,"user_tz":-60,"elapsed":9,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"bdcb53b9-3878-4e4a-9484-765a41714eb3"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.039095699270556936"]},"metadata":{},"execution_count":55}]},{"cell_type":"code","source":["tf_idf_df[\"given\"][\"Self-Organization of Associative Database and Its Applications\"]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vtxJpPbfNIHX","executionInfo":{"status":"ok","timestamp":1673211976195,"user_tz":-60,"elapsed":11,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"63000048-5574-4861-815b-3a68e6e1fb47"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.01261009742430314"]},"metadata":{},"execution_count":56}]},{"cell_type":"code","source":["main_sent[\"tf_idf\"]=0.\n","\n","# Get TF-IDF weight of words in sentences and compute the average value\n","from tqdm.auto import tqdm\n","\n","length = len(main_sent)\n","\n","for i in tqdm(range(length)):\n","  tfidfs = []\n","  title = main_sent[\"title\"][i]\n","  for term in main_sent[\"clean_sent_grams\"][i]:\n","    try:\n","      tfidfs.append( tf_idf_df[term][title] )\n","    except KeyError:\n","      tfidfs.append(0)\n","\n","  main_sent[\"tf_idf\"][i] = np.mean(tfidfs)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":136,"referenced_widgets":["1386d2bb1a894bd1a1e5ee13eaeee8fe","a1a31241bfc94166b91a460e68482f98","5777043fe84c4bec98400b4477c65ca5","5605f3036e464d66a0652bbf9113a4c3","c9b2625508fe4ed08929c0f22bbe2cab","9c2fe527a144414885224b2136bc5841","e90df5e04e694d4ab062620f9c4f0713","911269d9787a43b2882cb958483c1bee","c6d58aae167c4768b349b9d6dc3fc249","b8355747f8914de193a655c79bf125c6","c33fa73237594da7ac45a189208f2ae5"]},"id":"ka1g53axXC-M","executionInfo":{"status":"ok","timestamp":1673213675651,"user_tz":-60,"elapsed":649909,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"a199ee3a-7c20-45c3-fcf2-09f88d0b5884"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["  0%|          | 0/1178438 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1386d2bb1a894bd1a1e5ee13eaeee8fe"}},"metadata":{}},{"output_type":"stream","name":"stderr","text":["<ipython-input-69-71a9b0da481b>:16: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  main_sent[\"tf_idf\"][i] = np.mean(tfidfs)\n"]}]},{"cell_type":"code","source":["main_sent"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":902},"id":"cEtjg3i34r2O","executionInfo":{"status":"ok","timestamp":1673213675651,"user_tz":-60,"elapsed":8,"user":{"displayName":"Riccardo Porcedda","userId":"13682347931516633719"}},"outputId":"e1fb6f53-a127-4c7c-d90d-d04d5a7b49b7"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                                     title  \\\n","0        Self-Organization of Associative Database and ...   \n","1        Self-Organization of Associative Database and ...   \n","2        Self-Organization of Associative Database and ...   \n","3        Self-Organization of Associative Database and ...   \n","4        Self-Organization of Associative Database and ...   \n","...                                                    ...   \n","1178433  Correlation and Interpolation Networks for Rea...   \n","1178434  Correlation and Interpolation Networks for Rea...   \n","1178435  Correlation and Interpolation Networks for Rea...   \n","1178436  Correlation and Interpolation Networks for Rea...   \n","1178437  Correlation and Interpolation Networks for Rea...   \n","\n","                                                 sentences  \\\n","0                       Let a mapping f : X -+ Y be given.   \n","1        Here, X is a finite or infinite set, and Y is ...   \n","2        A learning machine observes any set of pairs (...   \n","3          (X x Y means the Cartesian product of X and Y.)   \n","4        And, it computes some estimate j : X -+ Y of f...   \n","...                                                    ...   \n","1178433  A distributed view-based representation is com...   \n","1178434  3-D constraints on facial shape are achieved t...   \n","1178435  With this framework we have been able to achei...   \n","1178436  A modeled face mimics the expression of a user...   \n","1178437  This system has promise as a new approach in t...   \n","\n","                                                clean_sent  sent_len  \\\n","0                              ['let', 'mapping', 'given']         3   \n","1        ['finite', 'infinite', 'set', 'another', 'fini...         7   \n","2        ['learning', 'machine', 'observes', 'set', 'pa...         7   \n","3                         ['mean', 'cartesian', 'product']         3   \n","4        ['computes', 'estimate', 'make', 'small', 'est...         7   \n","...                                                    ...       ...   \n","1178433  ['distributed', 'viewbased', 'representation',...        16   \n","1178434  ['constraint', 'facial', 'shape', 'achieved', ...        20   \n","1178435  ['framework', 'able', 'acheive', 'fast', 'robu...         9   \n","1178436  ['modeled', 'face', 'mimic', 'expression', 'us...        15   \n","1178437  ['system', 'promise', 'new', 'approach', 'inte...        11   \n","\n","         sent_pos                                   clean_sent_grams  \\\n","0        0.005051                              [let, mapping, given]   \n","1        0.010101  [finite_infinite, set, another, finite_infinit...   \n","2        0.015152  [learning, machine, observes, set, pair_sample...   \n","3        0.020202                          [mean, cartesian_product]   \n","4        0.025253  [computes, estimate, make, small, estimation_e...   \n","...           ...                                                ...   \n","1178433  0.968504  [distributed, viewbased, representation, compu...   \n","1178434  0.976378  [constraint, facial, shape, achieved, use, ana...   \n","1178435  0.984252  [framework, able, acheive, fast_robust, analys...   \n","1178436  0.992126  [modeled, face, mimic, expression, user, realt...   \n","1178437  1.000000  [system, promise, new_approach, interactive, a...   \n","\n","         topics_sim   ROUGE-2  summ_pos    tf_idf  \n","0          0.676909  0.000000      -1.0  0.049192  \n","1          0.474625  0.000000      -1.0  0.038056  \n","2          0.616646  0.000000      -1.0  0.048948  \n","3          0.516420  0.000000      -1.0  0.022411  \n","4          0.664286  0.000000      -1.0  0.034419  \n","...             ...       ...       ...       ...  \n","1178433    0.520661  0.272727       0.4  0.057450  \n","1178434    0.696025  0.222222       0.8  0.086652  \n","1178435    0.398139  0.100000       0.2  0.059963  \n","1178436    0.248271  0.000000      -1.0  0.106063  \n","1178437    0.746753  0.000000      -1.0  0.039870  \n","\n","[1178438 rows x 10 columns]"],"text/html":["\n","  <div id=\"df-8208791b-e5b2-4544-8375-ebe5d624239c\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>title</th>\n","      <th>sentences</th>\n","      <th>clean_sent</th>\n","      <th>sent_len</th>\n","      <th>sent_pos</th>\n","      <th>clean_sent_grams</th>\n","      <th>topics_sim</th>\n","      <th>ROUGE-2</th>\n","      <th>summ_pos</th>\n","      <th>tf_idf</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Let a mapping f : X -+ Y be given.</td>\n","      <td>['let', 'mapping', 'given']</td>\n","      <td>3</td>\n","      <td>0.005051</td>\n","      <td>[let, mapping, given]</td>\n","      <td>0.676909</td>\n","      <td>0.000000</td>\n","      <td>-1.0</td>\n","      <td>0.049192</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>Here, X is a finite or infinite set, and Y is ...</td>\n","      <td>['finite', 'infinite', 'set', 'another', 'fini...</td>\n","      <td>7</td>\n","      <td>0.010101</td>\n","      <td>[finite_infinite, set, another, finite_infinit...</td>\n","      <td>0.474625</td>\n","      <td>0.000000</td>\n","      <td>-1.0</td>\n","      <td>0.038056</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>A learning machine observes any set of pairs (...</td>\n","      <td>['learning', 'machine', 'observes', 'set', 'pa...</td>\n","      <td>7</td>\n","      <td>0.015152</td>\n","      <td>[learning, machine, observes, set, pair_sample...</td>\n","      <td>0.616646</td>\n","      <td>0.000000</td>\n","      <td>-1.0</td>\n","      <td>0.048948</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>(X x Y means the Cartesian product of X and Y.)</td>\n","      <td>['mean', 'cartesian', 'product']</td>\n","      <td>3</td>\n","      <td>0.020202</td>\n","      <td>[mean, cartesian_product]</td>\n","      <td>0.516420</td>\n","      <td>0.000000</td>\n","      <td>-1.0</td>\n","      <td>0.022411</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Self-Organization of Associative Database and ...</td>\n","      <td>And, it computes some estimate j : X -+ Y of f...</td>\n","      <td>['computes', 'estimate', 'make', 'small', 'est...</td>\n","      <td>7</td>\n","      <td>0.025253</td>\n","      <td>[computes, estimate, make, small, estimation_e...</td>\n","      <td>0.664286</td>\n","      <td>0.000000</td>\n","      <td>-1.0</td>\n","      <td>0.034419</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>1178433</th>\n","      <td>Correlation and Interpolation Networks for Rea...</td>\n","      <td>A distributed view-based representation is com...</td>\n","      <td>['distributed', 'viewbased', 'representation',...</td>\n","      <td>16</td>\n","      <td>0.968504</td>\n","      <td>[distributed, viewbased, representation, compu...</td>\n","      <td>0.520661</td>\n","      <td>0.272727</td>\n","      <td>0.4</td>\n","      <td>0.057450</td>\n","    </tr>\n","    <tr>\n","      <th>1178434</th>\n","      <td>Correlation and Interpolation Networks for Rea...</td>\n","      <td>3-D constraints on facial shape are achieved t...</td>\n","      <td>['constraint', 'facial', 'shape', 'achieved', ...</td>\n","      <td>20</td>\n","      <td>0.976378</td>\n","      <td>[constraint, facial, shape, achieved, use, ana...</td>\n","      <td>0.696025</td>\n","      <td>0.222222</td>\n","      <td>0.8</td>\n","      <td>0.086652</td>\n","    </tr>\n","    <tr>\n","      <th>1178435</th>\n","      <td>Correlation and Interpolation Networks for Rea...</td>\n","      <td>With this framework we have been able to achei...</td>\n","      <td>['framework', 'able', 'acheive', 'fast', 'robu...</td>\n","      <td>9</td>\n","      <td>0.984252</td>\n","      <td>[framework, able, acheive, fast_robust, analys...</td>\n","      <td>0.398139</td>\n","      <td>0.100000</td>\n","      <td>0.2</td>\n","      <td>0.059963</td>\n","    </tr>\n","    <tr>\n","      <th>1178436</th>\n","      <td>Correlation and Interpolation Networks for Rea...</td>\n","      <td>A modeled face mimics the expression of a user...</td>\n","      <td>['modeled', 'face', 'mimic', 'expression', 'us...</td>\n","      <td>15</td>\n","      <td>0.992126</td>\n","      <td>[modeled, face, mimic, expression, user, realt...</td>\n","      <td>0.248271</td>\n","      <td>0.000000</td>\n","      <td>-1.0</td>\n","      <td>0.106063</td>\n","    </tr>\n","    <tr>\n","      <th>1178437</th>\n","      <td>Correlation and Interpolation Networks for Rea...</td>\n","      <td>This system has promise as a new approach in t...</td>\n","      <td>['system', 'promise', 'new', 'approach', 'inte...</td>\n","      <td>11</td>\n","      <td>1.000000</td>\n","      <td>[system, promise, new_approach, interactive, a...</td>\n","      <td>0.746753</td>\n","      <td>0.000000</td>\n","      <td>-1.0</td>\n","      <td>0.039870</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1178438 rows × 10 columns</p>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8208791b-e5b2-4544-8375-ebe5d624239c')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-8208791b-e5b2-4544-8375-ebe5d624239c button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-8208791b-e5b2-4544-8375-ebe5d624239c');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":70}]},{"cell_type":"code","source":["main_sent.to_csv(\"drive/MyDrive/TM&S/data/main_grams_features.csv\", index=False)"],"metadata":{"id":"s_kOf9lURR3H"},"execution_count":null,"outputs":[]}],"metadata":{"colab":{"provenance":[],"toc_visible":true},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"2116adf179ba4a998cb0ef08675a27f3":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_4d270fea1c4e471aad6c62192dd814b9","IPY_MODEL_d1719e6e761945d680a42370aaf6c98c","IPY_MODEL_aa345f1512104e2096a346e8eb6f6364"],"layout":"IPY_MODEL_d52f071103ef4d59918abd8017c8a515"}},"4d270fea1c4e471aad6c62192dd814b9":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_ddfb9981d10b4bf08a2a329250da904f","placeholder":"​","style":"IPY_MODEL_cea6af79ae9e476aa45d5240ed3188f0","value":"100%"}},"d1719e6e761945d680a42370aaf6c98c":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_bfc6863e348c414da4c4046da8635d36","max":1178438,"min":0,"orientation":"horizontal","style":"IPY_MODEL_7e4ebb9ed6084520b78d2b25cd279500","value":1178438}},"aa345f1512104e2096a346e8eb6f6364":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a9716975110a486899cff1e783259e11","placeholder":"​","style":"IPY_MODEL_48d4199716444554a5d758b68e82c998","value":" 1178438/1178438 [00:30&lt;00:00, 36420.78it/s]"}},"d52f071103ef4d59918abd8017c8a515":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ddfb9981d10b4bf08a2a329250da904f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"cea6af79ae9e476aa45d5240ed3188f0":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"bfc6863e348c414da4c4046da8635d36":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7e4ebb9ed6084520b78d2b25cd279500":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"a9716975110a486899cff1e783259e11":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"48d4199716444554a5d758b68e82c998":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"479b89359a65468a8987973d9d7a64d3":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_d1eb86e2ef4d458a9c44bd4f6f160e88","IPY_MODEL_0c5ab31a7d6a42a88796c3d6ff3d30eb","IPY_MODEL_563267a481734c33b608ddf6c9fc74e0"],"layout":"IPY_MODEL_188f597ba4e34d79b50b2cf11282a36a"}},"d1eb86e2ef4d458a9c44bd4f6f160e88":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a47d825fee0c41678173fd0734899dc4","placeholder":"​","style":"IPY_MODEL_dd7cae919a94491e964a813b7d53393e","value":"100%"}},"0c5ab31a7d6a42a88796c3d6ff3d30eb":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_3ace9c3f2f674960b78e903a956da95a","max":1178438,"min":0,"orientation":"horizontal","style":"IPY_MODEL_be532305eaab4951b8039d3670d8cc5a","value":1178438}},"563267a481734c33b608ddf6c9fc74e0":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_683c3f4e26f546a0aa0e89e905ba238e","placeholder":"​","style":"IPY_MODEL_a6e0d9add4a1407abff4c3d4714fa629","value":" 1178438/1178438 [18:43&lt;00:00, 1134.93it/s]"}},"188f597ba4e34d79b50b2cf11282a36a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a47d825fee0c41678173fd0734899dc4":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"dd7cae919a94491e964a813b7d53393e":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"3ace9c3f2f674960b78e903a956da95a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"be532305eaab4951b8039d3670d8cc5a":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"683c3f4e26f546a0aa0e89e905ba238e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a6e0d9add4a1407abff4c3d4714fa629":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"1386d2bb1a894bd1a1e5ee13eaeee8fe":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_a1a31241bfc94166b91a460e68482f98","IPY_MODEL_5777043fe84c4bec98400b4477c65ca5","IPY_MODEL_5605f3036e464d66a0652bbf9113a4c3"],"layout":"IPY_MODEL_c9b2625508fe4ed08929c0f22bbe2cab"}},"a1a31241bfc94166b91a460e68482f98":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_9c2fe527a144414885224b2136bc5841","placeholder":"​","style":"IPY_MODEL_e90df5e04e694d4ab062620f9c4f0713","value":"100%"}},"5777043fe84c4bec98400b4477c65ca5":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_911269d9787a43b2882cb958483c1bee","max":1178438,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c6d58aae167c4768b349b9d6dc3fc249","value":1178438}},"5605f3036e464d66a0652bbf9113a4c3":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b8355747f8914de193a655c79bf125c6","placeholder":"​","style":"IPY_MODEL_c33fa73237594da7ac45a189208f2ae5","value":" 1178438/1178438 [10:49&lt;00:00, 1762.40it/s]"}},"c9b2625508fe4ed08929c0f22bbe2cab":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9c2fe527a144414885224b2136bc5841":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e90df5e04e694d4ab062620f9c4f0713":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"911269d9787a43b2882cb958483c1bee":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c6d58aae167c4768b349b9d6dc3fc249":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"b8355747f8914de193a655c79bf125c6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c33fa73237594da7ac45a189208f2ae5":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"nbformat":4,"nbformat_minor":0}