publication_tables.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "970c359a-ab35-4699-b52d-30ddf96b2148",
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "import sys\n",
    "\n",
    "# relative to project root\n",
    "sys.path.append(\"publication/\")\n",
    "from session_data import *\n",
    "\n",
    "# packages\n",
    "import pandas\n",
    "from IPython.display import display, display_html, display_latex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b345e836-9540-4d3f-af21-9fa656503586",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "  Exam Session ID Question Category  Question Number GPT Answer  \\\n0    bar-exam-001   Civil Procedure                1          D   \n1    bar-exam-001   Civil Procedure                2          D   \n2    bar-exam-001   Civil Procedure                3          C   \n3    bar-exam-001   Civil Procedure                4          C   \n4    bar-exam-001   Civil Procedure                5          C   \n\n  GPT Second Answer GPT Third Answer Correct Answer  Correct  Second Correct  \\\n0                 A                B              D     True           False   \n1                 B                A              D     True           False   \n2                 D                A              D    False            True   \n3                 D                B              A    False           False   \n4                 D                B              C     True           False   \n\n   Third Correct  Top Two Correct  Top Three Correct  Temperature  Max Tokens  \\\n0          False             True               True          0.0          16   \n1          False             True               True          0.0          16   \n2          False             True               True          0.0          16   \n3          False            False              False          0.0          16   \n4          False             True               True          0.0          16   \n\n   Top P  Best Of  Frequency Penalty  Presence Penalty  Session Duration  \n0    1.0        1                  0                 0        208.769812  \n1    1.0        1                  0                 0        208.769812  \n2    1.0        1                  0                 0        208.769812  \n3    1.0        1                  0                 0        208.769812  \n4    1.0        1                  0                 0        208.769812  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Exam Session ID</th>\n      <th>Question Category</th>\n      <th>Question Number</th>\n      <th>GPT Answer</th>\n      <th>GPT Second Answer</th>\n      <th>GPT Third Answer</th>\n      <th>Correct Answer</th>\n      <th>Correct</th>\n      <th>Second Correct</th>\n      <th>Third Correct</th>\n      <th>Top Two Correct</th>\n      <th>Top Three Correct</th>\n      <th>Temperature</th>\n      <th>Max Tokens</th>\n      <th>Top P</th>\n      <th>Best Of</th>\n      <th>Frequency Penalty</th>\n      <th>Presence Penalty</th>\n      <th>Session Duration</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>bar-exam-001</td>\n      <td>Civil Procedure</td>\n      <td>1</td>\n      <td>D</td>\n      <td>A</td>\n      <td>B</td>\n      <td>D</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>True</td>\n      <td>0.0</td>\n      <td>16</td>\n      <td>1.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>208.769812</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>bar-exam-001</td>\n      <td>Civil Procedure</td>\n      <td>2</td>\n      <td>D</td>\n      <td>B</td>\n      <td>A</td>\n      <td>D</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>True</td>\n      <td>0.0</td>\n      <td>16</td>\n      <td>1.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>208.769812</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>bar-exam-001</td>\n      <td>Civil Procedure</td>\n      <td>3</td>\n      <td>C</td>\n      <td>D</td>\n      <td>A</td>\n      <td>D</td>\n      <td>False</td>\n      <td>True</td>\n      <td>False</td>\n      <td>True</td>\n      <td>True</td>\n      <td>0.0</td>\n      <td>16</td>\n      <td>1.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>208.769812</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>bar-exam-001</td>\n      <td>Civil Procedure</td>\n      <td>4</td>\n      <td>C</td>\n      <td>D</td>\n      <td>B</td>\n      <td>A</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>0.0</td>\n      <td>16</td>\n      <td>1.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>208.769812</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>bar-exam-001</td>\n      <td>Civil Procedure</td>\n      <td>5</td>\n      <td>C</td>\n      <td>D</td>\n      <td>B</td>\n      <td>C</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>True</td>\n      <td>0.0</td>\n      <td>16</td>\n      <td>1.0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>208.769812</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read all session data\n",
    "session_df = get_session_data()\n",
    "session_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a24a8bde-bc71-496f-8d0a-bd1904556868",
   "metadata": {},
   "source": [
    "## Headline Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fb5d1057-9ec5-4356-9c6b-50cf0dc44a59",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lr}\n",
      " & Accuracy (%) \\\\\n",
      "Correct Rate & 49.970000 \\\\\n",
      "Top Two Correct Rate & 70.970000 \\\\\n",
      "Top Three Correct Rate & 87.750000 \\\\\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": "                        Accuracy (%)\nCorrect Rate                      50\nTop Two Correct Rate              71\nTop Three Correct Rate            88",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Accuracy (%)</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Correct Rate</th>\n      <td>50</td>\n    </tr>\n    <tr>\n      <th>Top Two Correct Rate</th>\n      <td>71</td>\n    </tr>\n    <tr>\n      <th>Top Three Correct Rate</th>\n      <td>88</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "performance_df = pandas.DataFrame({\n",
    "        \"Correct Rate\": session_df[\"Correct\"].mean() * 100.0,\n",
    "        \"Top Two Correct Rate\": session_df[\"Top Two Correct\"].mean() * 100.0,\n",
    "        \"Top Three Correct Rate\": session_df[\"Top Three Correct\"].mean() * 100.0\n",
    "}, index=[\"Accuracy (%)\"]).T\n",
    "\n",
    "with pandas.option_context(\"float_format\", \"{:2.0f}\".format):\n",
    "    print(performance_df.round(2).style.to_latex())\n",
    "    display(performance_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c3015df-158f-4ef0-ba3e-87dc3aeaf97f",
   "metadata": {},
   "source": [
    "## NCBE Rates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "9074f5f6-8f9b-400a-a56e-f7917a164103",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "                            Accuracy (%)\nCivil Procedure                     59.0\nConstitutional Law                  72.0\nContracts                           70.0\nCriminal Law and Procedure          71.0\nEvidence                            65.0\nReal Property                       65.0\nTorts                               71.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Accuracy (%)</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Civil Procedure</th>\n      <td>59.0</td>\n    </tr>\n    <tr>\n      <th>Constitutional Law</th>\n      <td>72.0</td>\n    </tr>\n    <tr>\n      <th>Contracts</th>\n      <td>70.0</td>\n    </tr>\n    <tr>\n      <th>Criminal Law and Procedure</th>\n      <td>71.0</td>\n    </tr>\n    <tr>\n      <th>Evidence</th>\n      <td>65.0</td>\n    </tr>\n    <tr>\n      <th>Real Property</th>\n      <td>65.0</td>\n    </tr>\n    <tr>\n      <th>Torts</th>\n      <td>71.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ncbe_df = pandas.DataFrame(pandas.Series(NCBE_CATEGORY_CORRECT_RATES) * 100.0, columns=[\"Accuracy (%)\"])\n",
    "ncbe_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49dec6fe-fc6e-44e4-addd-597019a41b91",
   "metadata": {},
   "source": [
    "## Accuracy by Question Category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e24d425f-6d59-4e04-9285-603d7c21310c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrr}\n",
      " & Correct Rate & Top Two Correct Rate & Top Three Correct Rate & NCBE Rate \\\\\n",
      "Evidence & 62.760000 & 84.470000 & 98.050000 & 65.000000 \\\\\n",
      "Torts & 61.650000 & 71.830000 & 93.860000 & 71.000000 \\\\\n",
      "Civil Procedure & 52.030000 & 62.680000 & 78.700000 & 59.000000 \\\\\n",
      "Constitutional Law & 49.020000 & 66.750000 & 86.830000 & 72.000000 \\\\\n",
      "Real Property & 44.960000 & 71.630000 & 84.800000 & 65.000000 \\\\\n",
      "Contracts & 44.720000 & 77.320000 & 85.850000 & 70.000000 \\\\\n",
      "Criminal Law and Procedure & 35.040000 & 62.110000 & 86.340000 & 71.000000 \\\\\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": "                            Correct Rate  Top Two Correct Rate  \\\nEvidence                              63                    84   \nTorts                                 62                    72   \nCivil Procedure                       52                    63   \nConstitutional Law                    49                    67   \nReal Property                         45                    72   \nContracts                             45                    77   \nCriminal Law and Procedure            35                    62   \n\n                            Top Three Correct Rate  NCBE Rate  \nEvidence                                        98         65  \nTorts                                           94         71  \nCivil Procedure                                 79         59  \nConstitutional Law                              87         72  \nReal Property                                   85         65  \nContracts                                       86         70  \nCriminal Law and Procedure                      86         71  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Correct Rate</th>\n      <th>Top Two Correct Rate</th>\n      <th>Top Three Correct Rate</th>\n      <th>NCBE Rate</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Evidence</th>\n      <td>63</td>\n      <td>84</td>\n      <td>98</td>\n      <td>65</td>\n    </tr>\n    <tr>\n      <th>Torts</th>\n      <td>62</td>\n      <td>72</td>\n      <td>94</td>\n      <td>71</td>\n    </tr>\n    <tr>\n      <th>Civil Procedure</th>\n      <td>52</td>\n      <td>63</td>\n      <td>79</td>\n      <td>59</td>\n    </tr>\n    <tr>\n      <th>Constitutional Law</th>\n      <td>49</td>\n      <td>67</td>\n      <td>87</td>\n      <td>72</td>\n    </tr>\n    <tr>\n      <th>Real Property</th>\n      <td>45</td>\n      <td>72</td>\n      <td>85</td>\n      <td>65</td>\n    </tr>\n    <tr>\n      <th>Contracts</th>\n      <td>45</td>\n      <td>77</td>\n      <td>86</td>\n      <td>70</td>\n    </tr>\n    <tr>\n      <th>Criminal Law and Procedure</th>\n      <td>35</td>\n      <td>62</td>\n      <td>86</td>\n      <td>71</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "performance_by_category_df = pandas.DataFrame({\n",
    "        \"Correct Rate\": session_df.groupby(\"Question Category\")[\"Correct\"].mean() * 100.0,\n",
    "        \"Top Two Correct Rate\": session_df.groupby(\"Question Category\")[\"Top Two Correct\"].mean() * 100.0,\n",
    "        \"Top Three Correct Rate\": session_df.groupby(\"Question Category\")[\"Top Three Correct\"].mean() * 100.0,\n",
    "        \"NCBE Rate\": ncbe_df[\"Accuracy (%)\"],\n",
    "})\\\n",
    "    .sort_values(\"Correct Rate\", ascending=False)\n",
    "\n",
    "\n",
    "with pandas.option_context(\"float_format\", \"{:2.0f}\".format):\n",
    "    print(performance_by_category_df.round(2).style.to_latex())\n",
    "    display(performance_by_category_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Hyperparameters - Temperature"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "a43319e0-5c42-4045-b8cd-d40ec424c0e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrr}\n",
      " & Correct Rate & Top Two Correct Rate & Top Three Correct Rate & Samples \\\\\n",
      "Temperature &  &  &  &  \\\\\n",
      "0.000000 & 49.860000 & 71.770000 & 89.000000 & 500.000000 \\\\\n",
      "0.500000 & 50.190000 & 71.050000 & 88.200000 & 1800.000000 \\\\\n",
      "1.000000 & 49.790000 & 70.650000 & 86.950000 & 1800.000000 \\\\\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": "             Correct Rate  Top Two Correct Rate  Top Three Correct Rate  \\\nTemperature                                                               \n0.00%              49.86%                71.77%                  89.00%   \n50.00%             50.19%                71.05%                  88.20%   \n100.00%            49.79%                70.65%                  86.95%   \n\n             Samples  \nTemperature           \n0.00%              5  \n50.00%            18  \n100.00%           18  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Correct Rate</th>\n      <th>Top Two Correct Rate</th>\n      <th>Top Three Correct Rate</th>\n      <th>Samples</th>\n    </tr>\n    <tr>\n      <th>Temperature</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0.00%</th>\n      <td>49.86%</td>\n      <td>71.77%</td>\n      <td>89.00%</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>50.00%</th>\n      <td>50.19%</td>\n      <td>71.05%</td>\n      <td>88.20%</td>\n      <td>18</td>\n    </tr>\n    <tr>\n      <th>100.00%</th>\n      <td>49.79%</td>\n      <td>70.65%</td>\n      <td>86.95%</td>\n      <td>18</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "performance_by_temperature_df = pandas.DataFrame({\n",
    "        \"Correct Rate\": session_df.groupby(\"Temperature\")[\"Correct\"].mean(),\n",
    "        \"Top Two Correct Rate\": session_df.groupby(\"Temperature\")[\"Top Two Correct\"].mean(),\n",
    "        \"Top Three Correct Rate\": session_df.groupby(\"Temperature\")[\"Top Three Correct\"].mean(),\n",
    "        \"Samples\": session_df.groupby(\"Temperature\")[\"Exam Session ID\"].nunique(),\n",
    "})\\\n",
    "    .sort_values(\"Temperature\", ascending=True)\n",
    "\n",
    "\n",
    "with pandas.option_context(\"float_format\", \"{:.2%}\".format):\n",
    "    print((100.0 * performance_by_temperature_df).round(2).style.to_latex())\n",
    "    display(performance_by_temperature_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Hyperparameters - Best Of"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "5df84467-9190-4d88-951d-f992e517bf66",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrr}\n",
      " & Correct Rate & Top Two Correct Rate & Top Three Correct Rate & Samples \\\\\n",
      "Best Of &  &  &  &  \\\\\n",
      "1 & 49.510000 & 70.590000 & 87.270000 & 1500.000000 \\\\\n",
      "2 & 50.270000 & 71.220000 & 88.170000 & 1400.000000 \\\\\n",
      "4 & 50.200000 & 71.130000 & 87.840000 & 1200.000000 \\\\\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": "         Correct Rate  Top Two Correct Rate  Top Three Correct Rate  Samples\nBest Of                                                                     \n1              49.51%                70.59%                  87.27%       15\n2              50.27%                71.22%                  88.17%       14\n4              50.20%                71.13%                  87.84%       12",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Correct Rate</th>\n      <th>Top Two Correct Rate</th>\n      <th>Top Three Correct Rate</th>\n      <th>Samples</th>\n    </tr>\n    <tr>\n      <th>Best Of</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>49.51%</td>\n      <td>70.59%</td>\n      <td>87.27%</td>\n      <td>15</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>50.27%</td>\n      <td>71.22%</td>\n      <td>88.17%</td>\n      <td>14</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>50.20%</td>\n      <td>71.13%</td>\n      <td>87.84%</td>\n      <td>12</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "performance_by_bestof_df = pandas.DataFrame({\n",
    "        \"Correct Rate\": session_df.groupby(\"Best Of\")[\"Correct\"].mean(),\n",
    "        \"Top Two Correct Rate\": session_df.groupby(\"Best Of\")[\"Top Two Correct\"].mean(),\n",
    "        \"Top Three Correct Rate\": session_df.groupby(\"Best Of\")[\"Top Three Correct\"].mean(),\n",
    "        \"Samples\": session_df.groupby(\"Best Of\")[\"Exam Session ID\"].nunique(),\n",
    "})\\\n",
    "    .sort_values(\"Best Of\", ascending=True)\n",
    "\n",
    "\n",
    "with pandas.option_context(\"float_format\", \"{:.2%}\".format):\n",
    "    print((100.0 * performance_by_bestof_df).round(2).style.to_latex())\n",
    "    display(performance_by_bestof_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Hyperparameter Surface"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct Rate\n"
     ]
    },
    {
     "data": {
      "text/plain": "Best Of             1         2         4\nTemperature                              \n0.0          0.494418  0.504785       NaN\n0.5          0.500000  0.500000  0.505582\n1.0          0.490431  0.504785  0.498405",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th>Best Of</th>\n      <th>1</th>\n      <th>2</th>\n      <th>4</th>\n    </tr>\n    <tr>\n      <th>Temperature</th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0.0</th>\n      <td>0.494418</td>\n      <td>0.504785</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>0.5</th>\n      <td>0.500000</td>\n      <td>0.500000</td>\n      <td>0.505582</td>\n    </tr>\n    <tr>\n      <th>1.0</th>\n      <td>0.490431</td>\n      <td>0.504785</td>\n      <td>0.498405</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct Rate - Standard Error of the Mean\n"
     ]
    },
    {
     "data": {
      "text/plain": "Best Of             1         2         4\nTemperature                              \n0.0          0.019983  0.024484       NaN\n0.5          0.014125  0.014125  0.014124\n1.0          0.014123  0.014125  0.014125",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th>Best Of</th>\n      <th>1</th>\n      <th>2</th>\n      <th>4</th>\n    </tr>\n    <tr>\n      <th>Temperature</th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0.0</th>\n      <td>0.019983</td>\n      <td>0.024484</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>0.5</th>\n      <td>0.014125</td>\n      <td>0.014125</td>\n      <td>0.014124</td>\n    </tr>\n    <tr>\n      <th>1.0</th>\n      <td>0.014123</td>\n      <td>0.014125</td>\n      <td>0.014125</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top Two Correct Rate\n"
     ]
    },
    {
     "data": {
      "text/plain": "Best Of             1         2         4\nTemperature                              \n0.0          0.716108  0.720096       NaN\n0.5          0.706539  0.712919  0.712121\n1.0          0.700159  0.708931  0.710526",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th>Best Of</th>\n      <th>1</th>\n      <th>2</th>\n      <th>4</th>\n    </tr>\n    <tr>\n      <th>Temperature</th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0.0</th>\n      <td>0.716108</td>\n      <td>0.720096</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>0.5</th>\n      <td>0.706539</td>\n      <td>0.712919</td>\n      <td>0.712121</td>\n    </tr>\n    <tr>\n      <th>1.0</th>\n      <td>0.700159</td>\n      <td>0.708931</td>\n      <td>0.710526</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top Three Correct Rate\n"
     ]
    },
    {
     "data": {
      "text/plain": "Best Of             1         2         4\nTemperature                              \n0.0          0.888357  0.892344       NaN\n0.5          0.877990  0.887560  0.880383\n1.0          0.859649  0.872408  0.876396",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th>Best Of</th>\n      <th>1</th>\n      <th>2</th>\n      <th>4</th>\n    </tr>\n    <tr>\n      <th>Temperature</th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0.0</th>\n      <td>0.888357</td>\n      <td>0.892344</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>0.5</th>\n      <td>0.877990</td>\n      <td>0.887560</td>\n      <td>0.880383</td>\n    </tr>\n    <tr>\n      <th>1.0</th>\n      <td>0.859649</td>\n      <td>0.872408</td>\n      <td>0.876396</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Samples\n"
     ]
    },
    {
     "data": {
      "text/plain": "Best Of        1    2    4\nTemperature               \n0.0          3.0  2.0  NaN\n0.5          6.0  6.0  6.0\n1.0          6.0  6.0  6.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th>Best Of</th>\n      <th>1</th>\n      <th>2</th>\n      <th>4</th>\n    </tr>\n    <tr>\n      <th>Temperature</th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0.0</th>\n      <td>3.0</td>\n      <td>2.0</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>0.5</th>\n      <td>6.0</td>\n      <td>6.0</td>\n      <td>6.0</td>\n    </tr>\n    <tr>\n      <th>1.0</th>\n      <td>6.0</td>\n      <td>6.0</td>\n      <td>6.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "performance_by_temp_bestof = pandas.DataFrame({\n",
    "    \"Correct Rate\": session_df.groupby([\"Temperature\", \"Best Of\"])[\"Correct\"].mean(),\n",
    "    \"Correct Rate SEM\": session_df.groupby([\"Temperature\", \"Best Of\"])[\"Correct\"].sem(),\n",
    "    \"Top Two Correct Rate\": session_df.groupby([\"Temperature\", \"Best Of\"])[\"Top Two Correct\"].mean(),\n",
    "    \"Top Three Correct Rate\": session_df.groupby([\"Temperature\", \"Best Of\"])[\"Top Three Correct\"].mean(),\n",
    "    \"Samples\": session_df.groupby([\"Temperature\", \"Best Of\"])[\"Exam Session ID\"].nunique(),\n",
    "})\n",
    "\n",
    "print(\"Correct Rate\")\n",
    "display(performance_by_temp_bestof[\"Correct Rate\"].unstack())\n",
    "\n",
    "print(\"Correct Rate - Standard Error of the Mean\")\n",
    "display(performance_by_temp_bestof[\"Correct Rate SEM\"].unstack())\n",
    "\n",
    "print(\"Top Two Correct Rate\")\n",
    "display(performance_by_temp_bestof[\"Top Two Correct Rate\"].unstack())\n",
    "\n",
    "print(\"Top Three Correct Rate\")\n",
    "display(performance_by_temp_bestof[\"Top Three Correct Rate\"].unstack())\n",
    "\n",
    "print(\"Samples\")\n",
    "display(performance_by_temp_bestof[\"Samples\"].unstack())"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2c8fe96-c374-45ed-871c-8bcae5d47f3a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}