Added PII and NIL HF models example

evidentlyai · Jun 14, 2024 · b4d5e90 · b4d5e90
1 parent e412e1f
commit b4d5e90
Showing 1 changed file with 65 additions and 11 deletions.
diff --git a/examples/how_to_questions/how_to_evaluate_llm_with_text_descriptors.ipynb b/examples/how_to_questions/how_to_evaluate_llm_with_text_descriptors.ipynb
@@ -230,7 +230,11 @@
     "        HuggingFaceModel(model=\"SamLowe/roberta-base-go_emotions\", params={\"label\": \"disappointment\"}, \n",
     "                         display_name=\"Hugging Face Disappointment for response\"), \n",
     "        HuggingFaceModel(model=\"SamLowe/roberta-base-go_emotions\", params={\"label\": \"optimism\"}, \n",
-    "                         display_name=\"Hugging Face Optimism for response\"),     \n",
+    "                         display_name=\"Hugging Face Optimism for response\"),\n",
+    "        HuggingFaceModel(model=\"MoritzLaurer/DeBERTa-v3-large-mnli-fever-anli-ling-wanli\", params={\"labels\": [\"HR\", \"finance\"], \"threshold\":0.5}, \n",
+    "                         display_name=\"Hugging Face Topic\"), \n",
+    "        HuggingFaceModel(model=\"lakshyakh93/deberta_finetuned_pii\", params={\"threshold\": 0.6},\n",
+    "                        display_name=\"Hugging Face PII for respone\"),\n",
     "    ])\n",
     "])\n",
     "\n",
@@ -388,6 +392,16 @@
     "test_suite"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8644398d-3c22-43cb-b0e8-136ef8c7abbc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_suite.datasets()[1]"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "2e5da017-975b-420f-863e-dbc25d82c458",
@@ -415,10 +429,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "\"\"\"ws = CloudWorkspace(\n",
-    "    \ttoken=\"YOUR_TOKEN_HERE\",\n",
-    "    \turl=\"https://app.evidently.cloud\"\n",
-    ")\"\"\""
+    "ws = CloudWorkspace(\n",
+    "    \ttoken=\"YOUR TOKEN HERE\",\n",
+    "    \turl=\"https://app.evidently.cloud/\"\n",
+    ")"
    ]
   },
   {
@@ -428,8 +442,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "#project = ws.create_project(\"Virtual assistant testing\", team_id=\"TEAM_ID_HERE\")\n",
-    "#project.description = \"A project to test NANs\""
+    "project = ws.create_project(\"Virtual assistant testing\", team_id=\"YOUR TEAM ID HERE\")\n",
+    "project.description = \"Project descriotion\""
    ]
   },
   {
@@ -453,16 +467,56 @@
     "    return test_suite"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12fe69ac-8f15-40be-bd20-185bef699b50",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def create_report(i: int):\n",
+    "    report = Report(metrics=[\n",
+    "        TextEvals(column_name=\"question\", descriptors=[\n",
+    "            Sentiment(display_name=\"Question sentiment\"),\n",
+    "            TextLength(display_name= \"Question length\"),\n",
+    "            OOV(display_name= \"Question out of vocabulary words\")\n",
+    "        ]),\n",
+    "        TextEvals(column_name=\"response\", descriptors=[\n",
+    "            Sentiment(display_name=\"Response sentiment\"),\n",
+    "            NonLetterCharacterPercentage(display_name=\"Non letter characters in response\"),\n",
+    "            SentenceCount(display_name=\"Sentence count in response\"),\n",
+    "            WordCount(display_name=\"Word count in response\")\n",
+    "        ])\n",
+    "    ],\n",
+    "        timestamp=datetime.now() + timedelta(days=i),\n",
+    "                   )\n",
+    "    \n",
+    "    report.run(reference_data=None, current_data=assistant_logs.iloc[20 * i : 20 * (i + 1), :], column_mapping=column_mapping)\n",
+    "    return report    "
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "247392a4-414f-4899-a2a0-09cb56698f33",
    "metadata": {},
    "outputs": [],
    "source": [
-    "for i in range(0, 3):\n",
+    "for i in range(0, 5):\n",
     "        test_suite = create_test_suite(i=i)\n",
-    "        #ws.add_test_suite(project.id, test_suite)"
+    "        ws.add_test_suite(project.id, test_suite)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f1c6e042-4b5d-400d-98a3-7a263943881e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for i in range(0, 5):\n",
+    "        report = create_report(i=i)\n",
+    "        ws.add_report(project.id, report)"
    ]
   },
   {
@@ -472,7 +526,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "\"\"\"project.dashboard.add_panel(\n",
+    "project.dashboard.add_panel(\n",
     "    DashboardPanelTestSuite(\n",
     "        title=\"Test results\",\n",
     "        filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),\n",
@@ -481,7 +535,7 @@
     "        time_agg=\"1D\",\n",
     "    )\n",
     ")\n",
-    "project.save()\"\"\""
+    "project.save()"
    ]
   },
   {