Example/llm tutorial (#1188)

examples/how_to_questions/how_to_evaluate_llm_with_text_descriptors.ipynb

@@ -26,6 +26,9 @@
  "from datetime import time\n",
  "from datetime import timedelta\n",
  "\n",
+ "import requests\n",
+ "from io import BytesIO\n",
+ "\n",
  "from sklearn import datasets, ensemble, model_selection"
  ]
  },
@@ -44,7 +47,7 @@
  "from evidently.report import Report\n",
  "from evidently.test_suite import TestSuite\n",
  "\n",
- "from evidently.metrics import ColumnSummaryMetric, ColumnDistributionMetric, ColumnDriftMetric, DataDriftTable, TextDescriptorsDistribution\n",
+ "from evidently.metrics import ColumnSummaryMetric, ColumnDistributionMetric, ColumnDriftMetric, DataDriftTable, TextDescriptorsDistribution, ColumnCategoryMetric\n",
  "from evidently.tests import TestColumnValueMin, TestColumnValueMean, TestCategoryShare, TestShareOfOutRangeValues\n",
  "\n",
  "from evidently.metric_preset import DataDriftPreset, DataQualityPreset, TextOverviewPreset, TextEvals\n",
@@ -79,18 +82,39 @@
  "# Load Data"
  ]
  },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e2038e2b-acdb-4844-a759-48170f1f539b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "response = requests.get(\"https://raw.githubusercontent.com/evidentlyai/evidently/main/examples/how_to_questions/chat_df.csv\")\n",
+ "csv_content = BytesIO(response.content)"
+ ]
+ },
  {
  "cell_type": "code",
  "execution_count": null,
  "id": "6dcad6c8-9abb-4c66-aa58-4e370baef072",
  "metadata": {},
  "outputs": [],
  "source": [
- "assistant_logs =  pd.read_csv('chat_df.csv', index_col=0, parse_dates=['start_time', 'end_time'])\n",
+ "assistant_logs = pd.read_csv(csv_content, index_col=0, parse_dates=['start_time', 'end_time'])\n",
  "assistant_logs.index = assistant_logs.start_time\n",
  "assistant_logs.index.rename('index', inplace=True)"
  ]
  },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "82975800-0f47-458e-a43e-43d4accbb446",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "pd.set_option('display.max_colwidth', None)"
+ ]
+ },
  {
  "cell_type": "code",
  "execution_count": null,
@@ -430,7 +454,7 @@
  "outputs": [],
  "source": [
  "ws = CloudWorkspace(\n",
- " \ttoken=\"YOUR TOKEN HERE\",\n",
+ " \ttoken=\"YOUR_TOKEN_HERE\",\n",
  " \turl=\"https://app.evidently.cloud/\"\n",
  ")"
  ]
@@ -442,8 +466,8 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "project = ws.create_project(\"Virtual assistant testing\", team_id=\"YOUR TEAM ID HERE\")\n",
- "project.description = \"Project descriotion\""
+ "project = ws.create_project(\"Project title\", team_id=\"YOUR_TEAM_ID_HERE\")\n",
+ "project.description = \"Project description\""
  ]
  },
  {
@@ -460,6 +484,8 @@
  " TestShareOfOutRangeValues(column_name=TextLength().on(\"question\"), left=30, right=100, lt=0.1),\n",
  " TestColumnValueMin(column_name=Sentiment().on(\"response\"), gt=0),\n",
  " TestColumnValueMean(column_name=OOV().on(\"response\"), lt=15),\n",
+ " TestCategoryShare(column_name = \"feedback\", category=\"downvote\", lt=0.1),\n",
+ " TestCategoryShare(column_name = IncludesWords(words_list=['salary']).on(\"response\"), category=False, lt=0.1), \n",
  " ],\n",
  " timestamp=datetime.now() + timedelta(days=i),\n",
  " )\n",
@@ -479,14 +505,15 @@
  " TextEvals(column_name=\"question\", descriptors=[\n",
  " Sentiment(display_name=\"Question sentiment\"),\n",
  " TextLength(display_name= \"Question length\"),\n",
- " OOV(display_name= \"Question out of vocabulary words\")\n",
+ " OOV(display_name= \"Question out of vocabulary words\"),\n",
  " ]),\n",
  " TextEvals(column_name=\"response\", descriptors=[\n",
  " Sentiment(display_name=\"Response sentiment\"),\n",
  " NonLetterCharacterPercentage(display_name=\"Non letter characters in response\"),\n",
  " SentenceCount(display_name=\"Sentence count in response\"),\n",
- " WordCount(display_name=\"Word count in response\")\n",
- " ])\n",
+ " WordCount(display_name=\"Word count in response\"),\n",
+ " ]),\n",
+ " ColumnCategoryMetric(column_name=IncludesWords(words_list=['salary']).for_column(\"response\"), category=True),\n",
  " ],\n",
  " timestamp=datetime.now() + timedelta(days=i),\n",
  " )\n",
@@ -511,7 +538,9 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "f1c6e042-4b5d-400d-98a3-7a263943881e",
- "metadata": {},
+ "metadata": {
+ "scrolled": true
+ },
  "outputs": [],
  "source": [
  "for i in range(0, 5):\n",
@@ -531,8 +560,6 @@
  " title=\"Test results\",\n",
  " filter=ReportFilter(metadata_values={}, tag_values=[], include_test_suites=True),\n",
  " size=WidgetSize.FULL,\n",
- " panel_type=TestSuitePanelType.DETAILED,\n",
- " time_agg=\"1D\",\n",
  " )\n",
  ")\n",
  "project.save()"