From ff3758451a743d6d72a5148bd28efff64f5aa795 Mon Sep 17 00:00:00 2001 From: Mikhail Sveshnikov Date: Wed, 10 Jul 2024 05:17:52 +0400 Subject: [PATCH] fix description for column_name (#1191) * fix description for column_name * add params to test panels * fix value order in distribution panels * same args only for same id items * mypy --- src/evidently/ui/dashboards/reports.py | 4 +-- src/evidently/ui/dashboards/test_suites.py | 10 +++--- src/evidently/ui/dashboards/utils.py | 37 +++++++++++++++------- src/evidently/ui/storage/utils.py | 17 ++++++++-- tests/ui/test_dashboards.py | 22 ++++++++++--- 5 files changed, 64 insertions(+), 26 deletions(-) diff --git a/src/evidently/ui/dashboards/reports.py b/src/evidently/ui/dashboards/reports.py index 654042bc38..788d2a6c78 100644 --- a/src/evidently/ui/dashboards/reports.py +++ b/src/evidently/ui/dashboards/reports.py @@ -25,8 +25,8 @@ from .utils import CounterAgg from .utils import HistBarMode from .utils import PlotType +from .utils import _get_hover_params from .utils import _get_metric_hover -from .utils import _get_metrics_hover_params if TYPE_CHECKING: from evidently.ui.base import DataStorage @@ -47,7 +47,7 @@ def build( points = data_storage.load_points(project_id, self.filter, self.values, timestamp_start, timestamp_end) # list[dict[metric, point]] all_metrics: Set[Metric] = set(m for data in points for m in data.keys()) - hover_params = _get_metrics_hover_params(all_metrics) + hover_params = _get_hover_params(all_metrics) fig = go.Figure(layout={"showlegend": True}) for val, metric_pts in zip(self.values, points): if len(metric_pts) == 0: diff --git a/src/evidently/ui/dashboards/test_suites.py b/src/evidently/ui/dashboards/test_suites.py index 0702ba40eb..a2c1c7c170 100644 --- a/src/evidently/ui/dashboards/test_suites.py +++ b/src/evidently/ui/dashboards/test_suites.py @@ -32,6 +32,7 @@ from .utils import TEST_COLORS from .utils import CounterAgg from .utils import TestSuitePanelType +from .utils import _get_hover_params from .utils import _get_test_hover from .utils import getattr_nested @@ -128,10 +129,9 @@ def _create_aggregate_fig(self, points: TestResultPoints): def _create_detailed_fig(self, points: TestResultPoints): dates = list(sorted(points.keys())) - tests = list(set(t for p in points.values() for t in p.keys())) - # date_to_test: Dict[datetime.datetime, Dict[Test, Test]] = { - # d: {t: t for t in tst.keys()} for d, tst in points.items() - # } + all_tests = set(t for p in points.values() for t in p.keys()) + tests = list(all_tests) + hover_params = _get_hover_params(all_tests) def get_description(test, date): description = points[date][test].description @@ -151,7 +151,7 @@ def get_color(test, date) -> Optional[str]: x=dates, y=[1 for _ in range(len(dates))], marker_color=[get_color(test, d) for d in dates], - hovertemplate=_get_test_hover(test), + hovertemplate=_get_test_hover(test.name, hover_params[test]), customdata=[get_description(test, d) for i, d in enumerate(dates)], showlegend=False, ) diff --git a/src/evidently/ui/dashboards/utils.py b/src/evidently/ui/dashboards/utils.py index 4a026692c8..904fd29989 100644 --- a/src/evidently/ui/dashboards/utils.py +++ b/src/evidently/ui/dashboards/utils.py @@ -4,8 +4,11 @@ from typing import Any from typing import Dict from typing import List +from typing import Optional from typing import Set from typing import Tuple +from typing import TypeVar +from typing import Union import plotly.io as pio @@ -117,17 +120,28 @@ def _get_metric_hover(params: List[str], value: "PanelValue"): return hover -def _get_metrics_hover_params(metrics: Set[Metric]) -> Dict[Metric, List[str]]: - if len(metrics) == 0: +def _hover_params_early_stop(obj: Any, paths: List[str]) -> Optional[List[Tuple[str, Any]]]: + if not isinstance(obj, ColumnName): + return None + column_name_str = obj.display_name or obj.name + return [(".".join(paths), column_name_str)] + + +TMT = TypeVar("TMT", bound=Union[Metric, Test]) + + +def _get_hover_params(items: Set[TMT]) -> Dict[TMT, List[str]]: + if len(items) == 0: return {} - metric_params: Dict[Metric, Set[Tuple[str, Any]]] = defaultdict(set) - for metric in metrics: - for path, value in iterate_obj_fields(metric, []): - metric_params[metric].add((path, value)) - same_args = set.intersection(*metric_params.values()) + params: Dict[str, Dict[TMT, Set[str]]] = defaultdict(lambda: defaultdict(set)) + for item in items: + for path, value in iterate_obj_fields(item, [], early_stop=_hover_params_early_stop): + params[item.get_id()][item].add(f"{path}: {value}") + same_args: Dict[str, Set[str]] = {k: set.intersection(*v.values()) for k, v in params.items()} return { - metric: [f"{pair[0]}: {pair[1]}" for pair in pairs if pair not in same_args] - for metric, pairs in metric_params.items() + item: [row for row in rows if row not in same_args[item_id]] + for item_id, p in params.items() + for item, rows in p.items() } @@ -142,8 +156,7 @@ def _get_metrics_hover_params(metrics: Set[Metric]) -> Dict[Metric, List[str]]: tests_colors_order = {ts: i for i, ts in enumerate(TEST_COLORS)} -def _get_test_hover(test: Test): - params = [f"{k}: {v}" for k, v in _flatten_params(test).items()] +def _get_test_hover(test_name: str, params: List[str]): params_join = "
".join(params) - hover = f"Timestamp: %{{x}}
{test.name}
{params_join}
%{{customdata}}
" + hover = f"Timestamp: %{{x}}
{test_name}
{params_join}
%{{customdata}}
" return hover diff --git a/src/evidently/ui/storage/utils.py b/src/evidently/ui/storage/utils.py index eb144a07c2..dc9f7d4bae 100644 --- a/src/evidently/ui/storage/utils.py +++ b/src/evidently/ui/storage/utils.py @@ -1,7 +1,9 @@ import json from typing import Any +from typing import Callable from typing import Iterator from typing import List +from typing import Optional from typing import Tuple from evidently._pydantic_compat import BaseModel @@ -10,18 +12,27 @@ from evidently.utils import NumpyEncoder -def iterate_obj_fields(obj: Any, paths: List[str]) -> Iterator[Tuple[str, Any]]: +def iterate_obj_fields( + obj: Any, paths: List[str], early_stop: Optional[Callable[[Any, List[str]], Optional[List[Tuple[str, Any]]]]] = None +) -> Iterator[Tuple[str, Any]]: + if early_stop is not None: + es = early_stop(obj, paths) + if es is not None: + yield from es + return if isinstance(obj, list): return if isinstance(obj, dict): - yield from (r for key, value in obj.items() for r in iterate_obj_fields(value, paths + [str(key)])) + yield from (r for key, value in obj.items() for r in iterate_obj_fields(value, paths + [str(key)], early_stop)) return if isinstance(obj, BaseResult) and obj.__config__.extract_as_obj: yield ".".join(paths), obj return if isinstance(obj, BaseModel): yield from ( - r for name, field in obj.__fields__.items() for r in iterate_obj_fields(getattr(obj, name), paths + [name]) + r + for name, field in obj.__fields__.items() + for r in iterate_obj_fields(getattr(obj, name), paths + [name], early_stop) ) return yield ".".join(paths), obj diff --git a/tests/ui/test_dashboards.py b/tests/ui/test_dashboards.py index 977c1cea93..cd199ef13e 100644 --- a/tests/ui/test_dashboards.py +++ b/tests/ui/test_dashboards.py @@ -4,6 +4,7 @@ import pytest from evidently._pydantic_compat import parse_obj_as +from evidently.base_metric import ColumnName from evidently.base_metric import InputData from evidently.base_metric import Metric from evidently.base_metric import MetricResult @@ -11,7 +12,7 @@ from evidently.descriptors import OOV from evidently.pydantic_utils import EvidentlyBaseModel from evidently.ui.dashboards import PanelValue -from evidently.ui.dashboards.utils import _get_metrics_hover_params +from evidently.ui.dashboards.utils import _get_hover_params from evidently.ui.dashboards.utils import getattr_nested @@ -71,9 +72,9 @@ def calculate(self, data: InputData) -> TResult: m2 = MyMetric(arg="1", n=Nested(f="2")) m3 = MyMetric(arg="2", n=Nested(f="1")) - assert _get_metrics_hover_params({m1}) == {m1: []} - assert _get_metrics_hover_params({m1, m2}) == {m1: ["n.f: 1"], m2: ["n.f: 2"]} - triple = _get_metrics_hover_params({m1, m2, m3}) + assert _get_hover_params({m1}) == {m1: []} + assert _get_hover_params({m1, m2}) == {m1: ["n.f: 1"], m2: ["n.f: 2"]} + triple = _get_hover_params({m1, m2, m3}) assert {m: set(lines) for m, lines in triple.items()} == { m1: { "n.f: 1", @@ -88,3 +89,16 @@ def calculate(self, data: InputData) -> TResult: "arg: 2", }, } + + +def test_metric_hover_template_column_name(): + class MyMetric(Metric[A]): + column_name: ColumnName + + def calculate(self, data: InputData) -> TResult: + return A(f="") + + m1 = MyMetric(column_name=ColumnName.from_any("col1")) + m2 = MyMetric(column_name=ColumnName.from_any("col2")) + + assert _get_hover_params({m1, m2}) == {m1: ["column_name: col1"], m2: ["column_name: col2"]}