flaky windows tests (#1198)

setup.cfg

@@ -105,3 +105,4 @@ testpaths=tests
 python_classes=*Test
 markers:
  slow: slow tests
+ asyncio: async tests

setup.py

@@ -97,6 +97,7 @@
  "httpx==0.24.1",
  "ruff==0.3.7",
  "pre-commit==3.5.0",
+ "pytest-asyncio==0.23.7",
  ],
  "llm": [
  "openai>=1.16.2",

src/evidently/collector/app.py

@@ -237,8 +237,8 @@ def reraise(_, exception: Exception):
  "service": Provide(lambda: service, use_cache=True, sync_to_thread=False),
  "storage": Provide(lambda: service.storage, use_cache=True, sync_to_thread=False),
  "parsed_json": Provide(parse_json, sync_to_thread=False),
- "service_config_path": Provide(lambda: config_path),
- "service_workspace": Provide(lambda: os.path.dirname(config_path)),
+ "service_config_path": Provide(lambda: config_path, sync_to_thread=False),
+ "service_workspace": Provide(lambda: os.path.dirname(config_path), sync_to_thread=False),
  },
  middleware=[auth_middleware_factory],
  guards=[is_authenticated],

src/evidently/pydantic_utils.py

@@ -119,7 +119,7 @@ def all_subclasses(cls: Type[T]) -> Set[Type[T]]:
 def register_type_alias(base_class: Type["PolymorphicModel"], classpath: str, alias: str):
  key = (base_class, alias)
 
- if key in TYPE_ALIASES and TYPE_ALIASES[key] != classpath:
+ if key in TYPE_ALIASES and TYPE_ALIASES[key] != classpath and "PYTEST_CURRENT_TEST" not in os.environ:
  warnings.warn(f"Duplicate key {key} in alias map")
  TYPE_ALIASES[key] = classpath
 
@@ -129,7 +129,7 @@ def register_loaded_alias(base_class: Type["PolymorphicModel"], cls: Type["Polym
  raise ValueError(f"Cannot register alias: {cls.__name__} is not subclass of {base_class.__name__}")
 
  key = (base_class, alias)
- if key in LOADED_TYPE_ALIASES and LOADED_TYPE_ALIASES[key] != cls:
+ if key in LOADED_TYPE_ALIASES and LOADED_TYPE_ALIASES[key] != cls and "PYTEST_CURRENT_TEST" not in os.environ:
  warnings.warn(f"Duplicate key {key} in alias map")
  LOADED_TYPE_ALIASES[key] = cls
 

src/evidently/report/report.py

@@ -111,7 +111,10 @@ def run(
  column_mapping,
  self.options.data_definition_options.categorical_features_cardinality,
  )
-
+ if METRIC_GENERATORS in self.metadata:
+ del self.metadata[METRIC_GENERATORS]
+ if METRIC_PRESETS in self.metadata:
+ del self.metadata[METRIC_PRESETS]
  # get each item from metrics/presets and add to metrics list
  # do it in one loop because we want to save metrics and presets order
  for item in self.metrics:

src/evidently/tracing/__init__.py

@@ -13,6 +13,7 @@
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
 from opentelemetry.sdk.trace.export import SpanExporter
+from requests import Response
 
 from evidently.ui.workspace.cloud import ACCESS_TOKEN_COOKIE
 from evidently.ui.workspace.cloud import CloudMetadataStorage
@@ -107,14 +108,15 @@ def _create_tracer_provider(
  )
 
  cloud = CloudMetadataStorage(_address, _api_key, ACCESS_TOKEN_COOKIE.key)
- datasets = cloud._request("/api/datasets", "GET").json()["datasets"]
+ datasets_response: Response = cloud._request("/api/datasets", "GET")
+ datasets = datasets_response.json()["datasets"]
  _export_id = None
  for dataset in datasets:
  if dataset["name"] == _export_name:
  _export_id = dataset["id"]
  break
  if _export_id is None:
- resp = cloud._request(
+ resp: Response = cloud._request(
  "/api/datasets/tracing",
  "POST",
  query_params={"team_id": _team_id},

src/evidently/ui/api/projects.py

@@ -30,6 +30,7 @@
 from evidently.ui.base import Permission
 from evidently.ui.base import Project
 from evidently.ui.base import ProjectManager
+from evidently.ui.base import SnapshotMetadata
 from evidently.ui.dashboards.base import DashboardPanel
 from evidently.ui.dashboards.reports import DashboardPanelCounter
 from evidently.ui.dashboards.reports import DashboardPanelDistribution
@@ -59,6 +60,21 @@ def list_reports(
  return reports
 
 
+@get("/{project_id:uuid}/snapshots", sync_to_thread=True)
+def list_snapshots(
+ project_id: Annotated[uuid.UUID, Parameter(title="id of project")],
+ project_manager: Annotated[ProjectManager, Dependency(skip_validation=True)],
+ log_event: Callable,
+ user_id: UserID,
+) -> List[SnapshotMetadata]:
+ project = project_manager.get_project(user_id, project_id)
+ if project is None:
+ raise HTTPException(status_code=404, detail="project not found")
+ snapshots = project_manager.list_snapshots(user_id, project_id)
+ log_event("list_snapshots", reports_count=len(snapshots))
+ return snapshots
+
+
 @get("/", sync_to_thread=True)
 def list_projects(
  project_manager: Annotated[ProjectManager, Dependency(skip_validation=True)],
@@ -236,6 +252,31 @@ def get_snapshot_data(
  return json.dumps(asdict(info), cls=NumpyEncoder)
 
 
+@get("/{project_id:uuid}/{snapshot_id:uuid}/metadata", sync_to_thread=True)
+def get_snapshot_metadata(
+ project_id: Annotated[uuid.UUID, Parameter(title="id of project")],
+ snapshot_id: Annotated[uuid.UUID, Parameter(title="id of snapshot")],
+ project_manager: Annotated[ProjectManager, Dependency(skip_validation=True)],
+ log_event: Callable,
+ user_id: UserID,
+) -> SnapshotMetadata:
+ project = project_manager.get_project(user_id, project_id)
+ if project is None:
+ raise HTTPException(status_code=404, detail="Project not found")
+ snapshot_meta = project.get_snapshot_metadata(snapshot_id)
+ if snapshot_meta is None:
+ raise HTTPException(status_code=404, detail="Snapshot not found")
+ log_event(
+ "get_snapshot_metadata",
+ snapshot_type="report" if snapshot_meta.is_report else "test_suite",
+ metric_presets=snapshot_meta.metadata.get(METRIC_PRESETS, []),
+ metric_generators=snapshot_meta.metadata.get(METRIC_GENERATORS, []),
+ test_presets=snapshot_meta.metadata.get(TEST_PRESETS, []),
+ test_generators=snapshot_meta.metadata.get(TEST_GENERATORS, []),
+ )
+ return snapshot_meta
+
+
 @get("/{project_id:uuid}/dashboard/panels", sync_to_thread=True)
 def list_project_dashboard_panels(
  project_id: Annotated[uuid.UUID, Parameter(title="id of project")],
@@ -253,7 +294,7 @@ def list_project_dashboard_panels(
 # We need this endpoint to export
 # some additional models to open api schema
 @get("/models/additional")
-def additional_models() -> (
+async def additional_models() -> (
  List[
  Union[
  DashboardInfoModel,
@@ -362,6 +403,8 @@ def create_projects_api(guard: Callable) -> Router:
  get_snapshot_download,
  list_project_dashboard_panels,
  project_dashboard,
+ list_snapshots,
+ get_snapshot_metadata,
  ],
  ),
  Router(

src/evidently/ui/components/security.py

@@ -79,9 +79,9 @@ class SimpleSecurity(SecurityComponent):
  def get_dependencies(self, ctx: ComponentContext) -> Dict[str, Provide]:
  return {
  "user_id": Provide(get_user_id),
- "security": Provide(self.get_security),
- "security_config": Provide(lambda: self),
- "auth_manager": Provide(lambda: NoopAuthManager()),
+ "security": Provide(self.get_security, sync_to_thread=False),
+ "security_config": Provide(lambda: self, sync_to_thread=False),
+ "auth_manager": Provide(lambda: NoopAuthManager(), sync_to_thread=False),
  }
 
 

src/evidently/ui/local_service.py

@@ -1,5 +1,9 @@
+import logging
+import time
+
 from litestar import Request
 from litestar import Response
+from litestar.logging import LoggingConfig
 
 from evidently.ui.api.projects import create_projects_api
 from evidently.ui.api.service import service_api
@@ -13,6 +17,7 @@
 from evidently.ui.components.storage import StorageComponent
 from evidently.ui.components.telemetry import TelemetryComponent
 from evidently.ui.config import AppConfig
+from evidently.ui.config import ConfigContext
 from evidently.ui.errors import EvidentlyServiceError
 
 
@@ -32,8 +37,56 @@ def get_route_handlers(self, ctx: ComponentContext):
 
  def apply(self, ctx: ComponentContext, builder: AppBuilder):
  super().apply(ctx, builder)
+ assert isinstance(ctx, ConfigContext)
  builder.exception_handlers[EvidentlyServiceError] = evidently_service_exception_handler
  builder.kwargs["debug"] = self.debug
+ if self.debug:
+ log_config = create_logging()
+ builder.kwargs["logging_config"] = LoggingConfig(**log_config)
+
+
+def create_logging() -> dict:
+ logging.Formatter.converter = time.gmtime
+ return {
+ "version": 1,
+ "log_exceptions": "always",
+ "disable_existing_loggers": False,
+ "formatters": {
+ "default": {
+ "()": "logging.Formatter",
+ "format": "%(asctime)s %(levelname)-8s %(name)-15s %(message)s",
+ "datefmt": "%Y-%m-%dT%H:%M:%SZ",
+ },
+ "access": {
+ "()": "logging.Formatter",
+ "format": "%(asctime)s %(levelname)-8s %(name)-15s %(message)s",
+ "datefmt": "%Y-%m-%dT%H:%M:%SZ",
+ },
+ "standard": {
+ "()": "logging.Formatter",
+ "format": "%(asctime)s %(levelname)-8s %(name)-15s %(message)s",
+ "datefmt": "%Y-%m-%dT%H:%M:%SZ",
+ },
+ },
+ "handlers": {
+ "default": {
+ "formatter": "default",
+ "class": "logging.StreamHandler",
+ "stream": "ext://sys.stdout",
+ },
+ "access": {
+ "formatter": "access",
+ "class": "logging.StreamHandler",
+ "stream": "ext://sys.stdout",
+ },
+ },
+ "loggers": {
+ "litestar": {"handlers": ["default"]},
+ "uvicorn": {"handlers": ["default"], "level": "INFO", "propagate": False},
+ "uvicorn.error": {"level": "INFO"},
+ "uvicorn.access": {"handlers": ["access"], "level": "INFO", "propagate": False},
+ },
+ }
 
 
 class LocalConfig(AppConfig):

src/evidently/ui/storage/local/watcher.py

@@ -119,5 +119,7 @@ def on_snapshot_event(self, event):
  or event.event_type == EVENT_TYPE_MOVED
  and not os.path.exists(event.src_path)
  ):
- del self.state.snapshots[pid][sid]
- del self.state.snapshot_data[pid][sid]
+ if pid in self.state.snapshots and sid in self.state.snapshots[pid]:
+ del self.state.snapshots[pid][sid]
+ if pid in self.state.snapshot_data and sid in self.state.snapshot_data[pid]:
+ del self.state.snapshot_data[pid][sid]

src/evidently/ui/workspace/cloud.py

@@ -2,13 +2,17 @@
 from typing import BinaryIO
 from typing import Dict
 from typing import List
+from typing import Literal
 from typing import NamedTuple
 from typing import Optional
+from typing import Type
 from typing import Union
+from typing import overload
 from uuid import UUID
 
 import pandas as pd
 from requests import HTTPError
+from requests import Response
 
 from evidently.ui.api.models import OrgModel
 from evidently.ui.api.models import TeamModel
@@ -24,6 +28,7 @@
 from evidently.ui.workspace.remote import NoopBlobStorage
 from evidently.ui.workspace.remote import NoopDataStorage
 from evidently.ui.workspace.remote import RemoteMetadataStorage
+from evidently.ui.workspace.remote import T
 from evidently.ui.workspace.view import WorkspaceView
 
 TOKEN_HEADER_NAME = "X-Evidently-Token"
@@ -84,17 +89,45 @@ def _prepare_request(
  r.cookies[self.token_cookie_name] = self.jwt_token
  return r
 
+ @overload
  def _request(
  self,
  path: str,
  method: str,
  query_params: Optional[dict] = None,
  body: Optional[dict] = None,
- response_model=None,
+ response_model: Type[T] = ...,
  cookies=None,
  headers: Dict[str, str] = None,
  form_data: bool = False,
- ):
+ ) -> T:
+ pass
+
+ @overload
+ def _request(
+ self,
+ path: str,
+ method: str,
+ query_params: Optional[dict] = None,
+ body: Optional[dict] = None,
+ response_model: Literal[None] = None,
+ cookies=None,
+ headers: Dict[str, str] = None,
+ form_data: bool = False,
+ ) -> Response:
+ pass
+
+ def _request(
+ self,
+ path: str,
+ method: str,
+ query_params: Optional[dict] = None,
+ body: Optional[dict] = None,
+ response_model: Optional[Type[T]] = None,
+ cookies=None,
+ headers: Dict[str, str] = None,
+ form_data: bool = False,
+ ) -> Union[Response, T]:
  try:
  res = super()._request(
  path=path,
@@ -142,7 +175,7 @@ def create_team(self, team: Team, org_id: OrgID = None) -> TeamModel:
  def add_dataset(
  self, file: BinaryIO, name: str, org_id: OrgID, team_id: TeamID, description: Optional[str]
  ) -> DatasetID:
- response = self._request(
+ response: Response = self._request(
  "/api/datasets/",
  "POST",
  body={"name": name, "description": description, "file": file},
@@ -152,7 +185,7 @@ def add_dataset(
  return DatasetID(response.json()["dataset_id"])
 
  def load_dataset(self, dataset_id: DatasetID) -> pd.DataFrame:
- response = self._request(f"/api/datasets/{dataset_id}/download", "GET")
+ response: Response = self._request(f"/api/datasets/{dataset_id}/download", "GET")
  return pd.read_parquet(BytesIO(response.content))