From 9e780190959c0ca786f3d58075da2185b7c74cf7 Mon Sep 17 00:00:00 2001 From: Sergiy Matusevych Date: Mon, 13 May 2024 14:08:11 -0700 Subject: [PATCH] Issue a warning on non-null optimization context (#735) Co-authored-by: Brian Kroth --- .../bayesian_optimizers/smac_optimizer.py | 9 ++++---- .../mlos_core/optimizers/flaml_optimizer.py | 4 ++-- .../mlos_core/optimizers/random_optimizer.py | 5 +++-- .../optimizers/bayesian_optimizers_test.py | 21 +++++++++++-------- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py index 9b5da7b38b..e072394853 100644 --- a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py +++ b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import Dict, List, Optional, Union, TYPE_CHECKING from tempfile import TemporaryDirectory +from warnings import warn import ConfigSpace import numpy.typing as npt @@ -244,7 +245,7 @@ def _register(self, configurations: pd.DataFrame, scores: pd.Series, context: Op from smac.runhistory import StatusType, TrialInfo, TrialValue # pylint: disable=import-outside-toplevel if context is not None: - raise NotImplementedError() + warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning) # Register each trial (one-by-one) for config, score in zip(self._to_configspace_configs(configurations), scores.tolist()): @@ -273,7 +274,7 @@ def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame: from smac.runhistory import TrialInfo # pylint: disable=import-outside-toplevel if context is not None: - raise NotImplementedError() + warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning) trial: TrialInfo = self.base_optimizer.ask() trial.config.is_valid_configuration() @@ -290,7 +291,7 @@ def surrogate_predict(self, configurations: pd.DataFrame, context: Optional[pd.D from smac.utils.configspace import convert_configurations_to_array # pylint: disable=import-outside-toplevel if context is not None: - raise NotImplementedError() + warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning) if self._space_adapter and not isinstance(self._space_adapter, IdentityAdapter): raise NotImplementedError() @@ -308,7 +309,7 @@ def surrogate_predict(self, configurations: pd.DataFrame, context: Optional[pd.D def acquisition_function(self, configurations: pd.DataFrame, context: Optional[pd.DataFrame] = None) -> npt.NDArray: if context is not None: - raise NotImplementedError() + warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning) if self._space_adapter: raise NotImplementedError() diff --git a/mlos_core/mlos_core/optimizers/flaml_optimizer.py b/mlos_core/mlos_core/optimizers/flaml_optimizer.py index ce27ae9639..07fa5fe3a2 100644 --- a/mlos_core/mlos_core/optimizers/flaml_optimizer.py +++ b/mlos_core/mlos_core/optimizers/flaml_optimizer.py @@ -88,7 +88,7 @@ def _register(self, configurations: pd.DataFrame, scores: pd.Series, Not Yet Implemented. """ if context is not None: - raise NotImplementedError() + warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning) for (_, config), score in zip(configurations.astype('O').iterrows(), scores): cs_config: ConfigSpace.Configuration = ConfigSpace.Configuration( self.optimizer_parameter_space, values=config.to_dict()) @@ -113,7 +113,7 @@ def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame: Pandas dataframe with a single row. Column names are the parameter names. """ if context is not None: - raise NotImplementedError() + warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning) config: dict = self._get_next_config() return pd.DataFrame(config, index=[0]) diff --git a/mlos_core/mlos_core/optimizers/random_optimizer.py b/mlos_core/mlos_core/optimizers/random_optimizer.py index 27421f687d..4b023ee8df 100644 --- a/mlos_core/mlos_core/optimizers/random_optimizer.py +++ b/mlos_core/mlos_core/optimizers/random_optimizer.py @@ -7,6 +7,7 @@ """ from typing import Optional +from warnings import warn import pandas as pd @@ -41,7 +42,7 @@ def _register(self, configurations: pd.DataFrame, scores: pd.Series, Not Yet Implemented. """ if context is not None: - raise NotImplementedError() + warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning) # should we pop them from self.pending_observations? def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame: @@ -61,7 +62,7 @@ def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame: """ if context is not None: # not sure how that works here? - raise NotImplementedError() + warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning) return pd.DataFrame(dict(self.optimizer_parameter_space.sample_configuration()), index=[0]) def register_pending(self, configurations: pd.DataFrame, diff --git a/mlos_core/mlos_core/tests/optimizers/bayesian_optimizers_test.py b/mlos_core/mlos_core/tests/optimizers/bayesian_optimizers_test.py index ef7aaf2fa1..70265be54e 100644 --- a/mlos_core/mlos_core/tests/optimizers/bayesian_optimizers_test.py +++ b/mlos_core/mlos_core/tests/optimizers/bayesian_optimizers_test.py @@ -17,26 +17,29 @@ from mlos_core.optimizers.bayesian_optimizers import BaseBayesianOptimizer +@pytest.mark.filterwarnings("error:Not Implemented") @pytest.mark.parametrize(('optimizer_class', 'kwargs'), [ *[(member.value, {}) for member in OptimizerType], ]) -def test_context_not_implemented_error(configuration_space: CS.ConfigurationSpace, - optimizer_class: Type[BaseOptimizer], kwargs: Optional[dict]) -> None: +def test_context_not_implemented_warning(configuration_space: CS.ConfigurationSpace, + optimizer_class: Type[BaseOptimizer], + kwargs: Optional[dict]) -> None: """ - Make sure we raise exceptions for the functionality that has not been implemented yet. + Make sure we raise warnings for the functionality that has not been implemented yet. """ if kwargs is None: kwargs = {} optimizer = optimizer_class(parameter_space=configuration_space, **kwargs) suggestion = optimizer.suggest() scores = pd.DataFrame({'score': [1]}) + context = pd.DataFrame([["something"]]) # test context not implemented errors - with pytest.raises(NotImplementedError): - optimizer.register(suggestion, scores['score'], context=pd.DataFrame([["something"]])) + with pytest.raises(UserWarning): + optimizer.register(suggestion, scores['score'], context=context) - with pytest.raises(NotImplementedError): - optimizer.suggest(context=pd.DataFrame([["something"]])) + with pytest.raises(UserWarning): + optimizer.suggest(context=context) if isinstance(optimizer, BaseBayesianOptimizer): - with pytest.raises(NotImplementedError): - optimizer.surrogate_predict(suggestion, context=pd.DataFrame([["something"]])) + with pytest.raises(UserWarning): + optimizer.surrogate_predict(suggestion, context=context)