Issue a warning on non-null optimization context (#735)

Co-authored-by: Brian Kroth <[email protected]>

mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py

@@ -11,6 +11,7 @@
 from pathlib import Path
 from typing import Dict, List, Optional, Union, TYPE_CHECKING
 from tempfile import TemporaryDirectory
+from warnings import warn
 
 import ConfigSpace
 import numpy.typing as npt
@@ -244,7 +245,7 @@ def _register(self, configurations: pd.DataFrame, scores: pd.Series, context: Op
         from smac.runhistory import StatusType, TrialInfo, TrialValue  # pylint: disable=import-outside-toplevel
 
         if context is not None:
-            raise NotImplementedError()
+            warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
 
         # Register each trial (one-by-one)
         for config, score in zip(self._to_configspace_configs(configurations), scores.tolist()):
@@ -273,7 +274,7 @@ def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
             from smac.runhistory import TrialInfo  # pylint: disable=import-outside-toplevel
 
         if context is not None:
-            raise NotImplementedError()
+            warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
 
         trial: TrialInfo = self.base_optimizer.ask()
         trial.config.is_valid_configuration()
@@ -290,7 +291,7 @@ def surrogate_predict(self, configurations: pd.DataFrame, context: Optional[pd.D
         from smac.utils.configspace import convert_configurations_to_array  # pylint: disable=import-outside-toplevel
 
         if context is not None:
-            raise NotImplementedError()
+            warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
         if self._space_adapter and not isinstance(self._space_adapter, IdentityAdapter):
             raise NotImplementedError()
 
@@ -308,7 +309,7 @@ def surrogate_predict(self, configurations: pd.DataFrame, context: Optional[pd.D
 
     def acquisition_function(self, configurations: pd.DataFrame, context: Optional[pd.DataFrame] = None) -> npt.NDArray:
         if context is not None:
-            raise NotImplementedError()
+            warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
         if self._space_adapter:
             raise NotImplementedError()
 

mlos_core/mlos_core/optimizers/flaml_optimizer.py

@@ -88,7 +88,7 @@ def _register(self, configurations: pd.DataFrame, scores: pd.Series,
             Not Yet Implemented.
         """
         if context is not None:
-            raise NotImplementedError()
+            warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
         for (_, config), score in zip(configurations.astype('O').iterrows(), scores):
             cs_config: ConfigSpace.Configuration = ConfigSpace.Configuration(
                 self.optimizer_parameter_space, values=config.to_dict())
@@ -113,7 +113,7 @@ def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
             Pandas dataframe with a single row. Column names are the parameter names.
         """
         if context is not None:
-            raise NotImplementedError()
+            warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
         config: dict = self._get_next_config()
         return pd.DataFrame(config, index=[0])
 

mlos_core/mlos_core/optimizers/random_optimizer.py

@@ -7,6 +7,7 @@
 """
 
 from typing import Optional
+from warnings import warn
 
 import pandas as pd
 
@@ -41,7 +42,7 @@ def _register(self, configurations: pd.DataFrame, scores: pd.Series,
             Not Yet Implemented.
         """
         if context is not None:
-            raise NotImplementedError()
+            warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
         # should we pop them from self.pending_observations?
 
     def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
@@ -61,7 +62,7 @@ def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
         """
         if context is not None:
             # not sure how that works here?
-            raise NotImplementedError()
+            warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
         return pd.DataFrame(dict(self.optimizer_parameter_space.sample_configuration()), index=[0])
 
     def register_pending(self, configurations: pd.DataFrame,

mlos_core/mlos_core/tests/optimizers/bayesian_optimizers_test.py

@@ -17,26 +17,29 @@
 from mlos_core.optimizers.bayesian_optimizers import BaseBayesianOptimizer
 
 
+@pytest.mark.filterwarnings("error:Not Implemented")
 @pytest.mark.parametrize(('optimizer_class', 'kwargs'), [
     *[(member.value, {}) for member in OptimizerType],
 ])
-def test_context_not_implemented_error(configuration_space: CS.ConfigurationSpace,
-                                       optimizer_class: Type[BaseOptimizer], kwargs: Optional[dict]) -> None:
+def test_context_not_implemented_warning(configuration_space: CS.ConfigurationSpace,
+                                         optimizer_class: Type[BaseOptimizer],
+                                         kwargs: Optional[dict]) -> None:
     """
-    Make sure we raise exceptions for the functionality that has not been implemented yet.
+    Make sure we raise warnings for the functionality that has not been implemented yet.
     """
     if kwargs is None:
         kwargs = {}
     optimizer = optimizer_class(parameter_space=configuration_space, **kwargs)
     suggestion = optimizer.suggest()
     scores = pd.DataFrame({'score': [1]})
+    context = pd.DataFrame([["something"]])
     # test context not implemented errors
-    with pytest.raises(NotImplementedError):
-        optimizer.register(suggestion, scores['score'], context=pd.DataFrame([["something"]]))
+    with pytest.raises(UserWarning):
+        optimizer.register(suggestion, scores['score'], context=context)
 
-    with pytest.raises(NotImplementedError):
-        optimizer.suggest(context=pd.DataFrame([["something"]]))
+    with pytest.raises(UserWarning):
+        optimizer.suggest(context=context)
 
     if isinstance(optimizer, BaseBayesianOptimizer):
-        with pytest.raises(NotImplementedError):
-            optimizer.surrogate_predict(suggestion, context=pd.DataFrame([["something"]]))
+        with pytest.raises(UserWarning):
+            optimizer.surrogate_predict(suggestion, context=context)