Enable Optimizer state and Multi-Fidelity passthrough in SMAC

.cspell.json

@@ -72,6 +72,7 @@
         "sklearn",
         "skopt",
         "smac",
+        "SOBOL",
         "sqlalchemy",
         "srcpaths",
         "subcmd",

mlos_bench/mlos_bench/optimizers/mlos_core_optimizer.py

@@ -180,7 +180,7 @@ def suggest(self) -> TunableGroups:
         tunables = super().suggest()
         if self._start_with_defaults:
             _LOG.info("Use default values for the first trial")
-        df_config = self._opt.suggest(defaults=self._start_with_defaults)
+        df_config, _ = self._opt.suggest(defaults=self._start_with_defaults)
         self._start_with_defaults = False
         _LOG.info("Iteration %d :: Suggest:\n%s", self._iter, df_config)
         return tunables.assign(

mlos_core/mlos_core/optimizers/README.md

@@ -0,0 +1,18 @@
+This is a directory that contains wrappers for different optimizers to integrate into MLOS.
+This is implemented though child classes for the `BaseOptimizer` class defined in `optimizer.py`.
+
+The main goal of these optimizers is to take a suggest configurations based on prior samples to find an optimum based on some objective. This process is interacted with through and ask and tell interface.
+
+The following defintions are useful for understanding the implementation
+- `configuration`: a vector representation of a configuration of a system to be evaluated.
+- `score`: the objective(s) associated with a configuration
+- `metadata`: additional information about the evaluation, such as the runtime budget used during evaluation.
+- `context`: additional information about the evaluation used to extend the internal model used for suggesting samples. This is not yet implemented.
+
+The interface for these classes can be described as follows:
+
+- `register`: this is a function that takes a configuration, a score, and, optionally, metadata about the evaluation to update the model for future evaluations.
+- `suggest`: this function returns a new confiugration for evaluation. Some optimizers will return additional metadata for evaluation, that should be used durin the register phase. This function can also optionally take context (not yet implemented), and an argument to force the function to return the default configuration.
+- `register_pending`: registers a configuration and metadata pair as pending to the optimizer.
+- `get_observations`: returns all observations reproted to the optimizer as a triplet of DataFrames (config, score, metadata).
+- `get_best_observations`: returns the best observation as A triplet of best (config, score, metadata) DataFrames.

mlos_core/mlos_core/optimizers/flaml_optimizer.py

@@ -6,7 +6,7 @@
 Contains the FlamlOptimizer class.
 """
 
-from typing import Dict, List, NamedTuple, Optional, Union
+from typing import Dict, List, NamedTuple, Optional, Tuple, Union
 from warnings import warn
 
 import ConfigSpace
@@ -86,7 +86,7 @@ def __init__(self, *,   # pylint: disable=too-many-arguments
         self._suggested_config: Optional[dict]
 
     def _register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
-                  context: Optional[pd.DataFrame] = None) -> None:
+                  metadata: Optional[pd.DataFrame] = None) -> None:
         """Registers the given configurations and scores.
 
         Parameters
@@ -97,11 +97,11 @@ def _register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
         scores : pd.DataFrame
             Scores from running the configurations. The index is the same as the index of the configurations.
 
-        context : None
+        metadata : None
             Not Yet Implemented.
         """
-        if context is not None:
-            warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
+        if metadata is not None:
+            warn(f"Not Implemented: Ignoring context {list(metadata.columns)}", UserWarning)
         for (_, config), (_, score) in zip(configurations.astype('O').iterrows(), scores.iterrows()):
             cs_config: ConfigSpace.Configuration = ConfigSpace.Configuration(
                 self.optimizer_parameter_space, values=config.to_dict())
@@ -112,7 +112,9 @@ def _register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
                 score=float(np.average(score.astype(float), weights=self._objective_weights)),
             )
 
-    def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
+    def _suggest(
+        self, context: Optional[pd.DataFrame] = None
+    ) -> Tuple[pd.DataFrame, Optional[pd.DataFrame]]:
         """Suggests a new configuration.
 
         Sampled at random using ConfigSpace.
@@ -130,10 +132,10 @@ def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
         if context is not None:
             warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
         config: dict = self._get_next_config()
-        return pd.DataFrame(config, index=[0])
+        return pd.DataFrame(config, index=[0]), None
 
     def register_pending(self, configurations: pd.DataFrame,
-                         context: Optional[pd.DataFrame] = None) -> None:
+                         metadata: Optional[pd.DataFrame] = None) -> None:
         raise NotImplementedError()
 
     def _target_function(self, config: dict) -> Union[dict, None]:

mlos_core/mlos_core/optimizers/optimizer.py

@@ -26,7 +26,7 @@ class BaseOptimizer(metaclass=ABCMeta):
 
     def __init__(self, *,
                  parameter_space: ConfigSpace.ConfigurationSpace,
-                 optimization_targets: List[str],
+                 optimization_targets: Optional[Union[str, List[str]]] = None,
                  objective_weights: Optional[List[float]] = None,
                  space_adapter: Optional[BaseSpaceAdapter] = None):
         """
@@ -57,8 +57,10 @@ def __init__(self, *,
 
         self._space_adapter: Optional[BaseSpaceAdapter] = space_adapter
         self._observations: List[Tuple[pd.DataFrame, pd.DataFrame, Optional[pd.DataFrame]]] = []
-        self._has_context: Optional[bool] = None
+        self._has_metadata: Optional[bool] = None
         self._pending_observations: List[Tuple[pd.DataFrame, Optional[pd.DataFrame]]] = []
+        self.delayed_config: Optional[pd.DataFrame] = None
+        self.delayed_metadata: Optional[pd.DataFrame] = None
 
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}(space_adapter={self.space_adapter})"
@@ -69,7 +71,7 @@ def space_adapter(self) -> Optional[BaseSpaceAdapter]:
         return self._space_adapter
 
     def register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
-                 context: Optional[pd.DataFrame] = None) -> None:
+                 metadata: Optional[pd.DataFrame] = None) -> None:
         """Wrapper method, which employs the space adapter (if any), before registering the configurations and scores.
 
         Parameters
@@ -78,34 +80,35 @@ def register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
             Dataframe of configurations / parameters. The columns are parameter names and the rows are the configurations.
         scores : pd.DataFrame
             Scores from running the configurations. The index is the same as the index of the configurations.
-
-        context : pd.DataFrame
-            Not Yet Implemented.
+        metadata : pd.DataFrame
+            Implementaton depends on instance.
         """
         # Do some input validation.
-        assert set(scores.columns) == set(self._optimization_targets), \
-            "Mismatched optimization targets."
-        assert self._has_context is None or self._has_context ^ (context is None), \
-            "Context must always be added or never be added."
+        if type(self._optimization_targets) is str:
+            assert self._optimization_targets in scores.columns, "Mismatched optimization targets."
+        if type(self._optimization_targets) is list:
+            assert set(scores.columns) >= set(self._optimization_targets), "Mismatched optimization targets."
+        assert self._has_metadata is None or self._has_metadata ^ (metadata is None), \
+            "Metadata must always be added or never be added."
         assert len(configurations) == len(scores), \
             "Mismatched number of configurations and scores."
-        if context is not None:
-            assert len(configurations) == len(context), \
-                "Mismatched number of configurations and context."
+        if metadata is not None:
+            assert len(configurations) == len(metadata), \
+                "Mismatched number of configurations and metadata."
         assert configurations.shape[1] == len(self.parameter_space.values()), \
             "Mismatched configuration shape."
-        self._observations.append((configurations, scores, context))
-        self._has_context = context is not None
+        self._observations.append((configurations, scores, metadata))
+        self._has_metadata = metadata is not None
 
         if self._space_adapter:
             configurations = self._space_adapter.inverse_transform(configurations)
             assert configurations.shape[1] == len(self.optimizer_parameter_space.values()), \
                 "Mismatched configuration shape after inverse transform."
-        return self._register(configurations, scores, context)
+        return self._register(configurations, scores, metadata)
 
     @abstractmethod
     def _register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
-                  context: Optional[pd.DataFrame] = None) -> None:
+                  metadata: Optional[pd.DataFrame] = None) -> None:
         """Registers the given configurations and scores.
 
         Parameters
@@ -115,12 +118,14 @@ def _register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
         scores : pd.DataFrame
             Scores from running the configurations. The index is the same as the index of the configurations.
 
-        context : pd.DataFrame
-            Not Yet Implemented.
+        metadata : pd.DataFrame
+            Implementaton depends on instance.
         """
         pass    # pylint: disable=unnecessary-pass # pragma: no cover
 
-    def suggest(self, context: Optional[pd.DataFrame] = None, defaults: bool = False) -> pd.DataFrame:
+    def suggest(
+        self, context: Optional[pd.DataFrame] = None, defaults: bool = False
+    ) -> Tuple[pd.DataFrame, Optional[pd.DataFrame]]:
         """
         Wrapper method, which employs the space adapter (if any), after suggesting a new configuration.
 
@@ -136,13 +141,25 @@ def suggest(self, context: Optional[pd.DataFrame] = None, defaults: bool = False
         -------
         configuration : pd.DataFrame
             Pandas dataframe with a single row. Column names are the parameter names.
+        metadata : pd.DataFrame
+            Pandas dataframe with a single row containing the metadata.
+            Column names are the budget, seed, and instance of the evaluation, if valid.
         """
         if defaults:
-            configuration = config_to_dataframe(self.parameter_space.get_default_configuration())
+            self.delayed_config, self.delayed_metadata = self._suggest(context)
+
+            configuration: pd.DataFrame = config_to_dataframe(
+                self.parameter_space.get_default_configuration()
+            )
+            metadata = self.delayed_metadata
             if self.space_adapter is not None:
                 configuration = self.space_adapter.inverse_transform(configuration)
         else:
-            configuration = self._suggest(context)
+            if self.delayed_config is None:
+                configuration, metadata = self._suggest(metadata)
+            else:
+                configuration, metadata = self.delayed_config, self.delayed_metadata
+                self.delayed_config, self.delayed_metadata = None, None
             assert len(configuration) == 1, \
                 "Suggest must return a single configuration."
             assert set(configuration.columns).issubset(set(self.optimizer_parameter_space)), \
@@ -151,10 +168,12 @@ def suggest(self, context: Optional[pd.DataFrame] = None, defaults: bool = False
             configuration = self._space_adapter.transform(configuration)
             assert set(configuration.columns).issubset(set(self.parameter_space)), \
                 "Space adapter produced a configuration that does not match the expected parameter space."
-        return configuration
+        return configuration, metadata
 
     @abstractmethod
-    def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
+    def _suggest(
+        self, context: Optional[pd.DataFrame] = None
+    ) -> Tuple[pd.DataFrame, Optional[pd.DataFrame]]:
         """Suggests a new configuration.
 
         Parameters
@@ -166,12 +185,16 @@ def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
         -------
         configuration : pd.DataFrame
             Pandas dataframe with a single row. Column names are the parameter names.
+
+        metadata : pd.DataFrame
+            Pandas dataframe with a single row containing the metadata.
+            Column names are the budget, seed, and instance of the evaluation, if valid.
         """
         pass    # pylint: disable=unnecessary-pass # pragma: no cover
 
     @abstractmethod
     def register_pending(self, configurations: pd.DataFrame,
-                         context: Optional[pd.DataFrame] = None) -> None:
+                         metadata: Optional[pd.DataFrame] = None) -> None:
         """Registers the given configurations as "pending".
         That is it say, it has been suggested by the optimizer, and an experiment trial has been started.
         This can be useful for executing multiple trials in parallel, retry logic, etc.
@@ -180,31 +203,29 @@ def register_pending(self, configurations: pd.DataFrame,
         ----------
         configurations : pd.DataFrame
             Dataframe of configurations / parameters. The columns are parameter names and the rows are the configurations.
-        context : pd.DataFrame
-            Not Yet Implemented.
         """
         pass    # pylint: disable=unnecessary-pass # pragma: no cover
 
     def get_observations(self) -> Tuple[pd.DataFrame, pd.DataFrame, Optional[pd.DataFrame]]:
         """
-        Returns the observations as a triplet of DataFrames (config, score, context).
+        Returns the observations as a triplet of DataFrames (config, score, metadata).
 
         Returns
         -------
         observations : Tuple[pd.DataFrame, pd.DataFrame, Optional[pd.DataFrame]]
-            A triplet of (config, score, context) DataFrames of observations.
+            A triplet of (config, score, metadata) DataFrames of observations.
         """
         if len(self._observations) == 0:
             raise ValueError("No observations registered yet.")
         configs = pd.concat([config for config, _, _ in self._observations]).reset_index(drop=True)
         scores = pd.concat([score for _, score, _ in self._observations]).reset_index(drop=True)
-        contexts = pd.concat([pd.DataFrame() if context is None else context
-                              for _, _, context in self._observations]).reset_index(drop=True)
-        return (configs, scores, contexts if len(contexts.columns) > 0 else None)
+        metadatas = pd.concat([pd.DataFrame() if metadata is None else metadata
+                              for _, _, metadata in self._observations]).reset_index(drop=True)
+        return (configs, scores, metadatas if len(metadatas.columns) > 0 else None)
 
     def get_best_observations(self, n_max: int = 1) -> Tuple[pd.DataFrame, pd.DataFrame, Optional[pd.DataFrame]]:
         """
-        Get the N best observations so far as a triplet of DataFrames (config, score, context).
+        Get the N best observations so far as a triplet of DataFrames (config, score, metadata).
         Default is N=1. The columns are ordered in ASCENDING order of the optimization targets.
         The function uses `pandas.DataFrame.nsmallest(..., keep="first")` method under the hood.
 
@@ -216,14 +237,14 @@ def get_best_observations(self, n_max: int = 1) -> Tuple[pd.DataFrame, pd.DataFr
         Returns
         -------
         observations : Tuple[pd.DataFrame, pd.DataFrame, Optional[pd.DataFrame]]
-            A triplet of best (config, score, context) DataFrames of best observations.
+            A triplet of best (config, score, metadata) DataFrames of best observations.
         """
         if len(self._observations) == 0:
             raise ValueError("No observations registered yet.")
-        (configs, scores, contexts) = self.get_observations()
+        (configs, scores, metadatas) = self.get_observations()
         idx = scores.nsmallest(n_max, columns=self._optimization_targets, keep="first").index
         return (configs.loc[idx], scores.loc[idx],
-                None if contexts is None else contexts.loc[idx])
+                None if metadatas is None else metadatas.loc[idx])
 
     def cleanup(self) -> None:
         """

mlos_core/mlos_core/optimizers/random_optimizer.py

@@ -6,7 +6,7 @@
 Contains the RandomOptimizer class.
 """
 
-from typing import Optional
+from typing import Optional, Tuple
 from warnings import warn
 
 import pandas as pd
@@ -25,7 +25,7 @@ class RandomOptimizer(BaseOptimizer):
     """
 
     def _register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
-                  context: Optional[pd.DataFrame] = None) -> None:
+                  metadata: Optional[pd.DataFrame] = None) -> None:
         """Registers the given configurations and scores.
 
         Doesn't do anything on the RandomOptimizer except storing configurations for logging.
@@ -38,14 +38,15 @@ def _register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
         scores : pd.DataFrame
             Scores from running the configurations. The index is the same as the index of the configurations.
 
-        context : None
-            Not Yet Implemented.
+        metadata : None
+            Metadata is ignored for random_optimizer.
         """
-        if context is not None:
-            warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
+        pass
         # should we pop them from self.pending_observations?
 
-    def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
+    def _suggest(
+        self, context: Optional[pd.DataFrame] = None
+    ) -> Tuple[pd.DataFrame, Optional[pd.DataFrame]]:
         """Suggests a new configuration.
 
         Sampled at random using ConfigSpace.
@@ -59,13 +60,17 @@ def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
         -------
         configuration : pd.DataFrame
             Pandas dataframe with a single row. Column names are the parameter names.
+
+        metadata : pd.DataFrame
+            Pandas dataframe with a single row containing the metadata.
+            Column names are the budget, seed, and instance of the evaluation, if valid.
         """
         if context is not None:
             # not sure how that works here?
             warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
-        return pd.DataFrame(dict(self.optimizer_parameter_space.sample_configuration()), index=[0])
+        return pd.DataFrame(dict(self.optimizer_parameter_space.sample_configuration()), index=[0]), None
 
     def register_pending(self, configurations: pd.DataFrame,
-                         context: Optional[pd.DataFrame] = None) -> None:
+                         metadata: Optional[pd.DataFrame] = None) -> None:
         raise NotImplementedError()
-        # self._pending_observations.append((configurations, context))
+        # self._pending_observations.append((configurations, metadata))

mlos_core/mlos_core/tests/optimizers/bayesian_optimizers_test.py

@@ -34,13 +34,10 @@ def test_context_not_implemented_warning(configuration_space: CS.ConfigurationSp
         optimization_targets=['score'],
         **kwargs
     )
-    suggestion = optimizer.suggest()
+    suggestion, _ = optimizer.suggest()
     scores = pd.DataFrame({'score': [1]})
     context = pd.DataFrame([["something"]])
 
-    with pytest.raises(UserWarning):
-        optimizer.register(suggestion, scores, context=context)
-
     with pytest.raises(UserWarning):
         optimizer.suggest(context=context)