Pass optional weights for optimization targets in mlos_core; implemen…

…t multi-target optimization for FLAML. (#738)

Summary of changes:
* Pass optional weights for optimization targets in mlos_core
* Implement weighted average for multi-objective optimization in FLAML
* Add more unit tests for multi-objective optimization on mlos_core side

Merge after ~#730~

---------

Co-authored-by: Brian Kroth <[email protected]>

mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py

@@ -30,6 +30,7 @@ class SmacOptimizer(BaseBayesianOptimizer):
     def __init__(self, *,  # pylint: disable=too-many-locals,too-many-arguments
                  parameter_space: ConfigSpace.ConfigurationSpace,
                  optimization_targets: List[str],
+                 objective_weights: Optional[List[float]] = None,
                  space_adapter: Optional[BaseSpaceAdapter] = None,
                  seed: Optional[int] = 0,
                  run_name: Optional[str] = None,
@@ -50,6 +51,9 @@ def __init__(self, *,  # pylint: disable=too-many-locals,too-many-arguments
         optimization_targets : List[str]
             The names of the optimization targets to minimize.
 
+        objective_weights : Optional[List[float]]
+            Optional list of weights of optimization targets.
+
         space_adapter : BaseSpaceAdapter
             The space adapter class to employ for parameter space transformations.
 
@@ -91,6 +95,7 @@ def __init__(self, *,  # pylint: disable=too-many-locals,too-many-arguments
         super().__init__(
             parameter_space=parameter_space,
             optimization_targets=optimization_targets,
+            objective_weights=objective_weights,
             space_adapter=space_adapter,
         )
 
@@ -193,9 +198,7 @@ def __init__(self, *,  # pylint: disable=too-many-locals,too-many-arguments
             random_design=random_design,
             config_selector=config_selector,
             multi_objective_algorithm=Optimizer_Smac.get_multi_objective_algorithm(
-                scenario,
-                # objective_weights=[1, 2],  # TODO: pass weights as constructor args
-            ),
+                scenario, objective_weights=self._objective_weights),
             overwrite=True,
             logging_level=False,  # Use the existing logger
         )

mlos_core/mlos_core/optimizers/flaml_optimizer.py

@@ -30,9 +30,13 @@ class FlamlOptimizer(BaseOptimizer):
     Wrapper class for FLAML Optimizer: A fast library for AutoML and tuning.
     """
 
+    # The name of an internal objective attribute that is calculated as a weighted average of the user provided objective metrics.
+    _METRIC_NAME = "FLAML_score"
+
     def __init__(self, *,   # pylint: disable=too-many-arguments
                  parameter_space: ConfigSpace.ConfigurationSpace,
                  optimization_targets: List[str],
+                 objective_weights: Optional[List[float]] = None,
                  space_adapter: Optional[BaseSpaceAdapter] = None,
                  low_cost_partial_config: Optional[dict] = None,
                  seed: Optional[int] = None):
@@ -46,7 +50,9 @@ def __init__(self, *,   # pylint: disable=too-many-arguments
 
         optimization_targets : List[str]
             The names of the optimization targets to minimize.
-            For FLAML it must be a list with a single element, e.g., `["score"]`.
+
+        objective_weights : Optional[List[float]]
+            Optional list of weights of optimization targets.
 
         space_adapter : BaseSpaceAdapter
             The space adapter class to employ for parameter space transformations.
@@ -61,13 +67,10 @@ def __init__(self, *,   # pylint: disable=too-many-arguments
         super().__init__(
             parameter_space=parameter_space,
             optimization_targets=optimization_targets,
+            objective_weights=objective_weights,
             space_adapter=space_adapter,
         )
 
-        if len(self._optimization_targets) != 1:
-            raise ValueError("FLAML does not support multi-target optimization")
-        self._flaml_optimization_target = self._optimization_targets[0]
-
         # Per upstream documentation, it is recommended to set the seed for
         # flaml at the start of its operation globally.
         if seed is not None:
@@ -99,14 +102,15 @@ def _register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
         """
         if context is not None:
             warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
-        for (_, config), score in zip(configurations.astype('O').iterrows(),
-                                      scores[self._flaml_optimization_target]):
+        for (_, config), (_, score) in zip(configurations.astype('O').iterrows(), scores.iterrows()):
             cs_config: ConfigSpace.Configuration = ConfigSpace.Configuration(
                 self.optimizer_parameter_space, values=config.to_dict())
             if cs_config in self.evaluated_samples:
                 warn(f"Configuration {config} was already registered", UserWarning)
-
-            self.evaluated_samples[cs_config] = EvaluatedSample(config=config.to_dict(), score=score)
+            self.evaluated_samples[cs_config] = EvaluatedSample(
+                config=config.to_dict(),
+                score=float(np.average(score.astype(float), weights=self._objective_weights)),
+            )
 
     def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
         """Suggests a new configuration.
@@ -147,11 +151,11 @@ def _target_function(self, config: dict) -> Union[dict, None]:
         Returns
         -------
         result: Union[dict, None]
-            Dictionary with a single key, `score`, if config already evaluated; `None` otherwise.
+            Dictionary with a single key, `FLAML_score`, if config already evaluated; `None` otherwise.
         """
         cs_config = normalize_config(self.optimizer_parameter_space, config)
         if cs_config in self.evaluated_samples:
-            return {self._flaml_optimization_target: self.evaluated_samples[cs_config].score}
+            return {self._METRIC_NAME: self.evaluated_samples[cs_config].score}
 
         self._suggested_config = dict(cs_config)  # Cleaned-up version of the config
         return None  # Returning None stops the process
@@ -194,7 +198,7 @@ def _get_next_config(self) -> dict:
             self._target_function,
             config=self.flaml_parameter_space,
             mode='min',
-            metric=self._flaml_optimization_target,
+            metric=self._METRIC_NAME,
             points_to_evaluate=points_to_evaluate,
             evaluated_rewards=evaluated_rewards,
             num_samples=len(points_to_evaluate) + 1,

mlos_core/mlos_core/optimizers/optimizer.py

@@ -27,6 +27,7 @@ class BaseOptimizer(metaclass=ABCMeta):
     def __init__(self, *,
                  parameter_space: ConfigSpace.ConfigurationSpace,
                  optimization_targets: List[str],
+                 objective_weights: Optional[List[float]] = None,
                  space_adapter: Optional[BaseSpaceAdapter] = None):
         """
         Create a new instance of the base optimizer.
@@ -37,6 +38,8 @@ def __init__(self, *,
             The parameter space to optimize.
         optimization_targets : List[str]
             The names of the optimization targets to minimize.
+        objective_weights : Optional[List[float]]
+            Optional list of weights of optimization targets.
         space_adapter : BaseSpaceAdapter
             The space adapter class to employ for parameter space transformations.
         """
@@ -48,6 +51,10 @@ def __init__(self, *,
             raise ValueError("Given parameter space differs from the one given to space adapter")
 
         self._optimization_targets = optimization_targets
+        self._objective_weights = objective_weights
+        if objective_weights is not None and len(objective_weights) != len(optimization_targets):
+            raise ValueError("Number of weights must match the number of optimization targets")
+
         self._space_adapter: Optional[BaseSpaceAdapter] = space_adapter
         self._observations: List[Tuple[pd.DataFrame, pd.DataFrame, Optional[pd.DataFrame]]] = []
         self._has_context: Optional[bool] = None

mlos_core/mlos_core/tests/optimizers/optimizer_multiobj_test.py

@@ -7,22 +7,49 @@
 """
 
 import logging
+from typing import List, Optional, Type
+
 import pytest
 
 import pandas as pd
 import numpy as np
 import ConfigSpace as CS
 
-from mlos_core.optimizers import OptimizerType, OptimizerFactory
+from mlos_core.optimizers import OptimizerType, BaseOptimizer
 
 from mlos_core.tests import SEED
 
-
 _LOG = logging.getLogger(__name__)
-_LOG.setLevel(logging.DEBUG)
 
 
-def test_multi_target_opt() -> None:
+@pytest.mark.parametrize(('optimizer_class', 'kwargs'), [
+    *[(member.value, {}) for member in OptimizerType],
+])
+def test_multi_target_opt_wrong_weights(optimizer_class: Type[BaseOptimizer], kwargs: dict) -> None:
+    """
+    Make sure that the optimizer raises an error if the number of objective weights
+    does not match the number of optimization targets.
+    """
+    with pytest.raises(ValueError):
+        optimizer_class(
+            parameter_space=CS.ConfigurationSpace(seed=SEED),
+            optimization_targets=['main_score', 'other_score'],
+            objective_weights=[1],
+            **kwargs
+        )
+
+
+@pytest.mark.parametrize(('objective_weights'), [
+    [2, 1],
+    [0.5, 0.5],
+    None,
+])
+@pytest.mark.parametrize(('optimizer_class', 'kwargs'), [
+    *[(member.value, {}) for member in OptimizerType],
+])
+def test_multi_target_opt(objective_weights: Optional[List[float]],
+                          optimizer_class: Type[BaseOptimizer],
+                          kwargs: dict) -> None:
     """
     Toy multi-target optimization problem to test the optimizers with
     mixed numeric types to ensure that original dtypes are retained.
@@ -32,7 +59,7 @@ def test_multi_target_opt() -> None:
     def objective(point: pd.DataFrame) -> pd.DataFrame:
         # mix of hyperparameters, optimal is to select the highest possible
         return pd.DataFrame({
-            "score": point.x + point.y,
+            "main_score": point.x + point.y,
             "other_score": point.x ** 2 + point.y ** 2,
         })
 
@@ -43,15 +70,11 @@ def objective(point: pd.DataFrame) -> pd.DataFrame:
     input_space.add_hyperparameter(
         CS.UniformFloatHyperparameter(name='y', lower=0.0, upper=5.0))
 
-    optimizer = OptimizerFactory.create(
+    optimizer = optimizer_class(
         parameter_space=input_space,
-        optimization_targets=['score', 'other_score'],
-        optimizer_type=OptimizerType.SMAC,
-        optimizer_kwargs={
-            # Test with default config.
-            'use_default_config': True,
-            # 'n_random_init': 10,
-        },
+        optimization_targets=['main_score', 'other_score'],
+        objective_weights=objective_weights,
+        **kwargs,
     )
 
     with pytest.raises(ValueError, match="No observations"):
@@ -75,15 +98,15 @@ def objective(point: pd.DataFrame) -> pd.DataFrame:
         # Test registering the suggested configuration with a score.
         observation = objective(suggestion)
         assert isinstance(observation, pd.DataFrame)
-        assert set(observation.columns) == {'score', 'other_score'}
+        assert set(observation.columns) == {'main_score', 'other_score'}
         optimizer.register(suggestion, observation)
 
     (best_config, best_score, best_context) = optimizer.get_best_observations()
     assert isinstance(best_config, pd.DataFrame)
     assert isinstance(best_score, pd.DataFrame)
     assert best_context is None
     assert set(best_config.columns) == {'x', 'y'}
-    assert set(best_score.columns) == {'score', 'other_score'}
+    assert set(best_score.columns) == {'main_score', 'other_score'}
     assert best_config.shape == (1, 2)
     assert best_score.shape == (1, 2)
 
@@ -92,6 +115,6 @@ def objective(point: pd.DataFrame) -> pd.DataFrame:
     assert isinstance(all_scores, pd.DataFrame)
     assert all_contexts is None
     assert set(all_configs.columns) == {'x', 'y'}
-    assert set(all_scores.columns) == {'score', 'other_score'}
+    assert set(all_scores.columns) == {'main_score', 'other_score'}
     assert all_configs.shape == (max_iterations, 2)
     assert all_scores.shape == (max_iterations, 2)