From 0b010293eef639c3feec8ffd06884c729676749e Mon Sep 17 00:00:00 2001 From: Sergiy Matusevych Date: Mon, 3 Jun 2024 15:24:15 -0700 Subject: [PATCH] Pass optional weights for optimization targets in mlos_core; implement multi-target optimization for FLAML. (#738) Summary of changes: * Pass optional weights for optimization targets in mlos_core * Implement weighted average for multi-objective optimization in FLAML * Add more unit tests for multi-objective optimization on mlos_core side Merge after ~#730~ --------- Co-authored-by: Brian Kroth --- .../bayesian_optimizers/smac_optimizer.py | 9 ++- .../mlos_core/optimizers/flaml_optimizer.py | 28 ++++++---- mlos_core/mlos_core/optimizers/optimizer.py | 7 +++ .../optimizers/optimizer_multiobj_test.py | 55 +++++++++++++------ 4 files changed, 68 insertions(+), 31 deletions(-) diff --git a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py index 2e58a8e057..8a433218fa 100644 --- a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py +++ b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py @@ -30,6 +30,7 @@ class SmacOptimizer(BaseBayesianOptimizer): def __init__(self, *, # pylint: disable=too-many-locals,too-many-arguments parameter_space: ConfigSpace.ConfigurationSpace, optimization_targets: List[str], + objective_weights: Optional[List[float]] = None, space_adapter: Optional[BaseSpaceAdapter] = None, seed: Optional[int] = 0, run_name: Optional[str] = None, @@ -50,6 +51,9 @@ def __init__(self, *, # pylint: disable=too-many-locals,too-many-arguments optimization_targets : List[str] The names of the optimization targets to minimize. + objective_weights : Optional[List[float]] + Optional list of weights of optimization targets. + space_adapter : BaseSpaceAdapter The space adapter class to employ for parameter space transformations. @@ -91,6 +95,7 @@ def __init__(self, *, # pylint: disable=too-many-locals,too-many-arguments super().__init__( parameter_space=parameter_space, optimization_targets=optimization_targets, + objective_weights=objective_weights, space_adapter=space_adapter, ) @@ -193,9 +198,7 @@ def __init__(self, *, # pylint: disable=too-many-locals,too-many-arguments random_design=random_design, config_selector=config_selector, multi_objective_algorithm=Optimizer_Smac.get_multi_objective_algorithm( - scenario, - # objective_weights=[1, 2], # TODO: pass weights as constructor args - ), + scenario, objective_weights=self._objective_weights), overwrite=True, logging_level=False, # Use the existing logger ) diff --git a/mlos_core/mlos_core/optimizers/flaml_optimizer.py b/mlos_core/mlos_core/optimizers/flaml_optimizer.py index 14b4433f6c..0ad3c2da29 100644 --- a/mlos_core/mlos_core/optimizers/flaml_optimizer.py +++ b/mlos_core/mlos_core/optimizers/flaml_optimizer.py @@ -30,9 +30,13 @@ class FlamlOptimizer(BaseOptimizer): Wrapper class for FLAML Optimizer: A fast library for AutoML and tuning. """ + # The name of an internal objective attribute that is calculated as a weighted average of the user provided objective metrics. + _METRIC_NAME = "FLAML_score" + def __init__(self, *, # pylint: disable=too-many-arguments parameter_space: ConfigSpace.ConfigurationSpace, optimization_targets: List[str], + objective_weights: Optional[List[float]] = None, space_adapter: Optional[BaseSpaceAdapter] = None, low_cost_partial_config: Optional[dict] = None, seed: Optional[int] = None): @@ -46,7 +50,9 @@ def __init__(self, *, # pylint: disable=too-many-arguments optimization_targets : List[str] The names of the optimization targets to minimize. - For FLAML it must be a list with a single element, e.g., `["score"]`. + + objective_weights : Optional[List[float]] + Optional list of weights of optimization targets. space_adapter : BaseSpaceAdapter The space adapter class to employ for parameter space transformations. @@ -61,13 +67,10 @@ def __init__(self, *, # pylint: disable=too-many-arguments super().__init__( parameter_space=parameter_space, optimization_targets=optimization_targets, + objective_weights=objective_weights, space_adapter=space_adapter, ) - if len(self._optimization_targets) != 1: - raise ValueError("FLAML does not support multi-target optimization") - self._flaml_optimization_target = self._optimization_targets[0] - # Per upstream documentation, it is recommended to set the seed for # flaml at the start of its operation globally. if seed is not None: @@ -99,14 +102,15 @@ def _register(self, configurations: pd.DataFrame, scores: pd.DataFrame, """ if context is not None: warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning) - for (_, config), score in zip(configurations.astype('O').iterrows(), - scores[self._flaml_optimization_target]): + for (_, config), (_, score) in zip(configurations.astype('O').iterrows(), scores.iterrows()): cs_config: ConfigSpace.Configuration = ConfigSpace.Configuration( self.optimizer_parameter_space, values=config.to_dict()) if cs_config in self.evaluated_samples: warn(f"Configuration {config} was already registered", UserWarning) - - self.evaluated_samples[cs_config] = EvaluatedSample(config=config.to_dict(), score=score) + self.evaluated_samples[cs_config] = EvaluatedSample( + config=config.to_dict(), + score=float(np.average(score.astype(float), weights=self._objective_weights)), + ) def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame: """Suggests a new configuration. @@ -147,11 +151,11 @@ def _target_function(self, config: dict) -> Union[dict, None]: Returns ------- result: Union[dict, None] - Dictionary with a single key, `score`, if config already evaluated; `None` otherwise. + Dictionary with a single key, `FLAML_score`, if config already evaluated; `None` otherwise. """ cs_config = normalize_config(self.optimizer_parameter_space, config) if cs_config in self.evaluated_samples: - return {self._flaml_optimization_target: self.evaluated_samples[cs_config].score} + return {self._METRIC_NAME: self.evaluated_samples[cs_config].score} self._suggested_config = dict(cs_config) # Cleaned-up version of the config return None # Returning None stops the process @@ -194,7 +198,7 @@ def _get_next_config(self) -> dict: self._target_function, config=self.flaml_parameter_space, mode='min', - metric=self._flaml_optimization_target, + metric=self._METRIC_NAME, points_to_evaluate=points_to_evaluate, evaluated_rewards=evaluated_rewards, num_samples=len(points_to_evaluate) + 1, diff --git a/mlos_core/mlos_core/optimizers/optimizer.py b/mlos_core/mlos_core/optimizers/optimizer.py index bd26619754..f1cedb85dc 100644 --- a/mlos_core/mlos_core/optimizers/optimizer.py +++ b/mlos_core/mlos_core/optimizers/optimizer.py @@ -27,6 +27,7 @@ class BaseOptimizer(metaclass=ABCMeta): def __init__(self, *, parameter_space: ConfigSpace.ConfigurationSpace, optimization_targets: List[str], + objective_weights: Optional[List[float]] = None, space_adapter: Optional[BaseSpaceAdapter] = None): """ Create a new instance of the base optimizer. @@ -37,6 +38,8 @@ def __init__(self, *, The parameter space to optimize. optimization_targets : List[str] The names of the optimization targets to minimize. + objective_weights : Optional[List[float]] + Optional list of weights of optimization targets. space_adapter : BaseSpaceAdapter The space adapter class to employ for parameter space transformations. """ @@ -48,6 +51,10 @@ def __init__(self, *, raise ValueError("Given parameter space differs from the one given to space adapter") self._optimization_targets = optimization_targets + self._objective_weights = objective_weights + if objective_weights is not None and len(objective_weights) != len(optimization_targets): + raise ValueError("Number of weights must match the number of optimization targets") + self._space_adapter: Optional[BaseSpaceAdapter] = space_adapter self._observations: List[Tuple[pd.DataFrame, pd.DataFrame, Optional[pd.DataFrame]]] = [] self._has_context: Optional[bool] = None diff --git a/mlos_core/mlos_core/tests/optimizers/optimizer_multiobj_test.py b/mlos_core/mlos_core/tests/optimizers/optimizer_multiobj_test.py index 888d07ff54..4ff5f157f7 100644 --- a/mlos_core/mlos_core/tests/optimizers/optimizer_multiobj_test.py +++ b/mlos_core/mlos_core/tests/optimizers/optimizer_multiobj_test.py @@ -7,22 +7,49 @@ """ import logging +from typing import List, Optional, Type + import pytest import pandas as pd import numpy as np import ConfigSpace as CS -from mlos_core.optimizers import OptimizerType, OptimizerFactory +from mlos_core.optimizers import OptimizerType, BaseOptimizer from mlos_core.tests import SEED - _LOG = logging.getLogger(__name__) -_LOG.setLevel(logging.DEBUG) -def test_multi_target_opt() -> None: +@pytest.mark.parametrize(('optimizer_class', 'kwargs'), [ + *[(member.value, {}) for member in OptimizerType], +]) +def test_multi_target_opt_wrong_weights(optimizer_class: Type[BaseOptimizer], kwargs: dict) -> None: + """ + Make sure that the optimizer raises an error if the number of objective weights + does not match the number of optimization targets. + """ + with pytest.raises(ValueError): + optimizer_class( + parameter_space=CS.ConfigurationSpace(seed=SEED), + optimization_targets=['main_score', 'other_score'], + objective_weights=[1], + **kwargs + ) + + +@pytest.mark.parametrize(('objective_weights'), [ + [2, 1], + [0.5, 0.5], + None, +]) +@pytest.mark.parametrize(('optimizer_class', 'kwargs'), [ + *[(member.value, {}) for member in OptimizerType], +]) +def test_multi_target_opt(objective_weights: Optional[List[float]], + optimizer_class: Type[BaseOptimizer], + kwargs: dict) -> None: """ Toy multi-target optimization problem to test the optimizers with mixed numeric types to ensure that original dtypes are retained. @@ -32,7 +59,7 @@ def test_multi_target_opt() -> None: def objective(point: pd.DataFrame) -> pd.DataFrame: # mix of hyperparameters, optimal is to select the highest possible return pd.DataFrame({ - "score": point.x + point.y, + "main_score": point.x + point.y, "other_score": point.x ** 2 + point.y ** 2, }) @@ -43,15 +70,11 @@ def objective(point: pd.DataFrame) -> pd.DataFrame: input_space.add_hyperparameter( CS.UniformFloatHyperparameter(name='y', lower=0.0, upper=5.0)) - optimizer = OptimizerFactory.create( + optimizer = optimizer_class( parameter_space=input_space, - optimization_targets=['score', 'other_score'], - optimizer_type=OptimizerType.SMAC, - optimizer_kwargs={ - # Test with default config. - 'use_default_config': True, - # 'n_random_init': 10, - }, + optimization_targets=['main_score', 'other_score'], + objective_weights=objective_weights, + **kwargs, ) with pytest.raises(ValueError, match="No observations"): @@ -75,7 +98,7 @@ def objective(point: pd.DataFrame) -> pd.DataFrame: # Test registering the suggested configuration with a score. observation = objective(suggestion) assert isinstance(observation, pd.DataFrame) - assert set(observation.columns) == {'score', 'other_score'} + assert set(observation.columns) == {'main_score', 'other_score'} optimizer.register(suggestion, observation) (best_config, best_score, best_context) = optimizer.get_best_observations() @@ -83,7 +106,7 @@ def objective(point: pd.DataFrame) -> pd.DataFrame: assert isinstance(best_score, pd.DataFrame) assert best_context is None assert set(best_config.columns) == {'x', 'y'} - assert set(best_score.columns) == {'score', 'other_score'} + assert set(best_score.columns) == {'main_score', 'other_score'} assert best_config.shape == (1, 2) assert best_score.shape == (1, 2) @@ -92,6 +115,6 @@ def objective(point: pd.DataFrame) -> pd.DataFrame: assert isinstance(all_scores, pd.DataFrame) assert all_contexts is None assert set(all_configs.columns) == {'x', 'y'} - assert set(all_scores.columns) == {'score', 'other_score'} + assert set(all_scores.columns) == {'main_score', 'other_score'} assert all_configs.shape == (max_iterations, 2) assert all_scores.shape == (max_iterations, 2)