First draft of multi-objective optimization (#1455)

* First draft of multi-objective optimization

Co-authored-by: Katharina Eggensperger <[email protected]>

* Feedback from Eddie

* Make metric internally always a list

* Fix most examples

* Take further feedback into account

* Fix unit tests

* Fix one more example

* Add multi-objective example

* Simplify internal interface

* Act on further feedback

* Fix bug

* Update cv_results_ for multi-objective sklearn compliance

* Update leaderboard for multi-objective optimization

* Include Feedback from Katharina

* Take offline feedback into account

* Take offline feedback into account

* Eddie's feedback

* Fix metadata generation unit test

* Test for metrics with the same name

* Fix?

* Test CV results

* Test leaderboard for multi-objective optimization

* Last batch of unit tests added

* Include Eddie's feedback

Co-authored-by: Katharina Eggensperger <[email protected]>

Author: mfeurer

autosklearn/automl.py

@@ -30,7 +30,7 @@
 from autosklearn.automl_common.common.utils.backend import Backend
 from autosklearn.constants import BINARY_CLASSIFICATION
 from autosklearn.ensembles.ensemble_selection import EnsembleSelection
-from autosklearn.metrics import Scorer, calculate_loss, calculate_score
+from autosklearn.metrics import Scorer, calculate_losses, calculate_scores
 from autosklearn.util.logging_ import get_named_client_logger
 from autosklearn.util.parallel import preload_modules
 
@@ -999,13 +999,13 @@ def compute_loss_per_model(self):
             # actually read the predictions and compute their respective loss
             try:
                 y_ensemble = self._read_np_fn(y_ens_fn)
-                loss = calculate_loss(
+                loss = calculate_losses(
                     solution=self.y_true_ensemble,
                     prediction=y_ensemble,
                     task_type=self.task_type,
-                    metric=self.metric,
+                    metrics=[self.metric],
                     scoring_functions=None,
-                )
+                )[self.metric.name]
 
                 if np.isfinite(self.read_losses[y_ens_fn]["ens_loss"]):
                     self.logger.debug(
@@ -1511,34 +1511,34 @@ def _add_ensemble_trajectory(self, train_pred, valid_pred, test_pred):
 
         performance_stamp = {
             "Timestamp": pd.Timestamp.now(),
-            "ensemble_optimization_score": calculate_score(
+            "ensemble_optimization_score": calculate_scores(
                 solution=self.y_true_ensemble,
                 prediction=train_pred,
                 task_type=self.task_type,
-                metric=self.metric,
+                metrics=[self.metric],
                 scoring_functions=None,
-            ),
+            )[self.metric.name],
         }
         if valid_pred is not None:
             # TODO: valid_pred are a legacy from competition manager
             # and this if never happens. Re-evaluate Y_valid support
-            performance_stamp["ensemble_val_score"] = calculate_score(
+            performance_stamp["ensemble_val_score"] = calculate_scores(
                 solution=self.y_valid,
                 prediction=valid_pred,
                 task_type=self.task_type,
-                metric=self.metric,
+                metrics=[self.metric],
                 scoring_functions=None,
-            )
+            )[self.metric.name]
 
         # In case test_pred was provided
         if test_pred is not None:
-            performance_stamp["ensemble_test_score"] = calculate_score(
+            performance_stamp["ensemble_test_score"] = calculate_scores(
                 solution=self.y_test,
                 prediction=test_pred,
                 task_type=self.task_type,
-                metric=self.metric,
+                metrics=[self.metric],
                 scoring_functions=None,
-            )
+            )[self.metric.name]
 
         self.ensemble_history.append(performance_stamp)
 

autosklearn/ensemble_builder.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Tuple, Union, cast
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import random
 from collections import Counter
@@ -8,7 +8,7 @@
 
 from autosklearn.constants import TASK_TYPES
 from autosklearn.ensembles.abstract_ensemble import AbstractEnsemble
-from autosklearn.metrics import Scorer, calculate_loss
+from autosklearn.metrics import Scorer, calculate_losses
 from autosklearn.pipeline.base import BasePipeline
 
 
@@ -164,18 +164,13 @@ def _fast(
                     out=fant_ensemble_prediction,
                 )
 
-                # calculate_loss is versatile and can return a dict of losses
-                # when scoring_functions=None, we know it will be a float
-                losses[j] = cast(
-                    float,
-                    calculate_loss(
-                        solution=labels,
-                        prediction=fant_ensemble_prediction,
-                        task_type=self.task_type,
-                        metric=self.metric,
-                        scoring_functions=None,
-                    ),
-                )
+                losses[j] = calculate_losses(
+                    solution=labels,
+                    prediction=fant_ensemble_prediction,
+                    task_type=self.task_type,
+                    metrics=[self.metric],
+                    scoring_functions=None,
+                )[self.metric.name]
 
             all_best = np.argwhere(losses == np.nanmin(losses)).flatten()
 
@@ -211,18 +206,13 @@ def _slow(self, predictions: List[np.ndarray], labels: np.ndarray) -> None:
             for j, pred in enumerate(predictions):
                 ensemble.append(pred)
                 ensemble_prediction = np.mean(np.array(ensemble), axis=0)
-                # calculate_loss is versatile and can return a dict of losses
-                # when scoring_functions=None, we know it will be a float
-                losses[j] = cast(
-                    float,
-                    calculate_loss(
-                        solution=labels,
-                        prediction=ensemble_prediction,
-                        task_type=self.task_type,
-                        metric=self.metric,
-                        scoring_functions=None,
-                    ),
-                )
+                losses[j] = calculate_losses(
+                    solution=labels,
+                    prediction=ensemble_prediction,
+                    task_type=self.task_type,
+                    metrics=[self.metric],
+                    scoring_functions=None,
+                )[self.metric.name]
                 ensemble.pop()
             best = np.nanargmin(losses)
             ensemble.append(predictions[best])