Merge pull request #161 from automl/development

Version 1.2.1

CHANGELOG.md

@@ -1,3 +1,24 @@
+# Version 1.2.1
+
+## Quality of Life
+- Runs now get displayed with their parent directory for better distinguishability.
+- Increase plot font sizes.
+- Add a simple loading bar functionality for longer runs.
+
+## General
+- Seed is now required in the Recorder.
+
+## Bug-Fixes
+- Use normalized LPI importance via variance instead of importance over mean (#152)
+- Return nan as importance values if variance is 0. for a hyperparameter / budget (#152)
+
+## Plugins
+- Show a run's hoover-text for the actual budget of a trial in Cost over Time with Combined budget (#154).
+- Use highest budget as default budget for Cost over Time instead of Combined.
+- Show best value / config for each objective instead of merged objective in Overview (#159).
+- Use chosen objective instead of merged objective to get the incumbent for the calculation of LPI importance (#159).
+- Add total runtime in overview (#155).
+
 # Version 1.2
 
 ## Plugins

Makefile

@@ -2,7 +2,7 @@
 # are usually completed in github actions.
 
 SHELL := /bin/bash
-VERSION := 1.2
+VERSION := 1.2.1
 
 NAME := DeepCAVE
 PACKAGE_NAME := deepcave
@@ -48,10 +48,10 @@ install:
 install-dev:
 	$(PIP) install -e ".[dev]"
 	pre-commit install
-	
+
 install-examples:
 	$(PIP) install -e ".[examples]"
-	
+
 check-black:
 	$(BLACK) ${SOURCE_DIR} --check || :
 	$(BLACK) ${EXAMPLES_DIR} --check || :
@@ -123,7 +123,7 @@ publish: clean build
 	$(PYTHON) -m twine upload --repository testpypi ${DIST}/*
 	@echo
 	@echo "Test with the following:"
-	@echo "* Create a new virtual environment to install the uplaoded distribution into"
+	@echo "* Create a new virtual environment to install the uploaded distribution into"
 	@echo "* Run the following:"
 	@echo "--- pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ ${PACKAGE_NAME}==${VERSION}"
 	@echo

deepcave/__init__.py

@@ -26,7 +26,7 @@
     "Source Code": "https://github.com/automl/deepcave",
 }
 copyright = f"Copyright {datetime.date.today().strftime('%Y')}, {author}"
-version = "1.2"
+version = "1.2.1"
 
 _exec_file = sys.argv[0]
 _exec_files = ["server.py", "worker.py", "sphinx-build"]

deepcave/config.py

@@ -57,6 +57,7 @@ class Config:
     FIGURE_MARGIN = dict(t=30, b=0, l=0, r=0)
     FIGURE_HEIGHT = "40vh"
     FIGURE_DOWNLOAD_SCALE = 4.0
+    FIGURE_FONT_SIZE = 20
 
     # Redis settings
     REDIS_PORT: int = 6379

deepcave/evaluators/fanova.py

@@ -20,6 +20,7 @@
 from deepcave.evaluators.epm.fanova_forest import FanovaForest
 from deepcave.runs import AbstractRun
 from deepcave.runs.objective import Objective
+from deepcave.utils.logs import get_logger
 
 
 class fANOVA:
@@ -50,6 +51,7 @@ def __init__(self, run: AbstractRun):
         self.cs = run.configspace
         self.hps = self.cs.get_hyperparameters()
         self.hp_names = self.cs.get_hyperparameter_names()
+        self.logger = get_logger(self.__class__.__name__)
 
     def calculate(
         self,
@@ -152,7 +154,14 @@ def get_importances(
                 )
 
                 if len(non_zero_idx[0]) == 0:
-                    raise RuntimeError("Encountered zero total variance in all trees.")
+                    self.logger.warning("Encountered zero total variance in all trees.")
+                    importances[sub_hp_ids] = (
+                        np.nan,
+                        np.nan,
+                        np.nan,
+                        np.nan,
+                    )
+                    continue
 
                 fractions_total = np.array(
                     [

deepcave/evaluators/lpi.py

@@ -95,7 +95,7 @@ def calculate(
 
         # Set variables
         self.continous_neighbors = continous_neighbors
-        self.incumbent, _ = self.run.get_incumbent(budget=budget)
+        self.incumbent, _ = self.run.get_incumbent(budget=budget, objectives=objectives)
         self.default = self.cs.get_default_configuration()
         self.incumbent_array = self.incumbent.get_array()
 
@@ -220,7 +220,10 @@ def calculate(
 
         # Normalize
         overall_var_per_tree = {
-            p: [t / sum_var_per_tree[idx] for idx, t in enumerate(trees)]
+            p: [
+                t / sum_var_per_tree[idx] if sum_var_per_tree[idx] != 0.0 else np.nan
+                for idx, t in enumerate(trees)
+            ]
             for p, trees in overall_var_per_tree.items()
         }
         self.variances = overall_var_per_tree
@@ -254,11 +257,11 @@ def get_importances(self, hp_names: List[str]) -> Dict[str, Tuple[float, float]]
             std = 0
 
             if hp_name in self.importances:
-                mean = self.importances[hp_name][0]
+                mean = np.mean(self.variances[hp_name])
                 std = np.var(self.variances[hp_name])
 
-            # Use this to quantify importance via variance
-            # mean = np.mean(overall_var_per_tree[hp_name])
+            # Use this to quantify importance via importance over mean value (not normalized to 1)
+            # mean = self.importances[hp_name][0]
 
             # Sometimes there is an ugly effect if default is better than
             # incumbent.

deepcave/plugins/__init__.py

@@ -1145,8 +1145,12 @@ def load_run_inputs(
         for run in runs:
             if check_run_compatibility(run):
                 try:
+                    run_path = run.path
+                    if run_path is not None:
+                        run_name = run_path.parent.name + "/" + run.name
+
                     values.append(run.id)
-                    labels.append(run.name)
+                    labels.append(run_name)
                     disabled.append(False)
                 except Exception:
                     pass

deepcave/plugins/budget/budget_correlation.py

@@ -311,6 +311,7 @@ def load_outputs(run, _, outputs) -> List[Any]:  # type: ignore
             yaxis=dict(title="Correlation"),
             margin=config.FIGURE_MARGIN,
             legend=dict(title="Budgets"),
+            font=dict(size=config.FIGURE_FONT_SIZE),
         )
 
         figure = go.Figure(data=traces, layout=layout)

deepcave/plugins/hyperparameter/importances.py

@@ -27,9 +27,12 @@
 from deepcave.runs import AbstractRun
 from deepcave.utils.cast import optional_int
 from deepcave.utils.layout import get_checklist_options, get_select_options, help_button
+from deepcave.utils.logs import get_logger
 from deepcave.utils.styled_plot import plt
 from deepcave.utils.styled_plotty import get_color, save_image
 
+logger = get_logger(__name__)
+
 
 class Importances(StaticPlugin):
     """
@@ -337,6 +340,8 @@ def process(run: AbstractRun, inputs: Dict[str, Any]) -> Dict[str, Any]:
             evaluator.calculate(objective, budget, n_trees=n_trees, seed=0)
 
             importances = evaluator.get_importances(hp_names)
+            if any(np.isnan(val) for value in importances.values() for val in value):
+                logger.warning(f"Nan encountered in importance values for budget {budget}.")
             data[budget_id] = importances
 
         return data  # type: ignore
@@ -458,6 +463,7 @@ def load_outputs(run, inputs, outputs) -> go.Figure:  # type: ignore
             legend={"title": "Budget"},
             margin=config.FIGURE_MARGIN,
             xaxis=dict(tickangle=-45),
+            font=dict(size=config.FIGURE_FONT_SIZE),
         )
         save_image(figure, "importances.pdf")
 

deepcave/plugins/hyperparameter/pdp.py

@@ -374,7 +374,7 @@ def get_output_layout(register: Callable) -> dcc.Graph:
 
     @staticmethod
     def get_pdp_figure(  # type: ignore
-        run, inputs, outputs, show_confidence, show_ice, title=None
+        run, inputs, outputs, show_confidence, show_ice, title=None, fontsize=None
     ) -> go.Figure:
         """
         Create a figure of the Partial Dependency Plot (PDP).
@@ -393,6 +393,8 @@ def get_pdp_figure(  # type: ignore
             Whether to show ice curves in the plot.
         title
             Title of the plot.
+        fontsize
+            Fontsize of the plot.
 
         Returns
         -------
@@ -471,6 +473,9 @@ def get_pdp_figure(  # type: ignore
             ]
 
             tickvals, ticktext = get_hyperparameter_ticks(hp1)
+            # Allow to pass a fontsize (necessary when leveraging PDP in Symbolic Explanation)
+            if fontsize is None:
+                fontsize = config.FIGURE_FONT_SIZE
             layout = go.Layout(
                 {
                     "xaxis": {
@@ -482,6 +487,7 @@ def get_pdp_figure(  # type: ignore
                         "title": objective_name,
                     },
                     "title": title,
+                    "font": dict(size=fontsize),
                 }
             )
         else:
@@ -509,6 +515,7 @@ def get_pdp_figure(  # type: ignore
                     yaxis=dict(tickvals=y_tickvals, ticktext=y_ticktext, title=hp2_name),
                     margin=config.FIGURE_MARGIN,
                     title=title,
+                    font=dict(size=fontsize),
                 )
             )
 

deepcave/plugins/hyperparameter/symbolic_explanations.py

@@ -577,6 +577,7 @@ def load_outputs(run, inputs, outputs) -> List[go.Figure]:  # type: ignore
                         "title": objective_name,
                     },
                     "title": expr,
+                    "font": dict(size=config.FIGURE_FONT_SIZE - 4),
                 }
             )
         else:
@@ -602,6 +603,7 @@ def load_outputs(run, inputs, outputs) -> List[go.Figure]:  # type: ignore
                     yaxis=dict(tickvals=y_tickvals, ticktext=y_ticktext, title=hp2_name),
                     margin=config.FIGURE_MARGIN,
                     title=expr,
+                    font=dict(size=config.FIGURE_FONT_SIZE - 4),
                 )
             )
 
@@ -614,7 +616,13 @@ def load_outputs(run, inputs, outputs) -> List[go.Figure]:  # type: ignore
             pdp_title = "Partial Dependency for comparison:"
 
         figure2 = PartialDependencies.get_pdp_figure(
-            run, inputs, outputs, show_confidence=False, show_ice=False, title=pdp_title
+            run,
+            inputs,
+            outputs,
+            show_confidence=False,
+            show_ice=False,
+            title=pdp_title,
+            fontsize=config.FIGURE_FONT_SIZE - 4,
         )
 
         return [figure1, figure2]

deepcave/plugins/objective/configuration_cube.py

@@ -416,7 +416,10 @@ def load_outputs(run, inputs, outputs) -> go.Figure:  # type: ignore
 
         if len(data) == 3:
             trace = go.Scatter3d(x=x, y=y, z=z, **scatter_kwargs)
-            layout = go.Layout({"scene": {**layout_kwargs}})
+            layout = go.Layout(
+                {"scene": {**layout_kwargs}},
+                font=dict(size=config.FIGURE_FONT_SIZE),
+            )
         else:
             if len(data) == 1:
                 y = [0 for _ in x]
@@ -425,7 +428,10 @@ def load_outputs(run, inputs, outputs) -> go.Figure:  # type: ignore
                 trace = go.Scatter(x=x, y=y, **scatter_kwargs)
             else:
                 trace = go.Scatter(x=[], y=[])
-            layout = go.Layout(**layout_kwargs)
+            layout = go.Layout(
+                **layout_kwargs,
+                font=dict(size=config.FIGURE_FONT_SIZE),
+            )
 
         figure = go.Figure(data=trace, layout=layout)
         figure.update_layout(dict(margin=config.FIGURE_MARGIN))

deepcave/plugins/objective/cost_over_time.py

@@ -225,7 +225,9 @@ def load_inputs(self) -> Dict[str, Any]:
             },
             "budget_id": {
                 "options": self.budget_options,
-                "value": self.budget_options[-1]["value"],
+                "value": self.budget_options[0]["value"]
+                if len(self.budget_options) == 1
+                else self.budget_options[-2]["value"],
             },
             "xaxis": {
                 "options": [
@@ -344,8 +346,7 @@ def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
                 continue
 
             objective = run.get_objective(inputs["objective_id"])
-            budget = run.get_budget(inputs["budget_id"])
-            config_ids = outputs[run.id]["config_ids"]
+            ids = outputs[run.id]["ids"]
             x = outputs[run.id]["times"]
             if inputs["xaxis"] == "trials":
                 x = outputs[run.id]["ids"]
@@ -360,9 +361,11 @@ def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
             hoverinfo = "skip"
             symbol = None
             mode = "lines"
-            if len(config_ids) > 0:
+            if len(run.history) > 0:
                 hovertext = [
-                    get_hovertext_from_config(run, config_id, budget) for config_id in config_ids
+                    get_hovertext_from_config(run, trial.config_id, trial.budget)
+                    for id, trial in enumerate(run.history)
+                    if id in ids
                 ]
                 hoverinfo = "text"
                 symbol = "circle"
@@ -423,6 +426,7 @@ def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
             xaxis=dict(title=xaxis_label, type=type),
             yaxis=dict(title=objective.name),
             margin=config.FIGURE_MARGIN,
+            font=dict(size=config.FIGURE_FONT_SIZE),
         )
 
         figure = go.Figure(data=traces, layout=layout)

deepcave/plugins/objective/parallel_coordinates.py

@@ -421,7 +421,10 @@ def load_outputs(run, inputs, outputs) -> go.Figure:  # type: ignore
                 dimensions=list([d for d in data.values()]),
                 labelangle=45,
             ),
-            layout=dict(margin=dict(t=150, b=50, l=100, r=0)),
+            layout=dict(
+                margin=dict(t=150, b=50, l=100, r=0),
+                font=dict(size=config.FIGURE_FONT_SIZE),
+            ),
         )
         save_image(figure, "parallel_coordinates.pdf")
 

deepcave/plugins/objective/pareto_front.py

@@ -446,7 +446,7 @@ def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
                     x_std += [points_std[point_idx][0]]
                     y_std += [points_std[point_idx][1]]
 
-            color = get_color(idx, alpha=0.1)
+            color = get_color(idx, alpha=0.5)
             color_pareto = get_color(idx)
 
             if show_all:
@@ -524,6 +524,7 @@ def load_outputs(runs, inputs, outputs) -> go.Figure:  # type: ignore
                 xaxis=dict(title=objective_1.name),
                 yaxis=dict(title=objective_2.name),
                 margin=config.FIGURE_MARGIN,
+                font=dict(size=config.FIGURE_FONT_SIZE),
             )
         else:
             layout = None
@@ -605,12 +606,11 @@ def load_mpl_outputs(runs, inputs, outputs):  # type: ignore
                     x += [points[point_idx][0]]
                     y += [points[point_idx][1]]
 
-            # , alpha=0.1)
             color = plt.get_color(idx)  # type: ignore
             color_pareto = plt.get_color(idx)  # type: ignore
 
             if show_all:
-                plt.scatter(x, y, color=color, marker="o", alpha=0.1, s=3)
+                plt.scatter(x, y, color=color, marker="o", s=3)
 
             # Check if hv or vh is needed
             objective_1 = run.get_objective(inputs["objective_id_1"])

deepcave/plugins/summary/configurations.py

@@ -418,7 +418,10 @@ def _get_objective_figure(
                     }
                 )
 
-        objective_layout = go.Layout(**layout_kwargs)
+        objective_layout = go.Layout(
+            **layout_kwargs,
+            font=dict(size=config.FIGURE_FONT_SIZE),
+        )
         objective_figure = go.Figure(data=objective_data, layout=objective_layout)
         save_image(objective_figure, "configure.pdf")
 
@@ -480,7 +483,8 @@ def _get_configspace_figure(
                 margin=dict(
                     t=150,
                     b=50,
-                )
+                ),
+                font=dict(size=config.FIGURE_FONT_SIZE),
             ),
         )