add global STABILITY_THRESHOLD to consistently parametrize across the codebase when materials count as thermodynamically stable

add figs/box-hull-dist.svelte generated by scripts/make_hull_dist_box_plot.py displayed on /si page
add copy buttons to code blocks
on /si page, add option to largest-error-scatter-select.svelte to show all figs in grid at once

Author: janosh

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ default_install_hook_types: [pre-commit, commit-msg]
 
 repos:
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.269
+    rev: v0.0.270
     hooks:
       - id: ruff
         args: [--fix]

data/mp/get_mp_energies.py

@@ -8,7 +8,7 @@
 from pymatviz.utils import annotate_metrics
 from tqdm import tqdm
 
-from matbench_discovery import today
+from matbench_discovery import STABILITY_THRESHOLD, today
 from matbench_discovery.data import DATA_FILES
 
 """
@@ -34,7 +34,6 @@
     "energy_above_hull",
     "decomposition_enthalpy",
     "energy_type",
-    "symmetry",
 }
 
 with MPRester(use_document_model=False) as mpr:
@@ -86,7 +85,9 @@
     alpha=0.1,
     xlim=[-5, 1],
     ylim=[-1, 1],
-    color=(df.decomposition_enthalpy > 0).map({True: "red", False: "blue"}),
+    color=(df.decomposition_enthalpy > STABILITY_THRESHOLD).map(
+        {True: "red", False: "blue"}
+    ),
     title=f"{today} - {len(df):,} MP entries",
 )
 

data/wbm/eda.py

@@ -12,7 +12,7 @@
 )
 from pymatviz.utils import save_fig
 
-from matbench_discovery import FIGS, PDF_FIGS, ROOT, today
+from matbench_discovery import FIGS, PDF_FIGS, ROOT, STABILITY_THRESHOLD, today
 from matbench_discovery import plots as plots
 from matbench_discovery.data import DATA_FILES, df_wbm
 from matbench_discovery.energy import mp_elem_reference_entries
@@ -124,8 +124,8 @@
 fig = df_hist.plot.area(x=x_label, y="count", backend="plotly", range_x=range_x)
 
 if col.startswith("e_above_hull"):
-    n_stable = sum(df_wbm[col] <= 0)
-    n_unstable = sum(df_wbm[col] > 0)
+    n_stable = sum(df_wbm[col] <= STABILITY_THRESHOLD)
+    n_unstable = sum(df_wbm[col] > STABILITY_THRESHOLD)
     assert n_stable + n_unstable == len(df_wbm.dropna())
 
     dummy_mae = (df_wbm[col] - df_wbm[col].mean()).abs().mean()

matbench_discovery/__init__.py

@@ -22,5 +22,8 @@
 # wandb <entity>/<project name> to record new runs to
 WANDB_PATH = "janosh/matbench-discovery"
 
+# threshold on hull distance for a material to be considered stable
+STABILITY_THRESHOLD = 0
+
 timestamp = f"{datetime.now():%Y-%m-%d@%H-%M-%S}"
 today = timestamp.split("@")[0]

matbench_discovery/metrics.py

@@ -6,6 +6,8 @@
 import pandas as pd
 from sklearn.metrics import r2_score
 
+from matbench_discovery import STABILITY_THRESHOLD
+
 """Functions to classify energy above convex hull predictions as true/false
 positive/negative and compute performance metrics.
 """
@@ -53,7 +55,7 @@ def classify_stable(
 def stable_metrics(
     each_true: Sequence[float],
     each_pred: Sequence[float],
-    stability_threshold: float = 0,
+    stability_threshold: float = STABILITY_THRESHOLD,
 ) -> dict[str, float]:
     """Get a dictionary of stability prediction metrics. Mostly binary classification
     metrics, but also MAE, RMSE and R2.
@@ -64,9 +66,12 @@ def stable_metrics(
         stability_threshold (float): Where to place stability threshold relative to
             convex hull in eV/atom, usually 0 or 0.1 eV. Defaults to 0.
 
-    Note: Should give equivalent classification metrics to sklearn.metrics.
-        classification_report(each_true > 0, each_pred > 0, output_dict=True) which
-        takes binary labels.
+    Note: Should give equivalent classification metrics to
+        sklearn.metrics.classification_report(
+            each_true > STABILITY_THRESHOLD,
+            each_pred > STABILITY_THRESHOLD,
+            output_dict=True,
+        )
 
     Returns:
         dict[str, float]: dictionary of classification metrics with keys DAF, Precision,

matbench_discovery/plots.py

@@ -22,6 +22,7 @@
 from pandas.io.formats.style import Styler
 from tqdm import tqdm
 
+from matbench_discovery import STABILITY_THRESHOLD
 from matbench_discovery.metrics import classify_stable
 
 __author__ = "Janosh Riebesell"
@@ -674,7 +675,7 @@ def cumulative_precision_recall(
     df_cum = pd.concat(dfs.values())
     # subselect rows for speed, plot has sufficient precision with 1k rows
     df_cum = df_cum.iloc[:: len(df_cum) // 1000 or 1]
-    n_stable = sum(e_above_hull_true <= 0)
+    n_stable = sum(e_above_hull_true <= STABILITY_THRESHOLD)
 
     if backend == "matplotlib":
         fig, axs = plt.subplots(

matbench_discovery/preds.py

@@ -6,7 +6,7 @@
 import pandas as pd
 from tqdm import tqdm
 
-from matbench_discovery import ROOT
+from matbench_discovery import ROOT, STABILITY_THRESHOLD
 from matbench_discovery.data import Files, df_wbm, glob_to_df
 from matbench_discovery.metrics import stable_metrics
 from matbench_discovery.plots import ev_per_atom, model_labels, quantity_labels
@@ -132,7 +132,7 @@ def load_df_wbm_with_preds(
 
 df_metrics = pd.DataFrame()
 df_metrics_10k = pd.DataFrame()  # look only at each model's 10k most stable predictions
-prevalence = (df_wbm[each_true_col] <= 0).mean()
+prevalence = (df_wbm[each_true_col] <= STABILITY_THRESHOLD).mean()
 
 df_metrics.index.name = "model"
 for model in PRED_FILES:

scripts/make_hull_dist_box_plot.py

@@ -0,0 +1,85 @@
+# %%
+import plotly.express as px
+import plotly.graph_objects as go
+import seaborn as sns
+from pymatviz.utils import save_fig
+
+from matbench_discovery import FIGS, PDF_FIGS, plots
+from matbench_discovery.preds import df_each_err, models
+
+__author__ = "Janosh Riebesell"
+__date__ = "2023-05-25"
+
+
+# %%
+ax = df_each_err[models].plot.box(
+    showfliers=False,
+    rot=90,
+    figsize=(12, 6),
+    # color="blue",
+    # different fill colors for each box
+    # patch_artist=True,
+    # notch=True,
+    # bootstrap=10000,
+    showmeans=True,
+    # meanline=True,
+)
+ax.axhline(0, linewidth=1, color="gray", linestyle="--")
+
+
+# %%
+ax = sns.violinplot(
+    data=df_each_err[models], inner="quartile", linewidth=0.3, palette="Set2", width=1
+)
+ax.set(ylim=(-0.9, 0.9))
+
+
+# %%
+px.box(
+    df_each_err[models].melt(),
+    x="variable",
+    y="value",
+    color="variable",
+    points=False,
+    hover_data={"variable": False},
+)
+
+
+# %%
+px.violin(
+    df_each_err[models].melt(),
+    x="variable",
+    y="value",
+    color="variable",
+    violinmode="overlay",
+    box=True,
+    # points="all",
+    hover_data={"variable": False},
+    width=1000,
+    height=500,
+)
+
+
+# %%
+fig = go.Figure()
+fig.layout.yaxis.title = plots.quantity_labels["e_above_hull_error"]
+fig.layout.margin = dict(l=0, r=0, b=0, t=0)
+
+for col in models:
+    val_min = df_each_err[col].quantile(0.05)
+    lower_box = df_each_err[col].quantile(0.25)
+    median = df_each_err[col].median()
+    upper_box = df_each_err[col].quantile(0.75)
+    val_max = df_each_err[col].quantile(0.95)
+
+    box_plot = go.Box(
+        y=[val_min, lower_box, median, upper_box, val_max],
+        name=col,
+        width=0.7,
+    )
+    fig.add_trace(box_plot)
+
+fig.layout.legend.update(orientation="h", y=1.15)
+fig.show()
+save_fig(fig, f"{FIGS}/box-hull-dist-errors.svelte")
+save_fig(fig, f"{PDF_FIGS}/box-hull-dist-errors.pdf")

scripts/prc_roc_curves_models.py

@@ -12,7 +12,7 @@
 from sklearn.metrics import auc, precision_recall_curve, roc_curve
 from tqdm import tqdm
 
-from matbench_discovery import FIGS, PDF_FIGS
+from matbench_discovery import FIGS, PDF_FIGS, STABILITY_THRESHOLD
 from matbench_discovery import plots as plots
 from matbench_discovery.preds import df_each_pred, df_preds, each_true_col, models
 
@@ -35,7 +35,7 @@
 for model in (pbar := tqdm(models, desc="Calculating ROC curves")):
     pbar.set_postfix_str(model)
     na_mask = df_preds[each_true_col].isna() | df_each_pred[model].isna()
-    y_true = (df_preds[~na_mask][each_true_col] <= 0).astype(int)
+    y_true = (df_preds[~na_mask][each_true_col] <= STABILITY_THRESHOLD).astype(int)
     y_pred = df_each_pred[model][~na_mask]
     fpr, tpr, thresholds = roc_curve(y_true, y_pred, pos_label=0)
     AUC = auc(fpr, tpr)
@@ -98,7 +98,7 @@
 for model in (pbar := tqdm(list(df_each_pred), desc="Calculating ROC curves")):
     pbar.set_postfix_str(model)
     na_mask = df_preds[each_true_col].isna() | df_each_pred[model].isna()
-    y_true = (df_preds[~na_mask][each_true_col] <= 0).astype(int)
+    y_true = (df_preds[~na_mask][each_true_col] <= STABILITY_THRESHOLD).astype(int)
     y_pred = df_each_pred[model][~na_mask]
     prec, recall, thresholds = precision_recall_curve(y_true, y_pred, pos_label=0)
     df_tmp = pd.DataFrame(

site/src/app.css

@@ -99,6 +99,7 @@ pre code {
   display: inline-block;
 }
 pre {
+  position: relative;
   border-radius: 4pt;
   font-size: 9.5pt;
   background-color: rgba(255, 255, 255, 0.05);

site/src/figs/box-hull-dist-errors.svelte

@@ -5,7 +5,7 @@
   import { repository } from '$site/package.json'
   import { CmdPalette } from 'svelte-multiselect'
   import Toc from 'svelte-toc'
-  import { GitHubCorner, PrevNext } from 'svelte-zoo'
+  import { CopyButton, GitHubCorner, PrevNext } from 'svelte-zoo'
   import '../app.css'
 
   const routes = Object.keys(import.meta.glob(`./*/+page.{svelte,md}`)).map(
@@ -45,6 +45,20 @@
     } else {
       document.documentElement.style.setProperty(`--main-max-width`, `50em`)
     }
+
+    for (const node of document.querySelectorAll('pre > code')) {
+      // skip if <pre> already contains a button (presumably for copy)
+      const pre = node.parentElement
+      if (!pre || pre.querySelector(`button`)) continue
+
+      new CopyButton({
+        target: pre,
+        props: {
+          content: node.textContent ?? '',
+          style: 'position: absolute; top: 1ex; right: 1ex;',
+        },
+      })
+    }
   })
 </script>
 

site/src/routes/+layout.svelte

@@ -9,7 +9,7 @@
   import type { ChemicalElement } from 'elementari'
   import { ColorBar, ColorScaleSelect, PeriodicTable, TableInset } from 'elementari'
   import Select from 'svelte-multiselect'
-  import { Toggle, Tooltip } from 'svelte-zoo'
+  import { Toggle } from 'svelte-zoo'
   import type { Snapshot } from './$types'
 
   const elem_counts = import.meta.glob(`./*-element-counts-{occu,comp}*.json`, {
@@ -59,12 +59,10 @@
         />
       </TableInset>
     </PeriodicTable>
-    <Tooltip
-      text="occurrence=(Fe: 1, O: 1), composition: Fe2O3=(Fe: 2, O: 3)"
-      style="display: inline-block; transform: translate(10cqw, 5ex);"
+    <label
+      for="count-mode"
+      style="display: inline-block; transform: translate(10cqw, 5ex);">Count Mode</label
     >
-      <label for="count-mode">Count Mode</label>
-    </Tooltip>
     <Select
       id="count-mode"
       bind:selected={count_mode}
@@ -74,6 +72,7 @@
     />
     <ColorScaleSelect bind:selected={color_scale} />
   </svelte:fragment>
+
   <svelte:fragment slot="mp-elements-heatmap">
     <PeriodicTable
       heatmap_values={mp_elem_counts}
@@ -94,6 +93,13 @@
         />
       </TableInset>
     </PeriodicTable>
+    <p>
+      The difference between count modes is best explained by example. <code
+        >occurrence</code
+      >
+      mode maps Fe2O3 to (Fe: 1, O: 1), <code>composition</code> mode maps it to (Fe: 2, O:
+      3).
+    </p>
   </svelte:fragment>
   <svelte:fragment slot="wbm-each-hist">
     {#if browser}