fix per-model KDE in scatter_e_above_hull_models.py, add color bar since color value is now meaningful

Author: janosh

citation.cff

@@ -26,6 +26,7 @@ authors:
   - given-names: Philipp
     family-names: Benner
     affiliation: German Federal Institute of Materials Research and Testing (BAM)
+    orcid: 0000-0002-0912-8137
     affil_key: 3
     github: https://github.com/pbenner
   - given-names: Kristin

models/mace/analyze_mace.py

@@ -0,0 +1,59 @@
+"""Investigate MACE energy underpredictions."""
+
+
+# %%
+import os
+
+import pandas as pd
+from pymatviz import density_scatter, ptable_heatmap_plotly, spacegroup_sunburst
+
+from matbench_discovery import plots as plots
+from matbench_discovery.data import df_wbm
+from matbench_discovery.preds import PRED_FILES
+
+__author__ = "Janosh Riebesell"
+__date__ = "2023-07-23"
+
+module_dir = os.path.dirname(__file__)
+id_col = "material_id"
+target_col = "e_form_per_atom_mp2020_corrected"
+pred_col = "e_form_per_atom_mace"
+
+
+# %%
+df_mace = pd.read_csv(PRED_FILES.MACE).set_index(id_col)
+df_mace[list(df_wbm)] = df_wbm
+
+wyckoff_col, spg_col = "wyckoff_spglib", "spacegroup"
+df_mace[spg_col] = df_wbm[wyckoff_col].str.split("_").str[2].astype(int)
+
+
+# %%
+density_scatter(df=df_mace, x=target_col, y=pred_col)
+
+
+# %%
+df_bad = df_mace.query(f"{target_col} - {pred_col} > 2")
+
+ax = density_scatter(df=df_bad, x=target_col, y=pred_col)
+ax.set(title=f"{len(df_bad):,} MACE severe energy underpredictions")
+
+
+# %%
+fig = ptable_heatmap_plotly(df_bad.formula)
+title = f"Elements in {len(df_bad):,} MACE severe energy underpredictions"
+fig.layout.title.update(text=title, x=0.4, y=0.95)
+fig.show()
+
+
+# %%
+fig = spacegroup_sunburst(df_bad[spg_col], title="MACE spacegroups")
+title = f"Spacegroup sunburst of {len(df_bad):,} MACE severe energy underpredictions"
+fig.layout.title.update(text=title, x=0.5)
+fig.show()
+
+
+"""
+Space groups of MACE underpredictions look unremarkable but unusually heavy in Silicon,
+Lanthanide and heavy alkali metals.
+"""

scripts/model_figs/scatter_e_above_hull_models.py

@@ -11,6 +11,7 @@
 import plotly.express as px
 import scipy.stats
 from pymatviz.utils import add_identity_line, bin_df_cols, save_fig
+from tqdm import tqdm
 
 from matbench_discovery import FIGS, PDF_FIGS
 from matbench_discovery.metrics import classify_stable
@@ -119,35 +120,52 @@
 # save_fig(fig, f"{img_path}.svelte")
 
 
-# %% plot all models in separate subplots
-n_cols = 2
-n_rows = math.ceil(len(models) / n_cols)
+# %%
+clr_col, cnt_col = "density", "counts"
+# compute KDE for each model's predictions separately
+for model in (pbar := tqdm(models)):
+    pbar.set_description(f"KDE for {model=}")
+
+    xy = df_preds[[each_true_col, model]].dropna().T
+    model_kde = scipy.stats.gaussian_kde(xy)
+
+    model_rows = df_bin[df_bin[facet_col] == model]
+    xy_binned = model_rows[[each_true_col, each_pred_col]].T
+    density = model_kde(xy_binned)
+    n_preds = len(df_preds[model].dropna())
+    df_bin.loc[model_rows.index, cnt_col] = density / density.sum() * n_preds
 
+df_bin[clr_col] = np.log1p(df_bin[cnt_col]).round(2)
 
-def get_density(xs: np.ndarray, ys: np.ndarray) -> np.ndarray:
-    """Get kernel density estimate for each (x, y) point."""
-    return scipy.stats.gaussian_kde([xs, ys])([xs, ys])
 
+# %% scatter plot of DFT vs predicted hull distance with each model in separate subplot
+n_cols = 2
+n_rows = math.ceil(len(models) / n_cols)
 
-# scatter plot of DFT vs predicted hull distance
 fig = px.scatter(
     df_bin,
     x=each_true_col,
     y=each_pred_col,
     facet_col=facet_col,
     facet_col_wrap=n_cols,
-    color=get_density(df_bin[each_true_col], df_bin[each_pred_col]),
+    color=clr_col,
     facet_col_spacing=0.02,
     facet_row_spacing=0.04,
     hover_data=hover_cols,
     hover_name=df_preds.index.name,
     # color=clf_col,
-    color_discrete_map=clf_color_map,
+    # color_discrete_map=clf_color_map,
     # opacity=0.4,
     range_x=(domain := (-4, 7)),
     range_y=domain,
     category_orders={facet_col: legend_order},
-    color_continuous_scale="turbo",
+    color_continuous_scale="turbo",  # "thermal"
+)
+
+# manually set colorbar ticks and labels (needed after log1p transform)
+tick_vals = [1, 10, 100, 1000, 10_000]
+fig.layout.coloraxis.colorbar.update(
+    tickvals=np.log1p(tick_vals), ticktext=list(map(str, tick_vals))
 )
 
 x_title = fig.layout.xaxis.title.text  # used in annotations below
@@ -229,7 +247,7 @@ def get_density(xs: np.ndarray, ys: np.ndarray) -> np.ndarray:
     **axis_titles,
 )
 fig.layout.height = 200 * n_rows
-fig.layout.coloraxis.showscale = False
+fig.layout.coloraxis.colorbar.update(orientation="h", thickness=9, len=0.5, y=1.05)
 # fig.layout.width = 1100
 fig.layout.margin.update(l=40, r=10, t=30, b=60)
 fig.update_xaxes(matches=None)

site/package.json

@@ -19,27 +19,27 @@
   "devDependencies": {
     "@iconify/svelte": "^3.1.4",
     "@rollup/plugin-yaml": "^4.1.1",
-    "@sveltejs/adapter-static": "^2.0.2",
-    "@sveltejs/kit": "^1.22.3",
+    "@sveltejs/adapter-static": "^2.0.3",
+    "@sveltejs/kit": "^1.22.4",
     "@sveltejs/vite-plugin-svelte": "^2.4.3",
-    "@typescript-eslint/eslint-plugin": "^6.2.0",
-    "@typescript-eslint/parser": "^6.2.0",
+    "@typescript-eslint/eslint-plugin": "^6.2.1",
+    "@typescript-eslint/parser": "^6.2.1",
     "d3-scale-chromatic": "^3.0.0",
     "elementari": "^0.2.2",
-    "eslint": "^8.45.0",
+    "eslint": "^8.46.0",
     "eslint-plugin-svelte": "^2.32.4",
-    "hastscript": "^7.2.0",
+    "hastscript": "^8.0.0",
     "highlight.js": "^11.8.0",
     "js-yaml": "^4.1.0",
     "katex": "^0.16.8",
     "mdsvex": "^0.11.0",
-    "prettier": "^3.0.0",
-    "prettier-plugin-svelte": "^3.0.1",
+    "prettier": "^3.0.1",
+    "prettier-plugin-svelte": "^3.0.3",
     "rehype-autolink-headings": "^6.1.1",
     "rehype-katex-svelte": "^1.2.0",
     "rehype-slug": "^5.1.0",
     "remark-math": "3.0.0",
-    "svelte": "^4.1.1",
+    "svelte": "^4.1.2",
     "svelte-check": "^3.4.6",
     "svelte-multiselect": "^10.1.0",
     "svelte-preprocess": "^5.0.4",
@@ -48,7 +48,7 @@
     "svelte2tsx": "^0.6.19",
     "tslib": "^2.6.1",
     "typescript": "5.1.6",
-    "vite": "^4.4.7"
+    "vite": "^4.4.8"
   },
   "prettier": {
     "semi": false,

site/src/app.css

@@ -174,10 +174,6 @@ strong a[aria-hidden='true'] {
   left: 0;
 }
 
-/* for /api/[slug] */
-kbd {
-  padding: 0 1ex 0 0;
-}
 aside.toc.desktop {
   position: fixed;
   top: 3em;

site/src/figs/each-scatter-models-5x2.svelte

@@ -8,7 +8,6 @@
   /* select all but first module h1s */
   :global(h1[id^='module-']:not(:nth-of-type(2))) {
     margin: 2em 0 0;
-    font-size: 2em !important;
   }
   :global(h1 > kbd) {
     font-size: 25pt;