fix outdated numbers in data/wbm/readme.md

rename figs/wbm-each-hist.(svg|svelte) to figs/hist-wbm-hull-dist.(svg|svelte)

Author: janosh

data/wbm/eda.py

@@ -12,7 +12,7 @@
 )
 from pymatviz.utils import save_fig
 
-from matbench_discovery import FIGS, PDF_FIGS, ROOT, STABILITY_THRESHOLD, today
+from matbench_discovery import FIGS, PDF_FIGS, ROOT, STABILITY_THRESHOLD
 from matbench_discovery import plots as plots
 from matbench_discovery.data import DATA_FILES, df_wbm
 from matbench_discovery.energy import mp_elem_reference_entries
@@ -35,14 +35,25 @@
 
 
 # %%
-for count_mode in ["occurrence", "composition"]:
-    wbm_elem_counts = count_elements(df_wbm.formula, count_mode=count_mode).astype(int)
+wbm_occu_counts = count_elements(df_wbm.formula, count_mode="occurrence").astype(int)
+wbm_comp_counts = count_elements(df_wbm.formula, count_mode="composition")
+
+mp_occu_counts = count_elements(df_mp.formula_pretty, count_mode="occurrence").astype(
+    int
+)
+mp_comp_counts = count_elements(df_mp.formula_pretty, count_mode="composition")
+
+all_counts = (
+    ("wbm", "occurrence", wbm_occu_counts),
+    ("wbm", "composition", wbm_comp_counts),
+    ("mp", "occurrence", mp_occu_counts),
+    ("mp", "composition", mp_comp_counts),
+)
 
-    wbm_elem_counts.to_json(f"{about_data_page}/wbm-element-counts-{count_mode}.json")
-    mp_elem_counts = count_elements(df_mp.formula_pretty, count_mode=count_mode).astype(
-        int
-    )
-    mp_elem_counts.to_json(f"{about_data_page}/mp-element-counts-{count_mode}.json")
+
+# %%
+for dataset, count_mode, elem_counts in all_counts:
+    elem_counts.to_json(f"{about_data_page}/{dataset}-element-counts-{count_mode}.json")
 
 
 # %% export element counts by WBM step to JSON
@@ -64,51 +75,21 @@
 
 
 # %%
-wbm_fig = ptable_heatmap_plotly(
-    wbm_elem_counts.drop("Xe"),
-    log=True,
-    colorscale="RdBu",
-    hover_props=dict(atomic_number="atomic number"),
-    hover_data=wbm_elem_counts,
-)
-
-title = "WBM Elements"
-wbm_fig.update_layout(
-    title=dict(text=title, x=0.35, y=0.9, font_size=20),
-    xaxis=dict(fixedrange=True),
-    yaxis=dict(fixedrange=True),
-    paper_bgcolor="rgba(0,0,0,0)",
-)
-wbm_fig.show()
-
-
-# %%
-wbm_fig.write_image(f"{module_dir}/figs/wbm-elements.svg", width=1000, height=500)
-# save_fig(wbm_fig, f"{FIGS}/wbm-elements.svelte")
-
-
-# %%
-mp_fig = ptable_heatmap_plotly(
-    mp_elem_counts[mp_elem_counts > 1],
-    log=True,
-    colorscale="RdBu",
-    hover_props=dict(atomic_number="atomic number"),
-    hover_data=mp_elem_counts,
-)
-
-title = "MP Elements"
-mp_fig.update_layout(
-    title=dict(text=title, x=0.35, y=0.9, font_size=20),
-    xaxis=dict(fixedrange=True),
-    yaxis=dict(fixedrange=True),
-    paper_bgcolor="rgba(0,0,0,0)",
-)
-mp_fig.show()
-
+for dataset, count_mode, elem_counts in all_counts:
+    ptable = ptable_heatmap_plotly(
+        elem_counts.drop("Xe")[elem_counts > 1],
+        font_size=11,
+        color_bar=dict(title=dict(text=f"WBM {count_mode} counts", font_size=24)),
+        # log=True,
+        # colorscale="cividis",
+        hover_props=dict(atomic_number="atomic number"),
+        hover_data=wbm_occu_counts,
+    )
 
-# %%
-mp_fig.write_image(f"{module_dir}/figs/{today}-mp-elements.svg", width=1000, height=500)
-# save_fig(mp_fig, f"{FIGS}/mp-elements.svelte")
+    ptable.layout.margin = dict(l=0, r=0, b=0, t=0)
+    ptable.show()
+    # save_fig(ptable, f"{module_dir}/figs/wbm-elements.svg", width=1000, height=500)
+    save_fig(ptable, f"{PDF_FIGS}/{dataset}-element-{count_mode}-counts.pdf")
 
 
 # %% histogram of energy above MP convex hull for WBM
@@ -148,8 +129,9 @@
 
 fig.show()
 
-save_fig(fig, f"{FIGS}/wbm-each-hist.svelte")
-save_fig(fig, "./figs/wbm-each-hist.svg", width=1000, height=500)
+# save_fig(fig, f"{FIGS}/hist-wbm-hull-dist.svelte")
+# save_fig(fig, "./figs/hist-wbm-hull-dist.svg", width=1000, height=500)
+save_fig(fig, f"{PDF_FIGS}/hist-wbm-hull-dist.pdf")
 
 
 # %%
@@ -254,6 +236,7 @@
 # %%
 fig = spacegroup_sunburst(df_wbm[spg_col], width=350, height=350, show_counts="percent")
 fig.layout.title.update(text="WBM Spacegroup Sunburst", x=0.5, font_size=14)
+fig.layout.margin = dict(l=0, r=0, t=30, b=0)
 fig.show()
 save_fig(fig, f"{FIGS}/spacegroup-sunburst-wbm.svelte")
 save_fig(fig, f"{PDF_FIGS}/spacegroup-sunburst-wbm.pdf")
@@ -262,6 +245,7 @@
 # %%
 fig = spacegroup_sunburst(df_mp[spg_col], width=350, height=350, show_counts="percent")
 fig.layout.title.update(text="MP Spacegroup Sunburst", x=0.5, font_size=14)
+fig.layout.margin = dict(l=0, r=0, t=30, b=0)
 fig.show()
 save_fig(fig, f"{FIGS}/spacegroup-sunburst-mp.svelte")
 save_fig(fig, f"{PDF_FIGS}/spacegroup-sunburst-mp.pdf")

data/wbm/figs/wbm-each-hist.svg data/wbm/figs/hist-wbm-hull-dist.svg

@@ -74,16 +74,16 @@ materialscloud:2021.68 includes a readme file with a description of the dataset,
 
 ## 🎯   Target Distribution
 
-The WBM test set has an energy above the MP convex hull distribution with **mean ± std = 0.02 ± 0.25 eV/atom**.
+Below is a histogram of hull distances of all WBM test set structures with respect to the Materials Project convex hull. This is the target distribution that the models are asked to predict.
 
 The dummy MAE of always predicting the test set mean is **0.12 eV/atom**.
 
-The number of stable materials (according to the MP convex hull which is spanned by the training data the models have access to) is **97k** out of **257k**, resulting in a dummy stability hit rate of **37%**.
+The number of stable materials (according to the MP convex hull which is spanned by the training data the models have access to) is **43k** out of **257k**, resulting in a dummy stability hit rate of **16.7%**.
 
 > Note: [According to the authors](https://www.nature.com/articles/s41524-020-00481-6#Sec2), the stability rate w.r.t. to the more complete hull constructed from the combined train and test set (MP + WBM) for the first 3 rounds of elemental substitution is 18,479 out of 189,981 crystals ($\approx$ 9.7%).
 
-<slot name="wbm-each-hist">
-  <img src="./figs/wbm-each-hist.svg" alt="WBM energy above MP convex hull distribution">
+<slot name="hist-wbm-hull-dist">
+  <img src="./figs/hist-wbm-hull-dist.svg" alt="WBM energy above MP convex hull distribution">
 </slot>
 
 ## 🧪 &thinsp; Chemical Diversity
@@ -104,6 +104,6 @@ Element counts for MP training set consisting of 146,323 `ComputedStructureEntri
 
 ## 📊 &thinsp; Symmetry Statistics
 
-Both the MP training and WBM test set have good coverage of all 7 crystal systems, triclinic crystals being the only notable exception at just 1% prevalence in WBM but still well represented in MP (15%). In MP, monoclinic (23%) and orthorhombic (21%) are most prevalent. In WBM, orthorhombic and tetragonal each make up 20%. Combined with the higher share of cubic structures in WBM (19% vs 14%), WBM structures have overall higher symmetry. This should benefit a model like Wrenformer reliant on symmetries to encode coarse-grained structural features. See [SI](/si#spacegroup-prevalence-in-wrenformer-failure-cases) for a failure case of this featurization.
+These sunburst diagrams show the spacegroup distribution of MP on the left and WBM on the right. Both have good coverage of all 7 crystal systems, the only exception being triclinic crystals which are just 1% of WBM but well represented in MP (15%). The 3 largest systems in MP are monoclinic, orthorhombic and triclinic vs orthorhombic, tetragonal and cubic in WBM. So WBM structures have overall higher symmetry which can benefit some models more than others. Wrenformer in particular uses symmetries as a coarse-grained description of the underlying structure. Its representations basically degrades to composition only on symmetry-less P1 structures. Given this spacegroup distribution, it should fare well on the WBM test set. The fact that Wrenformer is still outperformed by all interatomic potentials and some single-shot GNNs indicates the underlying methodology is unable to compete. See [SI](/si#spacegroup-prevalence-in-wrenformer-failure-cases) for a specific Wrenformer failure case.
 
 <slot name="spacegroup-sunbursts" />

data/wbm/readme.md

@@ -887,24 +887,24 @@ def df_to_pdf(
     except ImportError as exc:
         raise ImportError(
             "pdfkit not installed\nrun pip install pdfkit && brew install "
-            "homebrew/cask/wkhtmltopdf"
+            "homebrew/cask/wkhtmltopdf\n(brew is macOS only, use apt un linux))"
         ) from exc
 
+    pdfkit.from_string(styler.to_html(**kwargs), file_path)
+    if not crop:
+        return
     try:
         # needed to auto-crop large white margins from PDF
         # pip install pdfCropMargins
         from pdfCropMargins import crop as crop_pdf
-    except ImportError as exc:
-        raise ImportError(
-            "pdfCropMargins not installed\nrun pip install pdfCropMargins.\n"
-        ) from exc
 
-    pdfkit.from_string(styler.to_html(**kwargs), file_path)
-    try:
         # Remove PDF margins
         cropped_file_path, _exit_code, _stdout, _stderr = crop_pdf(
             ["--percentRetain", "0", file_path]
         )
         os.replace(cropped_file_path, file_path)
+    except ImportError as exc:
+        msg = "pdfCropMargins not installed\nrun pip install pdfCropMargins"
+        raise ImportError(msg) from exc
     except Exception as exc:
         raise RuntimeError("Error cropping PDF margins") from exc

matbench_discovery/plots.py

@@ -1,9 +1,9 @@
 <script lang="ts">
   import { browser } from '$app/environment'
   import FormEnergyHist from '$figs/hist-wbm-e-form-per-atom.svelte'
+  import HistWbmHullDist from '$figs/hist-wbm-hull-dist.svelte'
   import SpacegroupSunburstMp from '$figs/spacegroup-sunburst-mp.svelte'
   import SpacegroupSunburstWbm from '$figs/spacegroup-sunburst-wbm.svelte'
-  import WbmEachHist from '$figs/wbm-each-hist.svelte'
   import { PtableInset } from '$lib'
   import DataReadme from '$root/data/wbm/readme.md'
   import type { ChemicalElement } from 'elementari'
@@ -101,9 +101,9 @@
       3).
     </p>
   </svelte:fragment>
-  <svelte:fragment slot="wbm-each-hist">
+  <svelte:fragment slot="hist-wbm-hull-dist">
     {#if browser}
-      <WbmEachHist />
+      <HistWbmHullDist />
     {/if}
   </svelte:fragment>
   <div