move load_df_wbm_preds+PRED_FILES from matbench_discovery/{data->preds}.py

change model label 'Voronoi Random Forest' to 'Voronoi RF'

Author: janosh

matbench_discovery/data.py

@@ -13,7 +13,6 @@
 from tqdm import tqdm
 
 from matbench_discovery import ROOT
-from matbench_discovery.plots import model_labels
 
 df_wbm = pd.read_csv(f"{ROOT}/data/wbm/2022-10-19-wbm-summary.csv")
 df_wbm.index = df_wbm.material_id
@@ -39,7 +38,7 @@ def __init__(self) -> None:
         """Create a Files instance."""
         key_map = getattr(self, "_key_map", {})
         dct = {
-            key_map.get(key, key): f"{self._root}/{file}"  # type: ignore
+            key_map.get(key, key): f"{self._root}{file}"  # type: ignore
             for key, file in type(self).__dict__.items()
             if not key.startswith("_")
         }
@@ -52,7 +51,7 @@ class DataFiles(Files):
     See https://janosh.github.io/matbench-discovery/contribute for data descriptions.
     """
 
-    _root = f"{ROOT}/data"
+    _root = f"{ROOT}/data/"
 
     mp_computed_structure_entries = (
         "mp/2023-02-07-mp-computed-structure-entries.json.gz"
@@ -176,41 +175,6 @@ def load_train_test(
     return dfs
 
 
-class PredFiles(Files):
-    """Data files provided by Matbench Discovery.
-    See https://janosh.github.io/matbench-discovery/contribute for data descriptions.
-    """
-
-    _root = f"{ROOT}/models"
-    _key_map = model_labels
-
-    # CGCnn 10-member ensemble
-    cgcnn = "cgcnn/2023-01-26-test-cgcnn-wbm-IS2RE/cgcnn-ensemble-preds.csv"
-
-    # cgcnn 10-member ensemble with 5-fold training set perturbations
-    cgcnn_p = "cgcnn/2023-02-05-cgcnn-perturb=5.csv"
-
-    # magpie composition+voronoi tessellation structure features + sklearn random forest
-    voronoi_rf = "voronoi/2022-11-27-train-test/e-form-preds-IS2RE.csv"
-
-    # wrenformer 10-member ensemble
-    wrenformer = "wrenformer/2022-11-15-wrenformer-IS2RE-preds.csv"
-
-    # original megnet straight from publication, not re-trained
-    megnet = "megnet/2022-11-18-megnet-wbm-IS2RE/megnet-e-form-preds.csv"
-
-    # original m3gnet straight from publication, not re-trained
-    m3gnet = "m3gnet/2022-10-31-m3gnet-wbm-IS2RE.csv"
-
-    # m3gnet-relaxed structures fed into megnet for formation energy prediction
-    m3gnet_megnet = "m3gnet/2022-10-31-m3gnet-wbm-IS2RE.csv"
-    # bowsr optimizer coupled with original megnet
-    bowsr_megnet = "bowsr/2023-01-23-bowsr-megnet-wbm-IS2RE.csv"
-
-
-PRED_FILES = PredFiles()
-
-
 def glob_to_df(
     pattern: str,
     reader: Callable[[Any], pd.DataFrame] = None,
@@ -242,65 +206,3 @@ def glob_to_df(
         sub_dfs[file] = df
 
     return pd.concat(sub_dfs.values())
-
-
-def load_df_wbm_preds(
-    models: Sequence[str] = (*PRED_FILES,),
-    pbar: bool = True,
-    id_col: str = "material_id",
-    **kwargs: Any,
-) -> pd.DataFrame:
-    """Load WBM summary dataframe with model predictions from disk.
-
-    Args:
-        models (Sequence[str], optional): Model names must be keys of
-            matbench_discovery.data.PRED_FILES. Defaults to all models.
-        pbar (bool, optional): Whether to show progress bar. Defaults to True.
-        id_col (str, optional): Column to set as df.index. Defaults to "material_id".
-        **kwargs: Keyword arguments passed to glob_to_df().
-
-    Raises:
-        ValueError: On unknown model names.
-
-    Returns:
-        pd.DataFrame: WBM summary dataframe with model predictions.
-    """
-    if mismatch := ", ".join(set(models) - set(PRED_FILES)):
-        raise ValueError(f"Unknown models: {mismatch}")
-
-    dfs: dict[str, pd.DataFrame] = {}
-
-    for model_name in (bar := tqdm(models, disable=not pbar, desc="Loading preds")):
-        bar.set_postfix_str(model_name)
-        df = glob_to_df(PRED_FILES[model_name], pbar=False, **kwargs).set_index(id_col)
-        dfs[model_name] = df
-
-    df_out = df_wbm.copy()
-    for model_name, df in dfs.items():
-        model_key = model_name.lower().replace(" + ", "_").replace(" ", "_")
-        if f"e_form_per_atom_{model_key}" in df:
-            df_out[model_name] = df[f"e_form_per_atom_{model_key}"]
-
-        elif len(pred_cols := df.filter(like="_pred_ens").columns) > 0:
-            assert len(pred_cols) == 1
-            df_out[model_name] = df[pred_cols[0]]
-            if len(std_cols := df.filter(like="_std_ens").columns) > 0:
-                df_out[f"{model_name}_std"] = df[std_cols[0]]
-
-        elif len(pred_cols := df.filter(like=r"_pred_").columns) > 1:
-            # make sure we average the expected number of ensemble member predictions
-            assert len(pred_cols) == 10, f"{len(pred_cols) = }, expected 10"
-            df_out[model_name] = df[pred_cols].mean(axis=1)
-
-        elif "e_form_per_atom_voronoi_rf" in df:  # new voronoi
-            df_out[model_name] = df.e_form_per_atom_voronoi_rf
-
-        elif "e_form_pred" in df:  # old voronoi
-            df_out[model_name] = df.e_form_pred
-
-        else:
-            raise ValueError(
-                f"No pred col for {model_name=}, available cols={list(df)}"
-            )
-
-    return df_out

matbench_discovery/plots.py

@@ -56,7 +56,7 @@
     m3gnet="M3GNet",
     megnet="MEGNet",
     megnet_old="MEGNet Old",
-    voronoi_rf="Voronoi Random Forest",
+    voronoi_rf="Voronoi RF",
     wrenformer="Wrenformer",
     dft="DFT",
     wbm="WBM",

matbench_discovery/preds.py

@@ -1,9 +1,15 @@
 from __future__ import annotations
 
+from collections.abc import Sequence
+from typing import Any
+
 import pandas as pd
+from tqdm import tqdm
 
-from matbench_discovery.data import PRED_FILES, load_df_wbm_preds
+from matbench_discovery import ROOT
+from matbench_discovery.data import Files, glob_to_df
 from matbench_discovery.metrics import stable_metrics
+from matbench_discovery.plots import model_labels
 
 """Centralize data-loading and computing metrics for plotting scripts"""
 
@@ -14,6 +20,101 @@
 each_true_col = "e_above_hull_mp2020_corrected_ppd_mp"
 each_pred_col = "e_above_hull_pred"
 
+
+class PredFiles(Files):
+    """Data files provided by Matbench Discovery.
+    See https://janosh.github.io/matbench-discovery/contribute for data descriptions.
+    """
+
+    _root = f"{ROOT}/models/"
+    _key_map = model_labels  # remap model keys below to pretty plot labels (see Files)
+
+    # CGCnn 10-member ensemble
+    cgcnn = "cgcnn/2023-01-26-test-cgcnn-wbm-IS2RE/cgcnn-ensemble-preds.csv"
+
+    # cgcnn 10-member ensemble with 5-fold training set perturbations
+    cgcnn_p = "cgcnn/2023-02-05-cgcnn-perturb=5.csv"
+
+    # magpie composition+voronoi tessellation structure features + sklearn random forest
+    voronoi_rf = "voronoi/2022-11-27-train-test/e-form-preds-IS2RE.csv"
+
+    # wrenformer 10-member ensemble
+    wrenformer = "wrenformer/2022-11-15-wrenformer-IS2RE-preds.csv"
+
+    # original megnet straight from publication, not re-trained
+    megnet = "megnet/2022-11-18-megnet-wbm-IS2RE/megnet-e-form-preds.csv"
+    megnet_old = "megnet/2022-11-18-megnet-wbm-IS2RE/megnet-e-form-preds.csv"
+
+    # original m3gnet straight from publication, not re-trained
+    m3gnet = "m3gnet/2022-10-31-m3gnet-wbm-IS2RE.csv"
+
+    # m3gnet-relaxed structures fed into megnet for formation energy prediction
+    m3gnet_megnet = "m3gnet/2022-10-31-m3gnet-wbm-IS2RE.csv"
+    # bowsr optimizer coupled with original megnet
+    bowsr_megnet = "bowsr/2023-01-23-bowsr-megnet-wbm-IS2RE.csv"
+
+
+PRED_FILES = PredFiles()
+
+
+def load_df_wbm_preds(
+    models: Sequence[str] = (*PRED_FILES,),
+    pbar: bool = True,
+    id_col: str = "material_id",
+    **kwargs: Any,
+) -> pd.DataFrame:
+    """Load WBM summary dataframe with model predictions from disk.
+
+    Args:
+        models (Sequence[str], optional): Model names must be keys of
+            matbench_discovery.data.PRED_FILES. Defaults to all models.
+        pbar (bool, optional): Whether to show progress bar. Defaults to True.
+        id_col (str, optional): Column to set as df.index. Defaults to "material_id".
+        **kwargs: Keyword arguments passed to glob_to_df().
+
+    Raises:
+        ValueError: On unknown model names.
+
+    Returns:
+        pd.DataFrame: WBM summary dataframe with model predictions.
+    """
+    if mismatch := ", ".join(set(models) - set(PRED_FILES)):
+        raise ValueError(f"Unknown models: {mismatch}")
+
+    dfs: dict[str, pd.DataFrame] = {}
+
+    for model_name in (bar := tqdm(models, disable=not pbar, desc="Loading preds")):
+        bar.set_postfix_str(model_name)
+        df = glob_to_df(PRED_FILES[model_name], pbar=False, **kwargs).set_index(id_col)
+        dfs[model_name] = df
+
+    from matbench_discovery.data import df_wbm
+
+    df_out = df_wbm.copy()
+    for model_name, df in dfs.items():
+        model_key = model_name.lower().replace(" + ", "_").replace(" ", "_")
+        if (col := f"e_form_per_atom_{model_key}") in df:
+            df_out[model_name] = df[col]
+
+        elif pred_cols := list(df.filter(like="_pred_ens")):
+            assert len(pred_cols) == 1
+            df_out[model_name] = df[pred_cols[0]]
+            if std_cols := list(df.filter(like="_std_ens")):
+                df_out[f"{model_name}_std"] = df[std_cols[0]]
+
+        elif pred_cols := list(df.filter(like=r"_pred_")):
+            # make sure we average the expected number of ensemble member predictions
+            assert len(pred_cols) == 10, f"{len(pred_cols) = }, expected 10"
+            df_out[model_name] = df[pred_cols].mean(axis=1)
+
+        else:
+            raise ValueError(
+                f"No pred col for {model_name=}, available cols={list(df)}"
+            )
+
+    return df_out
+
+
 df_wbm = load_df_wbm_preds().round(3)
 
 

models/chgnet/metadata.yml

@@ -19,8 +19,8 @@ authors:
   - name: Christopher J. Bartel
     affiliation: University of Minnesota
     orcid: https://orcid.org/0000-0002-5198-5036
-  - name: Christopher J. Bartel
-    affiliation: Gerbrand Ceder
+  - name: Gerbrand Ceder
+    affiliation: UC Berkeley
     orcid: https://orcid.org/0000-0001-9275-3605
     email: [email protected]
 repo: https://github.com/CederGroupHub/chgnet
@@ -31,9 +31,10 @@ requirements:
   ase: 3.22.0
   pymatgen: 2022.10.22
   numpy: 1.24.0
-  pandas: 1.5.1
 trained_on_benchmark: false
 
 notes:
-  description: The Crystal Hamiltonian Graph Neural Network (CHGNet) is a universal GNN-based interatomic potential trained on energies, forces, stresses and magnetic moments from the MP trajectory dataset containing ∼1.5 million inorganic structures.
+  description: |
+    The Crystal Hamiltonian Graph Neural Network (CHGNet) is a universal GNN-based interatomic potential trained on energies, forces, stresses and magnetic moments from the MP trajectory dataset containing ∼1.5 million inorganic structures.
+    ![CHGNet Pipeline](https://user-images.githubusercontent.com/30958850/222842305-b6ed2468-8773-4e03-9de5-20c8e8de030e.svg)
   training: Using pre-trained model released with preprint. Training set unreleased until after review.

models/voronoi/metadata.yml

@@ -1,4 +1,4 @@
-model_name: Voronoi Random Forest
+model_name: Voronoi RF
 model_version: 1.1.2 # scikit learn version which implements the random forest
 matbench_discovery_version: 1.0
 date_added: "2022-11-26"

scripts/compile_metrics.py

@@ -17,9 +17,8 @@
 from tqdm import tqdm
 
 from matbench_discovery import FIGS, MODELS, ROOT, WANDB_PATH
-from matbench_discovery.data import PRED_FILES
 from matbench_discovery.plots import px
-from matbench_discovery.preds import df_metrics, df_wbm
+from matbench_discovery.preds import PRED_FILES, df_metrics, df_wbm
 
 __author__ = "Janosh Riebesell"
 __date__ = "2022-11-28"
@@ -35,7 +34,7 @@
             display_name={"$regex": "cgcnn-robust-formation_energy_per_atom"},
         ),
     ),
-    "Voronoi Random Forest": dict(
+    "Voronoi RF": dict(
         n_runs=68,
         filters=dict(
             created_at={"$gt": "2022-11-17", "$lt": "2022-11-28"},

scripts/rolling_mae_vs_hull_dist_wbm_batches.py

@@ -54,7 +54,7 @@
 
 
 # %% plotly
-model = "Wrenformer"  # ["M3GNet", "Wrenformer", "MEGNet", "Voronoi Random Forest"]
+model = "Wrenformer"  # ["M3GNet", "Wrenformer", "MEGNet", "Voronoi RF"]
 df_pivot = df_each_pred.pivot(columns=batch_col, values=model)
 
 # unstack two-level column index into new model column

site/src/lib/ModelCard.svelte

@@ -23,7 +23,7 @@
   const target = { target: `_blank`, rel: `noopener` }
 </script>
 
-<h2>
+<h2 id={model_name.toLowerCase().replaceAll(` `, `-`)}>
   {model_name}
   <button
     on:click={() => (show_details = !show_details)}
@@ -161,10 +161,7 @@
     font: inherit;
   }
   h3 {
-    margin: 1em 0 0;
-  }
-  div h3 {
-    margin: 0;
+    margin: 1ex 0 3pt;
   }
   ul {
     list-style: disc;
@@ -204,9 +201,6 @@
     flex-direction: column;
     max-height: 10em;
   }
-  section.metrics > h3 {
-    margin: 0;
-  }
   section.metrics > ul > li {
     font-weight: lighter;
     display: flex;

site/src/routes/models/2023-02-03-model-stats.json

@@ -1 +1 @@
-{"CGCNN":{"Run Time (h)":11.35,"GPU":1.0,"CPU":128.0,"Slurm Jobs":10.0,"DAF":1.97,"Precision":0.74,"Recall":0.79,"Accuracy":0.82,"F1":0.77,"TPR":0.79,"FPR":0.17,"TNR":0.83,"FNR":0.21,"MAE":0.13,"RMSE":0.23,"R2":0.16,"missing_preds":2,"missing_percent":"0.00%"},"Voronoi Random Forest":{"Run Time (h)":204.7,"GPU":0.0,"CPU":76.0,"Slurm Jobs":68.0,"DAF":1.55,"Precision":0.59,"Recall":0.79,"Accuracy":0.71,"F1":0.67,"TPR":0.79,"FPR":0.34,"TNR":0.66,"FNR":0.21,"MAE":0.14,"RMSE":0.21,"R2":0.32,"missing_preds":17,"missing_percent":"0.01%"},"Wrenformer":{"Run Time (h)":57.89,"GPU":1.0,"CPU":128.0,"Slurm Jobs":10.0,"DAF":1.78,"Precision":0.67,"Recall":0.91,"Accuracy":0.8,"F1":0.77,"TPR":0.91,"FPR":0.27,"TNR":0.73,"FNR":0.09,"MAE":0.11,"RMSE":0.18,"R2":0.46,"missing_preds":0,"missing_percent":"0.00%"},"MEGNet":{"Run Time (h)":3.44,"GPU":1.0,"CPU":128.0,"Slurm Jobs":1.0,"DAF":2.1,"Precision":0.8,"Recall":0.66,"Accuracy":0.81,"F1":0.72,"TPR":0.66,"FPR":0.1,"TNR":0.9,"FNR":0.34,"MAE":0.13,"RMSE":0.21,"R2":0.3,"missing_preds":0,"missing_percent":"0.00%"},"M3GNet":{"Run Time (h)":83.65,"GPU":0.0,"CPU":76.0,"Slurm Jobs":99.0,"DAF":1.88,"Precision":0.71,"Recall":0.93,"Accuracy":0.83,"F1":0.8,"TPR":0.93,"FPR":0.24,"TNR":0.76,"FNR":0.07,"MAE":0.07,"RMSE":0.11,"R2":0.79,"missing_preds":2569,"missing_percent":"1.00%"},"BOWSR + MEGNet":{"Run Time (h)":2776.57,"GPU":0.0,"CPU":32.0,"Slurm Jobs":500.0,"DAF":1.63,"Precision":0.59,"Recall":0.87,"Accuracy":0.73,"F1":0.7,"TPR":0.87,"FPR":0.36,"TNR":0.64,"FNR":0.13,"MAE":0.11,"RMSE":0.16,"R2":0.55,"missing_preds":6184,"missing_percent":"2.41%"},"M3GNet + MEGNet":{"Run Time (h)":87.09,"GPU":0.0,"CPU":76.0,"Slurm Jobs":99.0,"DAF":1.98,"Precision":0.74,"Recall":0.79,"Accuracy":0.82,"F1":0.76,"TPR":0.79,"FPR":0.17,"TNR":0.83,"FNR":0.21,"MAE":0.09,"RMSE":0.13,"R2":0.72,"missing_preds":2576,"missing_percent":"1.00%"}}
+{"CGCNN":{"Run Time (h)":11.35,"GPU":1.0,"CPU":128.0,"Slurm Jobs":10.0,"DAF":1.97,"Precision":0.74,"Recall":0.79,"Accuracy":0.82,"F1":0.77,"TPR":0.79,"FPR":0.17,"TNR":0.83,"FNR":0.21,"MAE":0.13,"RMSE":0.23,"R2":0.16,"missing_preds":2,"missing_percent":"0.00%"},"Voronoi RF":{"Run Time (h)":204.7,"GPU":0.0,"CPU":76.0,"Slurm Jobs":68.0,"DAF":1.55,"Precision":0.59,"Recall":0.79,"Accuracy":0.71,"F1":0.67,"TPR":0.79,"FPR":0.34,"TNR":0.66,"FNR":0.21,"MAE":0.14,"RMSE":0.21,"R2":0.32,"missing_preds":17,"missing_percent":"0.01%"},"Wrenformer":{"Run Time (h)":57.89,"GPU":1.0,"CPU":128.0,"Slurm Jobs":10.0,"DAF":1.78,"Precision":0.67,"Recall":0.91,"Accuracy":0.8,"F1":0.77,"TPR":0.91,"FPR":0.27,"TNR":0.73,"FNR":0.09,"MAE":0.11,"RMSE":0.18,"R2":0.46,"missing_preds":0,"missing_percent":"0.00%"},"MEGNet":{"Run Time (h)":3.44,"GPU":1.0,"CPU":128.0,"Slurm Jobs":1.0,"DAF":2.1,"Precision":0.8,"Recall":0.66,"Accuracy":0.81,"F1":0.72,"TPR":0.66,"FPR":0.1,"TNR":0.9,"FNR":0.34,"MAE":0.13,"RMSE":0.21,"R2":0.3,"missing_preds":0,"missing_percent":"0.00%"},"M3GNet":{"Run Time (h)":83.65,"GPU":0.0,"CPU":76.0,"Slurm Jobs":99.0,"DAF":1.88,"Precision":0.71,"Recall":0.93,"Accuracy":0.83,"F1":0.8,"TPR":0.93,"FPR":0.24,"TNR":0.76,"FNR":0.07,"MAE":0.07,"RMSE":0.11,"R2":0.79,"missing_preds":2569,"missing_percent":"1.00%"},"BOWSR + MEGNet":{"Run Time (h)":2776.57,"GPU":0.0,"CPU":32.0,"Slurm Jobs":500.0,"DAF":1.63,"Precision":0.59,"Recall":0.87,"Accuracy":0.73,"F1":0.7,"TPR":0.87,"FPR":0.36,"TNR":0.64,"FNR":0.13,"MAE":0.11,"RMSE":0.16,"R2":0.55,"missing_preds":6184,"missing_percent":"2.41%"},"M3GNet + MEGNet":{"Run Time (h)":87.09,"GPU":0.0,"CPU":76.0,"Slurm Jobs":99.0,"DAF":1.98,"Precision":0.74,"Recall":0.79,"Accuracy":0.82,"F1":0.76,"TPR":0.79,"FPR":0.17,"TNR":0.83,"FNR":0.21,"MAE":0.09,"RMSE":0.13,"R2":0.72,"missing_preds":2576,"missing_percent":"1.00%"}}