add figures/2022-11-17-m3gnet-wbm-hull-dist-hist-batches.pdf

add 2022-11-17-{wren,wrenformer}-rolling-mae-vs-hull-dist-wbm-batches

update plot script file paths

Author: janosh

matbench_discovery/plot_scripts/hist_classified_stable_vs_hull_dist_batches.py

@@ -36,7 +36,7 @@
     f"{ROOT}/data/2022-06-11-from-rhys/wren-mp-initial-structures.csv"
 ).set_index("material_id")
 dfs["m3gnet"] = pd.read_json(
-    f"{ROOT}/models/m3gnet/2022-08-16-m3gnet-wbm-IS2RE.json.gz"
+    f"{ROOT}/models/m3gnet/2022-10-31-m3gnet-wbm-IS2RE.json.gz"
 ).set_index("material_id")
 dfs["wrenformer"] = pd.read_csv(
     f"{ROOT}/models/wrenformer/mp/2022-09-20-wrenformer-e_form-ensemble-1-preds.csv"
@@ -55,7 +55,7 @@
     df["e_form_per_atom_pred"] = df[pred_cols].mean(axis=1)
 if "m3gnet" in dfs:
     df = dfs["m3gnet"]
-    df["e_form_per_atom_pred"] = df.e_form_m3gnet
+    df["e_form_per_atom_pred"] = df.e_form_per_atom_m3gnet
 if "bowsr_megnet" in dfs:
     df = dfs["bowsr_megnet"]
     df["e_form_per_atom_pred"] = df.e_form_per_atom_bowsr
@@ -106,7 +106,7 @@
 axs.flat[-1].set(title=f"Combined ({len(df.filter(like='e_').dropna()):,})")
 axs.flat[0].legend(frameon=False, loc="upper left")
 
-img_name = f"{today}-{model_name}-wbm-hull-dist-hist-{which_energy=}-{stability_crit=}"
+img_name = f"{today}-{model_name}-wbm-hull-dist-hist-batches"
 suptitle = img_name.replace("-", "/", 2).replace("-", " ")
 fig.suptitle(suptitle, y=1.07, fontsize=16)
 
@@ -117,9 +117,7 @@
 
 # %%
 pymatviz.density_scatter(
-    dfs["wren"].dropna().e_form_per_atom_pred, dfs["wren"].dropna().e_form_per_atom
-)
-
-pymatviz.density_scatter(
-    dfs["m3gnet"].dropna().e_form_per_atom_pred, dfs["m3gnet"].dropna().e_form_per_atom
+    df=dfs["m3gnet"].query("e_form_per_atom < 5"),
+    x="e_form_per_atom",
+    y="e_form_per_atom_pred",
 )

matbench_discovery/plot_scripts/precision_recall.py

@@ -24,7 +24,7 @@
     dfs[model_name] = df
 
 dfs["m3gnet"] = pd.read_json(
-    f"{ROOT}/models/m3gnet/2022-08-16-m3gnet-wbm-IS2RE.json.gz"
+    f"{ROOT}/models/m3gnet/2022-10-31-m3gnet-wbm-IS2RE.json.gz"
 ).set_index("material_id")
 
 dfs["wrenformer"] = pd.read_csv(

matbench_discovery/plot_scripts/rolling_mae_vs_hull_dist.py

@@ -25,7 +25,7 @@
 
 
 # %%
-rare = "all"
+# rare = "all"
 # from pymatgen.core import Composition
 # rare = "no-lanthanides"
 # df["contains_rare_earths"] = df.composition.map(
@@ -62,5 +62,5 @@
 fig.set_size_inches(10, 9)
 ax.legend(loc="lower right", frameon=False)
 
-img_path = f"{ROOT}/figures/{today}-rolling-mae-vs-hull-dist-{rare=}.pdf"
+img_path = f"{ROOT}/figures/{today}-rolling-mae-vs-hull-dist.pdf"
 # fig.savefig(img_path)

matbench_discovery/plot_scripts/rolling_mae_vs_hull_dist_wbm_batches.py

@@ -14,29 +14,31 @@
 
 
 # %%
-rare = "all"
-
 df_wren = pd.read_csv(
     f"{ROOT}/data/2022-06-11-from-rhys/wren-mp-initial-structures.csv"
 ).set_index("material_id")
 
 df_wrenformer = pd.read_csv(
-    f"{ROOT}/models/wrenformer/mp/2022-09-20-wrenformer-e_form-ensemble-1-preds.csv"
+    f"{ROOT}/models/wrenformer/mp/2022-11-15-wrenformer-IS2RE-preds.csv"
 ).set_index("material_id")
 
 
-df_wrenformer["e_above_hull_mp"] = df_wbm.e_above_hull_mp2020_corrected_ppd_mp
-assert df_wrenformer.e_above_hull_mp.isna().sum() == 0
+# %%
+model_name = "wren"
+df = {"wren": df_wren, "wrenformer": df_wrenformer}[model_name]
+
+df["e_above_hull_mp"] = df_wbm.e_above_hull_mp2020_corrected_ppd_mp
+assert df.e_above_hull_mp.isna().sum() == 0
 
-target_col = "e_form_per_atom"
-# target_col = "e_form_target"
+possible_targets = (
+    "e_form_per_atom_mp2020_corrected e_form_per_atom e_form_target".split()
+)
+target_col = next(filter(lambda x: x in df, possible_targets))
 
 # make sure we average the expected number of ensemble member predictions
-assert df_wrenformer.filter(regex=r"_pred_\d").shape[1] == 10
+assert df.filter(regex=r"_pred_\d").shape[1] == 10
 
-df_wrenformer["e_above_hull_pred"] = (
-    df_wrenformer.filter(regex=r"_pred_\d").mean(axis=1) - df_wrenformer[target_col]
-)
+df["e_above_hull_pred"] = df.filter(regex=r"_pred_\d").mean(axis=1) - df[target_col]
 
 
 # %%
@@ -45,13 +47,15 @@
 assert len(markers) == 5  # number of WBM rounds of element substitution
 
 for idx, marker in enumerate(markers, 1):
-    df = df_wrenformer[df_wrenformer.index.str.startswith(f"wbm-step-{idx}")]
-    title = f"Batch {idx} ({len(df.filter(like='e_').dropna()):,})"
-    assert 1e4 < len(df) < 1e5, print(f"{len(df) = :,}")
+    # select all rows from WBM step=idx
+    df_step = df[df.index.str.startswith(f"wbm-step-{idx}")]
+
+    title = f"Batch {idx} ({len(df_step.filter(like='e_').dropna()):,})"
+    assert 1e4 < len(df_step) < 1e5, print(f"{len(df_step) = :,}")
 
     rolling_mae_vs_hull_dist(
-        e_above_hull_pred=df.e_above_hull_pred,
-        e_above_hull_true=df.e_above_hull_mp,
+        e_above_hull_pred=df_step.e_above_hull_pred,
+        e_above_hull_true=df_step.e_above_hull_mp,
         ax=ax,
         label=title,
         marker=marker,
@@ -62,7 +66,8 @@
 
 
 ax.legend(loc="lower right", frameon=False)
+ax.set(title=f"{today} model={model_name}")
 
 
-img_path = f"{ROOT}/figures/{today}-rolling-mae-vs-hull-dist-wbm-batches-{rare=}.pdf"
-# fig.savefig(img_path)
+img_name = f"{today}-{model_name}-rolling-mae-vs-hull-dist-wbm-batches"
+fig.savefig(f"{ROOT}/figures/{img_name}.pdf")

models/m3gnet/eda_wbm_pre_vs_post_m3gnet_relaxation.py

@@ -30,7 +30,7 @@
 
 
 # %%
-is2re_path_old = f"{ROOT}/models/m3gnet/2022-08-16-m3gnet-wbm-IS2RE.json.gz"
+is2re_path_old = f"{ROOT}/models/m3gnet/2022-10-31-m3gnet-wbm-IS2RE.json.gz"
 df_m3gnet_is2re_old = pd.read_json(is2re_path_old).set_index("material_id")
 
 is2re_path = f"{ROOT}/models/m3gnet/2022-10-31-m3gnet-wbm-IS2RE.json.gz"
@@ -226,5 +226,5 @@
 # %% write df back to compressed JSON
 # filter out columns containing 'rs2re'
 # df_m3gnet_is2re.reset_index().filter(regex="^((?!rs2re).)*$").to_json(
-#     f"{ROOT}/models/m3gnet/2022-08-16-m3gnet-wbm-IS2RE-2.json.gz"
+#     f"{ROOT}/models/m3gnet/2022-10-31-m3gnet-wbm-IS2RE-2.json.gz"
 # ).set_index("material_id")

models/m3gnet/join_m3gnet_results.py

@@ -74,5 +74,5 @@
 out_path = f"{ROOT}/models/m3gnet/{today}-m3gnet-wbm-{task_type}.json.gz"
 df_m3gnet.reset_index().to_json(out_path, default_handler=as_dict_handler)
 
-# out_path = f"{ROOT}/models/m3gnet/2022-08-16-m3gnet-wbm-IS2RE.json.gz"
+# out_path = f"{ROOT}/models/m3gnet/2022-10-31-m3gnet-wbm-IS2RE.json.gz"
 # df_m3gnet = pd.read_json(out_path).set_index("material_id")