update wrenformer preds CSV path

remove models/**/__init__.py files
add figures/2022-11-18-wrenformer-wbm-hull-dist-hist-batches.pdf

Author: janosh

matbench_discovery/plot_scripts/hist_classified_stable_vs_hull_dist.py

@@ -31,7 +31,7 @@
 # %%
 df = pd.read_csv(
     # f"{ROOT}/data/2022-06-11-from-rhys/wren-mp-initial-structures.csv"
-    f"{ROOT}/models/wrenformer/mp/2022-09-20-wrenformer-e_form-ensemble-1-preds.csv"
+    f"{ROOT}/models/wrenformer/2022-11-15-wrenformer-IS2RE-preds.csv"
 ).set_index("material_id")
 
 df["e_above_hull"] = df_wbm.e_above_hull_mp2020_corrected_ppd_mp

matbench_discovery/plot_scripts/hist_classified_stable_vs_hull_dist_batches.py

@@ -39,46 +39,50 @@
     f"{ROOT}/models/m3gnet/2022-10-31-m3gnet-wbm-IS2RE.json.gz"
 ).set_index("material_id")
 dfs["wrenformer"] = pd.read_csv(
-    f"{ROOT}/models/wrenformer/mp/2022-09-20-wrenformer-e_form-ensemble-1-preds.csv"
+    f"{ROOT}/models/wrenformer/2022-11-15-wrenformer-IS2RE-preds.csv"
 ).set_index("material_id")
 dfs["bowsr_megnet"] = pd.read_json(
     f"{ROOT}/models/bowsr/2022-09-22-bowsr-megnet-wbm-IS2RE.json.gz"
 ).set_index("material_id")
 
 
 # %%
+pred_col = "e_form_per_atom_pred"
+target_col = "e_form_per_atom"
 if "wren" in dfs:
     df = dfs["wren"]
     pred_cols = df.filter(regex=r"_pred_\d").columns
     # make sure we average the expected number of ensemble member predictions
     assert len(pred_cols) == 10
-    df["e_form_per_atom_pred"] = df[pred_cols].mean(axis=1)
+    df[pred_col] = df[pred_cols].mean(axis=1)
 if "m3gnet" in dfs:
     df = dfs["m3gnet"]
-    df["e_form_per_atom_pred"] = df.e_form_per_atom_m3gnet
+    df[pred_col] = df.e_form_per_atom_m3gnet
 if "bowsr_megnet" in dfs:
     df = dfs["bowsr_megnet"]
-    df["e_form_per_atom_pred"] = df.e_form_per_atom_bowsr
+    df[pred_col] = df.e_form_per_atom_bowsr
+if "wrenformer" in dfs:
+    pred_col = "e_form_per_atom_mp2020_corrected_pred_ens"
 
 
 # %%
 which_energy: WhichEnergy = "true"
 stability_crit: StabilityCriterion = "energy"
 fig, axs = plt.subplots(2, 3, figsize=(18, 9))
 
-model_name = "m3gnet"
+model_name = "wrenformer"
 df = dfs[model_name]
 
 df["e_above_hull_mp"] = df_wbm.e_above_hull_mp2020_corrected_ppd_mp
-df["e_form_per_atom"] = df_wbm.e_form_per_atom_mp2020_corrected
+df[target_col] = df_wbm.e_form_per_atom_mp2020_corrected  # e_form targets
 
 
 for batch_idx, ax in zip(range(1, 6), axs.flat):
     batch_df = df[df.index.str.startswith(f"wbm-step-{batch_idx}-")]
     assert 1e4 < len(batch_df) < 1e5, print(f"{len(batch_df) = :,}")
 
     ax, metrics = hist_classified_stable_vs_hull_dist(
-        e_above_hull_pred=batch_df.e_form_per_atom_pred - batch_df.e_form_per_atom,
+        e_above_hull_pred=batch_df[pred_col] - batch_df.e_form_per_atom,
         e_above_hull_true=batch_df.e_above_hull_mp,
         which_energy=which_energy,
         stability_crit=stability_crit,
@@ -93,31 +97,28 @@
 
 
 ax, metrics = hist_classified_stable_vs_hull_dist(
-    e_above_hull_pred=df.e_form_per_atom_pred - df.e_form_per_atom,
+    e_above_hull_pred=df[pred_col] - df.e_form_per_atom,
     e_above_hull_true=df.e_above_hull_mp,
     which_energy=which_energy,
     stability_crit=stability_crit,
     ax=axs.flat[-1],
 )
 
 text = f"Enrichment\nFactor = {metrics['enrichment']:.3}"
-ax.text(0.02, 0.25, text, fontsize=16, transform=ax.transAxes)
+ax.text(0.02, 0.3, text, fontsize=16, transform=ax.transAxes)
 
-axs.flat[-1].set(title=f"Combined ({len(df.filter(like='e_').dropna()):,})")
+axs.flat[-1].set(title=f"All batches ({len(df.filter(like='e_').dropna()):,})")
 axs.flat[0].legend(frameon=False, loc="upper left")
 
-img_name = f"{today}-{model_name}-wbm-hull-dist-hist-batches"
-suptitle = img_name.replace("-", "/", 2).replace("-", " ")
-fig.suptitle(suptitle, y=1.07, fontsize=16)
+fig.suptitle(f"{today} {model_name}", y=1.07, fontsize=16)
 
 
 # %%
+img_name = f"{today}-{model_name}-wbm-hull-dist-hist-batches"
 ax.figure.savefig(f"{ROOT}/figures/{img_name}.pdf")
 
 
 # %%
 pymatviz.density_scatter(
-    df=dfs["m3gnet"].query("e_form_per_atom < 5"),
-    x="e_form_per_atom",
-    y="e_form_per_atom_pred",
+    df=dfs[model_name].query(f"{target_col} < 5"), x=target_col, y=pred_col
 )

matbench_discovery/plot_scripts/precision_recall.py

@@ -28,7 +28,7 @@
 ).set_index("material_id")
 
 dfs["wrenformer"] = pd.read_csv(
-    f"{ROOT}/models/wrenformer/mp/2022-09-20-wrenformer-e_form-ensemble-1-preds.csv"
+    f"{ROOT}/models/wrenformer/2022-11-15-wrenformer-IS2RE-preds.csv"
 ).set_index("material_id")
 
 dfs["bowsr_megnet"] = pd.read_json(

matbench_discovery/plot_scripts/rolling_mae_vs_hull_dist.py

@@ -18,7 +18,7 @@
 
 data_path = (
     f"{ROOT}/data/2022-06-11-from-rhys/wren-mp-initial-structures.csv"
-    # f"{ROOT}/models/wrenformer/mp/2022-09-20-wrenformer-e_form-ensemble-1-preds.csv"
+    # f"{ROOT}/models/wrenformer/2022-11-15-wrenformer-IS2RE-preds.csv"
 )
 df = pd.read_csv(data_path).set_index("material_id")
 legend_label = "Wren"

matbench_discovery/plot_scripts/rolling_mae_vs_hull_dist_wbm_batches.py

@@ -19,7 +19,7 @@
 ).set_index("material_id")
 
 df_wrenformer = pd.read_csv(
-    f"{ROOT}/models/wrenformer/mp/2022-11-15-wrenformer-IS2RE-preds.csv"
+    f"{ROOT}/models/wrenformer/2022-11-15-wrenformer-IS2RE-preds.csv"
 ).set_index("material_id")