diff --git a/data/figshare/1.0.0.json b/data/figshare/1.0.0.json index e026088d..91267a7d 100644 --- a/data/figshare/1.0.0.json +++ b/data/figshare/1.0.0.json @@ -1,4 +1,12 @@ { + "alignn_checkpoint": [ + "https://figshare.com/ndownloader/files/41233560", + "2023-06-02-pbenner-best-alignn-model.pth.zip" + ], + "mace_checkpoint": [ + "https://figshare.com/ndownloader/files/41565618", + "2023-07-14-mace-universal-2-big-128-6.model" + ], "mp_computed_structure_entries": [ "https://figshare.com/ndownloader/files/40344436", "2023-02-07-mp-computed-structure-entries.json.gz" diff --git a/data/mp/build_phase_diagram.py b/data/mp/build_phase_diagram.py index fd104846..7979dbe6 100644 --- a/data/mp/build_phase_diagram.py +++ b/data/mp/build_phase_diagram.py @@ -43,7 +43,7 @@ df = pd.read_json(data_path).set_index("material_id") # drop the structure, just load ComputedEntry, makes the PPD faster to build and load -mp_computed_entries = [ComputedEntry.from_dict(x) for x in tqdm(df.entry)] +mp_computed_entries = [ComputedEntry.from_dict(dct) for dct in tqdm(df.entry)] print(f"{len(mp_computed_entries) = :,} on {today}") # len(mp_computed_entries) = 146,323 on 2022-09-16 diff --git a/data/wbm/compare_cse_vs_ce_mp_2020_corrections.py b/data/wbm/compare_cse_vs_ce_mp_2020_corrections.py index 1959575e..81216656 100644 --- a/data/wbm/compare_cse_vs_ce_mp_2020_corrections.py +++ b/data/wbm/compare_cse_vs_ce_mp_2020_corrections.py @@ -28,10 +28,11 @@ ) cses = [ - ComputedStructureEntry.from_dict(x) for x in tqdm(df_cse.computed_structure_entry) + ComputedStructureEntry.from_dict(dct) + for dct in tqdm(df_cse.computed_structure_entry) ] -ces = [ComputedEntry.from_dict(x) for x in tqdm(df_cse.computed_structure_entry)] +ces = [ComputedEntry.from_dict(dct) for dct in tqdm(df_cse.computed_structure_entry)] warnings.filterwarnings(action="ignore", category=UserWarning, module="pymatgen") diff --git a/data/wbm/fetch_process_wbm_dataset.py b/data/wbm/fetch_process_wbm_dataset.py index acc00222..fea095cf 100644 --- a/data/wbm/fetch_process_wbm_dataset.py +++ b/data/wbm/fetch_process_wbm_dataset.py @@ -502,7 +502,8 @@ def fix_bad_struct_index_mismatch(material_id: str) -> str: assert mat_id == cse["entry_id"], f"{mat_id} != {cse['entry_id']}" df_wbm["cse"] = [ - ComputedStructureEntry.from_dict(x) for x in tqdm(df_wbm.computed_structure_entry) + ComputedStructureEntry.from_dict(dct) + for dct in tqdm(df_wbm.computed_structure_entry) ] # raw WBM ComputedStructureEntries have no energy corrections applied: assert all(cse.uncorrected_energy == cse.energy for cse in df_wbm.cse) @@ -640,6 +641,6 @@ def fix_bad_struct_index_mismatch(material_id: str) -> str: ).set_index("material_id") df_wbm["cse"] = [ - ComputedStructureEntry.from_dict(x) - for x in tqdm(df_wbm.computed_structure_entry) + ComputedStructureEntry.from_dict(dct) + for dct in tqdm(df_wbm.computed_structure_entry) ] diff --git a/matbench_discovery/__init__.py b/matbench_discovery/__init__.py index df54b599..18aaa22f 100644 --- a/matbench_discovery/__init__.py +++ b/matbench_discovery/__init__.py @@ -1,7 +1,6 @@ """Global variables used all across the matbench_discovery package.""" import os -import sys from datetime import datetime ROOT = os.path.dirname(os.path.dirname(__file__)) # repo root directory @@ -13,10 +12,6 @@ for directory in [FIGS, MODELS, FIGSHARE, PDF_FIGS]: os.makedirs(directory, exist_ok=True) -# whether a currently running slurm job is in debug mode -DEBUG = "DEBUG" in os.environ or ( - "slurm-submit" not in sys.argv and "SLURM_JOB_ID" not in os.environ -) # directory to store model checkpoints downloaded from wandb cloud storage CHECKPOINT_DIR = f"{ROOT}/wandb/checkpoints" # wandb / to record new runs to diff --git a/matbench_discovery/data.py b/matbench_discovery/data.py index 52474e02..aee5d5c0 100644 --- a/matbench_discovery/data.py +++ b/matbench_discovery/data.py @@ -232,6 +232,7 @@ def _on_not_found(self, key: str, msg: str) -> None: # type: ignore[override] if answer == "y": load(key) # download and cache data file + # TODO maybe set attrs to None and load file names from Figshare json mp_computed_structure_entries = ( "mp/2023-02-07-mp-computed-structure-entries.json.gz" ) @@ -246,6 +247,8 @@ def _on_not_found(self, key: str, msg: str) -> None: # type: ignore[override] "wbm/2022-10-19-wbm-computed-structure-entries+init-structs.json.bz2" ) wbm_summary = "wbm/2022-10-19-wbm-summary.csv.gz" + alignn_checkpoint = "2023-06-02-pbenner-best-alignn-model.pth.zip" + mace_checkpoint = "2023-07-14-mace-universal-2-big-128-6.model" # data files can be downloaded and cached with matbench_discovery.data.load() diff --git a/matbench_discovery/plots.py b/matbench_discovery/plots.py index 78f04416..5f0f011a 100644 --- a/matbench_discovery/plots.py +++ b/matbench_discovery/plots.py @@ -4,6 +4,7 @@ import math import os +import subprocess from collections import defaultdict from collections.abc import Sequence from pathlib import Path @@ -65,7 +66,7 @@ def unit(text: str) -> str: model_labels = dict( alignn="ALIGNN", alignn_pretrained="ALIGNN Pretrained", - bowsr_megnet="BOWSR + MEGNet", + bowsr_megnet="BOWSR", chgnet="CHGNet", chgnet_megnet="CHGNet + MEGNet", cgcnn_p="CGCNN+P", @@ -74,6 +75,7 @@ def unit(text: str) -> str: m3gnet="M3GNet", m3gnet_direct="M3GNet DIRECT", m3gnet_ms="M3GNet MS", + mace="MACE", megnet="MEGNet", voronoi_rf="Voronoi RF", wrenformer="Wrenformer", @@ -874,38 +876,81 @@ def df_to_svelte_table( def df_to_pdf( styler: Styler, file_path: str | Path, crop: bool = True, **kwargs: Any ) -> None: - """Export a pandas Styler to PDF. + """Export a pandas Styler to PDF with WeasyPrint. Args: styler (Styler): Styler object to export. - file_path (str): Path to save the PDF to. Requires pdfkit. - crop (bool): Whether to crop the PDF margins. Requires pdfCropMargins. Defaults - to True. + file_path (str): Path to save the PDF to. Requires WeasyPrint. + crop (bool): Whether to crop the PDF margins. Requires pdfCropMargins. + Defaults to True. **kwargs: Keyword arguments passed to Styler.to_html(). """ try: - # pdfkit used to export pandas Styler to PDF, requires: - # pip install pdfkit && brew install homebrew/cask/wkhtmltopdf - import pdfkit + from weasyprint import HTML except ImportError as exc: - raise ImportError( - "pdfkit not installed\nrun pip install pdfkit && brew install " - "homebrew/cask/wkhtmltopdf\n(brew is macOS only, use apt on linux)" - ) from exc - - pdfkit.from_string(styler.to_html(**kwargs), file_path) - if not crop: - return + msg = "weasyprint not installed\nrun pip install weasyprint" + raise ImportError(msg) from exc + + html_str = styler.to_html(**kwargs) + + # CSS to adjust layout and margins + html_str = f""" + + {html_str} + """ + + html = HTML(string=html_str) + + html.write_pdf(file_path) + + if crop: + normalize_and_crop_pdf(file_path) + + +def normalize_and_crop_pdf(file_path: str | Path) -> None: + """Normalize a PDF using Ghostscript and then crop it. + Without gs normalization, pdfCropMargins sometimes corrupts the PDF. + + Args: + file_path (str | Path): Path to the PDF file. + """ try: - # needed to auto-crop large white margins from PDF - # pip install pdfCropMargins - from pdfCropMargins import crop as crop_pdf + normalized_file_path = f"{file_path}_normalized.pdf" + from pdfCropMargins import crop + + # Normalize the PDF with Ghostscript + subprocess.run( + [ + "gs", + "-sDEVICE=pdfwrite", + "-dCompatibilityLevel=1.4", + "-dPDFSETTINGS=/default", + "-dNOPAUSE", + "-dQUIET", + "-dBATCH", + f"-sOutputFile={normalized_file_path}", + str(file_path), + ] + ) - # Remove PDF margins - cropped_file_path, _exit_code, _stdout, _stderr = crop_pdf( - ["--percentRetain", "0", file_path] + # Crop the normalized PDF + cropped_file_path, exit_code, stdout, stderr = crop( + ["--percentRetain", "0", normalized_file_path] ) - os.replace(cropped_file_path, file_path) + + if stderr: + print(f"pdfCropMargins {stderr=}") + # something went wrong, remove the cropped PDF + os.remove(cropped_file_path) + else: + # replace the original PDF with the cropped one + os.replace(cropped_file_path, str(file_path)) + + os.remove(normalized_file_path) + except ImportError as exc: msg = "pdfCropMargins not installed\nrun pip install pdfCropMargins" raise ImportError(msg) from exc diff --git a/matbench_discovery/preds.py b/matbench_discovery/preds.py index 5c17d854..097e1094 100644 --- a/matbench_discovery/preds.py +++ b/matbench_discovery/preds.py @@ -47,6 +47,9 @@ class PredFiles(Files): # m3gnet_direct = "m3gnet/2023-05-30-m3gnet-direct-wbm-IS2RE.csv.gz" # m3gnet_ms = "m3gnet/2023-06-01-m3gnet-manual-sampling-wbm-IS2RE.csv.gz" + # MACE trained on original M3GNet training set + mace = "mace/2023-07-23-mace-wbm-IS2RE-FIRE.csv.gz" + # original MEGNet straight from publication, not re-trained megnet = "megnet/2022-11-18-megnet-wbm-IS2RE.csv.gz" # CHGNet-relaxed structures fed into MEGNet for formation energy prediction @@ -106,8 +109,15 @@ def load_df_wbm_with_preds( df_out = df_wbm.copy() for model_name, df in dfs.items(): model_key = model_name.lower().replace(" + ", "_").replace(" ", "_") - if (col := f"e_form_per_atom_{model_key}") in df: - df_out[model_name] = df[col] + + cols = [col for col in df if col.startswith(f"e_form_per_atom_{model_key}")] + if cols: + if len(cols) > 1: + print( + f"Warning: multiple pred cols for {model_name=}, using {cols[0]!r} " + f"out of {cols=}" + ) + df_out[model_name] = df[cols[0]] elif pred_cols := list(df.filter(like="_pred_ens")): assert len(pred_cols) == 1 diff --git a/models/alignn/test_alignn.py b/models/alignn/test_alignn.py index 44f50770..30a703a2 100644 --- a/models/alignn/test_alignn.py +++ b/models/alignn/test_alignn.py @@ -17,7 +17,7 @@ from sklearn.metrics import r2_score from tqdm import tqdm -from matbench_discovery import DEBUG, today +from matbench_discovery import today from matbench_discovery.data import DATA_FILES, df_wbm from matbench_discovery.plots import wandb_scatter from matbench_discovery.slurm import slurm_submit @@ -36,7 +36,7 @@ input_col = "initial_structure" id_col = "material_id" device = "cuda" if torch.cuda.is_available() else "cpu" -job_name = f"{model_name}-wbm-{task_type}{'-debug' if DEBUG else ''}" +job_name = f"{model_name}-wbm-{task_type}" out_dir = os.getenv("SBATCH_OUTPUT", f"{module_dir}/{today}-{job_name}") @@ -85,15 +85,15 @@ assert input_col in df_in, f"{input_col=} not in {list(df_in)}" df_in[input_col] = [ - JarvisAtomsAdaptor.get_atoms(Structure.from_dict(x)) - for x in tqdm(df_in[input_col], leave=False, desc="Converting to JARVIS atoms") + JarvisAtomsAdaptor.get_atoms(Structure.from_dict(dct)) + for dct in tqdm(df_in[input_col], leave=False, desc="Converting to JARVIS atoms") ] # %% run_params = dict( data_path=data_path, - **{f"{dep}_version": version(dep) for dep in ("megnet", "numpy")}, + versions={dep: version(dep) for dep in ("megnet", "numpy")}, model_name=model_name, task_type=task_type, target_col=target_col, diff --git a/models/alignn/train_alignn.py b/models/alignn/train_alignn.py index 98874491..baa99d13 100644 --- a/models/alignn/train_alignn.py +++ b/models/alignn/train_alignn.py @@ -18,7 +18,7 @@ from torch.utils.data import DataLoader from tqdm import tqdm -from matbench_discovery import DEBUG, today +from matbench_discovery import today from matbench_discovery.data import DATA_FILES from matbench_discovery.slurm import slurm_submit @@ -35,7 +35,7 @@ input_col = "atoms" id_col = "material_id" device = "cuda" if torch.cuda.is_available() else "cpu" -job_name = f"train-{model_name}{'-debug' if DEBUG else ''}" +job_name = f"train-{model_name}" pred_col = "e_form_per_atom_alignn" @@ -49,7 +49,7 @@ slurm_vars = slurm_submit( job_name=job_name, # partition="perlmuttter", - account="matgen_g", + account="matgen", time="4:0:0", out_dir=out_dir, slurm_flags="--qos regular --constraint gpu --gpus 1", @@ -79,7 +79,7 @@ # %% run_params = dict( data_path=DATA_FILES.mp_energies, - **{f"{dep}_version": version(dep) for dep in ("alignn", "numpy", "torch", "dgl")}, + versions={dep: version(dep) for dep in ("alignn", "numpy", "torch", "dgl")}, model_name=model_name, target_col=target_col, df=dict(shape=str(df_in.shape), columns=", ".join(df_in)), diff --git a/models/bowsr/join_bowsr_results.py b/models/bowsr/join_bowsr_results.py index 2570eddd..ab510c2c 100644 --- a/models/bowsr/join_bowsr_results.py +++ b/models/bowsr/join_bowsr_results.py @@ -8,7 +8,6 @@ import pymatviz from tqdm import tqdm -from matbench_discovery import today from matbench_discovery.data import DATA_FILES __author__ = "Janosh Riebesell" @@ -66,7 +65,7 @@ # %% -out_path = f"{module_dir}/{today}-bowsr-megnet-wbm-{task_type}" +out_path = f"{module_dir}/{glob_pattern.split('/*')[0]}" df_bowsr = df_bowsr.round(4) # save energy and formation energy as fast-loading CSV df_bowsr.select_dtypes("number").to_csv(f"{out_path}.csv") diff --git a/models/bowsr/metadata.yml b/models/bowsr/metadata.yml index 6e0363e6..938cdf56 100644 --- a/models/bowsr/metadata.yml +++ b/models/bowsr/metadata.yml @@ -1,4 +1,4 @@ -model_name: BOWSR + MEGNet +model_name: BOWSR model_version: 2022.9.20 matbench_discovery_version: 1.0 date_added: "2022-11-17" diff --git a/models/bowsr/test_bowsr.py b/models/bowsr/test_bowsr.py index cbf8d22e..d857c8d1 100644 --- a/models/bowsr/test_bowsr.py +++ b/models/bowsr/test_bowsr.py @@ -14,7 +14,7 @@ from pymatgen.core import Structure from tqdm import tqdm -from matbench_discovery import DEBUG, timestamp, today +from matbench_discovery import timestamp, today from matbench_discovery.data import DATA_FILES, as_dict_handler from matbench_discovery.slurm import slurm_submit @@ -35,7 +35,7 @@ # post submission slurm_max_parallel = 100 energy_model = "megnet" -job_name = f"bowsr-{energy_model}-wbm-{task_type}{'-debug' if DEBUG else ''}" +job_name = f"bowsr-{energy_model}-wbm-{task_type}" out_dir = os.getenv("SBATCH_OUTPUT", f"{module_dir}/{today}-{job_name}") data_path = { @@ -66,11 +66,11 @@ out_path = f"{out_dir}/bowsr-preds-{slurm_array_task_id}.json.gz" if os.path.isfile(out_path): - raise SystemExit(f"{out_path = } already exists, exciting early") + raise SystemExit(f"{out_path=} already exists, exciting early") print(f"\nJob started running {timestamp}") print(f"{data_path = }") -print(f"{out_path = }") +print(f"{out_path=}") df_in: pd.DataFrame = np.array_split( pd.read_json(data_path).set_index("material_id"), slurm_array_task_count @@ -91,7 +91,7 @@ data_path=data_path, df=dict(shape=str(df_in.shape), columns=", ".join(df_in)), energy_model=energy_model, - **{f"{dep}_version": version(dep) for dep in ("maml", "numpy", energy_model)}, + versions={dep: version(dep) for dep in ("maml", "numpy", energy_model)}, optimize_kwargs=optimize_kwargs, task_type=task_type, slurm_vars=slurm_vars, @@ -110,7 +110,7 @@ structures = df_in[input_col].map(Structure.from_dict).to_dict() -for material_id in tqdm(structures, desc="Main loop", disable=None): +for material_id in tqdm(structures, desc="Relaxing", disable=None): structure = structures[material_id] if material_id in relax_results: continue @@ -125,8 +125,8 @@ try: struct_bowsr, energy_bowsr = optimizer.get_optimized_structure_and_energy() - except Exception as error: - print(f"Failed to relax {material_id}: {error}") + except Exception as exc: + print(f"Failed to relax {material_id}: {exc}") results = { f"e_form_per_atom_bowsr_{energy_model}": model.predict_energy(struct_bowsr), diff --git a/models/cgcnn/test_cgcnn.py b/models/cgcnn/test_cgcnn.py index 55b52cf8..ecdc64f7 100644 --- a/models/cgcnn/test_cgcnn.py +++ b/models/cgcnn/test_cgcnn.py @@ -2,7 +2,6 @@ from __future__ import annotations import os -import sys from importlib.metadata import version import pandas as pd @@ -14,7 +13,7 @@ from torch.utils.data import DataLoader from tqdm import tqdm -from matbench_discovery import CHECKPOINT_DIR, DEBUG, ROOT, WANDB_PATH, today +from matbench_discovery import CHECKPOINT_DIR, ROOT, WANDB_PATH, today from matbench_discovery.data import DATA_FILES, df_wbm from matbench_discovery.plots import wandb_scatter from matbench_discovery.slurm import slurm_submit @@ -29,8 +28,8 @@ """ task_type = "IS2RE" -debug = "slurm-submit" in sys.argv -job_name = f"test-cgcnn-wbm-{task_type}{'-debug' if DEBUG else ''}" +debug = False +job_name = f"test-cgcnn-wbm-{task_type}" module_dir = os.path.dirname(__file__) out_dir = os.getenv("SBATCH_OUTPUT", f"{module_dir}/{today}-{job_name}") @@ -49,7 +48,7 @@ "IS2RE": DATA_FILES.wbm_initial_structures, "RS2RE": DATA_FILES.wbm_computed_structure_entries, "IS2RE-debug": f"{ROOT}/data/wbm/2022-10-19-wbm-init-structs.json-1k-samples.bz2", -}[task_type + ("-debug" if DEBUG else "")] +}[task_type] input_col = {"IS2RE": "initial_structure", "RS2RE": "relaxed_structure"}[task_type] df = pd.read_json(data_path).set_index("material_id") @@ -85,7 +84,7 @@ run_params = dict( data_path=data_path, df=dict(shape=str(df.shape), columns=", ".join(df)), - **{f"{dep}_version": version(dep) for dep in ("aviary", "numpy", "torch")}, + versions={dep: version(dep) for dep in ("aviary", "numpy", "torch")}, ensemble_size=len(runs), task_type=task_type, target_col=e_form_col, diff --git a/models/cgcnn/train_cgcnn.py b/models/cgcnn/train_cgcnn.py index 218e875a..7e626c3b 100644 --- a/models/cgcnn/train_cgcnn.py +++ b/models/cgcnn/train_cgcnn.py @@ -11,7 +11,7 @@ from torch.utils.data import DataLoader from tqdm import tqdm, trange -from matbench_discovery import DEBUG, WANDB_PATH, timestamp, today +from matbench_discovery import WANDB_PATH, timestamp, today from matbench_discovery.data import DATA_FILES from matbench_discovery.slurm import slurm_submit from matbench_discovery.structure import perturb_structure @@ -32,7 +32,7 @@ # 0 for no perturbation, n>1 means train on n perturbations of each crystal # in the training set all assigned the same original target energy n_perturb = 0 -job_name = f"train-cgcnn-robust-{n_perturb=}{'-debug' if DEBUG else ''}" +job_name = f"train-cgcnn-robust-{n_perturb=}" print(f"{job_name=}") robust = "robust" in job_name.lower() ensemble_size = 10 @@ -107,7 +107,7 @@ run_params = dict( data_path=data_path, batch_size=batch_size, - **{f"{dep}_version": version(dep) for dep in ("aviary", "numpy", "torch")}, + versions={dep: version(dep) for dep in ("aviary", "numpy", "torch")}, train_df=dict(shape=str(train_data.df.shape), columns=", ".join(train_df)), test_df=dict(shape=str(test_data.df.shape), columns=", ".join(test_df)), slurm_vars=slurm_vars, diff --git a/models/chgnet/join_chgnet_results.py b/models/chgnet/join_chgnet_results.py index ea26fc41..c2b9e093 100644 --- a/models/chgnet/join_chgnet_results.py +++ b/models/chgnet/join_chgnet_results.py @@ -14,7 +14,6 @@ from pymatviz import density_scatter from tqdm import tqdm -from matbench_discovery import today from matbench_discovery.data import as_dict_handler from matbench_discovery.energy import get_e_form_per_atom from matbench_discovery.preds import df_preds, e_form_col @@ -64,7 +63,7 @@ # %% -out_path = f"{module_dir}/{today}-chgnet-wbm-{task_type}" +out_path = f"{module_dir}/{glob_pattern.split('/*')[0]}" df_chgnet = df_chgnet.round(4) df_chgnet.select_dtypes("number").to_csv(f"{out_path}.csv.gz") df_chgnet.reset_index().to_json(f"{out_path}.json.gz", default_handler=as_dict_handler) diff --git a/models/chgnet/test_chgnet.py b/models/chgnet/test_chgnet.py index cdefefb9..1d24e6c7 100644 --- a/models/chgnet/test_chgnet.py +++ b/models/chgnet/test_chgnet.py @@ -20,7 +20,7 @@ from pymatgen.core import Structure from tqdm import tqdm -from matbench_discovery import DEBUG, timestamp, today +from matbench_discovery import timestamp, today from matbench_discovery.data import DATA_FILES, as_dict_handler, df_wbm from matbench_discovery.plots import wandb_scatter from matbench_discovery.slurm import slurm_submit @@ -32,7 +32,7 @@ module_dir = os.path.dirname(__file__) # set large job array size for smaller data splits and faster testing/debugging slurm_array_task_count = 100 -job_name = f"chgnet-wbm-{task_type}{'-debug' if DEBUG else ''}" +job_name = f"chgnet-wbm-{task_type}" out_dir = os.getenv("SBATCH_OUTPUT", f"{module_dir}/{today}-{job_name}") slurm_vars = slurm_submit( @@ -51,7 +51,7 @@ out_path = f"{out_dir}/chgnet-preds-{slurm_array_task_id}.json.gz" if os.path.isfile(out_path): - raise SystemExit(f"{out_path = } already exists, exciting early") + raise SystemExit(f"{out_path=} already exists, exciting early") # %% @@ -70,7 +70,7 @@ run_params = dict( data_path=data_path, - **{f"{dep}_version": version(dep) for dep in ("chgnet", "numpy", "torch")}, + versions={dep: version(dep) for dep in ("chgnet", "numpy", "torch")}, task_type=task_type, df=dict(shape=str(df_in.shape), columns=", ".join(df_in)), slurm_vars=slurm_vars, @@ -91,21 +91,20 @@ structures = df_in[input_col].map(Structure.from_dict).to_dict() -for material_id in tqdm(structures, disable=None): +for material_id in tqdm(structures, desc="Relaxing", disable=None): if material_id in relax_results: continue try: relax_result = chgnet.relax( structures[material_id], verbose=False, steps=max_steps ) - except Exception as error: - print(f"Failed to relax {material_id}: {error}") - continue - relax_results[material_id] = { - "chgnet_structure": relax_result["final_structure"], - "chgnet_trajectory": relax_result["trajectory"].__dict__, - e_pred_col: relax_result["trajectory"].energies[-1], - } + relax_results[material_id] = { + "chgnet_structure": relax_result["final_structure"], + "chgnet_trajectory": relax_result["trajectory"].__dict__, + e_pred_col: relax_result["trajectory"].energies[-1], + } + except Exception as exc: + print(f"Failed to relax {material_id}: {exc}") # %% diff --git a/models/m3gnet/join_m3gnet_results.py b/models/m3gnet/join_m3gnet_results.py index 1ef88e39..5d4c2aaf 100644 --- a/models/m3gnet/join_m3gnet_results.py +++ b/models/m3gnet/join_m3gnet_results.py @@ -17,7 +17,6 @@ from pymatgen.entries.computed_entries import ComputedStructureEntry from tqdm import tqdm -from matbench_discovery import today from matbench_discovery.data import DATA_FILES, as_dict_handler from matbench_discovery.energy import get_e_form_per_atom @@ -60,7 +59,8 @@ ) df_cse["cse"] = [ - ComputedStructureEntry.from_dict(x) for x in tqdm(df_cse.computed_structure_entry) + ComputedStructureEntry.from_dict(dct) + for dct in tqdm(df_cse.computed_structure_entry) ] @@ -91,7 +91,7 @@ # %% -out_path = f"{module_dir}/{today}-m3gnet-{model_type}-wbm-{task_type}" +out_path = f"{module_dir}/{glob_pattern.split('/*')[0]}" df_m3gnet = df_m3gnet.round(4) df_m3gnet.select_dtypes("number").to_csv(f"{out_path}.csv.gz") df_m3gnet.reset_index().to_json(f"{out_path}.json.gz", default_handler=as_dict_handler) diff --git a/models/m3gnet/test_m3gnet.py b/models/m3gnet/test_m3gnet.py index 6f91a4e7..4210d2e5 100644 --- a/models/m3gnet/test_m3gnet.py +++ b/models/m3gnet/test_m3gnet.py @@ -20,7 +20,7 @@ from pymatgen.core import Structure from tqdm import tqdm -from matbench_discovery import DEBUG, ROOT, timestamp, today +from matbench_discovery import ROOT, timestamp, today from matbench_discovery.data import DATA_FILES, as_dict_handler from matbench_discovery.slurm import slurm_submit @@ -33,7 +33,7 @@ model_type: Literal["orig", "direct", "ms"] = "ms" # set large job array size for smaller data splits and faster testing/debugging slurm_array_task_count = 100 -job_name = f"m3gnet-{model_type}-wbm-{task_type}{'-debug' if DEBUG else ''}" +job_name = f"m3gnet-{model_type}-wbm-{task_type}" out_dir = os.getenv("SBATCH_OUTPUT", f"{module_dir}/{today}-{job_name}") slurm_vars = slurm_submit( @@ -55,7 +55,7 @@ out_path = f"{out_dir}/m3gnet-preds-{slurm_array_task_id}.json.gz" if os.path.isfile(out_path): - raise SystemExit(f"{out_path = } already exists, exciting early") + raise SystemExit(f"{out_path=} already exists, exciting early") warnings.filterwarnings(action="ignore", category=UserWarning, module="pymatgen") warnings.filterwarnings(action="ignore", category=UserWarning, module="tensorflow") @@ -76,7 +76,7 @@ run_params = dict( data_path=data_path, - **{f"{dep}_version": version(dep) for dep in ("m3gnet", "numpy")}, + versions={dep: version(dep) for dep in ("m3gnet", "numpy")}, task_type=task_type, df=dict(shape=str(df_in.shape), columns=", ".join(df_in)), slurm_vars=slurm_vars, @@ -92,7 +92,7 @@ checkpoint = f"{ROOT}/models/m3gnet/2023-05-26-DI-DFTstrictF10-TTRS-128U-442E" if model_type == "ms": checkpoint = f"{ROOT}/models/m3gnet/2023-05-26-MS-DFTstrictF10-128U-154E" -megnet = Relaxer(potential=checkpoint) # load pre-trained M3GNet model +m3gnet = Relaxer(potential=checkpoint) # load pre-trained M3GNet model relax_results: dict[str, dict[str, Any]] = {} input_col = {"IS2RE": "initial_structure", "RS2RE": "relaxed_structure"}[task_type] @@ -101,20 +101,18 @@ structures = df_in[input_col].map(Structure.from_dict).to_dict() -for material_id in tqdm(structures, disable=None): +for material_id in tqdm(structures, desc="Relaxing", disable=None): if material_id in relax_results: continue try: - relax_result = megnet.relax(structures[material_id]) - except Exception as error: - print(f"Failed to relax {material_id}: {error}") - continue - - relax_results[material_id] = { - f"m3gnet_{model_type}_structure": relax_result["final_structure"], - f"m3gnet_{model_type}_trajectory": relax_result["trajectory"].__dict__, - e_pred_col: relax_result["trajectory"].energies[-1], - } + relax_result = m3gnet.relax(structures[material_id]) + relax_results[material_id] = { + f"m3gnet_{model_type}_structure": relax_result["final_structure"], + f"m3gnet_{model_type}_trajectory": relax_result["trajectory"].__dict__, + e_pred_col: relax_result["trajectory"].energies[-1], + } + except Exception as exc: + print(f"Failed to relax {material_id}: {exc}") # %% diff --git a/models/mace/2023-07-22-mace-wbm-IS2RE.csv.gz b/models/mace/2023-07-22-mace-wbm-IS2RE.csv.gz new file mode 100644 index 00000000..4eeda233 Binary files /dev/null and b/models/mace/2023-07-22-mace-wbm-IS2RE.csv.gz differ diff --git a/models/mace/2023-07-23-mace-wbm-IS2RE-FIRE.csv.gz b/models/mace/2023-07-23-mace-wbm-IS2RE-FIRE.csv.gz new file mode 100644 index 00000000..812390b8 Binary files /dev/null and b/models/mace/2023-07-23-mace-wbm-IS2RE-FIRE.csv.gz differ diff --git a/models/mace/metadata.yml b/models/mace/metadata.yml new file mode 100644 index 00000000..75b9e95e --- /dev/null +++ b/models/mace/metadata.yml @@ -0,0 +1,42 @@ +model_name: MACE +model_version: 0.2.0-alpha +matbench_discovery_version: 1.0 +date_added: "2023-07-14" +date_published: "2022-05-13" +authors: + - name: Ilyes Batatia + affiliation: University of Cambridge + email: ilyes.batatia@ens-paris-saclay.fr + orcid: https://orcid.org/0000-0001-6915-9851 + - name: David P Kovacs + affiliation: University of Cambridge + orcid: https://orcid.org/0000-0002-0854-2635 + - name: Gregor Simm + affiliation: University of Cambridge + orcid: https://orcid.org/0000-0001-6815-352X + - name: Christoph Ortner + affiliation: University of Cambridge + orcid: https://orcid.org/0000-0003-1498-8120 + - name: Gabor Csanyi + affiliation: University of Cambridge + orcid: https://orcid.org/0000-0002-8180-2034 +repo: https://github.com/ACEsuit/mace +doi: https://doi.org/10.48550/arXiv.2205.06643 +preprint: https://arxiv.org/abs/2205.06643 +requirements: + torch: 2.0.1 + ase: 3.22.1 + pymatgen: 2023.7.14 + numpy: 1.25.0 +trained_for_benchmark: false + +hyperparams: + max_force: 0.05 + max_steps: 500 + ase_optimizer: FIRE + +notes: + description: | + The Many-body Atomic Convolutional Energies (MACE) is a higher-order equivariant message-passing neural network for fast and accurate force fields. + training: Using pre-trained model released with paper. Training set unspecified at time of writing. + corrections: None diff --git a/models/mace/readme.md b/models/mace/readme.md new file mode 100644 index 00000000..e5397e30 --- /dev/null +++ b/models/mace/readme.md @@ -0,0 +1,5 @@ +## MACE formation energy predictions on WBM test set + +This submission uses the [`2023-07-14-mace-universal-2-big-128-6.model`](https://figshare.com/ndownloader/files/41565618) checkpoint pre-trained on the [original M3GNet training set](https://figshare.com/articles/dataset/MPF_2021_2_8/19470599). + +MACE relaxed each test set structure until the maximum force in the training set dropped below 0.05 eV/Å or 500 optimization steps were reached, whichever occurred first. diff --git a/models/mace/test_mace.py b/models/mace/test_mace.py new file mode 100644 index 00000000..5917f31d --- /dev/null +++ b/models/mace/test_mace.py @@ -0,0 +1,156 @@ +# %% +from __future__ import annotations + +import os +from importlib.metadata import version +from typing import Any + +import numpy as np +import pandas as pd +import wandb +from ase.constraints import ExpCellFilter +from ase.optimize import FIRE, LBFGS +from mace.calculators.mace import MACECalculator +from pymatgen.core import Structure +from pymatgen.core.trajectory import Trajectory +from pymatgen.io.ase import AseAtomsAdaptor +from tqdm import tqdm + +from matbench_discovery import ROOT, timestamp, today +from matbench_discovery.data import DATA_FILES, as_dict_handler, df_wbm +from matbench_discovery.plots import wandb_scatter +from matbench_discovery.slurm import slurm_submit + +__author__ = "Janosh Riebesell" +__date__ = "2023-03-01" + +task_type = "IS2RE" # "RS2RE" +module_dir = os.path.dirname(__file__) +# set large job array size for smaller data splits and faster testing/debugging +slurm_array_task_count = 100 +ase_optimizer = "FIRE" +job_name = f"mace-wbm-{task_type}-{ase_optimizer}" +out_dir = os.getenv("SBATCH_OUTPUT", f"{module_dir}/{today}-{job_name}") +relax_cell = True + +slurm_vars = slurm_submit( + job_name=job_name, + out_dir=out_dir, + account="matgen", + time="12:0:0", + array=f"1-{slurm_array_task_count}", + slurm_flags="--qos regular --constraint gpu --gpus 1", + pre_cmd="module load pytorch/2.0.1; . ~/.venv/py311/bin/activate;", +) + + +# %% +slurm_array_task_id = int(os.getenv("SLURM_ARRAY_TASK_ID", "0")) +out_path = f"{out_dir}/mace-preds-{slurm_array_task_id}.json.gz" + +if os.path.isfile(out_path): + raise SystemExit(f"{out_path=} already exists, exciting early") + + +# %% +data_path = { + "RS2RE": DATA_FILES.wbm_computed_structure_entries, + "IS2RE": DATA_FILES.wbm_initial_structures, +}[task_type] +print(f"\nJob started running {timestamp}") +print(f"{data_path=}") +e_pred_col = "mace_energy" +max_steps = 500 +force_max = 0.05 # Run until the forces are smaller than this in eV/A + +df_in: pd.DataFrame = np.array_split( + pd.read_json(data_path).set_index("material_id"), slurm_array_task_count +)[slurm_array_task_id - 1] + +run_params = dict( + data_path=data_path, + versions={dep: version(dep) for dep in ("mace", "numpy", "torch")}, + task_type=task_type, + df=dict(shape=str(df_in.shape), columns=", ".join(df_in)), + slurm_vars=slurm_vars, + max_steps=max_steps, + relax_cell=relax_cell, + force_max=force_max, + ase_optimizer=ase_optimizer, +) + +run_name = f"{job_name}-{slurm_array_task_id}" +wandb.init(project="matbench-discovery", name=run_name, config=run_params) + + +# %% +checkpoint = f"{ROOT}/models/mace/2023-07-14-mace-universal-2-big-128-6.model" +# load MACE model pre-trained on M3GNet training set by original MACE authors +mace_calc = MACECalculator(checkpoint, device="cuda", default_dtype="float32") +relax_results: dict[str, dict[str, Any]] = {} +input_col = {"IS2RE": "initial_structure", "RS2RE": "relaxed_structure"}[task_type] + +if task_type == "RS2RE": + df_in[input_col] = [x["structure"] for x in df_in.computed_structure_entry] + +structs = df_in[input_col].map(Structure.from_dict).to_dict() + +for material_id in tqdm(structs, desc="Relaxing", disable=None): + if material_id in relax_results: + continue + try: + atoms = AseAtomsAdaptor.get_atoms(structs[material_id]) + atoms.calc = mace_calc + if relax_cell: + atoms = ExpCellFilter(atoms) + optim_cls = {"FIRE": FIRE, "LBFGS": LBFGS}[ase_optimizer] + optimizer = optim_cls(atoms, logfile="/dev/null") + + coords, lattices = [], [] + # attach observer functions to the optimizer + optimizer.attach(lambda: coords.append(atoms.get_positions())) # noqa: B023 + optimizer.attach(lambda: lattices.append(atoms.get_cell())) # noqa: B023 + + optimizer.run(fmax=force_max, steps=max_steps) + mace_traj = Trajectory( + species=structs[material_id].species, + coords=coords, + lattice=lattices, + constant_lattice=False, + ) + else: + mace_traj = None + mace_energy = atoms.get_potential_energy() + mace_struct = AseAtomsAdaptor.get_structure( + atoms.atoms if relax_cell else atoms + ) + + relax_results[material_id] = { + "mace_structure": mace_struct, + "mace_energy": mace_energy, + "mace_trajectory": mace_traj, # Add the trajectory to the results + } + except Exception as exc: + print(f"Failed to relax {material_id}: {exc}") + continue + + +# %% +df_out = pd.DataFrame(relax_results).T +df_out.index.name = "material_id" + +df_out.reset_index().to_json(out_path, default_handler=as_dict_handler) + + +# %% +df_wbm[e_pred_col] = df_out[e_pred_col] +table = wandb.Table( + dataframe=df_wbm.dropna()[ + ["uncorrected_energy", e_pred_col, "formula"] + ].reset_index() +) + +title = f"MACE {task_type} ({len(df_out):,})" +wandb_scatter(table, fields=dict(x="uncorrected_energy", y=e_pred_col), title=title) + +wandb.log_artifact(out_path, type=f"mace-wbm-{task_type}") diff --git a/models/megnet/test_megnet.py b/models/megnet/test_megnet.py index c6c8b637..6a759385 100644 --- a/models/megnet/test_megnet.py +++ b/models/megnet/test_megnet.py @@ -20,7 +20,7 @@ from sklearn.metrics import r2_score from tqdm import tqdm -from matbench_discovery import DEBUG, timestamp, today +from matbench_discovery import timestamp, today from matbench_discovery.data import DATA_FILES, df_wbm from matbench_discovery.plots import wandb_scatter from matbench_discovery.preds import PRED_FILES @@ -31,7 +31,7 @@ task_type = "chgnet_structure" module_dir = os.path.dirname(__file__) -job_name = f"megnet-wbm-{task_type}{'-debug' if DEBUG else ''}" +job_name = f"megnet-wbm-{task_type}" out_dir = os.getenv("SBATCH_OUTPUT", f"{module_dir}/{today}-{job_name}") slurm_array_task_count = 20 @@ -53,7 +53,7 @@ slurm_array_task_id = int(os.getenv("SLURM_ARRAY_TASK_ID", "0")) out_path = f"{out_dir}/megnet-e-form-preds.csv.gz" if os.path.isfile(out_path): - raise SystemExit(f"{out_path = } already exists, exciting early") + raise SystemExit(f"{out_path=} already exists, exciting early") data_path = { "IS2RE": DATA_FILES.wbm_initial_structures, @@ -75,7 +75,7 @@ # %% run_params = dict( data_path=data_path, - **{f"{dep}_version": version(dep) for dep in ("megnet", "numpy")}, + versions={dep: version(dep) for dep in ("megnet", "numpy")}, model_name=model_name, task_type=task_type, target_col=e_form_col, diff --git a/models/voronoi/join_voronoi_features.py b/models/voronoi/join_voronoi_features.py index 689abb6a..353064ff 100644 --- a/models/voronoi/join_voronoi_features.py +++ b/models/voronoi/join_voronoi_features.py @@ -39,5 +39,5 @@ # %% -out_path = f"{module_dir}/{date}-features-{data}.csv.bz2" +out_path = f"{module_dir}/{glob_pattern.split('-*')[0]}.csv.bz2" df_features.to_csv(out_path) diff --git a/models/voronoi/train_test_voronoi_rf.py b/models/voronoi/train_test_voronoi_rf.py index 20a878c9..63c9808a 100644 --- a/models/voronoi/train_test_voronoi_rf.py +++ b/models/voronoi/train_test_voronoi_rf.py @@ -12,7 +12,7 @@ from sklearn.metrics import r2_score from sklearn.pipeline import Pipeline -from matbench_discovery import DEBUG, today +from matbench_discovery import today from matbench_discovery.data import DATA_FILES, df_wbm, glob_to_df from matbench_discovery.plots import wandb_scatter from matbench_discovery.slurm import slurm_submit @@ -30,9 +30,9 @@ out_dir = f"{module_dir}/{today}-train-test" out_path = f"{out_dir}/e-form-preds-{task_type}.csv.gz" if os.path.isfile(out_path): - raise SystemExit(f"{out_path = } already exists, exciting early") + raise SystemExit(f"{out_path=} already exists, exciting early") -job_name = f"train-test-voronoi-rf{'-debug' if DEBUG else ''}" +job_name = "train-test-voronoi-rf" slurm_vars = slurm_submit( job_name=job_name, @@ -72,7 +72,7 @@ train_path=train_path, test_path=test_path, mp_energies_path=DATA_FILES.mp_energies, - **{f"{dep}_version": version(dep) for dep in ("scikit-learn", "matminer", "numpy")}, + versions={dep: version(dep) for dep in ("scikit-learn", "matminer", "numpy")}, model_name=model_name, train_target_col=train_e_form_col, test_target_col=test_e_form_col, diff --git a/models/voronoi/voronoi_featurize_dataset.py b/models/voronoi/voronoi_featurize_dataset.py index 964e9d25..972acdec 100644 --- a/models/voronoi/voronoi_featurize_dataset.py +++ b/models/voronoi/voronoi_featurize_dataset.py @@ -15,7 +15,7 @@ from pymatgen.core import Structure from tqdm import tqdm -from matbench_discovery import DEBUG, today +from matbench_discovery import today from matbench_discovery.data import DATA_FILES from matbench_discovery.slurm import slurm_submit from models.voronoi import featurizer @@ -33,7 +33,7 @@ input_col = "initial_structure" # input_col = "relaxed_structure" debug = "slurm-submit" in sys.argv -job_name = f"voronoi-features-{data_name}{'-debug' if DEBUG else ''}" +job_name = f"voronoi-features-{data_name}" module_dir = os.path.dirname(__file__) out_dir = os.getenv("SBATCH_OUTPUT", f"{module_dir}/{today}-{job_name}") slurm_array_task_count = 50 @@ -56,7 +56,7 @@ out_path = f"{out_dir}/{run_name}.csv.bz2" if os.path.isfile(out_path): - raise SystemExit(f"{out_path = } already exists, exciting early") + raise SystemExit(f"{out_path=} already exists, exciting early") print(f"{data_path=}") df_in: pd.DataFrame = np.array_split( @@ -82,7 +82,7 @@ input_col=input_col, slurm_vars=slurm_vars, out_path=out_path, - **{f"{dep}_version": version(dep) for dep in ("matminer", "numpy")}, + versions={dep: version(dep) for dep in ("matminer", "numpy")}, ) wandb.init(project="matbench-discovery", name=run_name, config=run_params) diff --git a/models/wrenformer/test_wrenformer.py b/models/wrenformer/test_wrenformer.py index 1399ea51..01176bf5 100644 --- a/models/wrenformer/test_wrenformer.py +++ b/models/wrenformer/test_wrenformer.py @@ -17,7 +17,7 @@ from aviary.wrenformer.data import df_to_in_mem_dataloader from aviary.wrenformer.model import Wrenformer -from matbench_discovery import CHECKPOINT_DIR, DEBUG, WANDB_PATH, today +from matbench_discovery import CHECKPOINT_DIR, WANDB_PATH, today from matbench_discovery.data import DATA_FILES from matbench_discovery.plots import wandb_scatter from matbench_discovery.slurm import slurm_submit @@ -29,7 +29,7 @@ task_type = "IS2RE" data_path = DATA_FILES.wbm_summary debug = "slurm-submit" in sys.argv -job_name = f"test-wrenformer-wbm-{task_type}{'-debug' if DEBUG else ''}" +job_name = f"test-wrenformer-wbm-{task_type}" module_dir = os.path.dirname(__file__) out_dir = os.getenv("SBATCH_OUTPUT", f"{module_dir}/{today}-{job_name}") @@ -74,7 +74,7 @@ run_params = dict( data_path=data_path, df=dict(shape=str(df.shape), columns=", ".join(df)), - **{f"{dep}_version": version(dep) for dep in ("aviary", "numpy", "torch")}, + versions={dep: version(dep) for dep in ("aviary", "numpy", "torch")}, ensemble_size=len(runs), task_type=task_type, target_col=e_form_col, diff --git a/models/wrenformer/train_wrenformer.py b/models/wrenformer/train_wrenformer.py index b3b79e71..f21acb14 100644 --- a/models/wrenformer/train_wrenformer.py +++ b/models/wrenformer/train_wrenformer.py @@ -8,7 +8,7 @@ import pandas as pd from aviary.train import df_train_test_split, train_wrenformer -from matbench_discovery import DEBUG, WANDB_PATH, timestamp, today +from matbench_discovery import WANDB_PATH, timestamp, today from matbench_discovery.data import DATA_FILES from matbench_discovery.slurm import slurm_submit @@ -23,7 +23,7 @@ # data_path = f"{ROOT}/data/2022-08-25-m3gnet-trainset-mp-2021-struct-energy.json.gz" # target_col = "mp_energy_per_atom" data_name = "m3gnet-trainset" if "m3gnet" in data_path else "mp" -job_name = f"train-wrenformer-robust-{data_name}{'-debug' if DEBUG else ''}" +job_name = f"train-wrenformer-robust-{data_name}" ensemble_size = 10 dataset = "mp" module_dir = os.path.dirname(__file__) @@ -59,7 +59,7 @@ run_params = dict( data_path=data_path, - **{f"{dep}_version": version(dep) for dep in ("aviary", "numpy", "torch")}, + versions={dep: version(dep) for dep in ("aviary", "numpy", "torch")}, batch_size=batch_size, train_df=dict(shape=train_df.shape, columns=", ".join(train_df)), test_df=dict(shape=test_df.shape, columns=", ".join(test_df)), diff --git a/pyproject.toml b/pyproject.toml index c73cb874..1d883184 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ running-models = [ ] 3d-structures = ["crystaltoolkit"] fetch-data = ["gdown"] -df-pdf-export = ["pdfCropMargins", "pdfkit"] +df-pdf-export = ["pdfCropMargins", "weasyprint"] [tool.setuptools.packages.find] include = ["matbench_discovery*"] diff --git a/scripts/compute_struct_fingerprints.py b/scripts/compute_struct_fingerprints.py index 945803ee..25443860 100644 --- a/scripts/compute_struct_fingerprints.py +++ b/scripts/compute_struct_fingerprints.py @@ -53,7 +53,7 @@ # %% out_path = f"{out_dir}/site-stats-{slurm_array_task_id}.json.gz" if os.path.isfile(out_path): - raise SystemExit(f"{out_path = } already exists, exciting early") + raise SystemExit(f"{out_path=} already exists, exciting early") print(f"\nJob started running {timestamp}") print(f"{out_path=}") diff --git a/scripts/model_figs/analyze_model_disagreement.py b/scripts/model_figs/analyze_model_disagreement.py index 4aef6df3..3ccbb9c0 100644 --- a/scripts/model_figs/analyze_model_disagreement.py +++ b/scripts/model_figs/analyze_model_disagreement.py @@ -4,6 +4,8 @@ # %% +import sys + import pandas as pd from crystal_toolkit.helpers.utils import hook_up_fig_with_struct_viewer from pymatviz.utils import add_identity_line, save_fig @@ -88,11 +90,14 @@ # %% struct viewer -app = hook_up_fig_with_struct_viewer( - fig, - df_cse, - "initial_structure", - # validate_id requires material_id to be hover_name - validate_id=lambda id: id.startswith(("wbm-", "mp-", "mvc-")), -) -app.run(port=8000) +# only run this in Jupyter Notebook +is_jupyter = "ipykernel" in sys.modules +if is_jupyter: + app = hook_up_fig_with_struct_viewer( + fig, + df_cse, + "initial_structure", + # validate_id requires material_id to be hover_name + validate_id=lambda id: id.startswith(("wbm-", "mp-", "mvc-")), + ) + app.run(port=8000) diff --git a/scripts/model_figs/cumulative_metrics.py b/scripts/model_figs/cumulative_metrics.py index 7d5e0c5d..d440b708 100644 --- a/scripts/model_figs/cumulative_metrics.py +++ b/scripts/model_figs/cumulative_metrics.py @@ -27,8 +27,8 @@ # %% -# metrics = ("Precision", "Recall") -metrics = ("MAE", "RMSE") +metrics = ("Precision", "Recall") +# metrics = ("MAE", "RMSE") range_y = { ("MAE", "RMSE"): (0, 0.5), ("Precision", "Recall"): (0, 1), diff --git a/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py b/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py index f24e61fb..b9388395 100644 --- a/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py +++ b/scripts/model_figs/hist_classified_stable_vs_hull_dist_models.py @@ -40,7 +40,7 @@ # %% backend: Final = "plotly" -n_cols = 3 +n_cols = 2 n_rows = math.ceil(len(models) // n_cols) which_energy: Final = "pred" kwds = ( diff --git a/scripts/model_figs/make_metrics_tables.py b/scripts/model_figs/make_metrics_tables.py index 2a7ab65d..12ec5434 100644 --- a/scripts/model_figs/make_metrics_tables.py +++ b/scripts/model_figs/make_metrics_tables.py @@ -45,21 +45,29 @@ df_metrics_10k["Dummy"] = dummy_metrics -# %% -ontology = { # (training type, test type, model type) +# %% for each model this ontology dict specifies +# (training type, test type, model class) +# RS2RE = relaxed structure to relaxed energy +# RP2RE = relaxed prototype to predicted energy +# IS2RE = initial structure to relaxed energy +# IS2E = initial structure to energy +# IS2RE-SR = initial structure to relaxed energy after ML structure relaxation +# S2EFS(M) = structure to energy, forces, stress, (magmoms) +ontology = { "ALIGNN": ("RS2RE", "IS2RE", "GNN"), - "ALIGNN Pretrained": ("RS2RE", "IS2RE", "GNN"), + # "ALIGNN Pretrained": ("RS2RE", "IS2RE", "GNN"), "CHGNet": ("S2EFSM", "IS2RE-SR", "UIP-GNN"), + "MACE": ("S2EFS", "IS2RE-SR", "UIP-GNN"), "M3GNet": ("S2EFS", "IS2RE-SR", "UIP-GNN"), "MEGNet": ("RS2RE", "IS2E", "GNN"), "CGCNN": ("RS2RE", "IS2E", "GNN"), "CGCNN+P": ("S2RE", "IS2RE", "GNN"), "Wrenformer": ("RP2RE", "IP2E", "Transformer"), - "BOWSR + MEGNet": ("RS2RE", "IS2RE-BO", "BO+GNN"), - "Voronoi RF": ("RS2RE", "IS2E", "Fingerprint+RF"), - "M3GNet + MEGNet": ("S2EFS", "IS2RE-SR", "UIP + GNN"), - "CHGNet + MEGNet": ("S2EFSM", "IS2RE-SR", "UIP + GNN"), - "Dummy": ("", "", "scikit-learn"), + "BOWSR": ("RS2RE", "IS2RE-BO", "BO-GNN"), + "Voronoi RF": ("RS2RE", "IS2E", "Fingerprint"), + "M3GNet->MEGNet": ("S2EFS", "IS2RE-SR", "UIP-GNN"), + "CHGNet->MEGNet": ("S2EFSM", "IS2RE-SR", "UIP-GNN"), + "Dummy": ("", "", ""), } ontology_cols = ["Trained", "Deployed", "Model Class"] df_ont = pd.DataFrame(ontology, index=ontology_cols) @@ -82,7 +90,7 @@ df_table = pd.concat([df, df_ont]).rename(index={"R2": R2_col}) df_table.index.name = "Model" - drop_models = ["CHGNet + MEGNet", "M3GNet + MEGNet"] + drop_models = ["CHGNet->MEGNet", "M3GNet->MEGNet"] if make_uip_megnet_comparison: drop_models = [*{*df_table} - {*drop_models, "MEGNet", "M3GNet", "CHGNet"}] label += "-uip-megnet-combos" @@ -117,7 +125,7 @@ styler.set_table_styles([dict(selector=sel, props=styles[sel]) for sel in styles]) styler.set_uuid("") - # export model metrics as styled HTML table and Svelte component + # export model metrics as styled HTML table and Svelte component # draw dotted line between classification and regression metrics df_to_svelte_table( styler, diff --git a/scripts/calc_wandb_model_runtimes.py b/scripts/model_figs/model_compute_cost.py similarity index 94% rename from scripts/calc_wandb_model_runtimes.py rename to scripts/model_figs/model_compute_cost.py index aeba003b..66e49964 100644 --- a/scripts/calc_wandb_model_runtimes.py +++ b/scripts/model_figs/model_compute_cost.py @@ -34,9 +34,10 @@ } test_run_filters: dict[str, tuple[int, str, str, str]] = { # model: (n_runs, display_name, created_gt, created_lt) - "BOWSR + MEGNet": (476, "bowsr-megnet", "2023-01-20", "2023-01-22"), + "BOWSR": (476, "bowsr-megnet", "2023-01-20", "2023-01-22"), "CHGNet": (100, "chgnet-wbm-IS2RE-", "2023-03-05", "2023-03-07"), "CGCNN": (1, "test-cgcnn-wbm-IS2RE", "2022-12-03", "2022-12-05"), + "MACE": (100, "mace-wbm-IS2RE-FIRE", "2023-07-22", "2023-07-24"), "M3GNet": (99, "m3gnet-wbm-IS2RE", "2022-10-31", "2022-11-01"), "MEGNet": (1, "megnet-wbm-IS2RE", "2022-11-17", "2022-11-19"), "Voronoi RF": (20, "voronoi-features-wbm", "2022-11-15", "2022-11-19"), @@ -49,7 +50,7 @@ # %% calculate total model run times from wandb logs # NOTE these model run times are pretty meaningless since some models were run on GPU -# (Wrenformer and CGCNN), others on CPU. Also BOWSR + MEGNet, M3GNet and MEGNet weren't +# (Wrenformer and CGCNN), others on CPU. Also BOWSR, M3GNet and MEGNet weren't # trained from scratch. Their run times only indicate the time needed to predict the # test set. @@ -129,7 +130,7 @@ .round(1) # maybe remove BOWSR since it used so much more compute time than the other models # that it makes the plot unreadable - # .drop(index="BOWSR + MEGNet") + # .drop(index="BOWSR") .reset_index(names=(model_col := "Model")) ) @@ -189,13 +190,13 @@ color=model_col, ) # reduce bar width -fig.update_traces(width=0.7) +fig.update_traces(width=0.8) title = f"All models: {df_stats[time_col].sum():.0f} h" -fig.layout.legend.update(x=0.98, y=0.98, xanchor="right", yanchor="top", title=title) +fig.layout.legend.update(title=title, orientation="h", xanchor="center", x=0.4, y=1.2) fig.layout.xaxis.title = "" fig.layout.margin.update(l=0, r=0, t=0, b=0) -# save_fig(fig, f"{FIGS}/model-run-times-bar.svelte") +save_fig(fig, f"{FIGS}/model-run-times-bar.svelte") pdf_fig = go.Figure(fig) # replace legend with annotation in PDF diff --git a/scripts/analyze_element_errors.py b/scripts/model_figs/per_element_errors.py similarity index 91% rename from scripts/analyze_element_errors.py rename to scripts/model_figs/per_element_errors.py index 45e030af..df27e022 100644 --- a/scripts/analyze_element_errors.py +++ b/scripts/model_figs/per_element_errors.py @@ -8,12 +8,12 @@ import pandas as pd import plotly.express as px from pymatgen.core import Composition, Element -from pymatviz import count_elements, ptable_heatmap_plotly +from pymatviz import ptable_heatmap_plotly from pymatviz.utils import bin_df_cols, df_ptable, save_fig from tqdm import tqdm -from matbench_discovery import FIGS, MODELS, PDF_FIGS -from matbench_discovery.data import DATA_FILES, df_wbm +from matbench_discovery import FIGS, MODELS, PDF_FIGS, ROOT +from matbench_discovery.data import df_wbm from matbench_discovery.preds import ( df_each_err, df_metrics, @@ -47,16 +47,15 @@ # df_frac_comp = df_frac_comp.dropna(axis=1, thresh=100) # remove Xe with only 1 entry -# %% -df_mp = pd.read_csv(DATA_FILES.mp_energies, na_filter=False).set_index("material_id") -# compute number of samples per element in training set +# %% compute number of samples per element in training set # counting element occurrences not weighted by composition, assuming model don't learn # much more about iron and oxygen from Fe2O3 than from FeO - -train_count_col = "MP Occurrences" -df_elem_err = count_elements(df_mp.formula_pretty, count_mode="occurrence").to_frame( - name=train_count_col +df_elem_err = pd.read_json( + f"{ROOT}/site/src/routes/about-the-data/mp-element-counts-occurrence.json", + typ="series", ) +train_count_col = "MP Occurrences" +df_elem_err = df_elem_err.reset_index(name=train_count_col).set_index("index") # %% @@ -87,7 +86,7 @@ df_struct_counts = df_struct_counts[df_struct_counts.sum(axis=1) > min_count] normalized = False if normalized: - df_struct_counts["MP"] /= len(df_mp) / 100 + df_struct_counts["MP"] /= len(df_preds) / 100 df_struct_counts["WBM"] /= len(df_wbm) / 100 y_col = "percent" if normalized else "count" fig = ( @@ -148,8 +147,8 @@ # %% expected_cols = { - *"ALIGNN, BOWSR + MEGNet, CGCNN, CGCNN+P, CHGNet, M3GNet, MEGNet, " - "MP Occurrences, Mean error all models, Test set standard deviation, Voronoi RF, " + *"ALIGNN, BOWSR, CGCNN, CGCNN+P, CHGNet, M3GNet, MEGNet, " + f"{train_count_col}, Mean error all models, {test_set_std_col}, Voronoi RF, " "Wrenformer".split(", ") } assert {*df_elem_err} >= expected_cols @@ -164,7 +163,7 @@ df_elem_err[elem_col] = [Element(el).long_name for el in df_elem_err.index] df_melt = df_elem_err.melt( - id_vars=["MP Occurrences", "Test set standard deviation", elem_col], + id_vars=[train_count_col, test_set_std_col, elem_col], value_name=(val_col := "Error"), var_name=(clr_col := "Model"), ignore_index=False, diff --git a/scripts/model_figs/roc_prc_curves_models.py b/scripts/model_figs/roc_prc_curves_models.py index 5399b5e3..c7d603c2 100644 --- a/scripts/model_figs/roc_prc_curves_models.py +++ b/scripts/model_figs/roc_prc_curves_models.py @@ -48,7 +48,7 @@ # %% -facetted = False +facet_plot = False kwds = dict( height=150 * len(df_roc[facet_col].unique()), color=color_col, @@ -62,7 +62,7 @@ df_roc.iloc[:: len(df_roc) // 500 or 1] .sort_values(["AUC", "FPR"], ascending=False) .plot, - "scatter" if facetted else "line", + "scatter" if facet_plot else "line", ) fig = plot_fn( @@ -74,13 +74,13 @@ range_y=(0, 1.02), hover_name=facet_col, hover_data={facet_col: False}, - **kwds if facetted else dict(color=facet_col, markers=True), + **kwds if facet_plot else dict(color=facet_col, markers=True, marker_size=3), ) for anno in fig.layout.annotations: anno.text = anno.text.split("=", 1)[1] # remove Model= from subplot titles -if not facetted: +if not facet_plot: fig.layout.legend.update(x=1, y=0, xanchor="right", title=None) fig.layout.coloraxis.colorbar.update(thickness=14, title_side="right") if n_cols == 2: @@ -95,7 +95,7 @@ fig.layout.margin.update(l=0, r=0, b=0, t=20, pad=0) fig.update_yaxes(matches=None) fig.show() -img_name = f"roc-models-{f'{n_rows}x{n_cols}' if facetted else 'all-in-one'}" +img_name = f"roc-models-{f'{n_rows}x{n_cols}' if facet_plot else 'all-in-one'}" # %% @@ -125,11 +125,16 @@ # %% +n_cols = 3 +n_rows = math.ceil(len(models) // n_cols) + fig = df_prc.iloc[:: len(df_roc) // 500 or 1].plot.scatter( x="Recall", y="Precision", facet_col=facet_col, - facet_col_wrap=2, + facet_col_wrap=n_cols, + facet_row_spacing=0.04, + facet_col_spacing=0.04, backend="plotly", height=150 * len(df_roc[facet_col].unique()), color=color_col, @@ -144,7 +149,7 @@ anno.text = anno.text.split("=", 1)[1] # remove Model= from subplot titles fig.layout.coloraxis.colorbar.update( - x=0.5, y=1.1, thickness=14, len=0.4, orientation="h" + x=0.5, y=1.03, thickness=14, len=0.4, orientation="h" ) fig.add_hline(y=0.5, line=line) fig.add_annotation( diff --git a/scripts/model_figs/scatter_e_above_hull_models.py b/scripts/model_figs/scatter_e_above_hull_models.py index 5918faa8..a26796a8 100644 --- a/scripts/model_figs/scatter_e_above_hull_models.py +++ b/scripts/model_figs/scatter_e_above_hull_models.py @@ -120,7 +120,7 @@ # %% plot all models in separate subplots domain = (-4, 7) -n_cols = 3 +n_cols = 2 n_rows = math.ceil(len(models) / n_cols) fig = px.scatter( @@ -130,7 +130,7 @@ facet_col=facet_col, facet_col_wrap=n_cols, facet_col_spacing=0.02, - facet_row_spacing=0.08, + facet_row_spacing=0.04, hover_data=hover_cols, hover_name=df_preds.index.name, color=clf_col, @@ -219,7 +219,7 @@ textangle=-90, **axis_titles, ) -# fig.layout.height = 1000 +fig.layout.height = 1000 # fig.layout.width = 1100 fig.layout.margin.update(l=40, r=10, t=10, b=50) fig.update_xaxes(matches=None) diff --git a/scripts/model_figs/update_all_model_figs.py b/scripts/model_figs/update_all_model_figs.py index 45822f3b..df54b8ff 100644 --- a/scripts/model_figs/update_all_model_figs.py +++ b/scripts/model_figs/update_all_model_figs.py @@ -17,7 +17,10 @@ # %% for file in glob(f"{module_dir}/*.py"): - if file == "run_all.py": + if file == __file__: # skip this file continue - print(f"Running {file}...") - runpy.run_path(file) + print(f"Running {file.split(os.path.sep)[-1]}...") + try: + runpy.run_path(file) + except Exception as exc: + print(f"{file!r} failed: {exc}") diff --git a/scripts/update_wandb_runs.py b/scripts/update_wandb_runs.py index 1d352a13..7d24cf62 100644 --- a/scripts/update_wandb_runs.py +++ b/scripts/update_wandb_runs.py @@ -15,7 +15,7 @@ # %% -filters = dict(display_name={"$regex": "voronoi-featurize"}) +filters = dict(display_name={"$regex": "mace-wbm-"}) runs = wandb.Api().runs(WANDB_PATH, filters=filters) print(f"matching runs: {len(runs)}") @@ -37,7 +37,9 @@ for idx, run in enumerate(runs, 1): old_config, new_config = run.config.copy(), run.config.copy() - new_display_name = run.display_name.replace("featurize", "features") + new_display_name = run.display_name.replace( + "mace-wbm-IS2RE-debug-", "mace-wbm-IS2RE-" + ) for x in ("IS2RE", "ES2RE"): if x in run.display_name: diff --git a/scripts/upload_to_figshare.py b/scripts/upload_to_figshare.py index 9f86cfbc..f0ef49a0 100644 --- a/scripts/upload_to_figshare.py +++ b/scripts/upload_to_figshare.py @@ -32,7 +32,17 @@ pyproject = tomllib.load(file)["project"] KEYWORDS = pyproject["keywords"] VERSION = pyproject["version"] -DESCRIPTION = pyproject["description"] +DESCRIPTION = f""" +These are the v{VERSION} data files for Matbench Discovery, +{pyproject['description'].lower()}. It contains relaxed structures of the MP +training set, initial+relaxed structures of the WBM test set, plus several checkpoints +for models trained on this data specifically for this benchmark. The full force field +training set containing 1.3M structures along with their energies, forces, stresses and +magmons is available at https://figshare.com/articles/dataset/23713842. +""".replace( + "\n", " " +).strip() +# https://figshare.com/articles/dataset/Materials_Project_Trjectory_MPtrj_Dataset/23713842 REFERENCES = list(pyproject["urls"].values()) TITLE = f"Matbench Discovery v{VERSION}" diff --git a/site/package.json b/site/package.json index 8daf8f02..c17b5e7c 100644 --- a/site/package.json +++ b/site/package.json @@ -20,34 +20,34 @@ "@iconify/svelte": "^3.1.4", "@rollup/plugin-yaml": "^4.1.1", "@sveltejs/adapter-static": "^2.0.2", - "@sveltejs/kit": "^1.22.1", - "@sveltejs/vite-plugin-svelte": "^2.4.2", - "@typescript-eslint/eslint-plugin": "^5.61.0", - "@typescript-eslint/parser": "^5.61.0", + "@sveltejs/kit": "^1.22.3", + "@sveltejs/vite-plugin-svelte": "^2.4.3", + "@typescript-eslint/eslint-plugin": "^6.2.0", + "@typescript-eslint/parser": "^6.2.0", "elementari": "^0.2.2", - "eslint": "^8.44.0", - "eslint-plugin-svelte": "^2.32.2", + "eslint": "^8.45.0", + "eslint-plugin-svelte": "^2.32.4", "hastscript": "^7.2.0", "highlight.js": "^11.8.0", "js-yaml": "^4.1.0", "katex": "^0.16.8", "mdsvex": "^0.11.0", "prettier": "^3.0.0", - "prettier-plugin-svelte": "^2.10.1", + "prettier-plugin-svelte": "^3.0.1", "rehype-autolink-headings": "^6.1.1", "rehype-katex-svelte": "^1.2.0", "rehype-slug": "^5.1.0", "remark-math": "3.0.0", - "svelte": "^4.0.5", - "svelte-check": "^3.4.5", - "svelte-multiselect": "^10.0.0", + "svelte": "^4.1.1", + "svelte-check": "^3.4.6", + "svelte-multiselect": "^10.1.0", "svelte-preprocess": "^5.0.4", "svelte-toc": "^0.5.5", - "svelte-zoo": "^0.4.8", + "svelte-zoo": "^0.4.9", "svelte2tsx": "^0.6.19", - "tslib": "^2.6.0", + "tslib": "^2.6.1", "typescript": "5.1.6", - "vite": "^4.4.2" + "vite": "^4.4.7" }, "prettier": { "semi": false, diff --git a/site/src/figs/box-hull-dist-errors.svelte b/site/src/figs/box-hull-dist-errors.svelte index 65d8aae6..b77196d1 100644 --- a/site/src/figs/box-hull-dist-errors.svelte +++ b/site/src/figs/box-hull-dist-errors.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/cumulative-mae-rmse.svelte b/site/src/figs/cumulative-mae-rmse.svelte index eece4c6c..361caece 100644 --- a/site/src/figs/cumulative-mae-rmse.svelte +++ b/site/src/figs/cumulative-mae-rmse.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/cumulative-precision-recall.svelte b/site/src/figs/cumulative-precision-recall.svelte index 6dca8f3a..3b01a9ae 100644 --- a/site/src/figs/cumulative-precision-recall.svelte +++ b/site/src/figs/cumulative-precision-recall.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/each-error-vs-least-prevalent-element-in-struct.svelte b/site/src/figs/each-error-vs-least-prevalent-element-in-struct.svelte index f3c8a197..78971d0b 100644 --- a/site/src/figs/each-error-vs-least-prevalent-element-in-struct.svelte +++ b/site/src/figs/each-error-vs-least-prevalent-element-in-struct.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/each-scatter-models-2x4.svelte b/site/src/figs/each-scatter-models-2x4.svelte deleted file mode 100644 index 274c4e0d..00000000 --- a/site/src/figs/each-scatter-models-2x4.svelte +++ /dev/null @@ -1 +0,0 @@ -
diff --git a/site/src/figs/each-scatter-models-3x3.svelte b/site/src/figs/each-scatter-models-3x3.svelte deleted file mode 100644 index 5b701720..00000000 --- a/site/src/figs/each-scatter-models-3x3.svelte +++ /dev/null @@ -1 +0,0 @@ -
diff --git a/site/src/figs/each-scatter-models-4x2.svelte b/site/src/figs/each-scatter-models-4x2.svelte deleted file mode 100644 index b05e6386..00000000 --- a/site/src/figs/each-scatter-models-4x2.svelte +++ /dev/null @@ -1 +0,0 @@ -
diff --git a/site/src/figs/each-scatter-models-5x2.svelte b/site/src/figs/each-scatter-models-5x2.svelte new file mode 100644 index 00000000..554da96b --- /dev/null +++ b/site/src/figs/each-scatter-models-5x2.svelte @@ -0,0 +1 @@ +
diff --git a/site/src/figs/element-prevalence-vs-error.svelte b/site/src/figs/element-prevalence-vs-error.svelte index b30f0021..a4c05e7c 100644 --- a/site/src/figs/element-prevalence-vs-error.svelte +++ b/site/src/figs/element-prevalence-vs-error.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/metrics-table-first-10k.svelte b/site/src/figs/metrics-table-first-10k.svelte index b5937210..708b9447 100644 --- a/site/src/figs/metrics-table-first-10k.svelte +++ b/site/src/figs/metrics-table-first-10k.svelte @@ -1,364 +1,349 @@ - + + + - +
- - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + +
ModelF1DAFPrecisionAccuracyMAERMSER2Model ClassModelF1DAFPrecisionAccuracyMAERMSER2Model Class
CHGNet0.914.970.830.830.070.100.82UIP-GNNCHGNet0.914.960.830.830.070.100.82UIP-GNN
M3GNet0.894.790.800.800.100.160.62UIP-GNN
M3GNet0.894.770.800.800.100.160.61UIP-GNNCGCNN0.763.660.610.610.170.240.27GNN
CGCNN0.763.660.610.610.170.240.26GNNALIGNN0.753.580.600.600.190.270.14GNN
ALIGNN0.753.570.590.590.190.270.12GNNCGCNN+P0.743.530.590.590.210.290.03GNN
CGCNN+P0.743.510.580.580.210.290.02GNNWrenformer0.743.520.590.590.190.250.22Transformer
Wrenformer0.743.510.580.580.190.250.21TransformerBOWSR0.693.140.520.520.270.33-1.10BO-GNN
BOWSR + MEGNet0.693.140.520.520.270.33-1.10BO+GNNMEGNet0.642.840.470.470.330.36-0.82GNN
MEGNet0.652.860.480.480.330.36-0.82GNNVoronoi RF0.582.450.410.410.360.43-0.81Fingerprint
Voronoi RF0.582.470.410.410.360.43-0.83Fingerprint+RFMACE0.471.860.310.310.981.27-10.71UIP-GNN
Dummy0.191.000.170.680.120.180.00scikit-learnDummy0.191.000.170.680.120.180.00
- - diff --git a/site/src/figs/metrics-table.svelte b/site/src/figs/metrics-table.svelte index 3e7b8195..88bb0a9b 100644 --- a/site/src/figs/metrics-table.svelte +++ b/site/src/figs/metrics-table.svelte @@ -1,469 +1,453 @@ - + + + - +
- - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + +
ModelF1DAFPrecisionAccuracyTPRTNRMAERMSER2Model ClassModelF1DAFPrecisionAccuracyTPRTNRMAERMSER2Model Class
CHGNet0.593.060.520.840.670.870.070.110.61UIP-GNNCHGNet0.583.060.520.840.660.880.070.110.61UIP-GNN
M3GNet0.572.670.450.800.770.810.070.110.60UIP-GNN
M3GNet0.582.660.450.800.790.800.070.120.59UIP-GNNALIGNN0.562.920.500.830.650.870.090.150.27GNN
ALIGNN0.572.870.490.820.660.860.090.150.27GNNMACE0.562.540.440.790.790.790.100.27-1.29UIP-GNN
MEGNet0.522.700.460.810.590.860.130.20-0.27GNNMEGNet0.512.700.460.810.570.860.130.20-0.28GNN
CGCNN0.522.620.450.810.600.850.140.23-0.61GNNCGCNN0.512.630.450.810.590.850.140.23-0.62GNN
CGCNN+P0.512.380.410.780.690.790.110.180.02GNNCGCNN+P0.512.400.410.780.670.800.110.180.03GNN
Wrenformer0.482.130.360.740.710.740.100.18-0.04TransformerWrenformer0.482.130.360.740.690.750.100.18-0.04Transformer
BOWSR + MEGNet0.441.900.320.680.740.670.110.160.15BO+GNNBOWSR0.441.910.320.680.740.670.120.160.14BO-GNN
Voronoi RF0.341.510.260.660.520.690.140.21-0.32Fingerprint+RFVoronoi RF0.341.510.260.670.510.700.140.21-0.31Fingerprint
Dummy0.191.000.170.680.230.770.120.180.00scikit-learnDummy0.191.000.170.680.230.770.120.180.00
- - diff --git a/site/src/figs/model-run-times-bar.svelte b/site/src/figs/model-run-times-bar.svelte index df18c7cc..d5acfa09 100644 --- a/site/src/figs/model-run-times-bar.svelte +++ b/site/src/figs/model-run-times-bar.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/roc-models-all-in-one.svelte b/site/src/figs/roc-models-all-in-one.svelte index ecd9e2a1..cceef1b0 100644 --- a/site/src/figs/roc-models-all-in-one.svelte +++ b/site/src/figs/roc-models-all-in-one.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/rolling-mae-vs-hull-dist-models.svelte b/site/src/figs/rolling-mae-vs-hull-dist-models.svelte index afda4092..2abd4e65 100644 --- a/site/src/figs/rolling-mae-vs-hull-dist-models.svelte +++ b/site/src/figs/rolling-mae-vs-hull-dist-models.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-actinides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-actinides.svelte index 20661c17..ddd0f763 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-actinides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-actinides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-all.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-all.svelte index 66d87851..e09f4320 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-all.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-all.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-borides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-borides.svelte index ad48d6ef..844ec248 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-borides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-borides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-carbides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-carbides.svelte index 95629693..57a8fd19 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-carbides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-carbides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-chalcogenides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-chalcogenides.svelte index 03084ccd..8b75df67 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-chalcogenides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-chalcogenides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-halides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-halides.svelte index 9497ee0e..eafa0f98 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-halides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-halides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-hydrides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-hydrides.svelte index d16f99e1..bf91276e 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-hydrides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-hydrides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-lanthanides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-lanthanides.svelte index 28c58ea2..58306c52 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-lanthanides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-lanthanides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-nitrides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-nitrides.svelte index 4664b900..e255d5fb 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-nitrides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-nitrides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxides.svelte index 5e66be09..ae47ce1e 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxynitrides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxynitrides.svelte index f6a50ae2..8afafd13 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxynitrides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-oxynitrides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-pnictides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-pnictides.svelte index 0dfcb2d4..af2bce00 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-pnictides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-pnictides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-sulfides.svelte b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-sulfides.svelte index 491a2fb8..6991d3c7 100644 --- a/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-sulfides.svelte +++ b/site/src/figs/scatter-largest-errors-models-mean-vs-true-hull-dist-sulfides.svelte @@ -1 +1 @@ -
+
diff --git a/site/src/routes/models/2023-02-03-model-stats.json b/site/src/routes/models/2023-02-03-model-stats.json index 9d042fba..c9f74573 100644 --- a/site/src/routes/models/2023-02-03-model-stats.json +++ b/site/src/routes/models/2023-02-03-model-stats.json @@ -1 +1 @@ -{"CGCNN":{"Run Time (h)":11.35,"GPU":1.0,"CPU":128.0,"Slurm Jobs":10.0,"DAF":1.97,"Precision":0.74,"Recall":0.79,"Accuracy":0.82,"F1":0.77,"TPR":0.79,"FPR":0.17,"TNR":0.83,"FNR":0.21,"MAE":0.13,"RMSE":0.23,"R2":0.16,"missing_preds":2,"missing_percent":"0.00%"},"Voronoi RF":{"Run Time (h)":204.7,"GPU":0.0,"CPU":76.0,"Slurm Jobs":68.0,"DAF":1.55,"Precision":0.59,"Recall":0.79,"Accuracy":0.71,"F1":0.67,"TPR":0.79,"FPR":0.34,"TNR":0.66,"FNR":0.21,"MAE":0.14,"RMSE":0.21,"R2":0.32,"missing_preds":17,"missing_percent":"0.01%"},"Wrenformer":{"Run Time (h)":57.89,"GPU":1.0,"CPU":128.0,"Slurm Jobs":10.0,"DAF":1.78,"Precision":0.67,"Recall":0.91,"Accuracy":0.8,"F1":0.77,"TPR":0.91,"FPR":0.27,"TNR":0.73,"FNR":0.09,"MAE":0.11,"RMSE":0.18,"R2":0.46,"missing_preds":0,"missing_percent":"0.00%"},"MEGNet":{"Run Time (h)":3.44,"GPU":1.0,"CPU":128.0,"Slurm Jobs":1.0,"DAF":2.1,"Precision":0.8,"Recall":0.66,"Accuracy":0.81,"F1":0.72,"TPR":0.66,"FPR":0.1,"TNR":0.9,"FNR":0.34,"MAE":0.13,"RMSE":0.21,"R2":0.3,"missing_preds":0,"missing_percent":"0.00%"},"M3GNet":{"Run Time (h)":83.65,"GPU":0.0,"CPU":76.0,"Slurm Jobs":99.0,"DAF":1.88,"Precision":0.71,"Recall":0.93,"Accuracy":0.83,"F1":0.8,"TPR":0.93,"FPR":0.24,"TNR":0.76,"FNR":0.07,"MAE":0.07,"RMSE":0.11,"R2":0.79,"missing_preds":2569,"missing_percent":"1.00%"},"BOWSR + MEGNet":{"Run Time (h)":2776.57,"GPU":0.0,"CPU":32.0,"Slurm Jobs":500.0,"DAF":1.63,"Precision":0.59,"Recall":0.87,"Accuracy":0.73,"F1":0.7,"TPR":0.87,"FPR":0.36,"TNR":0.64,"FNR":0.13,"MAE":0.11,"RMSE":0.16,"R2":0.55,"missing_preds":6184,"missing_percent":"2.41%"},"M3GNet + MEGNet":{"Run Time (h)":87.09,"GPU":0.0,"CPU":76.0,"Slurm Jobs":99.0,"DAF":1.98,"Precision":0.74,"Recall":0.79,"Accuracy":0.82,"F1":0.76,"TPR":0.79,"FPR":0.17,"TNR":0.83,"FNR":0.21,"MAE":0.09,"RMSE":0.13,"R2":0.72,"missing_preds":2576,"missing_percent":"1.00%"}} +{"CGCNN":{"Run Time (h)":11.35,"GPU":1.0,"CPU":128.0,"Slurm Jobs":10.0,"DAF":1.97,"Precision":0.74,"Recall":0.79,"Accuracy":0.82,"F1":0.77,"TPR":0.79,"FPR":0.17,"TNR":0.83,"FNR":0.21,"MAE":0.13,"RMSE":0.23,"R2":0.16,"missing_preds":2,"missing_percent":"0.00%"},"Voronoi RF":{"Run Time (h)":204.7,"GPU":0.0,"CPU":76.0,"Slurm Jobs":68.0,"DAF":1.55,"Precision":0.59,"Recall":0.79,"Accuracy":0.71,"F1":0.67,"TPR":0.79,"FPR":0.34,"TNR":0.66,"FNR":0.21,"MAE":0.14,"RMSE":0.21,"R2":0.32,"missing_preds":17,"missing_percent":"0.01%"},"Wrenformer":{"Run Time (h)":57.89,"GPU":1.0,"CPU":128.0,"Slurm Jobs":10.0,"DAF":1.78,"Precision":0.67,"Recall":0.91,"Accuracy":0.8,"F1":0.77,"TPR":0.91,"FPR":0.27,"TNR":0.73,"FNR":0.09,"MAE":0.11,"RMSE":0.18,"R2":0.46,"missing_preds":0,"missing_percent":"0.00%"},"MEGNet":{"Run Time (h)":3.44,"GPU":1.0,"CPU":128.0,"Slurm Jobs":1.0,"DAF":2.1,"Precision":0.8,"Recall":0.66,"Accuracy":0.81,"F1":0.72,"TPR":0.66,"FPR":0.1,"TNR":0.9,"FNR":0.34,"MAE":0.13,"RMSE":0.21,"R2":0.3,"missing_preds":0,"missing_percent":"0.00%"},"M3GNet":{"Run Time (h)":83.65,"GPU":0.0,"CPU":76.0,"Slurm Jobs":99.0,"DAF":1.88,"Precision":0.71,"Recall":0.93,"Accuracy":0.83,"F1":0.8,"TPR":0.93,"FPR":0.24,"TNR":0.76,"FNR":0.07,"MAE":0.07,"RMSE":0.11,"R2":0.79,"missing_preds":2569,"missing_percent":"1.00%"},"BOWSR":{"Run Time (h)":2776.57,"GPU":0.0,"CPU":32.0,"Slurm Jobs":500.0,"DAF":1.63,"Precision":0.59,"Recall":0.87,"Accuracy":0.73,"F1":0.7,"TPR":0.87,"FPR":0.36,"TNR":0.64,"FNR":0.13,"MAE":0.11,"RMSE":0.16,"R2":0.55,"missing_preds":6184,"missing_percent":"2.41%"},"M3GNet + MEGNet":{"Run Time (h)":87.09,"GPU":0.0,"CPU":76.0,"Slurm Jobs":99.0,"DAF":1.98,"Precision":0.74,"Recall":0.79,"Accuracy":0.82,"F1":0.76,"TPR":0.79,"FPR":0.17,"TNR":0.83,"FNR":0.21,"MAE":0.09,"RMSE":0.13,"R2":0.72,"missing_preds":2576,"missing_percent":"1.00%"}} diff --git a/site/src/routes/models/model-stats.json b/site/src/routes/models/model-stats.json index ca143ce5..0f2c4259 100644 --- a/site/src/routes/models/model-stats.json +++ b/site/src/routes/models/model-stats.json @@ -1 +1 @@ -{"CHGNet":{"F1":0.59,"DAF":3.06,"Precision":0.52,"Recall":0.67,"Accuracy":0.84,"TPR":0.67,"FPR":0.13,"TNR":0.87,"FNR":0.33,"TP":29561.0,"FP":27118.0,"TN":183325.0,"FN":14304.0,"MAE":0.07,"RMSE":0.11,"R2":0.61,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":142.48,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":100.0,"Run Time (h)":142.48,"missing_preds":2655,"missing_percent":"1.03%"},"M3GNet":{"F1":0.58,"DAF":2.66,"Precision":0.45,"Recall":0.79,"Accuracy":0.8,"TPR":0.79,"FPR":0.2,"TNR":0.8,"FNR":0.21,"TP":34556.0,"FP":41743.0,"TN":168956.0,"FN":9139.0,"MAE":0.07,"RMSE":0.12,"R2":0.6,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":83.65,"Test GPU":0.0,"Test CPU":76.0,"Test Slurm Jobs":99.0,"Run Time (h)":83.65,"missing_preds":2569,"missing_percent":"1.00%"},"ALIGNN":{"F1":0.57,"DAF":2.87,"Precision":0.49,"Recall":0.66,"Accuracy":0.82,"TPR":0.66,"FPR":0.14,"TNR":0.86,"FNR":0.34,"TP":29246.0,"FP":30061.0,"TN":182810.0,"FN":14846.0,"MAE":0.09,"RMSE":0.15,"R2":0.27,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":0,"missing_percent":"0.00%"},"MEGNet":{"F1":0.52,"DAF":2.7,"Precision":0.46,"Recall":0.59,"Accuracy":0.81,"TPR":0.59,"FPR":0.14,"TNR":0.86,"FNR":0.41,"TP":25892.0,"FP":29983.0,"TN":182888.0,"FN":18200.0,"MAE":0.13,"RMSE":0.2,"R2":-0.27,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":3.44,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":1.0,"Run Time (h)":3.44,"missing_preds":0,"missing_percent":"0.00%"},"CGCNN":{"F1":0.52,"DAF":2.62,"Precision":0.45,"Recall":0.6,"Accuracy":0.8,"TPR":0.6,"FPR":0.15,"TNR":0.85,"FNR":0.4,"TP":26552.0,"FP":32463.0,"TN":180406.0,"FN":17540.0,"MAE":0.14,"RMSE":0.23,"R2":-0.61,"Train Run Time (h)":11.35,"Train GPU":1.0,"Train CPU":128.0,"Train Slurm Jobs":10.0,"Test Run Time (h)":0.17,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":1.0,"Run Time (h)":11.52,"missing_preds":2,"missing_percent":"0.00%"},"CGCNN+P":{"F1":0.51,"DAF":2.38,"Precision":0.41,"Recall":0.69,"Accuracy":0.78,"TPR":0.69,"FPR":0.21,"TNR":0.79,"FNR":0.31,"TP":30321.0,"FP":43760.0,"TN":169109.0,"FN":13771.0,"MAE":0.11,"RMSE":0.18,"R2":0.02,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":2,"missing_percent":"0.00%"},"Wrenformer":{"F1":0.48,"DAF":2.13,"Precision":0.36,"Recall":0.71,"Accuracy":0.74,"TPR":0.71,"FPR":0.26,"TNR":0.74,"FNR":0.29,"TP":31381.0,"FP":54514.0,"TN":158357.0,"FN":12711.0,"MAE":0.1,"RMSE":0.18,"R2":-0.04,"Train Run Time (h)":57.89,"Train GPU":1.0,"Train CPU":128.0,"Train Slurm Jobs":10.0,"Test Run Time (h)":0.45,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":1.0,"Run Time (h)":58.34,"missing_preds":0,"missing_percent":"0.00%"},"BOWSR + MEGNet":{"F1":0.44,"DAF":1.9,"Precision":0.32,"Recall":0.74,"Accuracy":0.68,"TPR":0.74,"FPR":0.33,"TNR":0.67,"FNR":0.26,"TP":31607.0,"FP":68808.0,"TN":139481.0,"FN":10883.0,"MAE":0.11,"RMSE":0.16,"R2":0.15,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":2705.45,"Test GPU":0.0,"Test CPU":32.0,"Test Slurm Jobs":476.0,"Run Time (h)":2705.45,"missing_preds":6184,"missing_percent":"2.41%"},"Voronoi RF":{"F1":0.34,"DAF":1.51,"Precision":0.26,"Recall":0.52,"Accuracy":0.66,"TPR":0.52,"FPR":0.31,"TNR":0.69,"FNR":0.48,"TP":22885.0,"FP":65514.0,"TN":147341.0,"FN":21206.0,"MAE":0.14,"RMSE":0.21,"R2":-0.32,"Train Run Time (h)":173.78,"Train GPU":0.0,"Train CPU":76.0,"Train Slurm Jobs":48.0,"Test Run Time (h)":30.92,"Test GPU":0.0,"Test CPU":76.0,"Test Slurm Jobs":20.0,"Run Time (h)":204.7,"missing_preds":17,"missing_percent":"0.01%"}} +{"CHGNet":{"F1":0.58,"DAF":3.06,"Precision":0.52,"Recall":0.66,"Accuracy":0.84,"TPR":0.66,"FPR":0.12,"TNR":0.88,"FNR":0.34,"TP":28793.0,"FP":26230.0,"TN":184213.0,"FN":15072.0,"MAE":0.07,"RMSE":0.11,"R2":0.61,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":142.48,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":100.0,"Run Time (h)":142.48,"missing_preds":2655,"missing_percent":"1.03%"},"M3GNet":{"F1":0.57,"DAF":2.67,"Precision":0.45,"Recall":0.77,"Accuracy":0.8,"TPR":0.77,"FPR":0.19,"TNR":0.81,"FNR":0.23,"TP":33673.0,"FP":40425.0,"TN":170274.0,"FN":10022.0,"MAE":0.07,"RMSE":0.11,"R2":0.6,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":83.65,"Test GPU":0.0,"Test CPU":76.0,"Test Slurm Jobs":99.0,"Run Time (h)":83.65,"missing_preds":2569,"missing_percent":"1.00%"},"ALIGNN":{"F1":0.56,"DAF":2.92,"Precision":0.5,"Recall":0.65,"Accuracy":0.83,"TPR":0.65,"FPR":0.13,"TNR":0.87,"FNR":0.35,"TP":28598.0,"FP":28465.0,"TN":184406.0,"FN":15494.0,"MAE":0.09,"RMSE":0.15,"R2":0.27,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":0,"missing_percent":"0.00%"},"MACE":{"F1":0.56,"DAF":2.54,"Precision":0.44,"Recall":0.79,"Accuracy":0.79,"TPR":0.79,"FPR":0.21,"TNR":0.79,"FNR":0.21,"TP":34783.0,"FP":44954.0,"TN":167901.0,"FN":9309.0,"MAE":0.1,"RMSE":0.27,"R2":-1.29,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":111.9,"Test GPU":4.0,"Test CPU":64.0,"Test Slurm Jobs":100.0,"Run Time (h)":111.9,"missing_preds":16,"missing_percent":"0.01%"},"MEGNet":{"F1":0.51,"DAF":2.7,"Precision":0.46,"Recall":0.57,"Accuracy":0.81,"TPR":0.57,"FPR":0.14,"TNR":0.86,"FNR":0.43,"TP":25311.0,"FP":29343.0,"TN":183528.0,"FN":18781.0,"MAE":0.13,"RMSE":0.2,"R2":-0.28,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":3.44,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":1.0,"Run Time (h)":3.44,"missing_preds":0,"missing_percent":"0.00%"},"CGCNN":{"F1":0.51,"DAF":2.63,"Precision":0.45,"Recall":0.59,"Accuracy":0.81,"TPR":0.59,"FPR":0.15,"TNR":0.85,"FNR":0.41,"TP":25895.0,"FP":31475.0,"TN":181394.0,"FN":18197.0,"MAE":0.14,"RMSE":0.23,"R2":-0.62,"Train Run Time (h)":11.35,"Train GPU":1.0,"Train CPU":128.0,"Train Slurm Jobs":10.0,"Test Run Time (h)":0.17,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":1.0,"Run Time (h)":11.52,"missing_preds":2,"missing_percent":"0.00%"},"CGCNN+P":{"F1":0.51,"DAF":2.4,"Precision":0.41,"Recall":0.67,"Accuracy":0.78,"TPR":0.67,"FPR":0.2,"TNR":0.8,"FNR":0.33,"TP":29557.0,"FP":42282.0,"TN":170587.0,"FN":14535.0,"MAE":0.11,"RMSE":0.18,"R2":0.03,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":null,"Test GPU":null,"Test CPU":null,"Test Slurm Jobs":null,"Run Time (h)":0.0,"missing_preds":2,"missing_percent":"0.00%"},"Wrenformer":{"F1":0.48,"DAF":2.13,"Precision":0.36,"Recall":0.69,"Accuracy":0.74,"TPR":0.69,"FPR":0.25,"TNR":0.75,"FNR":0.31,"TP":30570.0,"FP":53088.0,"TN":159783.0,"FN":13522.0,"MAE":0.1,"RMSE":0.18,"R2":-0.04,"Train Run Time (h)":57.89,"Train GPU":1.0,"Train CPU":128.0,"Train Slurm Jobs":10.0,"Test Run Time (h)":0.45,"Test GPU":1.0,"Test CPU":128.0,"Test Slurm Jobs":1.0,"Run Time (h)":58.34,"missing_preds":0,"missing_percent":"0.00%"},"BOWSR":{"F1":0.44,"DAF":1.91,"Precision":0.32,"Recall":0.74,"Accuracy":0.68,"TPR":0.74,"FPR":0.33,"TNR":0.67,"FNR":0.26,"TP":31347.0,"FP":68140.0,"TN":140149.0,"FN":11143.0,"MAE":0.12,"RMSE":0.16,"R2":0.14,"Train Run Time (h)":null,"Train GPU":null,"Train CPU":null,"Train Slurm Jobs":null,"Test Run Time (h)":2705.45,"Test GPU":0.0,"Test CPU":32.0,"Test Slurm Jobs":476.0,"Run Time (h)":2705.45,"missing_preds":6184,"missing_percent":"2.41%"},"Voronoi RF":{"F1":0.34,"DAF":1.51,"Precision":0.26,"Recall":0.51,"Accuracy":0.66,"TPR":0.51,"FPR":0.3,"TNR":0.7,"FNR":0.49,"TP":22517.0,"FP":64431.0,"TN":148424.0,"FN":21574.0,"MAE":0.14,"RMSE":0.21,"R2":-0.31,"Train Run Time (h)":173.78,"Train GPU":0.0,"Train CPU":76.0,"Train Slurm Jobs":48.0,"Test Run Time (h)":30.92,"Test GPU":0.0,"Test CPU":76.0,"Test Slurm Jobs":20.0,"Run Time (h)":204.7,"missing_preds":17,"missing_percent":"0.01%"}} diff --git a/site/src/routes/models/per-element-each-errors.json b/site/src/routes/models/per-element-each-errors.json index e32d271d..dee374ba 100644 --- a/site/src/routes/models/per-element-each-errors.json +++ b/site/src/routes/models/per-element-each-errors.json @@ -1 +1 @@ -{"MP Occurrences":{"H":10409,"He":8,"Li":21576,"Be":1186,"B":6363,"C":9069,"N":11419,"O":81887,"F":12067,"Ne":1,"Na":12795,"Mg":19007,"Al":7803,"Si":12756,"P":16814,"S":15341,"Cl":6385,"Ar":2,"K":8034,"Ca":8422,"Sc":2686,"Ti":7721,"V":9703,"Cr":6774,"Mn":13966,"Fe":12889,"Co":11045,"Ni":8293,"Cu":9903,"Zn":6905,"Ga":5076,"Ge":5497,"As":3894,"Se":6509,"Br":3011,"Kr":15,"Rb":4599,"Sr":7297,"Y":5012,"Zr":3847,"Nb":5460,"Mo":5411,"Tc":696,"Ru":2453,"Rh":2512,"Pd":2985,"Ag":4109,"Cd":3396,"In":4400,"Sn":6157,"Sb":5784,"Te":4980,"I":3004,"Xe":157,"Cs":4071,"Ba":8338,"La":5599,"Ce":3271,"Pr":2894,"Nd":3234,"Pm":537,"Sm":2902,"Eu":2431,"Gd":1659,"Tb":2225,"Dy":2318,"Ho":2164,"Er":2208,"Tm":1752,"Yb":2121,"Lu":1634,"Hf":2729,"Ta":3407,"W":5121,"Re":1476,"Os":1229,"Ir":2053,"Pt":2444,"Au":2515,"Hg":2341,"Tl":2998,"Pb":3021,"Bi":4936,"Po":0,"At":0,"Rn":0,"Fr":0,"Ra":0,"Ac":304,"Th":1059,"Pa":286,"U":2439,"Np":410,"Pu":463,"Am":0,"Cm":0,"Bk":0,"Cf":0,"Es":0,"Fm":0,"Md":0,"No":0,"Lr":0,"Rf":0,"Db":0,"Sg":0,"Bh":0,"Hs":0,"Mt":0,"Ds":0,"Rg":0,"Cn":0,"Nh":0,"Fl":0,"Mc":0,"Lv":0,"Ts":0,"Og":0},"CHGNet":{"H":0.043,"He":null,"Li":0.0162,"Be":0.0316,"B":0.024,"C":0.0199,"N":0.0287,"O":0.0245,"F":0.0304,"Ne":null,"Na":0.0191,"Mg":0.0183,"Al":0.0248,"Si":0.0233,"P":0.0251,"S":0.0257,"Cl":0.0323,"Ar":null,"K":0.0147,"Ca":0.0182,"Sc":0.0203,"Ti":0.0193,"V":0.024,"Cr":0.0247,"Mn":0.0295,"Fe":0.0264,"Co":0.0191,"Ni":0.0178,"Cu":0.0177,"Zn":0.0222,"Ga":0.0222,"Ge":0.0214,"As":0.024,"Se":0.0274,"Br":0.033,"Kr":null,"Rb":0.0144,"Sr":0.0187,"Y":0.0199,"Zr":0.0231,"Nb":0.0261,"Mo":0.0188,"Tc":0.0353,"Ru":0.033,"Rh":0.0241,"Pd":0.0226,"Ag":0.0214,"Cd":0.0197,"In":0.0258,"Sn":0.0245,"Sb":0.0241,"Te":0.0358,"I":0.0338,"Xe":0.0003,"Cs":0.0152,"Ba":0.0152,"La":0.0185,"Ce":0.0181,"Pr":0.0176,"Nd":0.0173,"Pm":0.0292,"Sm":0.0172,"Eu":0.0239,"Gd":0.0201,"Tb":0.0178,"Dy":0.0174,"Ho":0.0174,"Er":0.0178,"Tm":0.018,"Yb":0.0186,"Lu":0.019,"Hf":0.0256,"Ta":0.0323,"W":0.0173,"Re":0.0236,"Os":0.0282,"Ir":0.029,"Pt":0.027,"Au":0.0253,"Hg":0.0256,"Tl":0.0261,"Pb":0.0257,"Bi":0.025,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0307,"Th":0.0214,"Pa":0.022,"U":0.0239,"Np":0.03,"Pu":0.0691,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"Test set standard deviation":{"H":0.1649,"He":null,"Li":0.1675,"Be":0.1417,"B":0.178,"C":0.2261,"N":0.2172,"O":0.2213,"F":0.2379,"Ne":null,"Na":0.1711,"Mg":0.1662,"Al":0.1418,"Si":0.1554,"P":0.1758,"S":0.1917,"Cl":0.1892,"Ar":null,"K":0.1896,"Ca":0.1619,"Sc":0.1519,"Ti":0.1446,"V":0.1888,"Cr":0.2026,"Mn":0.1885,"Fe":0.1793,"Co":0.1529,"Ni":0.1655,"Cu":0.1672,"Zn":0.155,"Ga":0.1398,"Ge":0.1471,"As":0.1558,"Se":0.1762,"Br":0.2457,"Kr":null,"Rb":0.2066,"Sr":0.1841,"Y":0.1489,"Zr":0.1462,"Nb":0.1674,"Mo":0.1611,"Tc":0.1802,"Ru":0.164,"Rh":0.1509,"Pd":0.1573,"Ag":0.17,"Cd":0.1718,"In":0.166,"Sn":0.1454,"Sb":0.1699,"Te":0.1876,"I":0.2091,"Xe":null,"Cs":0.2107,"Ba":0.1855,"La":0.1441,"Ce":0.1376,"Pr":0.14,"Nd":0.1389,"Pm":0.2869,"Sm":0.2033,"Eu":0.1858,"Gd":0.1924,"Tb":0.1466,"Dy":0.1443,"Ho":0.1406,"Er":0.2044,"Tm":0.1483,"Yb":0.1601,"Lu":0.1428,"Hf":0.1417,"Ta":0.226,"W":0.1781,"Re":0.1537,"Os":0.1818,"Ir":0.1752,"Pt":0.1649,"Au":0.1494,"Hg":0.1695,"Tl":0.1825,"Pb":0.2043,"Bi":0.3075,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.3074,"Th":0.1644,"Pa":0.4152,"U":0.2962,"Np":0.1903,"Pu":0.2224,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"M3GNet":{"H":0.0425,"He":null,"Li":0.0155,"Be":0.0327,"B":0.0268,"C":0.0219,"N":0.0304,"O":0.0279,"F":0.0303,"Ne":null,"Na":0.0145,"Mg":0.0182,"Al":0.028,"Si":0.0269,"P":0.0295,"S":0.0321,"Cl":0.0319,"Ar":null,"K":0.0144,"Ca":0.0159,"Sc":0.0202,"Ti":0.0204,"V":0.0232,"Cr":0.0271,"Mn":0.0293,"Fe":0.029,"Co":0.0205,"Ni":0.0197,"Cu":0.0184,"Zn":0.0224,"Ga":0.0248,"Ge":0.0258,"As":0.0282,"Se":0.0296,"Br":0.0331,"Kr":null,"Rb":0.0148,"Sr":0.0161,"Y":0.0176,"Zr":0.025,"Nb":0.0309,"Mo":0.0211,"Tc":0.0237,"Ru":0.0306,"Rh":0.0257,"Pd":0.0222,"Ag":0.0178,"Cd":0.0174,"In":0.0228,"Sn":0.0274,"Sb":0.0246,"Te":0.0337,"I":0.0321,"Xe":0.01,"Cs":0.016,"Ba":0.0155,"La":0.0164,"Ce":0.0185,"Pr":0.0162,"Nd":0.0163,"Pm":0.0229,"Sm":0.0167,"Eu":0.0205,"Gd":0.0187,"Tb":0.0171,"Dy":0.0177,"Ho":0.0173,"Er":0.0177,"Tm":0.0183,"Yb":0.0165,"Lu":0.0186,"Hf":0.0243,"Ta":0.0358,"W":0.0197,"Re":0.021,"Os":0.0275,"Ir":0.0298,"Pt":0.0286,"Au":0.0246,"Hg":0.0184,"Tl":0.0188,"Pb":0.0256,"Bi":0.0234,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0271,"Th":0.0236,"Pa":0.024,"U":0.0242,"Np":0.033,"Pu":0.0618,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"ALIGNN":{"H":0.0507,"He":null,"Li":0.0219,"Be":0.0296,"B":0.0319,"C":0.0296,"N":0.0444,"O":0.0567,"F":0.085,"Ne":null,"Na":0.0221,"Mg":0.0227,"Al":0.0292,"Si":0.0332,"P":0.0304,"S":0.0431,"Cl":0.0507,"Ar":null,"K":0.0211,"Ca":0.0206,"Sc":0.0239,"Ti":0.0332,"V":0.0422,"Cr":0.0312,"Mn":0.0324,"Fe":0.0361,"Co":0.028,"Ni":0.0276,"Cu":0.0236,"Zn":0.0256,"Ga":0.0248,"Ge":0.0277,"As":0.0263,"Se":0.0343,"Br":0.0516,"Kr":null,"Rb":0.021,"Sr":0.0202,"Y":0.0231,"Zr":0.0274,"Nb":0.0364,"Mo":0.0246,"Tc":0.0275,"Ru":0.0319,"Rh":0.0292,"Pd":0.0282,"Ag":0.0232,"Cd":0.0215,"In":0.0271,"Sn":0.0273,"Sb":0.0252,"Te":0.0388,"I":0.0469,"Xe":0.0107,"Cs":0.0215,"Ba":0.0211,"La":0.0248,"Ce":0.0251,"Pr":0.0217,"Nd":0.021,"Pm":0.0261,"Sm":0.0212,"Eu":0.0214,"Gd":0.0227,"Tb":0.0219,"Dy":0.0219,"Ho":0.0222,"Er":0.0219,"Tm":0.022,"Yb":0.019,"Lu":0.0227,"Hf":0.0252,"Ta":0.0459,"W":0.0186,"Re":0.0219,"Os":0.0274,"Ir":0.0332,"Pt":0.0297,"Au":0.0282,"Hg":0.0225,"Tl":0.0219,"Pb":0.0284,"Bi":0.0248,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0263,"Th":0.0552,"Pa":0.0283,"U":0.0289,"Np":0.0772,"Pu":0.0462,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"MEGNet":{"H":0.0691,"He":null,"Li":0.031,"Be":0.0454,"B":0.0312,"C":0.0415,"N":0.0695,"O":0.0629,"F":0.1945,"Ne":null,"Na":0.0329,"Mg":0.0322,"Al":0.0359,"Si":0.0427,"P":0.0416,"S":0.06,"Cl":0.1459,"Ar":null,"K":0.0333,"Ca":0.0261,"Sc":0.0322,"Ti":0.0533,"V":0.0749,"Cr":0.0467,"Mn":0.046,"Fe":0.0457,"Co":0.0422,"Ni":0.0377,"Cu":0.0302,"Zn":0.0318,"Ga":0.0361,"Ge":0.036,"As":0.038,"Se":0.0499,"Br":0.0496,"Kr":null,"Rb":0.0355,"Sr":0.0296,"Y":0.0327,"Zr":0.04,"Nb":0.0684,"Mo":0.0378,"Tc":0.0527,"Ru":0.0403,"Rh":0.0368,"Pd":0.0381,"Ag":0.031,"Cd":0.0281,"In":0.0359,"Sn":0.0349,"Sb":0.0337,"Te":0.0504,"I":0.0505,"Xe":0.1197,"Cs":0.0341,"Ba":0.0332,"La":0.0326,"Ce":0.0327,"Pr":0.0326,"Nd":0.031,"Pm":0.0274,"Sm":0.0292,"Eu":0.0303,"Gd":0.0327,"Tb":0.0321,"Dy":0.0312,"Ho":0.031,"Er":0.0301,"Tm":0.0299,"Yb":0.0251,"Lu":0.0304,"Hf":0.038,"Ta":0.0749,"W":0.0284,"Re":0.037,"Os":0.0386,"Ir":0.0398,"Pt":0.0409,"Au":0.0368,"Hg":0.0327,"Tl":0.0338,"Pb":0.0373,"Bi":0.0335,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.027,"Th":0.0402,"Pa":0.029,"U":0.0411,"Np":0.0451,"Pu":0.0699,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"CGCNN":{"H":0.0605,"He":null,"Li":0.0327,"Be":0.0496,"B":0.0464,"C":0.0484,"N":0.0713,"O":0.081,"F":0.0842,"Ne":null,"Na":0.0282,"Mg":0.0339,"Al":0.0449,"Si":0.0573,"P":0.0509,"S":0.0736,"Cl":0.0644,"Ar":null,"K":0.0236,"Ca":0.0278,"Sc":0.0386,"Ti":0.0727,"V":0.0982,"Cr":0.0507,"Mn":0.0436,"Fe":0.0591,"Co":0.0577,"Ni":0.0539,"Cu":0.0367,"Zn":0.038,"Ga":0.0392,"Ge":0.0447,"As":0.0451,"Se":0.0574,"Br":0.0565,"Kr":null,"Rb":0.0243,"Sr":0.0257,"Y":0.0287,"Zr":0.0461,"Nb":0.0719,"Mo":0.0522,"Tc":0.038,"Ru":0.0426,"Rh":0.0454,"Pd":0.0416,"Ag":0.0361,"Cd":0.03,"In":0.0353,"Sn":0.0393,"Sb":0.0383,"Te":0.0532,"I":0.0556,"Xe":0.0663,"Cs":0.0246,"Ba":0.0271,"La":0.0321,"Ce":0.0333,"Pr":0.026,"Nd":0.0264,"Pm":0.0368,"Sm":0.0282,"Eu":0.0237,"Gd":0.0332,"Tb":0.0274,"Dy":0.0265,"Ho":0.0277,"Er":0.0273,"Tm":0.0284,"Yb":0.0259,"Lu":0.0285,"Hf":0.0414,"Ta":0.0888,"W":0.0305,"Re":0.0318,"Os":0.0358,"Ir":0.0454,"Pt":0.0464,"Au":0.0393,"Hg":0.031,"Tl":0.0309,"Pb":0.0392,"Bi":0.0366,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0299,"Th":0.06,"Pa":0.0328,"U":0.0418,"Np":0.0808,"Pu":0.05,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"CGCNN+P":{"H":0.0566,"He":null,"Li":0.0232,"Be":0.0442,"B":0.0383,"C":0.0355,"N":0.0533,"O":0.0503,"F":0.0559,"Ne":null,"Na":0.0231,"Mg":0.0283,"Al":0.0415,"Si":0.0456,"P":0.0393,"S":0.0473,"Cl":0.0482,"Ar":null,"K":0.0213,"Ca":0.0253,"Sc":0.0308,"Ti":0.0492,"V":0.0602,"Cr":0.0417,"Mn":0.0452,"Fe":0.0512,"Co":0.0452,"Ni":0.0404,"Cu":0.0289,"Zn":0.0346,"Ga":0.034,"Ge":0.0373,"As":0.0357,"Se":0.0392,"Br":0.047,"Kr":null,"Rb":0.0207,"Sr":0.0242,"Y":0.0261,"Zr":0.0367,"Nb":0.0476,"Mo":0.0339,"Tc":0.0347,"Ru":0.0396,"Rh":0.036,"Pd":0.032,"Ag":0.0257,"Cd":0.0269,"In":0.0316,"Sn":0.0325,"Sb":0.0315,"Te":0.0455,"I":0.0452,"Xe":0.0013,"Cs":0.0209,"Ba":0.0228,"La":0.0268,"Ce":0.0258,"Pr":0.0223,"Nd":0.0226,"Pm":0.035,"Sm":0.0229,"Eu":0.0236,"Gd":0.0242,"Tb":0.0242,"Dy":0.0246,"Ho":0.0255,"Er":0.0253,"Tm":0.0263,"Yb":0.0234,"Lu":0.0272,"Hf":0.0329,"Ta":0.0547,"W":0.0261,"Re":0.0325,"Os":0.0351,"Ir":0.0413,"Pt":0.0386,"Au":0.0315,"Hg":0.0248,"Tl":0.027,"Pb":0.0353,"Bi":0.028,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0331,"Th":0.0402,"Pa":0.0274,"U":0.0365,"Np":0.1052,"Pu":0.044,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"Wrenformer":{"H":0.0616,"He":null,"Li":0.0208,"Be":0.043,"B":0.0473,"C":0.0383,"N":0.0544,"O":0.0632,"F":0.0817,"Ne":null,"Na":0.0259,"Mg":0.0265,"Al":0.036,"Si":0.0328,"P":0.0368,"S":0.0464,"Cl":0.0521,"Ar":null,"K":0.0273,"Ca":0.026,"Sc":0.0292,"Ti":0.0293,"V":0.0376,"Cr":0.0388,"Mn":0.0426,"Fe":0.046,"Co":0.0316,"Ni":0.0285,"Cu":0.0283,"Zn":0.0312,"Ga":0.0286,"Ge":0.0302,"As":0.0323,"Se":0.0394,"Br":0.0498,"Kr":null,"Rb":0.0267,"Sr":0.026,"Y":0.029,"Zr":0.0344,"Nb":0.0386,"Mo":0.0406,"Tc":0.0286,"Ru":0.0433,"Rh":0.035,"Pd":0.0303,"Ag":0.0229,"Cd":0.0277,"In":0.0323,"Sn":0.029,"Sb":0.0306,"Te":0.0434,"I":0.0456,"Xe":0.0533,"Cs":0.0274,"Ba":0.024,"La":0.0248,"Ce":0.0233,"Pr":0.0242,"Nd":0.0244,"Pm":0.0352,"Sm":0.0248,"Eu":0.0256,"Gd":0.0278,"Tb":0.0264,"Dy":0.0262,"Ho":0.0264,"Er":0.0268,"Tm":0.0271,"Yb":0.0271,"Lu":0.0257,"Hf":0.0359,"Ta":0.0428,"W":0.0447,"Re":0.0324,"Os":0.043,"Ir":0.0429,"Pt":0.0353,"Au":0.0347,"Hg":0.0241,"Tl":0.0264,"Pb":0.0308,"Bi":0.0256,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0305,"Th":0.0342,"Pa":0.0303,"U":0.04,"Np":0.0375,"Pu":0.0607,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"BOWSR + MEGNet":{"H":0.0745,"He":null,"Li":0.0263,"Be":0.0497,"B":0.0416,"C":0.0335,"N":0.0571,"O":0.0679,"F":0.0729,"Ne":null,"Na":0.0317,"Mg":0.0251,"Al":0.0376,"Si":0.0397,"P":0.0387,"S":0.072,"Cl":0.0575,"Ar":null,"K":0.0377,"Ca":0.0263,"Sc":0.0315,"Ti":0.0351,"V":0.0385,"Cr":0.04,"Mn":0.0409,"Fe":0.043,"Co":0.0321,"Ni":0.0284,"Cu":0.0321,"Zn":0.0302,"Ga":0.0298,"Ge":0.0308,"As":0.0344,"Se":0.0839,"Br":0.099,"Kr":null,"Rb":0.0365,"Sr":0.0275,"Y":0.0284,"Zr":0.0367,"Nb":0.0354,"Mo":0.0296,"Tc":0.034,"Ru":0.0446,"Rh":0.0365,"Pd":0.0373,"Ag":0.0319,"Cd":0.0249,"In":0.0315,"Sn":0.0312,"Sb":0.0286,"Te":0.0671,"I":0.0712,"Xe":0.134,"Cs":0.0385,"Ba":0.0275,"La":0.0249,"Ce":0.0258,"Pr":0.0246,"Nd":0.0246,"Pm":0.0283,"Sm":0.0259,"Eu":0.0287,"Gd":0.0283,"Tb":0.027,"Dy":0.0271,"Ho":0.0271,"Er":0.028,"Tm":0.0287,"Yb":0.0303,"Lu":0.0299,"Hf":0.0418,"Ta":0.049,"W":0.0256,"Re":0.0338,"Os":0.0424,"Ir":0.0385,"Pt":0.0391,"Au":0.0358,"Hg":0.0247,"Tl":0.0292,"Pb":0.0352,"Bi":0.032,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0301,"Th":0.0323,"Pa":0.0325,"U":0.0381,"Np":0.0323,"Pu":0.0469,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"Voronoi RF":{"H":0.0666,"He":null,"Li":0.0367,"Be":0.0579,"B":0.0535,"C":0.0442,"N":0.0881,"O":0.0888,"F":0.1127,"Ne":null,"Na":0.0444,"Mg":0.0367,"Al":0.0385,"Si":0.0403,"P":0.0454,"S":0.0569,"Cl":0.0788,"Ar":null,"K":0.0332,"Ca":0.0394,"Sc":0.0358,"Ti":0.0391,"V":0.0545,"Cr":0.0716,"Mn":0.0571,"Fe":0.062,"Co":0.044,"Ni":0.0348,"Cu":0.0405,"Zn":0.0412,"Ga":0.0339,"Ge":0.038,"As":0.0431,"Se":0.0554,"Br":0.0757,"Kr":null,"Rb":0.0329,"Sr":0.0401,"Y":0.034,"Zr":0.0376,"Nb":0.0462,"Mo":0.0366,"Tc":0.0514,"Ru":0.0605,"Rh":0.04,"Pd":0.0374,"Ag":0.0416,"Cd":0.0396,"In":0.0458,"Sn":0.0371,"Sb":0.0409,"Te":0.0597,"I":0.0756,"Xe":0.0127,"Cs":0.0316,"Ba":0.0351,"La":0.0321,"Ce":0.0362,"Pr":0.0342,"Nd":0.0331,"Pm":0.0445,"Sm":0.0339,"Eu":0.0415,"Gd":0.0355,"Tb":0.0357,"Dy":0.0353,"Ho":0.0361,"Er":0.0365,"Tm":0.0375,"Yb":0.0424,"Lu":0.033,"Hf":0.0429,"Ta":0.0548,"W":0.0361,"Re":0.035,"Os":0.0602,"Ir":0.0477,"Pt":0.04,"Au":0.0471,"Hg":0.0407,"Tl":0.05,"Pb":0.0579,"Bi":0.0463,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.051,"Th":0.0403,"Pa":0.0422,"U":0.0626,"Np":0.071,"Pu":0.0574,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"Mean error all models":{"H":0.0583,"He":null,"Li":0.0249,"Be":0.0426,"B":0.0379,"C":0.0348,"N":0.0552,"O":0.0586,"F":0.0839,"Ne":null,"Na":0.0269,"Mg":0.0269,"Al":0.0352,"Si":0.038,"P":0.0375,"S":0.0506,"Cl":0.0626,"Ar":null,"K":0.0251,"Ca":0.0251,"Sc":0.0291,"Ti":0.0393,"V":0.0506,"Cr":0.0413,"Mn":0.0407,"Fe":0.0443,"Co":0.0356,"Ni":0.0321,"Cu":0.0285,"Zn":0.0309,"Ga":0.0304,"Ge":0.0325,"As":0.0341,"Se":0.0462,"Br":0.0548,"Kr":null,"Rb":0.0251,"Sr":0.0254,"Y":0.0267,"Zr":0.0342,"Nb":0.0446,"Mo":0.0327,"Tc":0.0362,"Ru":0.0407,"Rh":0.0343,"Pd":0.0322,"Ag":0.028,"Cd":0.0262,"In":0.032,"Sn":0.0315,"Sb":0.0308,"Te":0.0475,"I":0.0507,"Xe":0.0454,"Cs":0.0255,"Ba":0.0246,"La":0.0259,"Ce":0.0265,"Pr":0.0244,"Nd":0.0241,"Pm":0.0317,"Sm":0.0245,"Eu":0.0266,"Gd":0.027,"Tb":0.0255,"Dy":0.0253,"Ho":0.0256,"Er":0.0257,"Tm":0.0262,"Yb":0.0254,"Lu":0.0261,"Hf":0.0342,"Ta":0.0532,"W":0.0274,"Re":0.0299,"Os":0.0376,"Ir":0.0386,"Pt":0.0362,"Au":0.0337,"Hg":0.0272,"Tl":0.0294,"Pb":0.0351,"Bi":0.0306,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0317,"Th":0.0386,"Pa":0.0298,"U":0.0374,"Np":0.0569,"Pu":0.0561,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null}} +{"MP Occurrences":{"H":10409,"He":8,"Li":21576,"Be":1186,"B":6363,"C":9069,"N":11419,"O":81887,"F":12067,"Ne":1,"Na":12795,"Mg":19007,"Al":7803,"Si":12756,"P":16814,"S":15341,"Cl":6385,"Ar":2,"K":8034,"Ca":8422,"Sc":2686,"Ti":7721,"V":9703,"Cr":6774,"Mn":13966,"Fe":12889,"Co":11045,"Ni":8293,"Cu":9903,"Zn":6905,"Ga":5076,"Ge":5497,"As":3894,"Se":6509,"Br":3011,"Kr":15,"Rb":4599,"Sr":7297,"Y":5012,"Zr":3847,"Nb":5460,"Mo":5411,"Tc":696,"Ru":2453,"Rh":2512,"Pd":2985,"Ag":4109,"Cd":3396,"In":4400,"Sn":6157,"Sb":5784,"Te":4980,"I":3004,"Xe":157,"Cs":4071,"Ba":8338,"La":5599,"Ce":3271,"Pr":2894,"Nd":3234,"Pm":537,"Sm":2902,"Eu":2431,"Gd":1659,"Tb":2225,"Dy":2318,"Ho":2164,"Er":2208,"Tm":1752,"Yb":2121,"Lu":1634,"Hf":2729,"Ta":3407,"W":5121,"Re":1476,"Os":1229,"Ir":2053,"Pt":2444,"Au":2515,"Hg":2341,"Tl":2998,"Pb":3021,"Bi":4936,"Po":0,"At":0,"Rn":0,"Fr":0,"Ra":0,"Ac":304,"Th":1059,"Pa":286,"U":2439,"Np":410,"Pu":463,"Am":0,"Cm":0,"Bk":0,"Cf":0,"Es":0,"Fm":0,"Md":0,"No":0,"Lr":0,"Rf":0,"Db":0,"Sg":0,"Bh":0,"Hs":0,"Mt":0,"Ds":0,"Rg":0,"Cn":0,"Nh":0,"Fl":0,"Mc":0,"Lv":0,"Ts":0,"Og":0},"Test set standard deviation":{"H":0.1649,"He":null,"Li":0.1675,"Be":0.1417,"B":0.178,"C":0.2261,"N":0.2172,"O":0.2213,"F":0.2379,"Ne":null,"Na":0.1711,"Mg":0.1662,"Al":0.1418,"Si":0.1554,"P":0.1758,"S":0.1917,"Cl":0.1892,"Ar":null,"K":0.1896,"Ca":0.1619,"Sc":0.1519,"Ti":0.1446,"V":0.1888,"Cr":0.2026,"Mn":0.1885,"Fe":0.1793,"Co":0.1529,"Ni":0.1655,"Cu":0.1672,"Zn":0.155,"Ga":0.1398,"Ge":0.1471,"As":0.1558,"Se":0.1762,"Br":0.2457,"Kr":null,"Rb":0.2066,"Sr":0.1841,"Y":0.1489,"Zr":0.1462,"Nb":0.1674,"Mo":0.1611,"Tc":0.1802,"Ru":0.164,"Rh":0.1509,"Pd":0.1573,"Ag":0.17,"Cd":0.1718,"In":0.166,"Sn":0.1454,"Sb":0.1699,"Te":0.1876,"I":0.2091,"Xe":null,"Cs":0.2107,"Ba":0.1855,"La":0.1441,"Ce":0.1376,"Pr":0.14,"Nd":0.1389,"Pm":0.2869,"Sm":0.2033,"Eu":0.1858,"Gd":0.1924,"Tb":0.1466,"Dy":0.1443,"Ho":0.1406,"Er":0.2044,"Tm":0.1483,"Yb":0.1601,"Lu":0.1428,"Hf":0.1417,"Ta":0.226,"W":0.1781,"Re":0.1537,"Os":0.1818,"Ir":0.1752,"Pt":0.1649,"Au":0.1494,"Hg":0.1695,"Tl":0.1825,"Pb":0.2043,"Bi":0.3075,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.3074,"Th":0.1644,"Pa":0.4152,"U":0.2962,"Np":0.1903,"Pu":0.2224,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"CHGNet":{"H":0.0429,"He":null,"Li":0.0163,"Be":0.0316,"B":0.024,"C":0.0199,"N":0.0286,"O":0.0245,"F":0.0304,"Ne":null,"Na":0.0193,"Mg":0.0184,"Al":0.0247,"Si":0.0233,"P":0.0251,"S":0.0259,"Cl":0.0323,"Ar":null,"K":0.0149,"Ca":0.0183,"Sc":0.0203,"Ti":0.0193,"V":0.024,"Cr":0.0247,"Mn":0.0295,"Fe":0.0264,"Co":0.0192,"Ni":0.0178,"Cu":0.0177,"Zn":0.0223,"Ga":0.0222,"Ge":0.0214,"As":0.024,"Se":0.0274,"Br":0.0363,"Kr":null,"Rb":0.0145,"Sr":0.0188,"Y":0.02,"Zr":0.023,"Nb":0.0261,"Mo":0.0188,"Tc":0.0353,"Ru":0.0329,"Rh":0.0242,"Pd":0.0226,"Ag":0.0216,"Cd":0.0198,"In":0.0257,"Sn":0.0244,"Sb":0.0241,"Te":0.0358,"I":0.0351,"Xe":0.0003,"Cs":0.0153,"Ba":0.0154,"La":0.0186,"Ce":0.0182,"Pr":0.0177,"Nd":0.0174,"Pm":0.0293,"Sm":0.0174,"Eu":0.0244,"Gd":0.0202,"Tb":0.0179,"Dy":0.0175,"Ho":0.0174,"Er":0.0177,"Tm":0.0181,"Yb":0.0188,"Lu":0.0189,"Hf":0.0254,"Ta":0.0321,"W":0.0173,"Re":0.0236,"Os":0.0282,"Ir":0.029,"Pt":0.0269,"Au":0.0254,"Hg":0.0257,"Tl":0.0262,"Pb":0.0258,"Bi":0.0263,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0308,"Th":0.0213,"Pa":0.0219,"U":0.024,"Np":0.0302,"Pu":0.0692,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"M3GNet":{"H":0.0423,"He":null,"Li":0.0155,"Be":0.0327,"B":0.0268,"C":0.0218,"N":0.0303,"O":0.0278,"F":0.0303,"Ne":null,"Na":0.0145,"Mg":0.0181,"Al":0.0279,"Si":0.0268,"P":0.0295,"S":0.032,"Cl":0.0318,"Ar":null,"K":0.0144,"Ca":0.0159,"Sc":0.0202,"Ti":0.0203,"V":0.0231,"Cr":0.027,"Mn":0.0292,"Fe":0.029,"Co":0.0205,"Ni":0.0196,"Cu":0.0184,"Zn":0.0223,"Ga":0.0248,"Ge":0.0258,"As":0.0281,"Se":0.0295,"Br":0.0318,"Kr":null,"Rb":0.0147,"Sr":0.016,"Y":0.0175,"Zr":0.0248,"Nb":0.0307,"Mo":0.021,"Tc":0.0237,"Ru":0.0305,"Rh":0.0256,"Pd":0.0221,"Ag":0.0178,"Cd":0.0174,"In":0.0226,"Sn":0.0273,"Sb":0.0245,"Te":0.0335,"I":0.0304,"Xe":0.01,"Cs":0.0158,"Ba":0.0155,"La":0.0163,"Ce":0.0185,"Pr":0.016,"Nd":0.0162,"Pm":0.0229,"Sm":0.0167,"Eu":0.0196,"Gd":0.0187,"Tb":0.017,"Dy":0.0176,"Ho":0.0172,"Er":0.0177,"Tm":0.0182,"Yb":0.0165,"Lu":0.0185,"Hf":0.0241,"Ta":0.0356,"W":0.0198,"Re":0.021,"Os":0.0275,"Ir":0.0297,"Pt":0.0285,"Au":0.0245,"Hg":0.0184,"Tl":0.0188,"Pb":0.0254,"Bi":0.0218,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0271,"Th":0.0235,"Pa":0.0241,"U":0.0241,"Np":0.0329,"Pu":0.0619,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"ALIGNN":{"H":0.0504,"He":null,"Li":0.0218,"Be":0.0295,"B":0.0319,"C":0.0296,"N":0.0443,"O":0.0567,"F":0.0849,"Ne":null,"Na":0.0221,"Mg":0.0227,"Al":0.0291,"Si":0.0331,"P":0.0304,"S":0.0432,"Cl":0.0507,"Ar":null,"K":0.0208,"Ca":0.0205,"Sc":0.0239,"Ti":0.0332,"V":0.0422,"Cr":0.0311,"Mn":0.0324,"Fe":0.0361,"Co":0.028,"Ni":0.0275,"Cu":0.0236,"Zn":0.0255,"Ga":0.0247,"Ge":0.0276,"As":0.0263,"Se":0.0343,"Br":0.0461,"Kr":null,"Rb":0.0206,"Sr":0.0202,"Y":0.023,"Zr":0.0273,"Nb":0.0364,"Mo":0.0246,"Tc":0.0274,"Ru":0.0318,"Rh":0.0291,"Pd":0.0282,"Ag":0.0231,"Cd":0.0214,"In":0.0269,"Sn":0.0272,"Sb":0.0252,"Te":0.0387,"I":0.0442,"Xe":0.0107,"Cs":0.0211,"Ba":0.0211,"La":0.0249,"Ce":0.025,"Pr":0.0216,"Nd":0.0209,"Pm":0.0261,"Sm":0.0211,"Eu":0.0201,"Gd":0.0226,"Tb":0.0219,"Dy":0.0218,"Ho":0.0221,"Er":0.0218,"Tm":0.0219,"Yb":0.0189,"Lu":0.0226,"Hf":0.0251,"Ta":0.0459,"W":0.0185,"Re":0.0219,"Os":0.0274,"Ir":0.0331,"Pt":0.0296,"Au":0.0281,"Hg":0.0224,"Tl":0.0218,"Pb":0.0282,"Bi":0.0235,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0264,"Th":0.0552,"Pa":0.0281,"U":0.0286,"Np":0.0769,"Pu":0.0461,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"MACE":{"H":0.0551,"He":null,"Li":0.0424,"Be":0.0373,"B":0.0274,"C":0.0285,"N":0.0378,"O":0.0376,"F":0.0652,"Ne":null,"Na":0.0539,"Mg":0.0395,"Al":0.0371,"Si":0.0465,"P":0.0414,"S":0.0358,"Cl":0.0607,"Ar":null,"K":0.0583,"Ca":0.0408,"Sc":0.0245,"Ti":0.0362,"V":0.0266,"Cr":0.0325,"Mn":0.0357,"Fe":0.0309,"Co":0.0223,"Ni":0.021,"Cu":0.0209,"Zn":0.0271,"Ga":0.0271,"Ge":0.025,"As":0.0246,"Se":0.0299,"Br":0.0674,"Kr":null,"Rb":0.0537,"Sr":0.0527,"Y":0.0386,"Zr":0.0298,"Nb":0.0303,"Mo":0.0215,"Tc":0.043,"Ru":0.0386,"Rh":0.0261,"Pd":0.025,"Ag":0.0199,"Cd":0.0235,"In":0.0292,"Sn":0.0255,"Sb":0.0249,"Te":0.034,"I":0.0488,"Xe":0.021,"Cs":0.0601,"Ba":0.058,"La":0.0723,"Ce":0.0297,"Pr":0.0328,"Nd":0.0302,"Pm":0.0568,"Sm":0.0331,"Eu":0.0318,"Gd":0.0265,"Tb":0.0258,"Dy":0.0366,"Ho":0.0318,"Er":0.0252,"Tm":0.0416,"Yb":0.0302,"Lu":0.0363,"Hf":0.0275,"Ta":0.0348,"W":0.0201,"Re":0.0284,"Os":0.0347,"Ir":0.0291,"Pt":0.0259,"Au":0.0246,"Hg":0.023,"Tl":0.0322,"Pb":0.0281,"Bi":0.0264,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0775,"Th":0.0335,"Pa":0.029,"U":0.0528,"Np":0.0703,"Pu":0.0577,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"MEGNet":{"H":0.0689,"He":null,"Li":0.0311,"Be":0.0454,"B":0.0312,"C":0.0416,"N":0.0694,"O":0.0633,"F":0.1944,"Ne":null,"Na":0.033,"Mg":0.0323,"Al":0.0359,"Si":0.0428,"P":0.0417,"S":0.0602,"Cl":0.1456,"Ar":null,"K":0.0336,"Ca":0.0263,"Sc":0.0324,"Ti":0.0535,"V":0.0751,"Cr":0.0468,"Mn":0.0461,"Fe":0.0458,"Co":0.0423,"Ni":0.0378,"Cu":0.0303,"Zn":0.0318,"Ga":0.0362,"Ge":0.0361,"As":0.038,"Se":0.0501,"Br":0.0576,"Kr":null,"Rb":0.0358,"Sr":0.0298,"Y":0.0329,"Zr":0.0402,"Nb":0.0687,"Mo":0.0379,"Tc":0.0528,"Ru":0.0403,"Rh":0.0369,"Pd":0.0381,"Ag":0.0312,"Cd":0.0281,"In":0.036,"Sn":0.0349,"Sb":0.0338,"Te":0.0506,"I":0.0547,"Xe":0.1197,"Cs":0.0344,"Ba":0.0335,"La":0.0328,"Ce":0.0329,"Pr":0.0328,"Nd":0.0311,"Pm":0.0275,"Sm":0.0294,"Eu":0.0309,"Gd":0.0328,"Tb":0.0323,"Dy":0.0313,"Ho":0.0312,"Er":0.0302,"Tm":0.0301,"Yb":0.0252,"Lu":0.0306,"Hf":0.038,"Ta":0.0752,"W":0.0284,"Re":0.037,"Os":0.0386,"Ir":0.0398,"Pt":0.0409,"Au":0.0369,"Hg":0.0328,"Tl":0.0339,"Pb":0.0374,"Bi":0.0359,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.027,"Th":0.0403,"Pa":0.0289,"U":0.0413,"Np":0.0452,"Pu":0.07,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"CGCNN":{"H":0.0606,"He":null,"Li":0.0328,"Be":0.0496,"B":0.0464,"C":0.0486,"N":0.0713,"O":0.0812,"F":0.0843,"Ne":null,"Na":0.0283,"Mg":0.034,"Al":0.0449,"Si":0.0574,"P":0.0509,"S":0.0739,"Cl":0.0644,"Ar":null,"K":0.0237,"Ca":0.0279,"Sc":0.0388,"Ti":0.0729,"V":0.0985,"Cr":0.0509,"Mn":0.0437,"Fe":0.0591,"Co":0.0578,"Ni":0.0539,"Cu":0.0368,"Zn":0.038,"Ga":0.0393,"Ge":0.0448,"As":0.0452,"Se":0.0576,"Br":0.0631,"Kr":null,"Rb":0.0244,"Sr":0.0258,"Y":0.0287,"Zr":0.0462,"Nb":0.0721,"Mo":0.0523,"Tc":0.0379,"Ru":0.0425,"Rh":0.0454,"Pd":0.0416,"Ag":0.0363,"Cd":0.0301,"In":0.0353,"Sn":0.0393,"Sb":0.0383,"Te":0.0533,"I":0.0575,"Xe":0.0663,"Cs":0.0248,"Ba":0.0273,"La":0.0323,"Ce":0.0333,"Pr":0.0261,"Nd":0.0266,"Pm":0.0369,"Sm":0.0283,"Eu":0.0233,"Gd":0.0333,"Tb":0.0275,"Dy":0.0266,"Ho":0.0278,"Er":0.0274,"Tm":0.0285,"Yb":0.026,"Lu":0.0285,"Hf":0.0414,"Ta":0.089,"W":0.0305,"Re":0.0318,"Os":0.0358,"Ir":0.0454,"Pt":0.0464,"Au":0.0394,"Hg":0.0311,"Tl":0.031,"Pb":0.0392,"Bi":0.0381,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.03,"Th":0.0601,"Pa":0.0327,"U":0.0418,"Np":0.0805,"Pu":0.05,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"CGCNN+P":{"H":0.0562,"He":null,"Li":0.0232,"Be":0.0442,"B":0.0383,"C":0.0354,"N":0.0532,"O":0.0502,"F":0.0559,"Ne":null,"Na":0.0231,"Mg":0.0283,"Al":0.0415,"Si":0.0455,"P":0.0392,"S":0.0474,"Cl":0.0481,"Ar":null,"K":0.0214,"Ca":0.0253,"Sc":0.0308,"Ti":0.0493,"V":0.0603,"Cr":0.0418,"Mn":0.0452,"Fe":0.0512,"Co":0.0452,"Ni":0.0404,"Cu":0.0288,"Zn":0.0345,"Ga":0.0339,"Ge":0.0373,"As":0.0357,"Se":0.0391,"Br":0.0472,"Kr":null,"Rb":0.0207,"Sr":0.0241,"Y":0.026,"Zr":0.0367,"Nb":0.0476,"Mo":0.0339,"Tc":0.0346,"Ru":0.0395,"Rh":0.036,"Pd":0.032,"Ag":0.0257,"Cd":0.0269,"In":0.0314,"Sn":0.0324,"Sb":0.0314,"Te":0.0454,"I":0.0435,"Xe":0.0013,"Cs":0.0208,"Ba":0.0229,"La":0.0269,"Ce":0.0256,"Pr":0.0222,"Nd":0.0225,"Pm":0.035,"Sm":0.0228,"Eu":0.0227,"Gd":0.0241,"Tb":0.0242,"Dy":0.0245,"Ho":0.0254,"Er":0.0252,"Tm":0.0262,"Yb":0.0234,"Lu":0.0272,"Hf":0.0327,"Ta":0.0546,"W":0.0261,"Re":0.0326,"Os":0.0351,"Ir":0.0413,"Pt":0.0385,"Au":0.0314,"Hg":0.0248,"Tl":0.027,"Pb":0.0352,"Bi":0.0276,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0331,"Th":0.0402,"Pa":0.0273,"U":0.0365,"Np":0.1048,"Pu":0.0439,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"Wrenformer":{"H":0.0612,"He":null,"Li":0.0208,"Be":0.043,"B":0.0473,"C":0.0382,"N":0.0543,"O":0.063,"F":0.0817,"Ne":null,"Na":0.0258,"Mg":0.0264,"Al":0.036,"Si":0.0328,"P":0.0368,"S":0.0463,"Cl":0.052,"Ar":null,"K":0.0272,"Ca":0.026,"Sc":0.0292,"Ti":0.0292,"V":0.0374,"Cr":0.0388,"Mn":0.0426,"Fe":0.046,"Co":0.0316,"Ni":0.0284,"Cu":0.0283,"Zn":0.0312,"Ga":0.0285,"Ge":0.0301,"As":0.0323,"Se":0.0394,"Br":0.0486,"Kr":null,"Rb":0.0267,"Sr":0.026,"Y":0.0289,"Zr":0.0343,"Nb":0.0384,"Mo":0.0405,"Tc":0.0286,"Ru":0.0432,"Rh":0.0349,"Pd":0.0302,"Ag":0.0228,"Cd":0.0277,"In":0.0322,"Sn":0.0289,"Sb":0.0305,"Te":0.0433,"I":0.0437,"Xe":0.0533,"Cs":0.0274,"Ba":0.024,"La":0.0247,"Ce":0.0232,"Pr":0.0242,"Nd":0.0244,"Pm":0.0352,"Sm":0.0249,"Eu":0.0245,"Gd":0.0277,"Tb":0.0264,"Dy":0.0261,"Ho":0.0263,"Er":0.0267,"Tm":0.0271,"Yb":0.0271,"Lu":0.0256,"Hf":0.0357,"Ta":0.0427,"W":0.0447,"Re":0.0325,"Os":0.043,"Ir":0.0428,"Pt":0.0352,"Au":0.0347,"Hg":0.0241,"Tl":0.0264,"Pb":0.0307,"Bi":0.0244,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0305,"Th":0.034,"Pa":0.0302,"U":0.04,"Np":0.0375,"Pu":0.0608,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"BOWSR":{"H":0.0746,"He":null,"Li":0.0263,"Be":0.0497,"B":0.0416,"C":0.0338,"N":0.057,"O":0.0676,"F":0.0728,"Ne":null,"Na":0.0318,"Mg":0.0251,"Al":0.0376,"Si":0.0397,"P":0.0387,"S":0.072,"Cl":0.0574,"Ar":null,"K":0.0381,"Ca":0.0265,"Sc":0.0316,"Ti":0.0351,"V":0.0386,"Cr":0.0401,"Mn":0.0409,"Fe":0.043,"Co":0.0322,"Ni":0.0284,"Cu":0.0321,"Zn":0.0302,"Ga":0.0299,"Ge":0.0309,"As":0.0344,"Se":0.0843,"Br":0.1156,"Kr":null,"Rb":0.037,"Sr":0.0275,"Y":0.0285,"Zr":0.0367,"Nb":0.0355,"Mo":0.0296,"Tc":0.034,"Ru":0.0446,"Rh":0.0364,"Pd":0.0373,"Ag":0.032,"Cd":0.0248,"In":0.0315,"Sn":0.0312,"Sb":0.0287,"Te":0.0674,"I":0.0797,"Xe":0.134,"Cs":0.0393,"Ba":0.0276,"La":0.025,"Ce":0.0259,"Pr":0.0248,"Nd":0.0248,"Pm":0.0283,"Sm":0.0261,"Eu":0.0272,"Gd":0.0284,"Tb":0.0272,"Dy":0.0273,"Ho":0.0273,"Er":0.0281,"Tm":0.0288,"Yb":0.0304,"Lu":0.03,"Hf":0.0418,"Ta":0.0489,"W":0.0257,"Re":0.0338,"Os":0.0423,"Ir":0.0384,"Pt":0.039,"Au":0.0357,"Hg":0.0247,"Tl":0.0293,"Pb":0.0353,"Bi":0.0296,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0301,"Th":0.0325,"Pa":0.0326,"U":0.0382,"Np":0.0325,"Pu":0.0471,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"Voronoi RF":{"H":0.0669,"He":null,"Li":0.0367,"Be":0.0579,"B":0.0535,"C":0.044,"N":0.0881,"O":0.0888,"F":0.1127,"Ne":null,"Na":0.0444,"Mg":0.0367,"Al":0.0385,"Si":0.0402,"P":0.0453,"S":0.0571,"Cl":0.0788,"Ar":null,"K":0.0332,"Ca":0.0395,"Sc":0.036,"Ti":0.0391,"V":0.0544,"Cr":0.0715,"Mn":0.057,"Fe":0.0619,"Co":0.0439,"Ni":0.0348,"Cu":0.0404,"Zn":0.0411,"Ga":0.0338,"Ge":0.038,"As":0.0431,"Se":0.0555,"Br":0.0812,"Kr":null,"Rb":0.033,"Sr":0.0401,"Y":0.0341,"Zr":0.0375,"Nb":0.0461,"Mo":0.0366,"Tc":0.0513,"Ru":0.0604,"Rh":0.0399,"Pd":0.0374,"Ag":0.0416,"Cd":0.0396,"In":0.0458,"Sn":0.037,"Sb":0.0409,"Te":0.0597,"I":0.0746,"Xe":0.0127,"Cs":0.0317,"Ba":0.0351,"La":0.0321,"Ce":0.0361,"Pr":0.0341,"Nd":0.0331,"Pm":0.0446,"Sm":0.0339,"Eu":0.0403,"Gd":0.0355,"Tb":0.0357,"Dy":0.0353,"Ho":0.0361,"Er":0.0366,"Tm":0.0376,"Yb":0.0425,"Lu":0.033,"Hf":0.0428,"Ta":0.0547,"W":0.0361,"Re":0.035,"Os":0.0601,"Ir":0.0476,"Pt":0.04,"Au":0.047,"Hg":0.0406,"Tl":0.0499,"Pb":0.0579,"Bi":0.0442,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.051,"Th":0.0402,"Pa":0.0423,"U":0.0623,"Np":0.0707,"Pu":0.0573,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null},"Mean error all models":{"H":0.0579,"He":null,"Li":0.0267,"Be":0.0421,"B":0.0368,"C":0.0341,"N":0.0534,"O":0.0564,"F":0.082,"Ne":null,"Na":0.0296,"Mg":0.0282,"Al":0.0353,"Si":0.0388,"P":0.0379,"S":0.0492,"Cl":0.0624,"Ar":null,"K":0.0285,"Ca":0.0267,"Sc":0.0287,"Ti":0.039,"V":0.0482,"Cr":0.0404,"Mn":0.0402,"Fe":0.043,"Co":0.0343,"Ni":0.031,"Cu":0.0277,"Zn":0.0305,"Ga":0.03,"Ge":0.0317,"As":0.0332,"Se":0.0447,"Br":0.0592,"Kr":null,"Rb":0.0281,"Sr":0.0282,"Y":0.0279,"Zr":0.0337,"Nb":0.0431,"Mo":0.0316,"Tc":0.0368,"Ru":0.0405,"Rh":0.0335,"Pd":0.0315,"Ag":0.0272,"Cd":0.0259,"In":0.0317,"Sn":0.0309,"Sb":0.0302,"Te":0.0462,"I":0.0512,"Xe":0.0429,"Cs":0.029,"Ba":0.0281,"La":0.0306,"Ce":0.0268,"Pr":0.0253,"Nd":0.0247,"Pm":0.0343,"Sm":0.0254,"Eu":0.0265,"Gd":0.027,"Tb":0.0256,"Dy":0.0265,"Ho":0.0263,"Er":0.0257,"Tm":0.0278,"Yb":0.0259,"Lu":0.0271,"Hf":0.0334,"Ta":0.0513,"W":0.0267,"Re":0.0298,"Os":0.0373,"Ir":0.0376,"Pt":0.0351,"Au":0.0328,"Hg":0.0268,"Tl":0.0297,"Pb":0.0344,"Bi":0.0298,"Po":null,"At":null,"Rn":null,"Fr":null,"Ra":null,"Ac":0.0364,"Th":0.0381,"Pa":0.0297,"U":0.0389,"Np":0.0582,"Pu":0.0563,"Am":null,"Cm":null,"Bk":null,"Cf":null,"Es":null,"Fm":null,"Md":null,"No":null,"Lr":null,"Rf":null,"Db":null,"Sg":null,"Bh":null,"Hs":null,"Mt":null,"Ds":null,"Rg":null,"Cn":null,"Nh":null,"Fl":null,"Mc":null,"Lv":null,"Ts":null,"Og":null}} diff --git a/site/src/routes/preprint/+page.md b/site/src/routes/preprint/+page.md index dacf3f6f..2f3287b3 100644 --- a/site/src/routes/preprint/+page.md +++ b/site/src/routes/preprint/+page.md @@ -1,6 +1,6 @@