Skip to content

Commit a4b37a3

Browse files
janoshjanosh
committed
SSoT for projects URLs in site/package.json
add site/src/routes/models/analyze_preds.py to display missing preds in ModelCard
1 parent 2f6fae3 commit a4b37a3

19 files changed

+107
-64
lines changed

‎.github/workflows/gh-pages.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ on:
44
pull_request:
55
branches: [main]
66
push:
7-
branches: [main, site]
7+
branches: [main]
88
workflow_dispatch:
99

1010
# set permissions of GITHUB_TOKEN to allow deployment to GitHub Pages

‎matbench_discovery/__init__.py

+8
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
"""Global variables used all across the matbench_discovery package."""
22

3+
import json
34
import os
45
import sys
56
from datetime import datetime
@@ -18,3 +19,10 @@
1819

1920
timestamp = f"{datetime.now():%Y-%m-%d@%H-%M-%S}"
2021
today = timestamp.split("@")[0]
22+
23+
# load URLs from package.json
24+
25+
with open(f"{ROOT}/site/package.json") as file:
26+
pkg = json.load(file)
27+
pypi_keys_to_npm = dict(Docs="homepage", Repo="repository", Package="package")
28+
URLs = {key: pkg[val] for key, val in pypi_keys_to_npm.items()}

‎matbench_discovery/data.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ def load_train_test(
6060
Recognized data keys are mp-computed-structure-entries, mp-elemental-ref-energies,
6161
mp-energies, mp-patched-phase-diagram, wbm-computed-structure-entries,
6262
wbm-initial-structures, wbm-summary. See
63-
https://matbench-discovery.janosh.dev/how-to-use for brief data descriptions.
63+
https://janosh.github.io/matbench-discovery/how-to-contribute for brief data descriptions.
6464
6565
Args:
6666
data_names (str | list[str], optional): Which parts of the MP/WBM dataset to load.
@@ -140,11 +140,11 @@ def load_train_test(
140140

141141
PRED_FILENAMES = {
142142
"CGCNN": "cgcnn/2022-11-23-test-cgcnn-wbm-IS2RE/cgcnn-ensemble-preds.csv",
143-
"Voronoi RF": "voronoi/2022-11-27-train-test/e-form-preds-IS2RE.csv",
143+
"Voronoi Random Forest": "voronoi/2022-11-27-train-test/e-form-preds-IS2RE.csv",
144144
"Wrenformer": "wrenformer/2022-11-15-wrenformer-IS2RE-preds.csv",
145145
"MEGNet": "megnet/2022-11-18-megnet-wbm-IS2RE/megnet-e-form-preds.csv",
146146
"M3GNet": "m3gnet/2022-10-31-m3gnet-wbm-IS2RE.csv",
147-
"BOWSR MEGNet": "bowsr/2022-11-22-bowsr-megnet-wbm-IS2RE.csv",
147+
"BOWSR MEGNet": "bowsr/2023-01-23-bowsr-megnet-wbm-IS2RE.csv",
148148
}
149149

150150

‎matbench_discovery/plots.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -612,7 +612,7 @@ def cumulative_precision_recall(
612612
# requires fixing index mismatch due to df sub-sampling above
613613
# customdata=dict(
614614
# df_cum.reset_index()
615-
# .pivot(index="index", columns="metric")["Voronoi RF above hull pred"]
615+
# .pivot(index="index", columns="metric")
616616
# .items()
617617
# ),
618618
**kwargs,

‎models/bowsr/join_bowsr_results.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@
5757
)
5858

5959

60+
# %% remove redundant column after sanity check
61+
df_bowsr = df_bowsr.drop(columns=[f"energy_bowsr_{energy_model}"])
62+
63+
6064
# %%
6165
pymatviz.density_scatter(
6266
x=df_bowsr.e_form_per_atom_bowsr_megnet,
@@ -71,5 +75,5 @@
7175
# save energy and formation energy as CSV for fast loading
7276
df_bowsr.select_dtypes("number").to_csv(out_path.replace(".json.gz", ".csv"))
7377

74-
# in_path = f"{ROOT}/models/bowsr/2022-11-22-bowsr-megnet-wbm-IS2RE.json.gz"
78+
in_path = f"{ROOT}/models/bowsr/2023-01-23-bowsr-megnet-wbm-IS2RE.json.gz"
7579
# df_bowsr = pd.read_json(in_path).set_index("material_id")

‎readme.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ Matbench Discovery
1313

1414
</h4>
1515

16-
Matbench Discovery is an [interactive leaderboard](https://matbench-discovery.janosh.dev/figures) and associated [PyPI package](https://pypi.org/project/matbench-discovery) for benchmarking ML energy models on a task designed to closely emulate a real-world computational materials discovery workflow. In it, these models take on the role of a triaging step prior to DFT to decide how to allocate limited compute budget for structure relaxations.
16+
Matbench Discovery is an [interactive leaderboard](https://janosh.github.io/matbench-discovery) and associated [PyPI package](https://pypi.org/project/matbench-discovery) for benchmarking ML energy models on a task designed to closely emulate a real-world computational materials discovery workflow. In it, these models take on the role of a triaging step prior to DFT to decide how to allocate limited compute budget for structure relaxations.
1717

1818
We welcome contributions that add new models to the leaderboard through [GitHub PRs](https://github.com/janosh/matbench-discovery/pulls). See the [usage and contributing guide](https://janosh.github.io/matbench-discovery/how-to-contribute) for details.
1919

‎scripts/cumulative_clf_metrics.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
# %%
1414
models = (
1515
# Wren, CGCNN IS2RE, CGCNN RS2RE, CGCNN
16-
"Voronoi RF, Wrenformer, MEGNet, M3GNet, BOWSR MEGNet"
16+
"Voronoi Random Forest, Wrenformer, MEGNet, M3GNet, BOWSR MEGNet"
1717
).split(", ")
1818

1919
df_wbm = load_df_wbm_with_preds(models=models).round(3)

‎scripts/hist_classified_stable_vs_hull_dist_models.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323

2424
# %%
2525
models = sorted(
26-
"CGCNN, Voronoi RF, Wrenformer, MEGNet, M3GNet, BOWSR MEGNet".split(", ")
26+
"CGCNN, Voronoi Random Forest, Wrenformer, MEGNet, M3GNet, BOWSR MEGNet".split(", ")
2727
)
2828
df_wbm = load_df_wbm_with_preds(models=models).round(3)
2929

‎scripts/metrics_table.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
display_name={"$regex": "cgcnn-robust-formation_energy_per_atom"},
2727
),
2828
),
29-
"Voronoi RF": dict(
29+
"Voronoi Random Forest": dict(
3030
n_runs=68,
3131
filters=dict(
3232
created_at={"$gt": "2022-11-17", "$lt": "2022-11-28"},
@@ -100,7 +100,7 @@
100100

101101

102102
# on 2022-11-28:
103-
# run_times = {'Voronoi RF': 739608,
103+
# run_times = {'Voronoi Random Forest': 739608,
104104
# 'Wrenformer': 208399,
105105
# 'MEGNet': 12396,
106106
# 'M3GNet': 301138,

‎scripts/rolling_mae_vs_hull_dist_all_models.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
# %%
1313
models = sorted(
14-
"Wrenformer, CGCNN, Voronoi RF, MEGNet, M3GNet, BOWSR MEGNet".split(", ")
14+
"Wrenformer, CGCNN, Voronoi Random Forest, MEGNet, M3GNet, BOWSR MEGNet".split(", ")
1515
)
1616
e_form_col = "e_form_per_atom_mp2020_corrected"
1717
e_above_hull_col = "e_above_hull_mp2020_corrected_ppd_mp"

‎scripts/scatter_e_above_hull_models.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
# %%
1717
print(f"loadable models: {list(PRED_FILENAMES)}")
1818
models = sorted(
19-
"CGCNN, Voronoi RF, Wrenformer, MEGNet, M3GNet, BOWSR MEGNet".split(", ")
19+
"CGCNN, Voronoi Random Forest, Wrenformer, MEGNet, M3GNet, BOWSR MEGNet".split(", ")
2020
)
2121
df_wbm = load_df_wbm_with_preds(models=models).round(3)
2222

‎site/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"author": "Janosh Riebesell <[email protected]>",
55
"homepage": "https://janosh.github.io/matbench-discovery",
66
"repository": "https://github.com/janosh/matbench-discovery",
7+
"package": "https://pypi.org/project/matbench-discovery",
78
"license": "MIT",
89
"type": "module",
910
"bugs": "https://github.com/janosh/matbench-discovery/issues",

‎site/src/app.css

+4
Original file line numberDiff line numberDiff line change
@@ -167,3 +167,7 @@ caption {
167167
:target {
168168
animation: highlight-scroll-target 3s;
169169
}
170+
171+
.pull-left {
172+
margin-left: calc(0.8 * (-50vw + 50cqw));
173+
}

‎site/src/lib/ModelCard.svelte

+53-36
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
<script lang="ts">
22
import { repository } from '$site/package.json'
33
import Icon from '@iconify/svelte'
4+
import { pretty_num } from 'sveriodic-table/labels'
45
import type { ModelMetadata } from './types'
56
67
export let key: string
78
export let data: ModelMetadata
89
910
const { model_name, repo, doi, preprint, url, date_added } = data
11+
const { missing_preds, test_set_size } = data
1012
</script>
1113

1214
<h2>{model_name}</h2>
@@ -21,45 +23,55 @@
2123
<p>
2224
Date added: {new Date(date_added).toISOString().split(`T`)[0]}
2325
&nbsp;&bull;&nbsp; Benchmark version: {data.matbench_discovery_version}
26+
&nbsp;&bull;&nbsp; Missing predictions:
27+
{pretty_num(missing_preds)}
28+
<small>({((100 * missing_preds) / test_set_size).toFixed(2)}%)</small>
2429
</p>
25-
<strong>Authors</strong>
26-
<section>
27-
<ul>
28-
{#each data.authors as { name, email, orcid, affiliation, url }}
29-
<li>
30-
<span title={affiliation}>{name}</span>
31-
{#if email}
32-
[<a href="mailto:{email}">email</a>]
33-
{/if}
34-
{#if orcid}
35-
[<a href={orcid}>Orcid</a>]
36-
{/if}
37-
{#if url}
38-
[<a href={url}>web</a>]
39-
{/if}
40-
</li>
41-
{/each}
42-
</ul>
43-
<strong>Package versions</strong>
44-
<ul>
45-
{#each Object.entries(data.requirements) as [name, version]}
46-
<li>
47-
{#if ![`aviary`].includes(name)}
48-
{@const href = `https://pypi.org/project/${name}/${version}`}
49-
{name}: <a {href}>{version}</a>
50-
{:else}
51-
{name}: {version}
52-
{/if}
53-
</li>
54-
{/each}
55-
</ul>
56-
</section>
30+
<div>
31+
<section>
32+
<h3>Authors</h3>
33+
<ul>
34+
{#each data.authors as { name, email, orcid, affiliation, url }}
35+
<li>
36+
<span title={affiliation}>{name}</span>
37+
{#if email}
38+
[<a href="mailto:{email}">email</a>]
39+
{/if}
40+
{#if orcid}
41+
[<a href={orcid}>Orcid</a>]
42+
{/if}
43+
{#if url}
44+
[<a href={url}>web</a>]
45+
{/if}
46+
</li>
47+
{/each}
48+
</ul>
49+
</section>
50+
<section>
51+
<h3>Package versions</h3>
52+
<ul>
53+
{#each Object.entries(data.requirements) as [name, version]}
54+
<li>
55+
{#if ![`aviary`].includes(name)}
56+
{@const href = `https://pypi.org/project/${name}/${version}`}
57+
{name}: <a {href}>{version}</a>
58+
{:else}
59+
{name}: {version}
60+
{/if}
61+
</li>
62+
{/each}
63+
</ul>
64+
</section>
65+
</div>
5766

5867
<!-- TODO add table with performance metrics (F1, Acc, Recall, Precision) for each model -->
5968
<style>
6069
h2 {
6170
margin: 5pt 0 1ex;
6271
}
72+
h3 {
73+
margin: 0;
74+
}
6375
ul {
6476
list-style-type: disc;
6577
}
@@ -71,11 +83,16 @@
7183
}
7284
nav > span {
7385
display: flex;
74-
gap: 0.5em;
86+
gap: 6pt;
7587
place-items: center;
7688
}
77-
strong {
78-
display: block;
79-
margin: 1em 0 5pt;
89+
div {
90+
display: flex;
91+
gap: 15pt;
92+
margin: 1em 0;
93+
justify-content: space-between;
94+
}
95+
small {
96+
font-weight: lighter;
8097
}
8198
</style>

‎site/src/routes/models/+page.server.ts

+11-5
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,19 @@
11
import type { ModelMetadata } from '$lib/types'
22
import { dirname } from 'path'
33
import type { PageServerLoad } from './$types'
4+
import analysis from './2023-01-23-pred-analysis.json'
45

56
export const load: PageServerLoad = async () => {
6-
const models: [string, ModelMetadata][] = Object.entries(
7-
import.meta.glob(`$root/models/**/metadata.yml`, {
8-
eager: true,
9-
})
10-
).map(([key, module]) => [dirname(key), module.default])
7+
const yml = import.meta.glob(`$root/models/**/metadata.yml`, {
8+
eager: true,
9+
})
10+
const models: [string, ModelMetadata][] = Object.entries(yml).map(
11+
([key, module]) => {
12+
const metadata = module.default as ModelMetadata
13+
const computed = analysis[metadata.model_name] ?? {}
14+
return [dirname(key), { ...metadata, ...computed }]
15+
}
16+
)
1117

1218
return { models }
1319
}

‎site/src/routes/models/+page.svelte

+6-4
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55
export let data: PageData
66
</script>
77

8-
<h1>Models</h1>
8+
<h1 class="pull-left">Models</h1>
99

10-
<ol>
11-
{#each data.models as [key, metadata], idx}
10+
<ol class="pull-left">
11+
{#each data.models as [key, metadata]}
1212
<li>
1313
<ModelCard {key} data={metadata} />
1414
</li>
@@ -20,9 +20,11 @@
2020
display: grid;
2121
gap: 2em;
2222
list-style: none;
23+
grid-template-columns: repeat(auto-fit, minmax(400px, 1fr));
2324
}
2425
ol > li {
2526
background-color: rgba(255, 255, 255, 0.05);
26-
padding: 3pt 9pt 5pt;
27+
padding: 3pt 10pt 7pt;
28+
border-radius: 3pt;
2729
}
2830
</style>

‎site/src/routes/paper/+page.svx

+1-3
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,6 @@ date: Jan 31, 2023
4949
import { References } from '$lib'
5050
import './heading-number.css' // CSS to auto-number headings
5151
import CumulativeClfMetrics from '$figs/2023-01-19-cumulative-clf-metrics.svelte'
52-
53-
const style = "margin-left: calc(0.8 * (-50vw + 50cqw));"
5452
</script>
5553

5654
# {title}<br><small>{subtitle}</small>
@@ -193,7 +191,7 @@ Our benchmark is designed to make [adding future models easy](/how-to-contribute
193191

194192
<div>
195193
{#if typeof document !== `undefined`}
196-
<CumulativeClfMetrics {style} />
194+
<CumulativeClfMetrics class="pull-left" />
197195
{/if}
198196
</div>
199197

‎tests/test_data.py

+5-2
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,11 @@ def test_load_train_test_doc_str() -> None:
111111
for name in DATA_FILENAMES:
112112
assert name in doc_str, f"Missing data {name=} in load_train_test() docstring"
113113

114-
# TODO refactor to load site URL from site/package.json for SSoT
115-
assert "https://matbench-discovery.janosh.dev" in doc_str
114+
route = "/how-to-contribute"
115+
from matbench_discovery import URLs
116+
117+
assert f"{URLs['Docs']}{route}" in doc_str
118+
assert os.path.isdir(f"{ROOT}/site/src/routes/{route}")
116119

117120

118121
@pytest.mark.skipif(website_down, reason=f"{RAW_REPO_URL} unreachable")

‎tests/test_plots.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
rolling_mae_vs_hull_dist,
1616
)
1717

18-
models = ["Wrenformer", "CGCNN", "Voronoi RF"]
18+
models = ["Wrenformer", "CGCNN", "Voronoi Random Forest"]
1919
df_wbm = load_df_wbm_with_preds(models=models, nrows=100)
2020
e_above_hull_col = "e_above_hull_mp2020_corrected_ppd_mp"
2121
e_form_col = "e_form_per_atom_mp2020_corrected"

0 commit comments

Comments
 (0)