add tests/test_models.py

link to pypi pages from site's model page with requirement lists

Author: janosh

data/wbm/readme.md

@@ -70,7 +70,7 @@ materialscloud:2021.68 includes a readme file with a description of the dataset,
 
 [wbm paper]: https://nature.com/articles/s41524-020-00481-6
 
-## Data Plots
+## 📊   Data Plots
 
 <caption>Heatmap of elemental prevalence in WBM dataset.</caption>
 <slot name="wbm-elements-log">

models/wrenformer/metadata.yml

@@ -3,13 +3,17 @@ model_version: 0.0.4 # the aviary version
 matbench_discovery_version: 1.0
 date_added: 2022-11-26
 authors:
-  - name: Rhys Goodall
-    affiliation: University of Cambridge
-    orcid: https://orcid.org/0000-0002-6589-1700
   - name: Janosh Riebesell
     affiliation: University of Cambridge, Lawrence Berkeley National Laboratory
     email: [email protected]
     orcid: https://orcid.org/0000-0001-5233-3462
+  - name: Rhys Goodall
+    affiliation: University of Cambridge
+    orcid: https://orcid.org/0000-0002-6589-1700
+  - name: Rokas
+    affiliation: University of Cambridge
+    url: https://hardingscholars.fund.cam.ac.uk/rokas-elijosius-2021-cohort
+    email: [email protected]
 repo: https://github.com/janosh/matbench-discovery
 doi: https://doi.org/10.1126/sciadv.abn4117
 preprint: https://arxiv.org/abs/2106.11132

site/src/app.css

@@ -18,9 +18,16 @@ body {
   font-family: -apple-system, BlinkMacSystemFont, Roboto, sans-serif;
   color: var(--text-color);
   line-height: 1.5;
+  margin: 0;
+}
+body > div {
+  display: flex;
+  min-height: 100vh;
+  flex-direction: column;
 }
 main {
   padding: calc(1ex + 2vw);
+  flex: 1;
 }
 button {
   color: var(--text-color);

site/src/routes/about-the-test-set/+page.svelte

@@ -7,7 +7,7 @@
   import elem_counts from './2022-12-30-wbm-element-counts.json'
 
   let log_color_scale = false
-  const heatmap_values = Object.values(elem_counts)
+  const heatmap_values: number[] = Object.values(elem_counts)
   const color_map = {
     200: `blue`,
     35_000: `green`,
@@ -30,6 +30,11 @@
         {#if active_element?.name}
           <strong>
             {active_element?.name}: {pretty_num(elem_counts[active_element?.symbol])}
+            <!-- compute percent of total -->
+            {#if elem_counts[active_element?.symbol] > 0}
+              {@const total = heatmap_values.reduce((a, b) => a + b, 0)}
+              ({pretty_num((elem_counts[active_element?.symbol] / total) * 100)}%)
+            {/if}
           </strong>
         {/if}
       </TableInset>

site/src/routes/models/+page.svelte

@@ -47,11 +47,17 @@
         <ul>
           {#each Object.entries(meta.requirements) as [name, version]}
             <li>
-              {name}: {version}
+              {#if ![`aviary`].includes(name)}
+                {@const href = `https://pypi.org/project/${name}/${version}`}
+                {name}: <a {href}>{version}</a>
+              {:else}
+                {name}: {version}
+              {/if}
             </li>
           {/each}
         </ul>
       </section>
+      <!-- TODO add table with performance metrics (F1, Acc, Recall, Precision) for each model -->
     </li>
   {/each}
 </ol>
@@ -67,16 +73,15 @@
     padding: 3pt 9pt 5pt;
   }
   ol > li > h2 {
-    display: inline;
+    margin: 5pt 0 1ex;
   }
   ul {
     list-style-type: disc;
   }
   nav {
     font-weight: 250;
-    display: inline-flex;
+    display: flex;
     gap: 5pt 1em;
-    margin: 0 1em;
     flex-wrap: wrap;
   }
   nav > span {

tests/test_models.py

@@ -0,0 +1,60 @@
+import os
+from datetime import date
+from glob import glob
+
+import yaml
+
+from matbench_discovery import ROOT
+
+MODEL_DIRS = glob(f"{ROOT}/models/*/")
+
+
+def test_model_dirs_have_metadata() -> None:
+    required = (
+        "authors",
+        "date_added",
+        "matbench_discovery_version",
+        "model_name",
+        "model_version",
+        "repo",
+    )
+    for model_dir in MODEL_DIRS:
+        md_file = f"{model_dir}metadata.yml"
+        assert os.path.isfile(md_file), f"Missing metadata file: {md_file}"
+
+        # make sure all required keys are non-empty
+        with open(md_file) as yml_file:
+            metadata = yaml.full_load(yml_file)
+
+        for key in required:
+            assert metadata.get(key), f"Empty {key=} in {md_file}"
+
+        authors, date_added, mbd_version, model_name, model_version, repo = (
+            metadata[key] for key in required
+        )
+
+        # make sure all keys are valid
+        assert (
+            3 < len(model_name) < 50
+        ), f"Invalid {model_name=} not between 3 and 50 characters"
+        assert (
+            1 < len(model_version) < 15
+        ), f"Invalid {model_version=} not between 1 and 15 characters"
+        # TODO increase max version when releasing new versions
+        assert (
+            1 <= mbd_version <= 1
+        ), f"Invalid matbench-discovery version: {mbd_version}"
+        assert isinstance(date_added, date), f"Invalid {date_added=} not a string"
+        assert (
+            isinstance(authors, list) and 1 < len(authors) < 30
+        ), "authors not list or not between 1 and 30 authors"
+        assert repo.startswith(
+            "https://"
+        ), f"Invalid {repo=} not starting with https://"
+
+
+def test_model_dirs_have_test_scripts() -> None:
+    for model_dir in MODEL_DIRS:
+        test_scripts = glob(f"{model_dir}*test_*.py")
+        test_nbs = glob(f"{model_dir}*test_*.ipynb")
+        assert len(test_scripts + test_nbs) > 0, f"Missing test file in {model_dir}"