Less test generation

core/Cargo.toml

@@ -88,12 +88,13 @@ strsim.workspace = true
 bumpalo = { workspace = true }
 
 [dev-dependencies]
-pretty_assertions.workspace = true
 assert_matches.workspace = true
 criterion.workspace = true
+glob.workspace = true
+libtest-mimic.workspace = true
 nickel-lang-utils = { workspace = true, features = ["pprof"] }
+pretty_assertions.workspace = true
 similar.workspace = true
-test-generator.workspace = true
 
 # Enable this to use flamegraphs
 # [profile.release]
@@ -131,5 +132,20 @@ harness = false
 name = "typecheck-nixpkgs-lib"
 harness = false
 
+[[test]]
+name = "manual"
+path = "tests/manual/main.rs"
+harness = false
+
+[[test]]
+name = "examples"
+path = "tests/examples/main.rs"
+harness = false
+
+[[test]]
+name = "integration"
+path = "tests/integration/main.rs"
+harness = false
+
 [lints.clippy]
 new_without_default = "allow"

core/src/files.rs

@@ -192,7 +192,7 @@ impl Files {
         self.files.get(id.0 as usize).ok_or(Error::FileMissing)
     }
 
-    pub(crate) fn filenames(&self) -> impl Iterator<Item = &OsStr> {
+    pub fn filenames(&self) -> impl Iterator<Item = &OsStr> {
         self.files.iter().map(|f| &*f.name)
     }
 }

core/tests/integration/contracts_fail.rs → core/tests/contracts_fail.rs

@@ -1,6 +1,8 @@
 use assert_matches::assert_matches;
-use codespan::Files;
-use nickel_lang_core::error::{Error, EvalError, IntoDiagnostics};
+use nickel_lang_core::{
+    error::{Error, EvalError, IntoDiagnostics},
+    files::Files,
+};
 
 use nickel_lang_utils::test_program::eval;
 
@@ -22,7 +24,7 @@ fn array() {
     // Check that reporting doesn't panic. Provide a dummy file database, as we won't report
     // the error message but just check that it can be built.
     let mut files = Files::new();
-    res.unwrap_err().into_diagnostics(&mut files, None);
+    res.unwrap_err().into_diagnostics(&mut files);
 
     let res = eval(
         "(%array/at% (\
@@ -39,7 +41,7 @@ fn array() {
         ),
         err => panic!("expected blame error, got {err:?}"),
     }
-    res.unwrap_err().into_diagnostics(&mut files, None);
+    res.unwrap_err().into_diagnostics(&mut files);
 }
 
 #[test]

core/tests/examples/main.rs

@@ -1,17 +1,24 @@
+use std::{path::Path, process::ExitCode};
+
 use assert_matches::assert_matches;
+use libtest_mimic::Arguments;
 use nickel_lang_core::error::{Error, EvalError, NullReporter};
 use nickel_lang_utils::{
     annotated_test::{read_annotated_test_case, TestCase},
     project_root::project_root,
     test_program::TestProgram,
 };
 use serde::Deserialize;
-use test_generator::test_resources;
 
-#[test_resources("examples/**/*.ncl")]
-fn check_example_file(path: &str) {
-    let test: TestCase<Expectation> =
-        read_annotated_test_case(path).expect("Failed to parse annotated program");
+pub fn main() -> ExitCode {
+    let args = Arguments::from_args();
+    let tests = nickel_lang_utils::path_tests::path_tests("examples/**/*.ncl", check_example_file);
+    libtest_mimic::run(&args, tests).exit_code()
+}
+
+fn check_example_file(_name: &str, path: &Path) {
+    let test: TestCase<Expectation> = read_annotated_test_case(path.as_os_str().to_str().unwrap())
+        .expect("Failed to parse annotated program");
 
     // `test_resources` uses paths relative to the workspace manifesty
     let mut p = TestProgram::new_from_file(

core/tests/integration/main.rs

@@ -1,5 +1,6 @@
-use std::{io::Cursor, thread};
+use std::{io::Cursor, path::Path, process::ExitCode, thread};
 
+use libtest_mimic::Arguments;
 use nickel_lang_core::{
     error::{
         Error, EvalError, ExportError, ExportErrorData, ImportError, NullReporter, ParseError,
@@ -14,18 +15,26 @@ use nickel_lang_utils::{
     test_program::TestProgram,
 };
 use serde::Deserialize;
-use test_generator::test_resources;
 
-mod contract_label_path;
-mod free_vars;
 mod pretty;
-mod query;
-mod stdlib_typecheck;
 
-#[test_resources("core/tests/integration/**/*.ncl")]
-fn check_annotated_nickel_file(path: &str) {
-    let test: TestCase<Test> =
-        read_annotated_test_case(path).expect("Failed to parse annotated program");
+pub fn main() -> ExitCode {
+    let args = Arguments::from_args();
+    let mut tests = nickel_lang_utils::path_tests::path_tests(
+        "core/tests/integration/**/*.ncl",
+        check_annotated_nickel_file,
+    );
+    tests.extend(nickel_lang_utils::path_tests::path_tests(
+        "core/tests/integration/inputs/imports/imported/import_parent.ncl",
+        check_from_dir,
+    ));
+    tests.extend(pretty::tests());
+    libtest_mimic::run(&args, tests).exit_code()
+}
+
+fn check_annotated_nickel_file(_name: &str, path: &Path) {
+    let test: TestCase<Test> = read_annotated_test_case(path.as_os_str().to_str().unwrap())
+        .expect("Failed to parse annotated program");
 
     // By default, cargo runs tests with a 2MB stack, which is easy to overflow. To avoid this we
     // run the tests with an increased stack size.
@@ -44,10 +53,9 @@ fn check_annotated_nickel_file(path: &str) {
 // Like check_annotated_nickel_file, but runs the test from the directory of
 // the test file itself (and opens the test file with a relative path). This
 // is mainly for integration testing path normalization.
-#[test_resources("core/tests/integration/inputs/imports/imported/import_parent.ncl")]
-fn check_from_dir(path: &str) {
-    let test: TestCase<Test> =
-        read_annotated_test_case(path).expect("Failed to parse annotated program");
+fn check_from_dir(_name: &str, path: &Path) {
+    let test: TestCase<Test> = read_annotated_test_case(path.as_os_str().to_str().unwrap())
+        .expect("Failed to parse annotated program");
 
     let path = project_root().join(path);
     let dir = std::env::current_dir().unwrap();

core/tests/integration/pretty.rs

@@ -1,3 +1,4 @@
+use libtest_mimic::{Failed, Trial};
 use nickel_lang_core::mk_app;
 use nickel_lang_core::term::{make, StrChunk, Term, UnaryOp};
 use nickel_lang_utils::{
@@ -6,7 +7,21 @@ use nickel_lang_utils::{
 };
 
 use std::io::Read;
-use test_generator::test_resources;
+use std::path::Path;
+
+pub fn tests() -> Vec<Trial> {
+    let mut tests = nickel_lang_utils::path_tests::path_tests(
+        "core/tests/integration/inputs/**/*.ncl",
+        |_, path| check_idempotent(path),
+    );
+    tests.extend(nickel_lang_utils::path_tests::path_tests(
+        "core/stdlib/*.ncl",
+        |_, path| check_idempotent(path),
+    ));
+    tests.push(Trial::test("str_vs_strchunks", str_vs_strchunks));
+    tests.push(Trial::test("negative_numbers", negative_numbers));
+    tests
+}
 
 #[track_caller]
 fn diff(s1: &str, s2: &str) {
@@ -25,7 +40,7 @@ fn diff(s1: &str, s2: &str) {
 }
 
 #[track_caller]
-fn check_idempotent(path: &str) {
+fn check_idempotent(path: &Path) {
     let mut buffer = String::new();
     let mut file = std::fs::File::open(project_root().join(path)).expect("Failed to open file");
 
@@ -42,29 +57,18 @@ fn check_idempotent(path: &str) {
     }
 }
 
-#[test_resources("core/tests/integration/inputs/**/*.ncl")]
-fn pretty_integration_tests(path: &str) {
-    check_idempotent(path)
-}
-
-#[test_resources("core/stdlib/*.ncl")]
-fn pretty_standard_library(path: &str) {
-    check_idempotent(path)
-}
-
-#[test]
-fn str_vs_strchunks() {
+fn str_vs_strchunks() -> Result<(), Failed> {
     assert_eq!(
         format!("{}", Term::Str("string".into())),
         format!(
             "{}",
             Term::StrChunks(vec![StrChunk::Literal("string".to_string())])
         )
     );
+    Ok(())
 }
 
-#[test]
-fn negative_numbers() {
+fn negative_numbers() -> Result<(), Failed> {
     eval(format!(
         "{}",
         mk_app!(
@@ -73,4 +77,5 @@ fn negative_numbers() {
         )
     ))
     .unwrap();
+    Ok(())
 }

core/tests/manual/main.rs

@@ -1,19 +1,19 @@
-use std::{fs::File, io::read_to_string, iter::once};
+use std::{fs::File, io::read_to_string, iter::once, path::Path, process::ExitCode};
 
 use codespan_reporting::term::termcolor::NoColor;
 use comrak::{
     arena_tree::{Node, NodeEdge},
     nodes::{Ast, AstNode, NodeCodeBlock, NodeValue},
     parse_document, ComrakOptions,
 };
+use libtest_mimic::Arguments;
 use nickel_lang_core::{
     error,
     eval::cache::CacheImpl,
     repl::{EvalResult, Repl, ReplImpl},
 };
 use nickel_lang_utils::{project_root::project_root, test_program};
 use pretty_assertions::assert_str_eq;
-use test_generator::test_resources;
 use typed_arena::Arena;
 
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
@@ -139,6 +139,7 @@ fn check_error_report(actual: impl AsRef<str>, expected: MessageExpectation) {
 
 fn check_repl(content: String) {
     use error::report::{report_with, ErrorFormat};
+    dbg!(&content);
 
     let mut repl = ReplImpl::<CacheImpl>::new(std::io::sink());
     repl.load_stdlib().unwrap();
@@ -157,6 +158,7 @@ fn check_repl(content: String) {
                 (Err(e), ReplResult::Error(expected)) => {
                     let mut error = NoColor::new(Vec::<u8>::new());
                     let mut files = repl.cache_mut().sources.files().clone();
+                    dbg!(files.filenames().collect::<Vec<_>>());
                     report_with(&mut error, &mut files, e, ErrorFormat::Text);
 
                     check_error_report(String::from_utf8(error.into_inner()).unwrap(), expected);
@@ -210,8 +212,13 @@ fn nickel_code_blocks<'a>(document: &'a AstNode<'a>) -> impl Iterator<Item = Cod
     })
 }
 
-#[test_resources("doc/manual/*.md")]
-fn check_manual_snippets(path: &str) {
+pub fn main() -> ExitCode {
+    let args = Arguments::from_args();
+    let tests = nickel_lang_utils::path_tests::path_tests("doc/manual/*.md", check_manual_snippets);
+    libtest_mimic::run(&args, tests).exit_code()
+}
+
+fn check_manual_snippets(_name: &str, path: &Path) {
     let contents = read_to_string(File::open(project_root().join(path)).unwrap()).unwrap();
     let arena = Arena::new();
     let snippets = nickel_code_blocks(parse_document(&arena, &contents, &ComrakOptions::default()));

lsp/lsp-harness/Cargo.toml

@@ -20,5 +20,4 @@ assert_cmd.workspace = true
 [dev-dependencies]
 assert_cmd.workspace = true
 insta.workspace = true
-test-generator.workspace = true
 nickel-lang-utils.workspace = true