[red-knot] Add GitHub PR annotations when mdtests fail in CI

.github/workflows/ci.yaml

@@ -36,6 +36,8 @@ jobs:
       code: ${{ steps.check_code.outputs.changed }}
       # Flag that is raised when any code that affects the fuzzer is changed
       fuzz: ${{ steps.check_fuzzer.outputs.changed }}
+      # Flag that is set to "true" when code related to red-knot changes.
+      red_knot: ${{ steps.check_red_knot.outputs.changed }}
 
       # Flag that is set to "true" when code related to the playground changes.
       playground: ${{ steps.check_playground.outputs.changed }}
@@ -166,6 +168,29 @@ jobs:
               echo "changed=true" >> "$GITHUB_OUTPUT"
           fi
 
+      - name: Check if the red-knot code changed
+        id: check_red_knot
+        env:
+          MERGE_BASE: ${{ steps.merge_base.outputs.sha }}
+        run: |
+          if git diff --quiet "${MERGE_BASE}...HEAD" -- \
+            ':Cargo.toml' \
+            ':Cargo.lock' \
+            ':crates/red_knot*/**' \
+            ':crates/ruff_db/**' \
+            ':crates/ruff_annotate_snippets/**' \
+            ':crates/ruff_python_ast/**' \
+            ':crates/ruff_python_parser/**' \
+            ':crates/ruff_python_trivia/**' \
+            ':crates/ruff_source_file/**' \
+            ':crates/ruff_text_size/**' \
+            ':.github/workflows/ci.yaml' \
+          ; then
+              echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+              echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
   cargo-fmt:
     name: "cargo fmt"
     runs-on: ubuntu-latest
@@ -408,6 +433,31 @@ jobs:
         run: cargo binstall cargo-fuzz --force  --disable-strategies crate-meta-data --no-confirm
       - run: cargo fuzz build -s none
 
+  mdtest-github-format:
+    name: "red-knot mdtest (GitHub annotations)"
+    runs-on: ubuntu-latest
+    needs: determine_changes
+    if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-test') && needs.determine_changes.outputs.red_knot == 'true' && github.event_name == 'pull_request' }}
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          persist-credentials: false
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2
+      - name: "Install Rust toolchain"
+        run: rustup show
+      - name: "Install mold"
+        uses: rui314/setup-mold@v1
+      - name: "Install cargo insta"
+        uses: taiki-e/install-action@6aca1cfa12ef3a6b98ee8c70e0171bfa067604f5 # v2
+        with:
+          tool: cargo-insta
+      - name: "Run mdtest"
+        shell: bash
+        env:
+          NO_COLOR: 1
+        run: cargo test --features=mdtest_github_output_format -p red_knot_python_semantic --test mdtest
+
   fuzz-parser:
     name: "fuzz parser"
     runs-on: ubuntu-latest

crates/red_knot_python_semantic/Cargo.toml

@@ -60,6 +60,7 @@ quickcheck_macros = { version = "1.0.0" }
 
 [features]
 serde = ["ruff_db/serde", "dep:serde", "ruff_python_ast/serde"]
+mdtest_github_output_format = []
 
 [lints]
 workspace = true

crates/red_knot_python_semantic/resources/mdtest/type_properties/is_equivalent_to.md

@@ -143,7 +143,7 @@
 def f3(a1: int, /, *args1: int, **kwargs2: int) -> None: ...
 def f4(a2: int, /, *args2: int, **kwargs1: int) -> None: ...
 
-static_assert(is_equivalent_to(CallableTypeOf[f3], CallableTypeOf[f4]))
+static_assert(not is_equivalent_to(CallableTypeOf[f3], CallableTypeOf[f4]))
 ```
 
 Putting it all together, the following two callables are equivalent:
@@ -186,7 +186,7 @@
 def f3(): ...
 def f4() -> None: ...
 
-static_assert(not is_equivalent_to(Callable[[], int], Callable[[], None]))
+static_assert(not is_equivalent_to(Callable[[], int], Callable[[], None]))  # revealed: int
 static_assert(not is_equivalent_to(CallableTypeOf[f3], CallableTypeOf[f3]))
 static_assert(not is_equivalent_to(CallableTypeOf[f3], CallableTypeOf[f4]))
 static_assert(not is_equivalent_to(CallableTypeOf[f4], CallableTypeOf[f3]))
@@ -228,6 +228,7 @@
 static_assert(not is_equivalent_to(CallableTypeOf[f10], CallableTypeOf[f11]))
 static_assert(not is_equivalent_to(CallableTypeOf[f11], CallableTypeOf[f10]))
 static_assert(not is_equivalent_to(CallableTypeOf[f11], CallableTypeOf[f11]))
+reveal_type(f9)
 ```
 
 When the default value for a parameter is present only in one of the callable type:

crates/red_knot_python_semantic/tests/mdtest.rs

@@ -1,5 +1,6 @@
 use camino::Utf8Path;
 use dir_test::{dir_test, Fixture};
+use red_knot_test::OutputFormat;
 
 /// See `crates/red_knot_test/README.md` for documentation on these tests.
 #[dir_test(
@@ -18,12 +19,19 @@ fn mdtest(fixture: Fixture<&str>) {
 
     let test_name = test_name("mdtest", absolute_fixture_path);
 
+    let output_format = if cfg!(feature = "mdtest_github_output_format") {
+        OutputFormat::GitHub
+    } else {
+        OutputFormat::Cargo
+    };
+
     red_knot_test::run(
         absolute_fixture_path,
         relative_fixture_path,
         &snapshot_path,
         short_title,
         &test_name,
+        output_format,
     );
 }
 

crates/red_knot_test/src/lib.rs

@@ -34,6 +34,7 @@ pub fn run(
     snapshot_path: &Utf8Path,
     short_title: &str,
     test_name: &str,
+    output_format: OutputFormat,
 ) {
     let source = std::fs::read_to_string(absolute_fixture_path).unwrap();
     let suite = match test_parser::parse(short_title, &source) {
@@ -59,7 +60,10 @@ pub fn run(
 
         if let Err(failures) = run_test(&mut db, relative_fixture_path, snapshot_path, &test) {
             any_failures = true;
-            println!("\n{}\n", test.name().bold().underline());
+
+            if output_format.is_cargo() {
+                println!("\n{}\n", test.name().bold().underline());
+            }
 
             let md_index = LineIndex::from_source_text(&source);
 
@@ -72,21 +76,31 @@ pub fn run(
                         source_map.to_absolute_line_number(relative_line_number);
 
                     for failure in failures {
-                        let line_info =
-                            format!("{relative_fixture_path}:{absolute_line_number}").cyan();
-                        println!("  {line_info} {failure}");
+                        match output_format {
+                            OutputFormat::Cargo => {
+                                let line_info =
+                                    format!("{relative_fixture_path}:{absolute_line_number}")
+                                        .cyan();
+                                println!("  {line_info} {failure}");
+                            }
+                            OutputFormat::GitHub => println!(
+                                "::error file={absolute_fixture_path},line={absolute_line_number}::{failure}"
+                            ),
+                        }
                     }
                 }
             }
 
             let escaped_test_name = test.name().replace('\'', "\\'");
 
-            println!(
-                "\nTo rerun this specific test, set the environment variable: {MDTEST_TEST_FILTER}='{escaped_test_name}'",
-            );
-            println!(
-                "{MDTEST_TEST_FILTER}='{escaped_test_name}' cargo test -p red_knot_python_semantic --test mdtest -- {test_name}",
-            );
+            if output_format.is_cargo() {
+                println!(
+                    "\nTo rerun this specific test, set the environment variable: {MDTEST_TEST_FILTER}='{escaped_test_name}'",
+                );
+                println!(
+                    "{MDTEST_TEST_FILTER}='{escaped_test_name}' cargo test -p red_knot_python_semantic --test mdtest -- {test_name}",
+                );
+            }
         }
     }
 
@@ -95,6 +109,23 @@ pub fn run(
     assert!(!any_failures, "Some tests failed.");
 }
 
+/// Defines the format in which mdtest should print an error to the terminal
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum OutputFormat {
+    /// The format `cargo test` should use by default.
+    Cargo,
+    /// A format that will provide annotations from GitHub Actions
+    /// if mdtest fails on a PR.
+    /// See <https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-error-message>
+    GitHub,
+}
+
+impl OutputFormat {
+    const fn is_cargo(self) -> bool {
+        matches!(self, OutputFormat::Cargo)
+    }
+}
+
 fn run_test(
     db: &mut db::Db,
     relative_fixture_path: &Utf8Path,