Add msan CI job

[ryanofsky]

I've been struggling to add a memory sanitizer job as part of #186.

I've having problems getting MemorySanitizer to be usable in nix though. Opening this issue to document the problem and avoid spending too much more time on this.

Using -fsanitize=memory alone doesn't work because although the build succeeds, running any program shows false positive errors unless other dependencies are also built with -fsanitize. (From https://github.com/google/sanitizers/wiki/memorysanitizer "It is critical that you should build all the code in your program (including libraries it uses, in particular, C++ standard library) with MSan.")

So I tried to add support for this in shell.nix :

shell.nix

{ pkgs ? import <nixpkgs> {}
, crossPkgs ? pkgs
, enableClang  ? false         # use clang instead of gcc
, enableLibcxx ? false         # use libc++ instead of libstdc++
, enableMsan   ? false         # build all C/C++ code with -fsanitize=memory
, enableTools  ? false         # enable clang-tools like clang-tidy
}:

let
  lib  = pkgs.lib;
  llvm = crossPkgs.llvmPackages_20;
  addFlags = p: p.overrideAttrs (old: {
    env.CXXFLAGS = (old.env.CXXFLAGS or "") + " -fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer";
  });
  capnproto = addFlags (crossPkgs.capnproto.override { clangStdenv = stdenv; });
  libcxx    = addFlags llvm.libcxx;
  clang     = if enableLibcxx then llvm.libcxxClang else llvm.clang;
  stdenv    = if enableClang then stdenvClang else crossPkgs.stdenv;
  stdenvClang = if enableLibcxx then llvm.libcxxStdenv else llvm.stdenv;
  clang-tools = llvm.clang-tools.override { inherit enableLibcxx; };
in (crossPkgs.mkShell.override { stdenv = stdenv; }) {

  buildInputs = [
    capnproto
  ] ++ lib.optionals enableLibcxx [libcxx]
    ++ lib.optionals enableMsan [llvm.compiler-rt];

  nativeBuildInputs = with pkgs; [
    cmake
    include-what-you-use
    ninja
  ] ++ lib.optionals enableClang [
    clang
  ] ++ lib.optionals enableTools [
    clang-tools
  ];

  # Tell IWYU where its libc++ mapping lives
  IWYU_MAPPING_FILE = if enableLibcxx then "${llvm.libcxx.dev}/include/c++/v1/libcxx.imp" else null;

  shellHook = lib.optionalString enableMsan ''
    export MSAN_SYMBOLIZER_PATH=${llvm.bintools}/bin/llvm-symbolizer
  '';
}

Build libc++ and cap'n proto with

nix-shell shell.nix --arg enableClang true --arg enableLibcxx true --arg enableMsan true

But this results in errors from cmake building libcxx which I don't understand:

       >     clang++: warning: argument unused during compilation: '-rtlib=compiler-rt' [-Wunused-command-line-argument]
       >     [2/2] : && /nix/store/xmbmym1mayyr2327lkap5yihq3g88z3x-clang-wrapper-20.1.5/bin/clang++ -fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer  CMakeFiles/cmTC_de5b8.dir/testCXXCompiler.cxx.o -o cmTC_de5b8   && :
       >     FAILED: cmTC_de5b8
       >     : && /nix/store/xmbmym1mayyr2327lkap5yihq3g88z3x-clang-wrapper-20.1.5/bin/clang++ -fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer  CMakeFiles/cmTC_de5b8.dir/testCXXCompiler.cxx.o -o cmTC_de5b8   && :
       >     /nix/store/v63bxfiacw082c7ijshf60alvvrpfxsq-binutils-2.44/bin/ld: cannot find /nix/store/xmbmym1mayyr2327lkap5yihq3g88z3x-clang-wrapper-20.1.5/resource-root/lib/x86_64-unknown-linux-gnu/libclang_rt.msan.a: No such file or directory
       >     /nix/store/v63bxfiacw082c7ijshf60alvvrpfxsq-binutils-2.44/bin/ld: cannot find /nix/store/xmbmym1mayyr2327lkap5yihq3g88z3x-clang-wrapper-20.1.5/resource-root/lib/x86_64-unknown-linux-gnu/libclang_rt.msan_cxx.a: No such file or directory
       >     clang++: error: linker command failed with exit code 1 (use -v to see invocation)
       >     ninja: build stopped: subcommand failed.

It seems like missing msan libraries are present in the compiler-rt-libc package but the clang-wrapper package doesn't seem to link to them, so need to figure that out or take a different approach if this is not the right one.