integration-test: Implement running on VMs

This allows integration tests to be run using qemu at an
arbitrary kernel version.

Note that before this change we were only testing fedora 38
which used 6.2. Now we're testing 6.1 kernels. The tests all
pass. Older kernel versions were attempted, but the tests don't
all pass. Later work can add more kernel versions to test.

Author: tamird

.github/workflows/ci.yml

@@ -139,8 +139,14 @@ jobs:
             --target ${{ matrix.target }} \
             -Z build-std=core
 
-  build-integration-test:
-    runs-on: ubuntu-22.04
+  run-integration-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        runner:
+          - macos-12
+          - ubuntu-22.04
+    runs-on: ${{ matrix.runner }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -150,13 +156,12 @@ jobs:
         with:
           toolchain: nightly
           components: rust-src
+          targets: aarch64-unknown-linux-musl,x86_64-unknown-linux-musl
 
       - uses: Swatinem/rust-cache@v2
 
-      - name: bpf-linker
-        run: cargo install bpf-linker --git https://github.com/aya-rs/bpf-linker.git
-
-      - name: Install dependencies
+      - name: Install prerequisites
+        if: runner.os == 'Linux'
         # ubuntu-22.04 comes with clang 14[0] which doesn't include support for signed and 64bit
         # enum values which was added in clang 15[1].
         #
@@ -171,63 +176,78 @@ jobs:
           set -euxo pipefail
           wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
           echo deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy main | sudo tee /etc/apt/sources.list.d/llvm.list
-          sudo apt-get update
-          sudo apt-get -y install clang gcc-multilib llvm
+          sudo apt update
+          sudo apt -y install clang gcc-multilib llvm locate qemu-system-{arm,x86}
 
-      - name: Build
+      - name: bpf-linker
+        if: runner.os == 'Linux'
+        run: cargo install bpf-linker --git https://github.com/aya-rs/bpf-linker.git
+
+      - name: Install prerequisites
+        if: runner.os == 'macOS'
+        # The clang shipped on macOS doesn't support BPF, so we need LLVM from brew.
+        #
+        # We also need LLVM for bpf-linker, see comment below.
         run: |
           set -euxo pipefail
-          mkdir -p integration-test-binaries
-          # See https://doc.rust-lang.org/cargo/reference/profiles.html for the
-          # names of the builtin profiles. Note that dev builds "debug" targets.
-          cargo xtask build-integration-test --cargo-arg=--profile=dev | xargs -I % cp % integration-test-binaries/dev
-          cargo xtask build-integration-test --cargo-arg=--profile=release | xargs -I % cp % integration-test-binaries/release
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: integration-test-binaries
-          path: integration-test-binaries
+          brew install qemu dpkg pkg-config llvm
+          echo /usr/local/opt/llvm/bin >> $GITHUB_PATH
 
-  run-integration-test:
-    runs-on: macos-latest
-    needs: ["build-integration-test"]
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          sparse-checkout: |
-            test/run.sh
-            test/cloud-localds
+      - name: bpf-linker
+        if: runner.os == 'macOS'
+        # NB: rustc doesn't ship libLLVM.so on macOS, so disable proxying (default feature).
+        run: cargo install bpf-linker --git https://github.com/aya-rs/bpf-linker.git --no-default-features
 
-      - name: Install Pre-requisites
+      - name: Download debian kernels
+        if: runner.arch == 'ARM64'
         run: |
-          brew install qemu gnu-getopt coreutils cdrtools
-
-      - name: Cache tmp files
-        uses: actions/cache@v3
-        with:
-          path: |
-            .tmp/*.qcow2
-            .tmp/test_rsa
-            .tmp/test_rsa.pub
-          key: tmp-files-${{ hashFiles('test/run.sh') }}
-
-      - uses: actions/download-artifact@v3
-        with:
-          name: integration-test-binaries
-          path: integration-test-binaries
-
-      - name: Run integration tests
+          set -euxo pipefail
+          mkdir -p test/.tmp/debian-kernels/arm64
+          # NB: a 4.19 kernel image for arm64 was not available.
+          # TODO(https://github.com/aya-rs/aya/pull/725): enable tests on kernels before 6.0.
+          # linux-image-5.10.0-23-cloud-arm64-unsigned_5.10.179-3_arm64.deb \
+          printf '%s\0' \
+            linux-image-6.1.0-10-cloud-arm64-unsigned_6.1.38-2_arm64.deb \
+            linux-image-6.4.0-1-cloud-arm64-unsigned_6.4.4-2_arm64.deb \
+          | xargs -0 -t -P0 -I {} wget -nd -q -P test/.tmp/debian-kernels/arm64 ftp://ftp.us.debian.org/debian/pool/main/l/linux/{}
+
+      - name: Download debian kernels
+        if: runner.arch == 'X64'
         run: |
           set -euxo pipefail
-          find integration-test-binaries -type f -exec chmod +x {} \;
-          test/run.sh integration-test-binaries
+          mkdir -p test/.tmp/debian-kernels/amd64
+          # TODO(https://github.com/aya-rs/aya/pull/725): enable tests on kernels before 6.0.
+          # linux-image-4.19.0-21-cloud-amd64-unsigned_4.19.249-2_amd64.deb \
+          # linux-image-5.10.0-23-cloud-amd64-unsigned_5.10.179-3_amd64.deb \
+          printf '%s\0' \
+            linux-image-6.1.0-10-cloud-amd64-unsigned_6.1.38-2_amd64.deb \
+            linux-image-6.4.0-1-cloud-amd64-unsigned_6.4.4-2_amd64.deb \
+          | xargs -0 -t -P0 -I {} wget -nd -q -P test/.tmp/debian-kernels/amd64 ftp://ftp.us.debian.org/debian/pool/main/l/linux/{}
+
+      - name: Alias gtar as tar
+        if: runner.os == 'macOS'
+        # macOS tar doesn't support --wildcards which we use below.
+        run: mkdir tar-is-gtar && ln -s "$(which gtar)" tar-is-gtar/tar && echo "$PWD"/tar-is-gtar >> $GITHUB_PATH
+
+      - name: Extract debian kernels
+        run: |
+          set -euxo pipefail
+          find test/.tmp -name '*.deb' -print0 | xargs -t -0 -I {} \
+            sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp --wildcards --extract '*vmlinuz*' --file -"
+
+      - name: Run integration tests
+        run: find test/.tmp -name 'vmlinuz-*' | xargs -t cargo xtask integration-test vm
 
   # Provides a single status check for the entire build workflow.
   # This is used for merge automation, like Mergify, since GH actions
   # has no concept of "when all status checks pass".
   # https://docs.mergify.com/conditions/#validating-all-status-checks
   build-workflow-complete:
-    needs: ["lint", "build-test-aya", "build-test-aya-bpf", "run-integration-test"]
+    needs:
+      - lint
+      - build-test-aya
+      - build-test-aya-bpf
+      - run-integration-test
     runs-on: ubuntu-latest
     steps:
       - name: Build Complete

Cargo.toml

@@ -6,6 +6,7 @@ members = [
     "aya-log-parser",
     "aya-obj",
     "aya-tool",
+    "init",
     "test/integration-test",
     "xtask",
 
@@ -29,6 +30,7 @@ default-members = [
     "aya-log-parser",
     "aya-obj",
     "aya-tool",
+    "init",
     # test/integration-test is omitted; including it in this list causes `cargo test` to run its
     # tests, and that doesn't work unless they've been built with `cargo xtask`.
     "xtask",
@@ -72,6 +74,7 @@ lazy_static = { version = "1", default-features = false }
 libc = { version = "0.2.105", default-features = false }
 log = { version = "0.4", default-features = false }
 netns-rs = { version = "0.1", default-features = false }
+nix = { version = "0.26.2", default-features = false }
 num_enum = { version = "0.6", default-features = false }
 object = { version = "0.31", default-features = false }
 parking_lot = { version = "0.12.0", default-features = false }

bpf/aya-log-ebpf/src/lib.rs

@@ -1,10 +1,9 @@
 #![no_std]
 #![warn(clippy::cast_lossless, clippy::cast_sign_loss)]
 
-use aya_bpf::{
-    macros::map,
-    maps::{PerCpuArray, PerfEventByteArray},
-};
+#[cfg(target_arch = "bpf")]
+use aya_bpf::macros::map;
+use aya_bpf::maps::{PerCpuArray, PerfEventByteArray};
 pub use aya_log_common::{write_record_header, Level, WriteToBuf, LOG_BUF_CAPACITY};
 pub use aya_log_ebpf_macros::{debug, error, info, log, trace, warn};
 
@@ -15,11 +14,19 @@ pub struct LogBuf {
 }
 
 #[doc(hidden)]
-#[map]
+// This cfg_attr prevents compilation failures on macOS where the generated section name doesn't
+// meet mach-o's requirements. We wouldn't ordinarily build this crate for macOS, but we do so
+// because the integration-test crate depends on this crate transitively. See comment in
+// test/integration-test/Cargo.toml.
+#[cfg_attr(target_arch = "bpf", map)]
 pub static mut AYA_LOG_BUF: PerCpuArray<LogBuf> = PerCpuArray::with_max_entries(1, 0);
 
 #[doc(hidden)]
-#[map]
+// This cfg_attr prevents compilation failures on macOS where the generated section name doesn't
+// meet mach-o's requirements. We wouldn't ordinarily build this crate for macOS, but we do so
+// because the integration-test crate depends on this crate transitively. See comment in
+// test/integration-test/Cargo.toml.
+#[cfg_attr(target_arch = "bpf", map)]
 pub static mut AYA_LOGS: PerfEventByteArray = PerfEventByteArray::new(0);
 
 #[doc(hidden)]

init/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "init"
+version = "0.1.0"
+authors = ["Tamir Duberstein <[email protected]>"]
+edition = "2021"
+publish = false
+
+[dependencies]
+anyhow = { workspace = true, features = ["std"] }
+nix = { workspace = true, features = ["fs", "mount", "reboot"] }

init/src/main.rs

@@ -0,0 +1,161 @@
+use anyhow::Context as _;
+
+#[derive(Debug)]
+struct Errors(Vec<anyhow::Error>);
+
+impl std::fmt::Display for Errors {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let Self(errors) = self;
+        for (i, error) in errors.iter().enumerate() {
+            if i != 0 {
+                writeln!(f)?;
+            }
+            write!(f, "{:?}", error)?;
+        }
+        Ok(())
+    }
+}
+
+impl std::error::Error for Errors {}
+
+fn run() -> anyhow::Result<()> {
+    const RXRXRX: nix::sys::stat::Mode = nix::sys::stat::Mode::empty()
+        .union(nix::sys::stat::Mode::S_IRUSR)
+        .union(nix::sys::stat::Mode::S_IXUSR)
+        .union(nix::sys::stat::Mode::S_IRGRP)
+        .union(nix::sys::stat::Mode::S_IXGRP)
+        .union(nix::sys::stat::Mode::S_IROTH)
+        .union(nix::sys::stat::Mode::S_IXOTH);
+
+    struct Mount {
+        source: &'static str,
+        target: &'static str,
+        fstype: &'static str,
+        flags: nix::mount::MsFlags,
+        data: Option<&'static str>,
+        target_mode: Option<nix::sys::stat::Mode>,
+    }
+
+    for Mount {
+        source,
+        target,
+        fstype,
+        flags,
+        data,
+        target_mode,
+    } in [
+        Mount {
+            source: "proc",
+            target: "/proc",
+            fstype: "proc",
+            flags: nix::mount::MsFlags::empty(),
+            data: None,
+            target_mode: Some(RXRXRX),
+        },
+        Mount {
+            source: "sysfs",
+            target: "/sys",
+            fstype: "sysfs",
+            flags: nix::mount::MsFlags::empty(),
+            data: None,
+            target_mode: Some(RXRXRX),
+        },
+        Mount {
+            source: "debugfs",
+            target: "/sys/kernel/debug",
+            fstype: "debugfs",
+            flags: nix::mount::MsFlags::empty(),
+            data: None,
+            target_mode: None,
+        },
+        Mount {
+            source: "bpffs",
+            target: "/sys/fs/bpf",
+            fstype: "bpf",
+            flags: nix::mount::MsFlags::empty(),
+            data: None,
+            target_mode: None,
+        },
+    ] {
+        match target_mode {
+            None => {
+                // Must exist.
+                let nix::sys::stat::FileStat { st_mode, .. } = nix::sys::stat::stat(target)
+                    .with_context(|| format!("stat({target}) failed"))?;
+                let s_flag = nix::sys::stat::SFlag::from_bits_truncate(st_mode);
+
+                if !s_flag.contains(nix::sys::stat::SFlag::S_IFDIR) {
+                    anyhow::bail!("{target} is not a directory");
+                }
+            }
+            Some(target_mode) => {
+                // Must not exist.
+                nix::unistd::mkdir(target, target_mode)
+                    .with_context(|| format!("mkdir({target}) failed"))?;
+            }
+        }
+        nix::mount::mount(Some(source), target, Some(fstype), flags, data).with_context(|| {
+            format!("mount({source}, {target}, {fstype}, {flags:?}, {data:?}) failed")
+        })?;
+    }
+
+    // By contract we run everything in /bin and assume they're rust test binaries.
+    //
+    // If the user requested command line arguments, they're named init.arg={}.
+
+    // Read kernel parameters from /proc/cmdline. They're space separated on a single line.
+    let cmdline = std::fs::read_to_string("/proc/cmdline")
+        .with_context(|| "read_to_string(/proc/cmdline) failed")?;
+    let args = cmdline
+        .split_whitespace()
+        .filter_map(|parameter| {
+            parameter
+                .strip_prefix("init.arg=")
+                .map(std::ffi::OsString::from)
+        })
+        .collect::<Vec<_>>();
+
+    // Iterate files in /bin.
+    let read_dir = std::fs::read_dir("/bin").context("read_dir(/bin) failed")?;
+    let errors = read_dir
+        .filter_map(|entry| {
+            match (|| {
+                let entry = entry.context("read_dir(/bin) failed")?;
+                let path = entry.path();
+                let status = std::process::Command::new(&path)
+                    .args(&args)
+                    .status()
+                    .with_context(|| format!("failed to execute {}", path.display()))?;
+
+                if status.code() == Some(0) {
+                    Ok(())
+                } else {
+                    Err(anyhow::anyhow!("{} failed: {status:?}", path.display()))
+                }
+            })() {
+                Ok(()) => None,
+                Err(err) => Some(err),
+            }
+        })
+        .collect::<Vec<_>>();
+    if errors.is_empty() {
+        Ok(())
+    } else {
+        Err(Errors(errors).into())
+    }
+}
+
+fn main() {
+    match run() {
+        Ok(()) => {
+            println!("init: success");
+        }
+        Err(err) => {
+            println!("{err:?}");
+            println!("init: failure");
+        }
+    }
+    let _: std::convert::Infallible =
+        nix::sys::reboot::reboot(nix::sys::reboot::RebootMode::RB_POWER_OFF)
+            .expect("reboot failed");
+}