Merge branch 'pr/rebase-with-upstream' into main

Author: mkravchik

.github/ISSUE_TEMPLATE/bug_report.md

@@ -13,6 +13,8 @@ Thank you for making LibAFL better!
 
 **Describe the bug**
 A clear and concise description of what the bug is.
+If you want to present the backtrace, don't forget to run with `errors_backtrace` feature and log from `RUST_LOG`
+In addition, please tell us what is your fuzzer's Cargo.toml
 
 **To Reproduce**
 Steps to reproduce the behavior:

.github/workflows/build_and_test.yml

@@ -9,6 +9,7 @@ on:
   merge_group:
 env:
   CARGO_TERM_COLOR: always
+  CARGO_NET_GIT_FETCH_WITH_CLI: true
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -116,7 +117,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/workflows/ubuntu-prepare
       - uses: Swatinem/rust-cache@v2
-      - name: Add nightly rustfmt and clippy
+      - name: Add nightly clippy
         run: rustup toolchain install nightly --component miri --allow-downgrade
       # --- miri undefined behavior test --
       - name: Run miri tests
@@ -135,22 +136,19 @@ jobs:
         run: sudo apt purge llvm* clang* lld* lldb* opt*
       - name: Install and cache deps
         run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
-      - name: Add nightly rustfmt and clippy
-        run: rustup toolchain install nightly --component rustfmt --component clippy --component miri --allow-downgrade
+      - name: Add nightly clippy
+        run: rustup toolchain install nightly --component clippy --component miri --allow-downgrade
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
         with: { shared-key: "ubuntu" }
       - name: Install LLVM and Clang
-        uses: KyleMayes/install-llvm-action@v1
+        uses: KyleMayes/install-llvm-action@v2
         with:
           directory: ${{ runner.temp }}/llvm
           version: 17
-      # ---- format check ----
       # pcguard edges and pcguard hitcounts are not compatible and we need to build them seperately
       - name: Check pcguard edges
         run: cargo check --features=sancov_pcguard_edges
-      - name: Run clang-format style check for C/C++ programs.
-        run: clang-format -n -Werror --style=file $(find . -type f \( -name '*.cpp' -o -iname '*.hpp' -o -name '*.cc' -o -name '*.cxx' -o -name '*.cc' -o -name '*.h' \) | grep -v '/target/' | grep -v 'libpng-1\.6\.37' | grep -v 'stb_image\.h' | grep -v 'dlmalloc\.c' | grep -v 'QEMU-Nyx')
       - name: run shellcheck
         run: shellcheck ./scripts/*.sh
       # ---- build normal and examples ----
@@ -171,36 +169,22 @@ jobs:
 
       - name: Install and cache deps
         run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev
-      - name: Add nightly rustfmt and clippy
+      - name: Add nightly clippy
         run: rustup toolchain install nightly --component clippy --allow-downgrade && rustup default nightly
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
         with: { shared-key: "ubuntu" }
       - name: Install LLVM and Clang
-        uses: KyleMayes/install-llvm-action@v1
+        uses: KyleMayes/install-llvm-action@v2
         with:
           directory: ${{ runner.temp }}/llvm
           version: 17
       - name: Run clippy
         run: ./scripts/clippy.sh
-
     # --- test embedding the libafl_libfuzzer_runtime library
     # Fix me plz
     # - name: Test Build libafl_libfuzzer with embed
     #  run: cargo +nightly test --features=embed-runtime --manifest-path libafl_libfuzzer/Cargo.toml
-  ubuntu-check-nightly:
-    runs-on: ubuntu-22.04
-    needs: ubuntu
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/workflows/ubuntu-prepare
-      - uses: Swatinem/rust-cache@v2
-        with: { shared-key: "ubuntu" }
-      # ---- build and feature check ----
-      # cargo-hack's --feature-powerset would be nice here but libafl has a too many knobs
-      - name: Check nightly features
-        run: cargo +nightly check --features=agpl && cargo +nightly check --features=nautilus
-
 
   ubuntu-check:
     runs-on: ubuntu-22.04
@@ -217,7 +201,6 @@ jobs:
       # cargo-hack's --feature-powerset would be nice here but libafl has a too many knobs
       - name: Check each feature
         # Skipping `python` as it has to be built with the `maturin` tool
-        # `agpl`, `nautilus` require nightly
         # `sancov_pcguard_edges` is tested seperatelyc
         run: python3 ./scripts/parallellize_cargo_check.py ${{ matrix.instance_idx }}
 
@@ -247,7 +230,7 @@ jobs:
       - name: Remove existing clang and LLVM
         run: sudo apt purge llvm* clang*
       - name: Install LLVM and Clang
-        uses: KyleMayes/install-llvm-action@v1
+        uses: KyleMayes/install-llvm-action@v2
         with:
           directory: ${{ runner.temp }}/llvm
           version: 17
@@ -272,8 +255,15 @@ jobs:
           override: true
           components: rustfmt
       - uses: actions/checkout@v3
+      - name: Remove existing clang and LLVM
+        run: sudo apt purge llvm* clang*
+      - name: Install LLVM and Clang
+        uses: KyleMayes/install-llvm-action@v2
+        with:
+          directory: ${{ runner.temp }}/llvm
+          version: 17
       - name: Format Check
-        run: cargo fmt -- --check
+        run: ./scripts/fmt_all.sh check
 
   fuzzers-preflight:
     runs-on: ubuntu-latest
@@ -344,6 +334,8 @@ jobs:
           - ./fuzzers/backtrace_baby_fuzzers/forkserver_executor
           - ./fuzzers/backtrace_baby_fuzzers/c_code_with_inprocess_executor
           - ./fuzzers/backtrace_baby_fuzzers/rust_code_with_fork_executor
+          - ./fuzzers/toolbox
+          # - ./fuzzers/frida_simple_exe
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
@@ -399,7 +391,7 @@ jobs:
           profile: minimal
           toolchain: nightly
           override: true
-          components: rustfmt, rust-src
+          components: rust-src
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
       - name: Add targets
@@ -419,7 +411,7 @@ jobs:
           profile: minimal
           toolchain: nightly
           override: true
-          components: rustfmt, clippy, rust-src
+          components: clippy, rust-src
       - uses: actions/checkout@v3
       - uses: Swatinem/rust-cache@v2
       - name: Add targets
@@ -488,11 +480,11 @@ jobs:
           profile: minimal
           toolchain: stable
       - uses: actions/checkout@v3
+      - uses: ./.github/workflows/windows-tester-prepare      
       - uses: Swatinem/rust-cache@v2
-      - name: Run clippy
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
+      - name: Run real clippy, not the fake one
+        shell: pwsh
+        run: .\scripts\clippy.ps1
 
   macos:
     runs-on: macOS-latest
@@ -501,8 +493,8 @@ jobs:
         with:
           profile: minimal
           toolchain: stable
-      - name: Add nightly rustfmt and clippy
-        run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade && rustup default nightly
+      - name: Add nightly clippy
+        run: rustup toolchain install nightly --component clippy --allow-downgrade && rustup default nightly
       - name: Install deps
         run: brew install z3 gtk+3
       - name: Install cxxbridge
@@ -516,7 +508,7 @@ jobs:
       - name: Run Tests
         run: cargo test
       - name: Clippy
-        run: cargo +nightly clippy --tests --all
+        run: cargo +nightly clippy --tests --all --exclude libafl_nyx --exclude symcc_runtime --exclude runtime_test
 
   ios:
     runs-on: macOS-latest

.github/workflows/fuzzer-tester-prepare/action.yml

@@ -13,12 +13,12 @@ runs:
       with:
         profile: minimal
         toolchain: stable
-    - name: Add stable rustfmt and clippy
+    - name: Add stable clippy
       shell: bash
-      run: rustup toolchain install stable --component rustfmt --component clippy --allow-downgrade
-    - name: Add nightly rustfmt and clippy
+      run: rustup toolchain install stable --component clippy --allow-downgrade
+    - name: Add nightly clippy
       shell: bash
-      run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade
+      run: rustup toolchain install nightly --component clippy --allow-downgrade
     - name: Add no_std toolchain
       shell: bash
       run: rustup toolchain install nightly-x86_64-unknown-linux-gnu ; rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu
@@ -30,7 +30,7 @@ runs:
       shell: bash
       run: sudo apt purge -y llvm* clang*
     - name: Install LLVM and Clang
-      uses: KyleMayes/install-llvm-action@v1
+      uses: KyleMayes/install-llvm-action@v2
       with:
         directory: ${{ runner.temp }}/llvm
         version: 17

.github/workflows/qemu-fuzzer-tester-prepare/action.yml

@@ -16,18 +16,18 @@ runs:
       with:
         profile: minimal
         toolchain: stable
-    - name: Add stable rustfmt and clippy
+    - name: Add stable clippy
       shell: bash
-      run: rustup toolchain install stable --component rustfmt --component clippy --allow-downgrade
-    - name: Add nightly rustfmt and clippy
+      run: rustup toolchain install stable --component clippy --allow-downgrade
+    - name: Add nightly clippy
       shell: bash
-      run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade
+      run: rustup toolchain install nightly --component clippy --allow-downgrade
     - name: Remove obsolete llvm (Linux)
       if: runner.os == 'Linux'
       shell: bash
       run: sudo apt purge -y llvm* clang*
     - name: Install LLVM and Clang
-      uses: KyleMayes/install-llvm-action@v1
+      uses: KyleMayes/install-llvm-action@v2
       with:
         directory: ${{ runner.temp }}/llvm
         version: 17

.github/workflows/ubuntu-prepare/action.yml

@@ -21,7 +21,7 @@ runs:
       shell: bash
       run: rustup toolchain install nightly --allow-downgrade
     - name: Install LLVM and Clang
-      uses: KyleMayes/install-llvm-action@v1
+      uses: KyleMayes/install-llvm-action@v2
       with:
         directory: ${{ runner.temp }}/llvm
         version: 17

.github/workflows/windows-tester-prepare/action.yml

@@ -9,9 +9,6 @@ runs:
         toolchain: stable
     - uses: actions/checkout@v3
     - uses: Swatinem/rust-cache@v2
-    - name: Windows Build
-      shell: pwsh
-      run: cargo build --verbose
     - name: Build docs
       shell: pwsh
       run: cargo doc

.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: local
+    hooks:
+      - id: fmt
+        name: fmt
+        entry: scripts/fmt_all.sh check
+        language: script

CONTRIBUTING.md

@@ -0,0 +1,18 @@
+# How to Contribute to LibAFL
+
+For bugs, feel free to open issues or contact us directly. Thank you for your support. <3
+
+## Pull Request guideline
+
+Even though we will gladly assist you in finishing up your PR, try to
+- keep all the crates compiling with *stable* rust (hide the eventual non-stable code under [`cfg`s](https://github.com/AFLplusplus/LibAFL/blob/main/libafl/build.rs#L26))
+- run `cargo +nightly fmt` on your code before pushing
+- check the output of `cargo clippy --all` or `./clippy.sh`
+- run `cargo build --no-default-features` to check for `no_std` compatibility (and possibly add `#[cfg(feature = "std")]`) to hide parts of your code.
+
+Some of the parts in this list may be hard, don't be afraid to open a PR if you cannot fix them by yourself, so we can help.
+
+### Pre-commit hooks
+
+Some of these checks can be performed automatically during commit using [pre-commit](https://pre-commit.com/).
+Once the package is installed, simply run `pre-commit install` to enable the hooks, the checks will run automatically before the commit becomes effective.

Cargo.toml

@@ -35,6 +35,7 @@ exclude = [
     "libafl_qemu/libafl_qemu_sys",
     "utils/noaslr",
     "utils/gdb_qemu",
+    "utils/libafl_fmt",
     "scripts",
 ]
 

README.md

@@ -116,15 +116,7 @@ The best-tested fuzzer is [`./fuzzers/libfuzzer_libpng`](./fuzzers/libfuzzer_lib
 
 ## Contributing
 
-For bugs, feel free to open issues or contact us directly. Thank you for your support. <3
-
-Even though we will gladly assist you in finishing up your PR, try to
-- keep all the crates compiling with *stable* rust (hide the eventual non-stable code under [`cfg`s](https://github.com/AFLplusplus/LibAFL/blob/main/libafl/build.rs#L26))
-- run `cargo +nightly fmt` on your code before pushing
-- check the output of `cargo clippy --all` or `./clippy.sh`
-- run `cargo build --no-default-features` to check for `no_std` compatibility (and possibly add `#[cfg(feature = "std")]`) to hide parts of your code.
-
-Some of the parts in this list may be hard, don't be afraid to open a PR if you cannot fix them by yourself, so we can help.
+Please check out [CONTRIBUTING.md](CONTRIBUTING.md) for the contributing guideline.
 
 ## Cite
 
@@ -157,11 +149,3 @@ Unless you explicitly state otherwise, any contribution intentionally submitted
 for inclusion in this crate by you, as defined in the Apache-2.0 license, shall
 be dual licensed as above, without any additional terms or conditions.
 </sub>
-
-<br>
-
-<sub>
-Dependencies under more restrictive licenses, such as GPL or AGPL, can be enabled
-using the respective feature in each crate when it is present, such as the
-'agpl' feature of the libafl crate.
-</sub>

fuzzers/baby_fuzzer_nautilus/rust-toolchain

@@ -13,10 +13,10 @@ use libafl::{
     feedbacks::{CrashFeedback, MaxMapFeedback},
     fuzzer::{Fuzzer, StdFuzzer},
     inputs::{BytesInput, HasTargetBytes},
-    mutators::{StdScheduledMutator, StringCategoryRandMutator, StringSubcategoryRandMutator},
+    mutators::{StdScheduledMutator, UnicodeCategoryRandMutator, UnicodeSubcategoryRandMutator},
     observers::StdMapObserver,
     schedulers::QueueScheduler,
-    stages::{mutational::StdMutationalStage, StringIdentificationStage},
+    stages::{mutational::StdMutationalStage, UnicodeIdentificationStage},
     state::StdState,
     Evaluator,
 };
@@ -121,14 +121,14 @@ pub fn main() {
 
     // Setup a mutational stage with a basic bytes mutator
     let mutator = StdScheduledMutator::new(tuple_list!(
-        StringCategoryRandMutator,
-        StringSubcategoryRandMutator,
-        StringSubcategoryRandMutator,
-        StringSubcategoryRandMutator,
-        StringSubcategoryRandMutator
+        UnicodeCategoryRandMutator,
+        UnicodeSubcategoryRandMutator,
+        UnicodeSubcategoryRandMutator,
+        UnicodeSubcategoryRandMutator,
+        UnicodeSubcategoryRandMutator
     ));
     let mut stages = tuple_list!(
-        StringIdentificationStage::new(),
+        UnicodeIdentificationStage::new(),
         StdMutationalStage::transforming(mutator)
     );
 

fuzzers/baby_fuzzer_unicode/src/main.rs

@@ -1,6 +1,6 @@
 #![no_main]
 
-use libfuzzer_sys::fuzz_target;
 use cargo_fuzz_test::do_thing;
+use libfuzzer_sys::fuzz_target;
 
 fuzz_target!(|data: &[u8]| do_thing(data));

fuzzers/cargo_fuzz/fuzz/fuzz_targets/fuzz_target_1.rs

@@ -1,5 +1,5 @@
 [package]
-name = "fuzzbench"
+name = "dynamic_analysis"
 version = "0.12.0"
 authors = ["Andrea Fioraldi <[email protected]>", "Dominik Maier <[email protected]>"]
 edition = "2021"
@@ -22,18 +22,18 @@ strip = true
 
 [build-dependencies]
 cc = { version = "1.0", features = ["parallel"] }
-which = "4.4"
+which = "6.0"
 
 [dependencies]
-env_logger = "0.10"
+env_logger = "0.11"
 once_cell = "1.19"
 libafl = { path = "../../libafl/" }
 libafl_bolts = { path = "../../libafl_bolts/" }
 libafl_targets = { path = "../../libafl_targets/", features = ["sancov_pcguard_hitcounts", "sancov_cmplog", "libfuzzer", "function-logging"] }
 # TODO Include it only when building cc
 libafl_cc = { path = "../../libafl_cc/" }
 clap = { version = "4.0", features = ["default"] }
-nix = { version = "0.27", features = ["fs"] }
+nix = { version = "0.29", features = ["fs"] }
 mimalloc = { version = "*", default-features = false }
 
 [lib]