Use CMake in llvm_mode (#61)

- Use CMake to compile in llvm_mode
- Improve custom function context

Author: spinpx

.cirrus.yml

@@ -4,6 +4,10 @@ angora_build_task:
     cargo_cache:
         folder: $CARGO_HOME/registry
         #    fingerprint_script: cat Cargo.lock
-    build_script: ./build/build.sh
+    build_script: 
+        - apt-get update
+        - apt-get install -y cmake
+        - cmake --version
+        - ./build/build.sh
     test_script: cargo test
     before_cache_script: rm -rf $CARGO_HOME/registry/index

.gitignore

@@ -11,10 +11,11 @@ Cargo.lock
 GPATH
 GRTAGS
 GTAGS
+GSYMS
 output*/
 *.tar.gz
 *.tar.xz
 auto/
 /bin/
 obj-intel64/
-.DS_Store
+.DS_Store

Dockerfile

@@ -2,7 +2,7 @@ FROM ubuntu:16.04
 
 RUN apt-get update && \
     apt-get -y upgrade && \
-    apt-get install -y git build-essential wget zlib1g-dev golang-go python-pip python-dev build-essential && \
+    apt-get install -y git build-essential wget zlib1g-dev golang-go python-pip python-dev build-essential cmake && \
     apt-get clean
 
 

angora_fuzzer

@@ -9,7 +9,7 @@ if [ -z ${LOG_TYPE} ]
 then LOG_TYPE="info"
 fi
 
-envs="RUST_BACKTRACE=1 RUST_LOG=${LOG_TYPE}"
+envs="RUST_BACKTRACE=1 RUST_LOG=${LOG_TYPE} ANGORA_BIN_DIR=${FUZZ_DIR}/bin"
 fuzzer="${FUZZ_DIR}/target/${BUILD_TYPE}/fuzzer"
 
 cmd="$envs $fuzzer $*"

build/build.sh

@@ -1,20 +1,33 @@
 #!/bin/bash
+BIN_PATH=$(readlink -f "$0")
+ROOT_DIR=$(dirname $(dirname $BIN_PATH))
 
 set -euxo pipefail
 
 if ! [ -x "$(command -v llvm-config)"  ]; then
-    ./build/install_llvm.sh
+    ${ROOT_DIR}/build/install_llvm.sh
     export PATH=${HOME}/clang+llvm/bin:$PATH
     export LD_LIBRARY_PATH=${HOME}/clang+llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
+    export CC=clang
+    export CXX=clang++
 fi
 
+PREFIX=${PREFIX:-${ROOT_DIR}/bin/}
+
 cargo build
 cargo build --release
 
-PREFIX=bin/
+rm -rf ${PREFIX}
 mkdir -p ${PREFIX}
-cp target/release/*.a ${PREFIX}
+mkdir -p ${PREFIX}/lib
 cp target/release/fuzzer ${PREFIX}
+cp target/release/*.a ${PREFIX}/lib
 
+cd llvm_mode
+rm -rf build
+mkdir -p build
+cd build
+cmake -DCMAKE_INSTALL_PREFIX=${PREFIX} -DCMAKE_BUILD_TYPE=Release ..
+make # VERBOSE=1 
+make install # VERBOSE=1
 
-cd llvm_mode && make

build/install_pin_mode.sh

@@ -9,4 +9,4 @@ PREFIX=/ ./install_pin.sh
 make
 cp env.init /opt/
 cd ..
-make
+make OBJDIR=../bin/lib/

build/install_tools.sh

@@ -5,6 +5,6 @@ set -euxo pipefail
 #wllvm and gllvm
 pip install --upgrade pip==9.0.3
 pip install wllvm
-mkdir /go
+mkdir ${HOME}/go
 go get github.com/SRI-CSL/gllvm/cmd/...
 

common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "angora_common"
-version = "1.2.0"
+version = "1.2.1"
 authors = ["sp1npx <[email protected]>"]
 edition = "2018"
 

common/src/config.rs

@@ -10,8 +10,8 @@ pub const PREFER_FAST_COND: bool = true;
 
 // ************ Resources ****************
 // branch.rs
-pub const MAP_LENGTH: usize = 20;
-pub const BRANCHES_SIZE: usize = 1 << MAP_LENGTH;
+pub const MAP_SIZE_POW2: usize = 20;
+pub const BRANCHES_SIZE: usize = 1 << MAP_SIZE_POW2;
 
 // executor.rs:
 pub const TMOUT_SKIP: usize = 3;

common/src/defs.rs

@@ -1,6 +1,8 @@
 use std;
 // -- envs
 pub static DISABLE_CPU_BINDING_VAR: &str = "ANGORA_DISABLE_CPU_BINDING";
+pub static ANGORA_BIN_DIR: &str = "ANGORA_BIN_DIR";
+
 // executor.rs
 pub static TRACK_OUTPUT_VAR: &str = "ANGORA_TRACK_OUTPUT";
 pub static COND_STMT_ENV_VAR: &str = "ANGORA_COND_STMT_SHM_ID";
@@ -34,12 +36,6 @@ pub const UNREACHABLE: u64 = std::u64::MAX;
 
 // ** Cond Type
 // < 0xFF: simple if
-pub const COND_BASIC_MASK: u32 = 0xFF;
-pub const COND_SIGN_MASK: u32 = 0x100;
-pub const COND_BOOL_MASK: u32 = 0x200;
-// pub const COND_CALL_MASK: u32 = 0x400;
-// pub const COND_CALL_REV_MASK: u32 = 0xFBFF;
-
 // http://llvm.org/doxygen/InstrTypes_8h_source.html
 // Opcode              U L G E    Intuitive operation
 pub const COND_FCMP_FALSE: u32 = 0;
@@ -87,6 +83,12 @@ pub const COND_ICMP_SLT_OP: u32 = 40;
 pub const COND_ICMP_SLE_OP: u32 = 41;
 pub const COND_SW_OP: u32 = 0x00FF;
 
+pub const COND_BASIC_MASK: u32 = 0xFF;
+pub const COND_SIGN_MASK: u32 = 0x100;
+pub const COND_BOOL_MASK: u32 = 0x200;
+// pub const COND_CALL_MASK: u32 = 0x400;
+// pub const COND_CALL_REV_MASK: u32 = 0xFBFF;
+
 pub const COND_MAX_EXPLORE_OP: u32 = 0x4000 - 1;
 pub const COND_MAX_EXPLOIT_OP: u32 = 0x5000 - 1;
 

docs/build_target.md

@@ -102,7 +102,7 @@ Example: how to custom `crc32` function in `zlib` library. (see `llvm_mode/exter
 ./angora/tools/gen_library_abilist.sh /usr/lib/x86_64-linux-gnu/libz.so  custom > zlib_abilist.txt
 
 export ANGORA_TAINT_RULE_LIST=/path-to/zlib_abilist.txt
-# write your custom function, e.g. llvm_mode/external_lib/zlib-func.c
+# write your custom function, e.g. llvm_mode/external_lib/zlib-func.c and llvm_mode/external_lib/zlib_abilist.txt 
 # compile it and 
 export ANGORA_TAINT_CUSTOM_RULE=/path-to/zlib-func.o
 ```
@@ -113,15 +113,15 @@ Use `USE_DFSAN=1 make` to build them.
 
 ## Build C++ program and C++ standard library
 - C++ program: CXX=/path-to-angora/bin/angora-clang++ or -DCMAKE_CXX_COMPILER=...
-- C++ standard library: we have built one under ubuntu 16.04 64bits in llvm_mode/libcxx_dfsan. You can built it by yourself with the following commands and move the libraries to llvm_mode/libcxx_dfsan directory and bin/. (run libcxx_dfsan/compile.sh)
+- C++ standard library: we have built one under ubuntu 16.04 64bits in llvm_mode/libcxx. You can built it by yourself with the following commands and move the libraries to llvm_mode/libcxx directory and bin/lib. (run libcxx_dfsan/compile.sh)
 
 ```
 # http://lists.llvm.org/pipermail/cfe-dev/2015-January/040876.html
 # install cmake ninja and download LLVM&CLANG source code
 CC=~/angora/bin/angora-clang CXX=~/angora/bin/angora-clang++ cmake -G Ninja ../llvm  -DLIBCXXABI_ENABLE_SHARED=NO -DLIBCXX_ENABLE_SHARED=NO -DLLVM_FORCE_USE_OLD_TOOLCHAIN=YES -DLIBCXX_CXX_ABI=libcxxabi
 USE_DFSAN=1 ninja cxx cxxabi
-# move them to llvm_mode/libcxx_dfsan
+# move them to llvm_mode/libcxx and bin/lib
 ```
 
 ## Add taints in input functions
-Angora models most input functions in `llvm_mode/io-func.c`. But it doesn't support some input functions like `scanf` or other input function in external libraries. You can add taints by yourself by the approach described in *Model an external library*. For example, program `who` use `getutxent` to read input, and we add taints in `__dfsw_getutxent` in `io-func.c` file.
+Angora models most input functions in `llvm_mode/external_lib/io-func.c`. But it doesn't support some input functions like `scanf` or other input function in external libraries. You can add taints by yourself by the approach described in *Model an external library*. For example, program `who` use `getutxent` to read input, and we add taints in `__dfsw_getutxent` in `io-func.c` file.

docs/configuration.md

@@ -1,6 +1,6 @@
 # Configuration Files
 
-- `llvm_mode/config.h`: Configuration file for llvm pass.
-- `llvm_mode/custom/angora_abilist.txt` : Taint propagation rules for functions in libraries in llvm mode.
-- `llvm_mode/custom/exploitation_list.txt` : Security sensitive functions or instructions in llvm mode.
+- `llvm_mode/include/defs.h`: Configuration and definition file for llvm pass.
+- `llvm_mode/rules/angora_abilist.txt` : Taint propagation rules for functions in libraries in llvm mode.
+- `llvm_mode/rules/exploitation_list.txt` : Security sensitive functions or instructions in llvm mode.
 - `common/src/config.rs`: Configuration file for fuzzer.

docs/environment_variables.md

@@ -6,8 +6,8 @@
 - `ANGORA_CUSTOM_FN_CONTEXT=k` : Use only the last k ( 0 <= k <= 32) function call location as the context, e.g. `ANGORA_CUSTOM_FN_CONTEXT=8`. Angora disables context if k is 0.
 - `ANGORA_GEN_ID_RANDOM=1` : Generate ids for predicates randomly instead of the hash of their locations.
 - `ANGORA_OUTPUT_COND_LOC=1` : (Debug option) Output the location of each predicate during compiling.
-- `ANGORA_TAINT_CUSTOM_RULE=/path/to/object` : object contains those proxy function (how to propagate taints), e.g. `ANGORA_TAINT_CUSTOM_RULE=~/angora/bin/zlib-func.o` . You should add it as custom type in the file passed by `ANGORA_TAINT_RULE_LIST` first.
-- `ANGORA_TAINT_RULE_LIST=/path/to/list` : DataFlowSanitizer’s [ABI list](https://clang.llvm.org/docs/DataFlowSanitizer.html), e.g. `ANGORA_TAINT_RULE_LIST=~/angora/bin/extra_list.txt`.
+- `ANGORA_TAINT_CUSTOM_RULE=/path/to/object` : object contains those proxy function (how to propagate taints), e.g. `ANGORA_TAINT_CUSTOM_RULE=~/angora/bin/lib/zlib-func.o` . You should add it as custom type in the file passed by `ANGORA_TAINT_RULE_LIST` first.
+- `ANGORA_TAINT_RULE_LIST=/path/to/list` : DataFlowSanitizer’s [ABI list](https://clang.llvm.org/docs/DataFlowSanitizer.html), e.g. `ANGORA_TAINT_RULE_LIST=~/angora/bin/rules/zlib_abilist.txt`.
 
 # Environment variables for running
 

docs/exploitation.md

@@ -1,6 +1,6 @@
 # Exploitation
 
-As BuzzFuzz, Angora supports finding which input bytes were processed by "attack point" we defined by taint tracking. You can add your custom "attack point" in `llvm_mode/custom/exploitation_list.txt`, and then recompile the tested program.
+As BuzzFuzz, Angora supports finding which input bytes were processed by "attack point" we defined by taint tracking. You can add your custom "attack point" in `llvm_mode/rules/exploitation_list.txt`, and then recompile the tested program.
 
 ```
 # the 2th(start from 0) argument of function memeset is an attack point

docs/lava-who-fix.md

@@ -27,7 +27,7 @@ base and bound information for allcoations. This only requires modifications to
 the `__dfsw_*alloc()` functions. The HashMap insertion, deletion and querying 
 operations are implemented in Rust. The HashMap uses base pointer values as the
 key and bound values as the value. Minimal instrumentation is required for this 
-approach. The source code can be found in the repository under the `llvm_mode`
+approach. The source code can be found in the repository under the `llvm_mode/external_lib`
 directory. 
 
 ## Path Coverage (Unsolved)

docs/overview.md

@@ -34,6 +34,7 @@ and precise input generation which significantly increases input coverage.
   - `src/stats`: Statistical chart.
   - `src/branches`: Branch counting.
 - `llvm_mode`: Includes source code for instrumenting compilers and DFSan, the taint tracking framework.
+- `pin_mode`: Includes source code for instrumenting based on Intel Pin.
 - `runtime`: Taint tracking runtime library for target program.
 - `runtime_fast`: Branch and constraint information collection library for target program.
 - `tests`: Sample tests to evaluate fuzzer performance.

docs/pin_mode.md

@@ -15,7 +15,7 @@ export LIBDFT_PATH=/path-to-libdft64
 ## Build Pin mode
 ```
 cd pin_mode
-make
+make OBJDIR=../bin/lib/
 ```
 
 ## Build a target program

docs/troubleshoot.md

@@ -19,4 +19,4 @@ directory is populated. Otherwise no branches can be found.
 
 - Multiple inconsistent warnings. It caused by the fast and track programs has different behaviors. If most constraints are inconsistent, ensure they are compiled with the same environment. Otherwise, report us.
 
-- Density is too large (> 10%). Please increase `MAP_SIZE_POW2` in `llvm_mode/config.h` and `MAP_LENGTH` in `common/src/config.rs`. Or disable function-call context by compiling with `ANGORA_DISABLE_CONTEXT=1` or `ANGORA_DIRECT_FN_CONTEXT=1` environment variable.
+- Density is too large (> 10%). Please increase `MAP_SIZE_POW2` in `common/src/config.rs`. Or disable function-call context(density > 50%) by compiling with `ANGORA_CUSTOM_FN_CONTEXT=k` (k is an integer and 0 <= k <= 32) environment variable. Angora disables context if k is 0.

docs/usage.md

@@ -1,7 +1,7 @@
 # Angora usage
 ```
 # /path-to-angora/angora_fuzzer --help
-angora-fuzzer 1.2.0
+angora-fuzzer 1.2.1
 fuzz some program
 
 USAGE:

fuzzer/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "angora"
-version = "1.2.0"
+version = "1.2.1"
 authors = ["spinpx <[email protected]>"]
 edition = "2018"
 
@@ -19,7 +19,7 @@ byteorder = "1.2"
 chrono = "0.4"
 priority-queue = "0.6"
 num_cpus = "1.0"
-derive_more = "0.14"
+derive_more = "0.15"
 colored = "1.6"
 serde="1.0"
 serde_derive = "1.0"

fuzzer/src/command.rs

@@ -1,4 +1,5 @@
 use crate::{check_dep, search, tmpfs};
+use angora_common::defs;
 use std::{
     env,
     path::{Path, PathBuf},
@@ -105,27 +106,19 @@ impl CommandOpt {
         let track_bin;
         let mut track_args = Vec::<String>::new();
         if mode.is_pin_mode() {
-            // ugly
-            let exe_path = env::current_exe().unwrap();
-            let project_dir = exe_path
-                .parent()
-                .unwrap()
-                .parent()
-                .unwrap()
-                .parent()
-                .unwrap();
-
+            let project_bin_dir = env::var(defs::ANGORA_BIN_DIR).expect("Please set ANGORA_PROJ_DIR");
+            
             let pin_root =
                 env::var(PIN_ROOT_VAR).expect("You should set the environment of PIN_ROOT!");
             let pin_bin = format!("{}/{}", pin_root, "pin");
             track_bin = pin_bin.to_string();
-            let pin_tool = project_dir
-                .join("pin_mode")
-                .join("obj-intel64")
+            let pin_tool = Path::new(&project_bin_dir)
+                .join("lib")
                 .join("pin_track.so")
                 .to_str()
                 .unwrap()
                 .to_owned();
+
             track_args.push(String::from("-t"));
             track_args.push(pin_tool);
             track_args.push(String::from("--"));

fuzzer/src/stats/chart.rs

@@ -126,7 +126,7 @@ impl ChartStats {
 impl fmt::Display for ChartStats {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         if self.density.0 > 10.0 {
-            warn!("Density is too large (> 10%). Please increase `MAP_SIZE_POW2` in `llvm_mode/config.h` and `MAP_LENGTH` in `common/src/config.rs`. Or disable function-call context(density > 50%) by compiling with `ANGORA_CUSTOM_FN_CONTEXT=k` (k is an integer and 0 <= k <= 32) environment variable. Angora disables context if k is 0.");
+            warn!("Density is too large (> 10%). Please increase `MAP_SIZE_POW2` in and `common/src/config.rs`. Or disable function-call context(density > 50%) by compiling with `ANGORA_CUSTOM_FN_CONTEXT=k` (k is an integer and 0 <= k <= 32) environment variable. Angora disables context if k is 0.");
         }
 
         if self.search.multiple_inconsist() {

llvm_mode/CMakeLists.txt

@@ -0,0 +1,19 @@
+cmake_minimum_required(VERSION 3.4)
+
+project(angora_llvm_mode VERSION 1.2.1 LANGUAGES C CXX ASM)
+
+include_directories(include)
+include_directories(dfsan_rt)
+include_directories(../runtime/include)
+
+set(ANGORA_BIN_DIR ".")
+set(ANGORA_LIB_DIR "lib")
+set(ANGORA_RULE_DIR "rules")
+set(ANGORA_PASS_DIR "pass")
+
+add_subdirectory(compiler)
+add_subdirectory(pass)
+add_subdirectory(rules)
+add_subdirectory(libcxx)
+add_subdirectory(external_lib)
+add_subdirectory(dfsan_rt)