fuzzamoto-cli: Add Stability command for showing unstable functions

Dockerfile.stability

@@ -0,0 +1,103 @@
+FROM debian:bookworm
+
+# ------ Coverage for fuzzamoto ------
+
+ARG LLVM_V=19
+
+# Install LLVM toolchain & Bitcoin Core build dependencies
+# vim & tmux are useful
+RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget && \
+    wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc && \
+    apt-add-repository "deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-${LLVM_V} main" && \
+    apt-add-repository "deb-src http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-${LLVM_V} main" && \
+    apt-get update && apt install -y --no-install-recommends \
+    build-essential \
+    clang-${LLVM_V} \
+    cmake \
+    curl \
+    git \
+    libclang-rt-${LLVM_V}-dev \
+    lld-${LLVM_V} \
+    llvm-${LLVM_V} \
+    llvm-${LLVM_V}-dev \
+    patch \
+    python3 \
+    tmux \
+    vim
+
+# Install rust and tools
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+RUN rustup install nightly && rustup default nightly
+
+# ------ Build Bitcoin Core ------
+
+# Build Bitcoin Core
+ARG OWNER=bitcoin
+ARG REPO=bitcoin
+ARG BRANCH=master
+RUN git clone --depth 1 --branch $BRANCH https://github.com/$OWNER/$REPO.git
+
+ENV CC=clang-${LLVM_V}
+ENV CXX=clang++-${LLVM_V}
+
+ENV SOURCES_PATH=/tmp/bitcoin-depends
+RUN make -C bitcoin/depends NO_QT=1 NO_ZMQ=1 NO_USDT=1 download-linux SOURCES_PATH=$SOURCES_PATH
+# Keep extracted source
+RUN sed -i --regexp-extended '/.*rm -rf .*extract_dir.*/d' ./bitcoin/depends/funcs.mk && \
+    make -C ./bitcoin/depends DEBUG=1 NO_QT=1 NO_ZMQ=1 NO_USDT=1 \
+      SOURCES_PATH=$SOURCES_PATH \
+      AR=llvm-ar-${LLVM_V} NM=llvm-nm-${LLVM_V} RANLIB=llvm-ranlib-${LLVM_V} STRIP=llvm-strip-${LLVM_V} \
+      -j$(nproc)
+
+COPY ./target-patches/bitcoin-core-aggressive-rng.patch bitcoin/
+
+RUN cd bitcoin/ && \
+      git apply bitcoin-core-aggressive-rng.patch
+
+RUN cd bitcoin/ && cmake -B build_fuzz_cov \
+      --toolchain ./depends/$(./depends/config.guess)/toolchain.cmake \
+      -DAPPEND_CFLAGS="-fprofile-instr-generate -fcoverage-mapping" \
+      -DAPPEND_CXXFLAGS="-fprofile-instr-generate -fcoverage-mapping" \
+      -DAPPEND_LDFLAGS="-fprofile-instr-generate -fcoverage-mapping -fuse-ld=lld-${LLVM_V}" \
+      -DAPPEND_CPPFLAGS="-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION"
+
+RUN cmake --build bitcoin/build_fuzz_cov -j$(nproc) --target bitcoind
+
+WORKDIR /fuzzamoto/fuzzamoto-nyx-sys
+COPY ./fuzzamoto-nyx-sys/Cargo.toml .
+COPY ./fuzzamoto-nyx-sys/src/ src/
+COPY ./fuzzamoto-nyx-sys/build.rs .
+
+WORKDIR /fuzzamoto/fuzzamoto
+COPY ./fuzzamoto/Cargo.toml .
+COPY ./fuzzamoto/src/ src/
+
+WORKDIR /fuzzamoto/fuzzamoto-cli
+COPY ./fuzzamoto-cli/Cargo.toml .
+COPY ./fuzzamoto-cli/src/ src/
+
+WORKDIR /fuzzamoto/fuzzamoto-ir
+COPY ./fuzzamoto-ir/Cargo.toml .
+COPY ./fuzzamoto-ir/src/ src/
+
+WORKDIR /fuzzamoto/fuzzamoto-libafl
+COPY ./fuzzamoto-libafl/Cargo.toml .
+COPY ./fuzzamoto-libafl/src/ src/
+
+WORKDIR /fuzzamoto/fuzzamoto-scenarios
+COPY ./fuzzamoto-scenarios/Cargo.toml .
+COPY ./fuzzamoto-scenarios/bin/ bin/
+
+WORKDIR /fuzzamoto
+COPY ./Cargo.toml .
+RUN mkdir .cargo && cargo vendor > .cargo/config
+
+ENV BITCOIND_PATH=/bitcoin/build_fuzz_cov/bin/bitcoind
+RUN cargo build -p fuzzamoto-scenarios --bins -p fuzzamoto-cli --verbose --features "reproduce" --release
+
+RUN git clone https://github.com/tokatoka/llvm-profparser.git /llvm-profparser && cd /llvm-profparser && cargo build --release
+ENV LLVM_V=${LLVM_V}
+
+ENTRYPOINT ["/fuzzamoto/target/release/fuzzamoto-cli", "stability", "--output", "/mnt/output", \
+            "--corpus", "/mnt/corpus", "--llvm-profparser", "/llvm-profparser/target/release/profparser", "--unstable-testcases", "/mnt/unstable-testcases", "--bitcoind", "/bitcoin/build_fuzz_cov/bin/bitcoind", "--scenario"]

fuzzamoto-cli/src/commands/mod.rs

@@ -1,7 +1,9 @@
 pub mod coverage;
 pub mod init;
 pub mod ir;
+pub mod stability;
 
 pub use coverage::CoverageCommand;
 pub use init::InitCommand;
 pub use ir::IrCommand;
+pub use stability::StabilityCommand;

fuzzamoto-cli/src/commands/stability.rs

@@ -0,0 +1,172 @@
+use crate::error::CliError;
+use crate::error::Result;
+use crate::utils::{file_ops, process};
+use std::fs::File;
+use std::io::{self, BufRead, BufReader};
+use std::path::{Path, PathBuf};
+use std::process::Command;
+pub struct StabilityCommand;
+
+impl StabilityCommand {
+    fn read_lines<P: AsRef<Path>>(path: P) -> io::Result<Vec<String>> {
+        let file = File::open(path)?;
+        let reader = BufReader::new(file);
+
+        let mut lines = Vec::new();
+        for line in reader.lines() {
+            lines.push(line?);
+        }
+
+        Ok(lines)
+    }
+
+    pub fn execute(
+        output: PathBuf,
+        corpus: PathBuf,
+        unstable_testcases: PathBuf,
+        llvm_profparser: PathBuf,
+        bitcoind: PathBuf,
+        scenario: PathBuf,
+    ) -> Result<()> {
+        file_ops::ensure_file_exists(&bitcoind)?;
+        file_ops::ensure_file_exists(&scenario)?;
+
+        let corpus_files = file_ops::read_dir_files(&corpus)?;
+        let unstable = Self::read_lines(unstable_testcases)?;
+
+        let unstable_files: Vec<PathBuf> = corpus_files
+            .iter()
+            .filter(|f| unstable.contains(&f.file_name().unwrap().to_string_lossy().to_string()))
+            .cloned()
+            .collect();
+
+        // Run scenario for each corpus file
+        for corpus_file in unstable_files {
+            if let Err(e) = Self::run_one_input(
+                &output,
+                &corpus_file,
+                &llvm_profparser,
+                &bitcoind,
+                &scenario,
+            ) {
+                log::error!("Failed to run input ({:?}): {}", corpus_file, e);
+            }
+        }
+        Self::generate_coverage_report(&output, &bitcoind)?;
+        Ok(())
+    }
+
+    fn run_profparser(
+        llvm_profparser: &PathBuf,
+        output: &PathBuf,
+        base: &PathBuf,
+        other: &PathBuf,
+    ) -> Result<()> {
+        let _status = Command::new(llvm_profparser)
+            .arg("merge")
+            .arg("--output")
+            .arg(output)
+            .arg("--input")
+            .arg(base)
+            .arg("--input")
+            .arg(other)
+            .status()?;
+        Ok(())
+    }
+
+    fn run_one_input(
+        output: &PathBuf,
+        input: &PathBuf,
+        llvm_profparser: &PathBuf,
+        bitcoind: &PathBuf,
+        scenario: &PathBuf,
+    ) -> Result<()> {
+        log::info!("Running scenario with input: {}", input.display());
+
+        let mut profraw_files: Vec<PathBuf> = Vec::new();
+        for iter in 0..11 {
+            let profraw_file = output.join(format!(
+                "{}.coverage.profraw.{}",
+                input.file_name().unwrap().to_str().unwrap(),
+                iter
+            ));
+            profraw_files.push(profraw_file.clone());
+
+            let env_vars = vec![
+                ("LLVM_PROFILE_FILE", profraw_file.to_str().unwrap()),
+                ("FUZZAMOTO_INPUT", input.to_str().unwrap()),
+                ("RUST_LOG", "debug"),
+            ];
+            process::run_scenario_command(scenario, bitcoind, &env_vars)?;
+        }
+
+        let base = profraw_files.first().unwrap().clone();
+        profraw_files.remove(0);
+        for (iter, other) in profraw_files.iter().enumerate() {
+            let summary = output.join(format!(
+                "{}.{}.coverage.profraw.diff",
+                input.file_name().unwrap().to_str().unwrap(),
+                iter
+            ));
+            Self::run_profparser(llvm_profparser, &summary, &base, &other)?;
+        }
+
+        Ok(())
+    }
+
+    fn generate_coverage_report(output: &PathBuf, bitcoind: &PathBuf) -> Result<()> {
+        // Find all profraw files
+        let mut profraw_files = Vec::new();
+        for entry in std::fs::read_dir(output)? {
+            let path = entry?.path();
+            if let Some(file_name) = path.file_name().and_then(|s| s.to_str()) {
+                if file_name.contains("coverage.profraw.diff") {
+                    profraw_files.push(path.to_str().unwrap().to_string());
+                }
+            }
+        }
+
+        if profraw_files.is_empty() {
+            return Err(CliError::ProcessError("No profraw files found".to_string()));
+        }
+
+        // Merge profraw files
+        let coverage_profdata = output.join("coverage.profdata");
+        let coverage_profdata_str = coverage_profdata.to_str().unwrap();
+
+        let mut merge_args = vec!["merge", "-sparse"];
+        let profraw_refs: Vec<&str> = profraw_files.iter().map(|s| s.as_str()).collect();
+        merge_args.extend(profraw_refs);
+        merge_args.extend(["-o", coverage_profdata_str]);
+
+        let merge_cmd = process::get_llvm_command("llvm-profdata");
+        process::run_command_with_status(&merge_cmd, &merge_args, None)?;
+
+        // Generate HTML report
+        let coverage_report_dir = output.join("coverage-report");
+        let coverage_report_str = coverage_report_dir.to_str().unwrap();
+        let instr_profile_arg = format!("-instr-profile={}", coverage_profdata_str);
+        let output_dir_arg = format!("-output-dir={}", coverage_report_str);
+
+        let show_args = vec![
+            "show",
+            bitcoind.to_str().unwrap(),
+            &instr_profile_arg,
+            "-format=html",
+            "-show-directory-coverage",
+            "-show-branches=count",
+            &output_dir_arg,
+            "-Xdemangler=c++filt",
+        ];
+
+        let show_cmd = process::get_llvm_command("llvm-cov");
+        process::run_command_with_status(&show_cmd, &show_args, None)?;
+
+        log::info!(
+            "Coverage report generated in: {}",
+            coverage_report_dir.display()
+        );
+
+        Ok(())
+    }
+}

fuzzamoto-cli/src/main.rs

@@ -69,6 +69,28 @@ enum Commands {
         scenario: PathBuf,
     },
 
+    /// Create a summary describing which functions are unstable
+    Stability {
+        #[arg(long, help = "Path to the output file for the unstability summary")]
+        output: PathBuf,
+        #[arg(long, help = "Path to the input corpus directory")]
+        corpus: PathBuf,
+        #[arg(long, help = "Path to the list of unstable corpus entries")]
+        unstable_testcases: PathBuf,
+        #[arg(long, help = "Path to the llvm-profparser")]
+        llvm_profparser: PathBuf,
+        #[arg(
+            long,
+            help = "Path to the bitcoind binary that should be copied into the share directory"
+        )]
+        bitcoind: PathBuf,
+        #[arg(
+            long,
+            help = "Path to the fuzzamoto scenario binary that should be copied into the share directory"
+        )]
+        scenario: PathBuf,
+    },
+
     /// Fuzzamoto intermediate representation (IR) commands
     IR {
         #[command(subcommand)]
@@ -113,6 +135,21 @@ fn main() -> Result<()> {
             bitcoind.clone(),
             scenario.clone(),
         ),
+        Commands::Stability {
+            output,
+            unstable_testcases,
+            llvm_profparser,
+            corpus,
+            bitcoind,
+            scenario,
+        } => StabilityCommand::execute(
+            output.clone(),
+            corpus.clone(),
+            unstable_testcases.clone(),
+            llvm_profparser.clone(),
+            bitcoind.clone(),
+            scenario.clone(),
+        ),
         Commands::IR { command } => IrCommand::execute(command),
     }
 }

fuzzamoto-cli/src/utils/process.rs

@@ -67,7 +67,7 @@ pub fn run_scenario_command(
         cmd.env(key, value);
     }
 
-    cmd.stdout(Stdio::inherit()).stderr(Stdio::inherit());
+    cmd.stdout(Stdio::null()).stderr(Stdio::null());
 
     let status = cmd.status()?;
 

fuzzamoto-libafl/src/instance.rs

@@ -430,7 +430,12 @@ where
         let continue_minimizing = RefCell::new(1u64);
 
         let probing = ProbingStage::new(&stdout_observer_handle);
-        let stability = StabilityCheckStage::new(&map_observer_handle, &map_feedback_name, 8);
+        let stability_path = self
+            .options
+            .output_dir(self.client_description.core_id())
+            .join("unstable_testcases.txt");
+        let stability =
+            StabilityCheckStage::new(&map_observer_handle, &map_feedback_name, 8, &stability_path);
         let mut stages = tuple_list!(
             ClosureStage::new(|_a: &mut _, _b: &mut _, _c: &mut _, _d: &mut _| {
                 // Always try minimizing at least for one pass