Merge pull request 'Parallel fuzzing' (#18) from parallel-fuzz into main

Reviewed-on: https://git.ts.mattnite.net/mattnite/cairn/pulls/18

actions/cairn-zig-fuzz-afl/action.yml

@@ -24,7 +24,7 @@ inputs:
         varint_decode
     required: true
   fuzz_binary:
-    description: 'Binary name in zig-out/bin/ (auto-detected if only one)'
+    description: 'Binary name in the build output bin/ directory (auto-detected if only one)'
     required: false
     default: ''
   seed_dir:
@@ -90,65 +90,114 @@ runs:
         set -eu
 
         echo "Cairn CLI version: $(cairn version)"
+        NCPU=$(nproc 2>/dev/null || echo 1)
+        echo "Available cores: ${NCPU}"
 
-        TOTAL_CRASHES=0
+        RESULTS_DIR=$(mktemp -d)
-        TARGET_NUM=0
+        trap 'rm -rf "${RESULTS_DIR}"' EXIT
-        ACTIVE_RUN_ID=""
 
-        # Single EXIT trap that finishes whatever run is active.
+        # ── Collect target names ──
-        cleanup() {
+        TARGET_NAMES=()
-          if [ -n "${ACTIVE_RUN_ID}" ]; then
+        while IFS= read -r line; do
-            echo "Finishing run ${ACTIVE_RUN_ID} (cleanup)..."
+          line=$(echo "${line}" | sed 's/#.*//' | xargs)
-            cairn run finish -server "${CAIRN_SERVER}" -id "${ACTIVE_RUN_ID}" || true
+          [ -z "${line}" ] && continue
-            ACTIVE_RUN_ID=""
+          TARGET_NAMES+=("${line}")
-          fi
+        done <<EOF
-        }
+        ${TARGETS}
-        trap cleanup EXIT
+        EOF
 
-        # ── Iterate over each target name ──
+        TARGET_COUNT=${#TARGET_NAMES[@]}
-        while IFS= read -r FUZZ_TARGET; do
+        echo "Targets: ${TARGET_COUNT}, parallel slots: ${NCPU}"
-          # Skip empty lines and comments
-          FUZZ_TARGET=$(echo "${FUZZ_TARGET}" | sed 's/#.*//' | xargs)
-          [ -z "${FUZZ_TARGET}" ] && continue
 
-          TARGET_NUM=$((TARGET_NUM + 1))
+        # ── Phase 1: Build all targets sequentially ──
-          BUILD_ARGS="fuzz -Dfuzz-target=${FUZZ_TARGET}"
         echo ""
         echo "=========================================="
-          echo "Target ${TARGET_NUM}: ${FUZZ_TARGET} (zig build ${BUILD_ARGS})"
+        echo "Building ${TARGET_COUNT} target(s)"
         echo "=========================================="
 
-          # ── Ensure Cairn target ──
+        declare -A TARGET_BINS
+        for i in "${!TARGET_NAMES[@]}"; do
+          FUZZ_TARGET="${TARGET_NAMES[$i]}"
+          NUM=$((i + 1))
+          WORK="work/${FUZZ_TARGET}"
+          mkdir -p "${WORK}"
+
+          echo "[${NUM}/${TARGET_COUNT}] Building ${FUZZ_TARGET}..."
+
+          # Each target gets its own zig-out to avoid binary name collisions.
+          ZIG_OUT="${WORK}/zig-out"
+          rm -rf "${ZIG_OUT}"
+          zig build fuzz -Dfuzz-target="${FUZZ_TARGET}" --prefix "${ZIG_OUT}"
+
+          # Locate fuzz binary.
+          if [ -n "${FUZZ_BINARY}" ]; then
+            FUZZ_BIN="${ZIG_OUT}/bin/${FUZZ_BINARY}"
+          else
+            FUZZ_BIN=$(find "${ZIG_OUT}/bin" -maxdepth 1 -type f -executable)
+            BIN_COUNT=$(echo "${FUZZ_BIN}" | wc -l)
+            if [ "${BIN_COUNT}" -eq 0 ] || [ -z "${FUZZ_BIN}" ]; then
+              echo "ERROR: No executable found in ${ZIG_OUT}/bin/"
+              exit 1
+            elif [ "${BIN_COUNT}" -gt 1 ]; then
+              echo "ERROR: Multiple executables in ${ZIG_OUT}/bin/, specify fuzz_binary input"
+              exit 1
+            fi
+          fi
+
+          if [ ! -x "${FUZZ_BIN}" ]; then
+            echo "ERROR: Fuzz binary not found or not executable: ${FUZZ_BIN}"
+            exit 1
+          fi
+
+          TARGET_BINS["${FUZZ_TARGET}"]="${FUZZ_BIN}"
+          echo "  Built: ${FUZZ_BIN}"
+        done
+
+        # ── Phase 2: Prepare seeds and Cairn targets (sequential, network I/O) ──
+        echo ""
+        echo "=========================================="
+        echo "Preparing corpus for ${TARGET_COUNT} target(s)"
+        echo "=========================================="
+
+        declare -A CAIRN_TARGET_IDS
+        declare -A RUN_IDS
+
+        for i in "${!TARGET_NAMES[@]}"; do
+          FUZZ_TARGET="${TARGET_NAMES[$i]}"
+          NUM=$((i + 1))
+          FUZZ_BIN="${TARGET_BINS[${FUZZ_TARGET}]}"
+          SEEDS="work/${FUZZ_TARGET}/seeds"
+          mkdir -p "${SEEDS}"
+
+          echo "[${NUM}/${TARGET_COUNT}] Preparing ${FUZZ_TARGET}..."
+
+          # Ensure Cairn target.
           CAIRN_TARGET_ID=$(cairn target ensure \
             -server "${CAIRN_SERVER}" \
             -repo "${REPO}" \
             -owner "${OWNER}" \
             -name "${FUZZ_TARGET}" \
             -type fuzz)
-          echo "Cairn target ID: ${CAIRN_TARGET_ID}"
+          CAIRN_TARGET_IDS["${FUZZ_TARGET}"]="${CAIRN_TARGET_ID}"
 
-          # ── Start a run ──
+          # Start a run.
           RUN_ID=$(cairn run start \
             -server "${CAIRN_SERVER}" \
             -target-id "${CAIRN_TARGET_ID}" \
             -commit "${COMMIT}")
-          ACTIVE_RUN_ID="${RUN_ID}"
+          RUN_IDS["${FUZZ_TARGET}"]="${RUN_ID}"
-          echo "Run ID: ${RUN_ID}"
+          echo "  Target ID: ${CAIRN_TARGET_ID}, Run ID: ${RUN_ID}"
 
-          # ── Download existing corpus ──
+          # Copy per-target seeds from repo.
-          SEEDS="afl-seeds-${TARGET_NUM}"
-          rm -rf "${SEEDS}"
-          mkdir -p "${SEEDS}"
-
-          # Copy per-target seeds from repo if available.
           if [ -n "${SEED_DIR}" ] && [ -d "${SEED_DIR}/${FUZZ_TARGET}" ]; then
             REPO_SEED_COUNT=$(find "${SEED_DIR}/${FUZZ_TARGET}" -maxdepth 1 -type f | wc -l)
             if [ "${REPO_SEED_COUNT}" -gt 0 ]; then
               cp "${SEED_DIR}/${FUZZ_TARGET}"/* "${SEEDS}/" 2>/dev/null || true
-              echo "Copied ${REPO_SEED_COUNT} repo seeds from ${SEED_DIR}/${FUZZ_TARGET}"
+              echo "  Copied ${REPO_SEED_COUNT} repo seeds"
             fi
           fi
 
+          # Download existing corpus.
           DL_START=$(date +%s)
           cairn corpus download \
             -server "${CAIRN_SERVER}" \
@@ -156,64 +205,53 @@ runs:
             -dir "${SEEDS}" || true
           DL_ELAPSED=$(( $(date +%s) - DL_START ))
           DL_COUNT=$(find "${SEEDS}" -maxdepth 1 -type f | wc -l)
-          echo "Downloaded ${DL_COUNT} corpus entries (${DL_ELAPSED}s)"
+          echo "  Corpus: ${DL_COUNT} entries (${DL_ELAPSED}s)"
 
           if [ "$(find "${SEEDS}" -maxdepth 1 -type f | wc -l)" -eq 0 ]; then
             printf 'A' > "${SEEDS}/seed-0"
           fi
 
-          # ── Build ──
+          # Minimize corpus.
-          rm -rf zig-out
-          zig build ${BUILD_ARGS}
-
-          # ── Locate fuzz binary ──
-          if [ -n "${FUZZ_BINARY}" ]; then
-            FUZZ_BIN="zig-out/bin/${FUZZ_BINARY}"
-          else
-            BIN_COUNT=$(find zig-out/bin -maxdepth 1 -type f -executable 2>/dev/null | wc -l)
-            if [ "${BIN_COUNT}" -eq 0 ]; then
-              echo "ERROR: No executable found in zig-out/bin/"
-              exit 1
-            elif [ "${BIN_COUNT}" -gt 1 ]; then
-              echo "ERROR: Multiple executables in zig-out/bin/, specify fuzz_binary input"
-              find zig-out/bin -maxdepth 1 -type f -executable
-              exit 1
-            fi
-            FUZZ_BIN=$(find zig-out/bin -maxdepth 1 -type f -executable)
-          fi
-
-          if [ ! -x "${FUZZ_BIN}" ]; then
-            echo "ERROR: Fuzz binary not found or not executable: ${FUZZ_BIN}"
-            exit 1
-          fi
-          echo "Fuzz binary: ${FUZZ_BIN}"
-
-          # ── Minimize corpus ──
           SEED_COUNT=$(find "${SEEDS}" -maxdepth 1 -type f | wc -l)
           if [ "${SEED_COUNT}" -gt 1 ]; then
-            echo "Minimizing corpus (${SEED_COUNT} inputs)..."
+            echo "  Minimizing corpus (${SEED_COUNT} inputs)..."
             CMIN_START=$(date +%s)
-            MINIMIZED="afl-cmin-${TARGET_NUM}"
+            MINIMIZED="work/${FUZZ_TARGET}/minimized"
             rm -rf "${MINIMIZED}"
             mkdir -p "${MINIMIZED}"
             if afl-cmin -i "${SEEDS}" -o "${MINIMIZED}" -- "${FUZZ_BIN}" >/dev/null 2>&1; then
               CMIN_ELAPSED=$(( $(date +%s) - CMIN_START ))
               MINIMIZED_COUNT=$(find "${MINIMIZED}" -maxdepth 1 -type f | wc -l)
-              echo "Corpus minimized: ${SEED_COUNT} -> ${MINIMIZED_COUNT} inputs (${CMIN_ELAPSED}s)"
+              echo "  Minimized: ${SEED_COUNT} -> ${MINIMIZED_COUNT} inputs (${CMIN_ELAPSED}s)"
               rm -rf "${SEEDS}"
               mv "${MINIMIZED}" "${SEEDS}"
             else
-              echo "afl-cmin failed, using unminimized corpus"
+              echo "  afl-cmin failed, using unminimized corpus"
               rm -rf "${MINIMIZED}"
             fi
           fi
+        done
+
+        # ── Phase 3: Fuzz all targets in parallel ──
+        echo ""
+        echo "=========================================="
+        echo "Fuzzing ${TARGET_COUNT} target(s) in parallel (${NCPU} cores, ${DURATION}s each)"
+        echo "=========================================="
+
+        # Function that runs in a background subshell per target.
+        fuzz_target() {
+          local FUZZ_TARGET="$1"
+          local FUZZ_BIN="$2"
+          local SEEDS="$3"
+          local FINDINGS="$4"
+          local RESULT_FILE="$5"
+          local SEED_COUNT="$6"
 
-          # ── Run AFL++ ──
-          FINDINGS="findings-${TARGET_NUM}"
-          rm -rf "${FINDINGS}"
           mkdir -p "${FINDINGS}"
+          echo "[${FUZZ_TARGET}] Starting AFL++ (${SEED_COUNT} seeds, ${DURATION}s)"
 
-          AFL_EXIT=0
+          local FUZZ_START=$(date +%s)
+          local AFL_EXIT=0
           {
             AFL_NO_UI=1 \
             AFL_SKIP_CPUFREQ=1 \
@@ -226,16 +264,75 @@ runs:
               ${EXTRA_AFL_ARGS} \
               -- "${FUZZ_BIN}"
           } >/dev/null 2>&1 || AFL_EXIT=$?
+          local FUZZ_ELAPSED=$(( $(date +%s) - FUZZ_START ))
 
-          if [ "${AFL_EXIT}" -eq 0 ]; then
+          # Summary from fuzzer_stats.
-            echo "AFL++ exited normally (completed run)"
+          local STATS_FILE="${FINDINGS}/default/fuzzer_stats"
-          elif [ "${AFL_EXIT}" -eq 1 ]; then
+          if [ -f "${STATS_FILE}" ]; then
-            echo "AFL++ exited after reaching duration limit (${DURATION}s)"
+            local EXECS=$(grep -oP 'execs_done\s*:\s*\K\d+' "${STATS_FILE}" || echo "?")
+            local PATHS=$(grep -oP 'corpus_count\s*:\s*\K\d+' "${STATS_FILE}" || echo "?")
+            local CRASHES=$(grep -oP 'saved_crashes\s*:\s*\K\d+' "${STATS_FILE}" || echo "?")
+            local SPEED=$(grep -oP 'execs_per_sec\s*:\s*\K[\d.]+' "${STATS_FILE}" || echo "?")
+            local COVERAGE=$(grep -oP 'bitmap_cvg\s*:\s*\K[\d.]+%' "${STATS_FILE}" || echo "?")
+            echo "[${FUZZ_TARGET}] Finished in ${FUZZ_ELAPSED}s: ${EXECS} execs (${SPEED}/s), ${PATHS} paths, ${CRASHES} crashes, ${COVERAGE} coverage (exit ${AFL_EXIT})"
           else
-            echo "AFL++ exited with code ${AFL_EXIT}"
+            echo "[${FUZZ_TARGET}] AFL++ exited with code ${AFL_EXIT} after ${FUZZ_ELAPSED}s"
           fi
 
-          # ── Upload crashes ──
+          # Count crashes for the result file.
+          local CRASH_COUNT=0
+          local CRASH_DIR="${FINDINGS}/default/crashes"
+          if [ -d "${CRASH_DIR}" ]; then
+            CRASH_COUNT=$(find "${CRASH_DIR}" -maxdepth 1 -type f -name 'id:*' | wc -l)
+          fi
+          echo "${CRASH_COUNT}" > "${RESULT_FILE}"
+        }
+
+        FUZZ_PHASE_START=$(date +%s)
+        PIDS=()
+        for i in "${!TARGET_NAMES[@]}"; do
+          FUZZ_TARGET="${TARGET_NAMES[$i]}"
+          FUZZ_BIN="${TARGET_BINS[${FUZZ_TARGET}]}"
+          SEEDS="work/${FUZZ_TARGET}/seeds"
+          FINDINGS="work/${FUZZ_TARGET}/findings"
+          RESULT_FILE="${RESULTS_DIR}/${FUZZ_TARGET}"
+          SEED_COUNT=$(find "${SEEDS}" -maxdepth 1 -type f | wc -l)
+
+          fuzz_target "${FUZZ_TARGET}" "${FUZZ_BIN}" "${SEEDS}" "${FINDINGS}" "${RESULT_FILE}" "${SEED_COUNT}" &
+          PIDS+=($!)
+
+          # Limit parallelism to available cores.
+          if [ "${#PIDS[@]}" -ge "${NCPU}" ]; then
+            wait "${PIDS[0]}"
+            PIDS=("${PIDS[@]:1}")
+          fi
+        done
+
+        # Wait for remaining jobs.
+        for pid in "${PIDS[@]}"; do
+          wait "${pid}"
+        done
+        FUZZ_PHASE_ELAPSED=$(( $(date +%s) - FUZZ_PHASE_START ))
+        echo "Fuzzing phase completed in ${FUZZ_PHASE_ELAPSED}s"
+
+        # ── Phase 4: Upload results sequentially ──
+        echo ""
+        echo "=========================================="
+        echo "Uploading results"
+        echo "=========================================="
+
+        TOTAL_CRASHES=0
+        for i in "${!TARGET_NAMES[@]}"; do
+          FUZZ_TARGET="${TARGET_NAMES[$i]}"
+          NUM=$((i + 1))
+          FUZZ_BIN="${TARGET_BINS[${FUZZ_TARGET}]}"
+          RUN_ID="${RUN_IDS[${FUZZ_TARGET}]}"
+          CAIRN_TARGET_ID="${CAIRN_TARGET_IDS[${FUZZ_TARGET}]}"
+          FINDINGS="work/${FUZZ_TARGET}/findings"
+
+          echo "[${NUM}/${TARGET_COUNT}] ${FUZZ_TARGET}..."
+
+          # Upload crashes.
           CRASH_DIR="${FINDINGS}/default/crashes"
           if [ -d "${CRASH_DIR}" ]; then
             for crash_file in "${CRASH_DIR}"/id:*; do
@@ -255,14 +352,13 @@ runs:
               REPLAY_OUTPUT=$(timeout 10 "${FUZZ_BIN}" < "${crash_file}" 2>&1 || true)
               if [ -n "${REPLAY_OUTPUT}" ]; then
                 STACK_TRACE="${REPLAY_OUTPUT}"
-                # Extract a concise crash message from the first meaningful line.
                 FIRST_LINE=$(echo "${REPLAY_OUTPUT}" | grep -m1 -iE 'panic|error|fault|abort|overflow|undefined|sanitizer|SUMMARY' || true)
                 if [ -n "${FIRST_LINE}" ]; then
                   CRASH_MSG="${FIRST_LINE}"
                 fi
               fi
 
-              echo "Uploading crash: ${CRASH_NAME}"
+              echo "  Uploading crash: ${CRASH_NAME}"
               set -- -server "${CAIRN_SERVER}" -repo "${REPO}" -owner "${OWNER}" \
                 -commit "${COMMIT}" -run-id "${RUN_ID}" -type fuzz -file "${crash_file}" \
                 -kind crash \
@@ -283,47 +379,42 @@ runs:
             done
           fi
 
-          # ── Minimize and upload corpus ──
+          # Minimize and upload corpus.
           QUEUE_DIR="${FINDINGS}/default/queue"
           if [ -d "${QUEUE_DIR}" ]; then
             QUEUE_COUNT=$(find "${QUEUE_DIR}" -maxdepth 1 -type f -name 'id:*' | wc -l)
             if [ "${QUEUE_COUNT}" -gt 0 ]; then
-              UPLOAD_DIR="corpus-upload-${TARGET_NUM}"
+              UPLOAD_DIR="work/${FUZZ_TARGET}/corpus-upload"
               rm -rf "${UPLOAD_DIR}"
               mkdir -p "${UPLOAD_DIR}"
               CMIN_START=$(date +%s)
               if afl-cmin -i "${QUEUE_DIR}" -o "${UPLOAD_DIR}" -- "${FUZZ_BIN}" >/dev/null 2>&1; then
                 CMIN_ELAPSED=$(( $(date +%s) - CMIN_START ))
                 UPLOAD_COUNT=$(find "${UPLOAD_DIR}" -maxdepth 1 -type f | wc -l)
-                echo "Corpus for upload minimized: ${QUEUE_COUNT} -> ${UPLOAD_COUNT} entries (${CMIN_ELAPSED}s)"
+                echo "  Corpus minimized: ${QUEUE_COUNT} -> ${UPLOAD_COUNT} entries (${CMIN_ELAPSED}s)"
               else
-                echo "afl-cmin failed, uploading full queue"
+                echo "  afl-cmin failed, uploading full queue"
                 rm -rf "${UPLOAD_DIR}"
                 UPLOAD_DIR="${QUEUE_DIR}"
                 UPLOAD_COUNT="${QUEUE_COUNT}"
               fi
-              echo "Uploading corpus (${UPLOAD_COUNT} entries)..."
+              echo "  Uploading corpus (${UPLOAD_COUNT} entries)..."
               cairn corpus upload \
                 -server "${CAIRN_SERVER}" \
                 -target-id "${CAIRN_TARGET_ID}" \
                 -run-id "${RUN_ID}" \
                 -dir "${UPLOAD_DIR}"
-              rm -rf "corpus-upload-${TARGET_NUM}"
             fi
           fi
 
-          # ── Finish run ──
+          # Finish run.
           cairn run finish -server "${CAIRN_SERVER}" -id "${RUN_ID}" || true
-          ACTIVE_RUN_ID=""
+        done
-
-        done <<EOF
-        ${TARGETS}
-        EOF
 
         # ── Final report ──
         echo ""
         echo "=========================================="
-        echo "Fuzzed ${TARGET_NUM} target(s), ${TOTAL_CRASHES} total crash(es)"
+        echo "Fuzzed ${TARGET_COUNT} target(s), ${TOTAL_CRASHES} total crash(es)"
         echo "=========================================="
 
         if [ "${TOTAL_CRASHES}" -gt 0 ]; then