From 1329b073efefe2c5fc9c9b5bcda7373132c5deb1 Mon Sep 17 00:00:00 2001 From: Jou Ho <43765840+jouho@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:12:35 -0700 Subject: [PATCH] chore: remove make fuzz and AFL fuzz (#4808) --- Makefile | 16 ---- codebuild/bin/install_default_dependencies.sh | 6 -- codebuild/bin/install_libFuzzer.sh | 55 ------------ codebuild/bin/s2n_codebuild.sh | 2 - codebuild/spec/buildspec_ubuntu_fuzz_afl.yml | 52 ----------- .../spec/buildspec_ubuntu_fuzz_artifacts.yml | 45 ---------- lib/Makefile | 4 - s2n.mk | 38 -------- tests/Makefile | 6 -- tests/fuzz/LD_PRELOAD/Makefile | 31 ------- tests/fuzz/Makefile | 86 ------------------- tests/fuzz/Readme.md | 39 ++++++--- tests/fuzz/runFuzzTest.sh | 42 +-------- 13 files changed, 31 insertions(+), 391 deletions(-) delete mode 100755 codebuild/bin/install_libFuzzer.sh delete mode 100644 codebuild/spec/buildspec_ubuntu_fuzz_afl.yml delete mode 100644 codebuild/spec/buildspec_ubuntu_fuzz_artifacts.yml delete mode 100644 tests/fuzz/LD_PRELOAD/Makefile delete mode 100644 tests/fuzz/Makefile diff --git a/Makefile b/Makefile index f16c403983d..42e55aeac90 100644 --- a/Makefile +++ b/Makefile @@ -72,22 +72,6 @@ valgrind: bin pedantic_valgrind: bin $(MAKE) -C tests pedantic_valgrind -.PHONY : fuzz -ifeq ($(shell uname),Linux) -fuzz : fuzz-linux -else -fuzz : fuzz-osx -endif - -.PHONY : fuzz-osx -fuzz-osx : - @echo "\033[33;1mSKIPPED\033[0m Fuzzing is not supported on \"$$(uname -mprs)\" at this time." - -.PHONY : fuzz-linux -fuzz-linux : export S2N_UNSAFE_FUZZING_MODE = 1 -fuzz-linux : bin - $(MAKE) -C tests fuzz - .PHONY : coverage coverage: run-lcov run-genhtml diff --git a/codebuild/bin/install_default_dependencies.sh b/codebuild/bin/install_default_dependencies.sh index dbb9a814169..b7f11a8f166 100755 --- a/codebuild/bin/install_default_dependencies.sh +++ b/codebuild/bin/install_default_dependencies.sh @@ -23,12 +23,6 @@ if [[ "$TESTS" == "fuzz" || "$TESTS" == "ALL" || "$LATEST_CLANG" == "true" ]]; t codebuild/bin/install_clang.sh "$(mktemp -d)" "$LATEST_CLANG_INSTALL_DIR" "$OS_NAME" > /dev/null ; fi -# Download and Install LibFuzzer with latest clang -if [[ "$TESTS" == "fuzz" || "$TESTS" == "ALL" ]]; then - mkdir -p "$LIBFUZZER_INSTALL_DIR" || true - PATH=$LATEST_CLANG_INSTALL_DIR/bin:$PATH codebuild/bin/install_libFuzzer.sh "$(mktemp -d)" "$LIBFUZZER_INSTALL_DIR" "$OS_NAME" ; -fi - # Download and Install Openssl 1.1.1 if [[ ("$S2N_LIBCRYPTO" == "openssl-1.1.1") || ( "$TESTS" == "integrationv2" || "$TESTS" == "ALL" ) ]]; then if [[ ! -x "$OPENSSL_1_1_1_INSTALL_DIR/bin/openssl" ]]; then diff --git a/codebuild/bin/install_libFuzzer.sh b/codebuild/bin/install_libFuzzer.sh deleted file mode 100755 index 414c6196450..00000000000 --- a/codebuild/bin/install_libFuzzer.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://aws.amazon.com/apache2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. -# -set -ex - -usage() { - echo "install_libFuzzer.sh download_dir install_dir os_name" - exit 1 -} - -if [ "$#" -ne "3" ]; then - usage -fi - -LIBFUZZER_DOWNLOAD_DIR=$1 -LIBFUZZER_INSTALL_DIR=$2 -export PLATFORM=$3 - -mkdir -p "$LIBFUZZER_DOWNLOAD_DIR" -cd "$LIBFUZZER_DOWNLOAD_DIR" - -git clone https://chromium.googlesource.com/chromium/llvm-project/llvm/lib/Fuzzer -cd Fuzzer -git checkout 651ead -cd .. - -echo "Compiling LibFuzzer..." -clang++ -c -g -v -O2 -lstdc++ -std=c++11 Fuzzer/*.cpp -IFuzzer -ar ruv libFuzzer.a Fuzzer*.o - -echo "Copying libFuzzer.a to $LIBFUZZER_INSTALL_DIR" -mkdir -p "$LIBFUZZER_INSTALL_DIR"/lib && cp libFuzzer.a "$LIBFUZZER_INSTALL_DIR"/lib - -# Run AFL instead of libfuzzer if AFL_FUZZ is set. Not compatible with fuzz coverage. -if [[ "$AFL_FUZZ" == "true" && "$FUZZ_COVERAGE" != "true" ]]; then - # Clusterfuzz's bash script changed from AFL to AFL++ on April 1, 2021; this - # commit (ac5ac9e4604ea03cfd643185ad1e3800e952ea44) pins the script to an older version - # of Clusterfuzz until we support AFL++. - mkdir -p "$LIBFUZZER_INSTALL_DIR" && curl https://raw.githubusercontent.com/google/clusterfuzz/ac5ac9e4604ea03cfd643185ad1e3800e952ea44/docs/setting-up-fuzzing/build_afl.bash > "$LIBFUZZER_INSTALL_DIR"/build_afl.bash - chmod +x "$LIBFUZZER_INSTALL_DIR"/build_afl.bash - cd "$LIBFUZZER_INSTALL_DIR" - "$LIBFUZZER_INSTALL_DIR"/build_afl.bash - cd - -fi diff --git a/codebuild/bin/s2n_codebuild.sh b/codebuild/bin/s2n_codebuild.sh index 64a53a57b12..c7790c9921d 100755 --- a/codebuild/bin/s2n_codebuild.sh +++ b/codebuild/bin/s2n_codebuild.sh @@ -127,8 +127,6 @@ if [[ "$TESTS" == "ALL" || "$TESTS" == "integrationv2" ]]; then run_integration_ if [[ "$TESTS" == "ALL" || "$TESTS" == "crt" ]]; then ./codebuild/bin/build_aws_crt_cpp.sh $(mktemp -d) $(mktemp -d); fi if [[ "$TESTS" == "ALL" || "$TESTS" == "sharedandstatic" ]]; then ./codebuild/bin/test_install_shared_and_static.sh $(mktemp -d); fi if [[ "$TESTS" == "ALL" || "$TESTS" == "dynamicload" ]]; then ./codebuild/bin/test_dynamic_load.sh $(mktemp -d); fi -# This can be removed after CI starts using buildspec_fuzz.yml -if [[ "$TESTS" == "ALL" || "$TESTS" == "fuzz" ]]; then (make clean && make fuzz) ; fi if [[ "$TESTS" == "sawHMAC" ]] && [[ "$OS_NAME" == "linux" ]]; then make -C tests/saw/ tmp/verify_HMAC.log ; fi if [[ "$TESTS" == "sawDRBG" ]]; then make -C tests/saw tmp/verify_drbg.log ; fi if [[ "$TESTS" == "ALL" || "$TESTS" == "tls" ]]; then make -C tests/saw tmp/verify_handshake.log ; fi diff --git a/codebuild/spec/buildspec_ubuntu_fuzz_afl.yml b/codebuild/spec/buildspec_ubuntu_fuzz_afl.yml deleted file mode 100644 index 20fd1c53caa..00000000000 --- a/codebuild/spec/buildspec_ubuntu_fuzz_afl.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use -# this file except in compliance with the License. A copy of the License is -# located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing permissions and -# limitations under the License. -version: 0.2 - -env: - variables: - # CODEBUILD_ is a reserved namespace. - CB_BIN_DIR: "./codebuild/bin" - -phases: - install: - runtime-versions: - python: 3.x - pre_build: - commands: - - | - if [ -d "third-party-src" ]; then - cd third-party-src; - fi - - $CB_BIN_DIR/install_ubuntu_dependencies.sh - - TESTS=fuzz $CB_BIN_DIR/install_default_dependencies.sh - - mkdir tests/fuzz/results - - mount -t tmpfs -o size=8096m ramdisk tests/fuzz/results - - ln -s test-deps/libfuzzer fuzz_dependencies - - ln -s $(which afl-fuzz) / - build: - commands: - - AFL_NO_UI=true TESTS=fuzz AFL_FUZZ=true make fuzz -artifacts: - files: - - "./tests/fuzz/results/$FUZZ_TESTS/*" - name: afl-fuzz-stats-$FUZZ_TESTS-$(date +%Y%m%d)-$CODEBUILD_BUILD_NUMBER - discard-paths: no - secondary-artifacts: - logs: - files: - - "./tests/fuzz/results/$FUZZ_TESTS/crashes/*" - - "./tests/fuzz/results/$FUZZ_TESTS/hangs/*" - - "./tests/fuzz/results/$FUZZ_TESTS/queue/*" - name: aflfuzz-corpus-$FUZZ_TESTS-$(date +%Y%m%d)-$CODEBUILD_BUILD_NUMBER - discard-paths: no diff --git a/codebuild/spec/buildspec_ubuntu_fuzz_artifacts.yml b/codebuild/spec/buildspec_ubuntu_fuzz_artifacts.yml deleted file mode 100644 index 7bab1662935..00000000000 --- a/codebuild/spec/buildspec_ubuntu_fuzz_artifacts.yml +++ /dev/null @@ -1,45 +0,0 @@ ---- -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use -# this file except in compliance with the License. A copy of the License is -# located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing permissions and -# limitations under the License. -version: 0.2 -env: - variables: - # CODEBUILD_ is a reserved namespace. - CB_BIN_DIR: "./codebuild/bin" - -phases: - pre_build: - commands: - - | - if [ -d "third-party-src" ]; then - cd third-party-src; - ln -s /usr/local $CODEBUILD_SRC_DIR/third-party-src/test-deps; - fi - - ln -s /usr/local $CODEBUILD_SRC_DIR/test-deps - - touch tests/fuzz/placeholder_results.txt tests/fuzz/placeholder_output.txt - build: - commands: - - $CB_BIN_DIR/s2n_codebuild.sh -artifacts: - files: - - "./tests/fuzz/corpus/$FUZZ_TESTS/*" - name: fuzz-corpus-$FUZZ_TESTS-$(date +%Y%m%d)-$CODEBUILD_BUILD_NUMBER - discard-paths: no - secondary-artifacts: - logs: - files: - - "./tests/fuzz/**/*_results.txt" - - "./tests/fuzz/**/*_output.txt" - - "./coverage/fuzz/**" - name: fuzz-cov-logs-$FUZZ_TESTS-$(date +%Y%m%d)-$CODEBUILD_BUILD_NUMBER - discard-paths: no diff --git a/lib/Makefile b/lib/Makefile index bea08e291b0..d8e71639d85 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -20,10 +20,6 @@ all: libs2n.a libs2n.so libs2n.dylib include ../s2n.mk -ifeq ($(FUZZ_COVERAGE), true) - FUZZCOV_FLAGS = -fprofile-instr-generate -fcoverage-mapping -endif - libs2n.a: ${OBJS} $(AR) cru libs2n.a ${OBJS} $(RANLIB) libs2n.a diff --git a/s2n.mk b/s2n.mk index 70fdfa1b53e..0195388eec2 100644 --- a/s2n.mk +++ b/s2n.mk @@ -57,23 +57,6 @@ COVERAGE_CFLAGS = -fprofile-arcs -ftest-coverage COVERAGE_LDFLAGS = --coverage LDFLAGS = -z relro -z now -z noexecstack -FUZZ_CFLAGS = -fsanitize-coverage=trace-pc-guard -fsanitize=address,undefined,leak - -# Define FUZZ_COVERAGE - to be used for generating coverage reports on fuzz tests -# !!! NOT COMPATIBLE WITH S2N_COVERAGE !!! -ifeq ($(FUZZ_COVERAGE), true) - FUZZ_CFLAGS += -fprofile-instr-generate -fcoverage-mapping -else - ifeq ($(S2N_COVERAGE), true) - DEFAULT_CFLAGS += ${COVERAGE_CFLAGS} - LIBS += ${COVERAGE_LDFLAGS} - endif -endif - -ifdef FUZZ_TIMEOUT_SEC - DEFAULT_CFLAGS += -DS2N_FUZZ_TESTING=1 -endif - # Add a flag to disable stack protector for alternative libcs without # libssp. ifneq ($(NO_STACK_PROTECTOR), 1) @@ -134,27 +117,6 @@ endif LLVM_GCOV_MARKER_FILE=${COVERAGE_DIR}/use-llvm-gcov.tmp -ifeq ($(S2N_UNSAFE_FUZZING_MODE),1) - # Override compiler to clang if fuzzing, since gcc does not support as many sanitizer flags as clang - CC=clang - - # Create a marker file so that later invocations of make can pick the right COV_TOOL by default - $(shell touch "${LLVM_GCOV_MARKER_FILE}") - - # Turn on debugging and fuzzing flags when S2N_UNSAFE_FUZZING_MODE is enabled to give detailed stack traces in case - # an error occurs while fuzzing. - CFLAGS += ${DEFAULT_CFLAGS} ${DEBUG_CFLAGS} ${FUZZ_CFLAGS} - - # Filter out the visibility settings if we are fuzzing - CFLAGS := $(filter-out -fvisibility=hidden,$(CFLAGS)) - CFLAGS := $(filter-out -DS2N_EXPORTS,$(CFLAGS)) - DEFAULT_CFLAGS := $(filter-out -fvisibility=hidden,$(DEFAULT_CFLAGS)) - DEFAULT_CFLAGS := $(filter-out -DS2N_EXPORTS,$(DEFAULT_CFLAGS)) - CPPFLAGS := $(filter-out -fvisibility=hidden,$(CPPFLAGS)) - CPPFLAGS := $(filter-out -DS2N_EXPORTS,$(CPPFLAGS)) - -endif - # Disable strict-prototypes check in clang ifneq '' '$(findstring clang,$(CC))' CFLAGS += -Wno-strict-prototypes diff --git a/tests/Makefile b/tests/Makefile index a20706063c8..d7affd989a2 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -43,11 +43,6 @@ libs: integrationv2: ${MAKE} -C integrationv2 -.PHONY : fuzz -fuzz: - ${MAKE} -C testlib - ${MAKE} -C fuzz - .PHONY : viz viz: ${MAKE} -C viz @@ -59,7 +54,6 @@ clean: decruft ${MAKE} -C testlib decruft ${MAKE} -C LD_PRELOAD decruft ${MAKE} -C unit clean - ${MAKE} -C fuzz clean ${MAKE} -C viz clean ${MAKE} -C saw decruft diff --git a/tests/fuzz/LD_PRELOAD/Makefile b/tests/fuzz/LD_PRELOAD/Makefile deleted file mode 100644 index 5bdfa5d6712..00000000000 --- a/tests/fuzz/LD_PRELOAD/Makefile +++ /dev/null @@ -1,31 +0,0 @@ -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://aws.amazon.com/apache2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. -# - -SRCS=$(wildcard *.c) -OVERRIDES=$(SRCS:.c=) - -.PHONY : all -all : $(OVERRIDES) - -include ../../../s2n.mk - -CRUFT += $(wildcard *.so) - -LD_PRELOAD_CFLAGS = -Wno-unreachable-code -O0 -I$(LIBCRYPTO_ROOT)/include/ -I../../../ -I../../../api/ - -$(OVERRIDES):: - # Don't include Sanitizer/Fuzz compiler flags since when the LD_PRELOAD shared object is Preloaded, the Sanitizer init - # functions won't have been loaded yet, causing undefined symbol errors. - ${CC} ${DEFAULT_CFLAGS} ${DEBUG_CFLAGS} ${LD_PRELOAD_CFLAGS} -shared -fPIC $@.c -o $@.so -ldl diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile deleted file mode 100644 index d702ba39326..00000000000 --- a/tests/fuzz/Makefile +++ /dev/null @@ -1,86 +0,0 @@ -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://aws.amazon.com/apache2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. -# - -SRCS=$(wildcard *.c) -OBJS=$(SRCS:.c=.o) -TESTS=$(SRCS:.c=) -CRYPTO_LDFLAGS = -L$(LIBCRYPTO_ROOT)/lib -SHELL:=/bin/bash - - -ifndef LIBFUZZER_ROOT - export LIBFUZZER_ROOT = $(shell echo "../../fuzz_dependencies") -endif - -ifndef FUZZ_TIMEOUT_SEC - export FUZZ_TIMEOUT_SEC=120 -endif - -ifndef CORPUS_UPLOAD_LOC - export CORPUS_UPLOAD_LOC="none" -endif - -ifndef ARTIFACT_UPLOAD_LOC - export ARTIFACT_UPLOAD_LOC="none" -endif - -ifndef FUZZ_TESTS - export FUZZ_TESTS=${TESTS} -endif - -.PHONY : all -all : run_tests - -include ../../s2n.mk - -CRUFT += $(wildcard *_test) $(wildcard fuzz-*.log) $(wildcard *_test_output.txt) $(wildcard *_test_results.txt) $(wildcard LD_PRELOAD/*.so) $(wildcard *.prof*) - -# We do not warn on unused results (-Wno-unused-result) because we expect that -# many of the fuzz test inputs will not be valid and operations will not succeed. -CFLAGS += -Wno-unreachable-code -Wno-unused-result -O0 -I$(LIBCRYPTO_ROOT)/include/ -I../ -LIBS += -L../testlib/ -ltests2n -L../../lib/ -ls2n -LDFLAGS += $(LIBFUZZER_ROOT)/lib/libFuzzer.a -lstdc++ -LDFLAGS += ${CRYPTO_LDFLAGS} ${LIBS} ${CRYPTO_LIBS} -lm -ldl -lrt -pthread - -DYLD_LIBRARY_PATH="../../lib/:../testlib/:$(LIBCRYPTO_ROOT)/lib:$$DYLD_LIBRARY_PATH" -LD_LIBRARY_PATH="../../lib/:../testlib/:$(LIBCRYPTO_ROOT)/lib:$$LD_LIBRARY_PATH" - -ifdef AFL_FUZZ - ifneq ($(FUZZ_COVERAGE), true) - CFLAGS += $(LIBFUZZER_ROOT)/FuzzingEngine.a - endif -endif - - -ld-preload : - ${MAKE} -C LD_PRELOAD - -$(TESTS):: - @${CC} ${CFLAGS} $@.c -o $@ ${LDFLAGS} - -run_tests:: $(FUZZ_TESTS) ld-preload - { set -e; \ - for test_name in ${FUZZ_TESTS} ; do \ - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}; \ - export DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}; \ - export LIBCRYPTO_ROOT=${LIBCRYPTO_ROOT}; \ - ./runFuzzTest.sh $${test_name} ${FUZZ_TIMEOUT_SEC} ${CORPUS_UPLOAD_LOC} ${ARTIFACT_UPLOAD_LOC}; done; \ - } - ./calcTotalCov.sh - -.PHONY : clean -clean: decruft - ${MAKE} -C LD_PRELOAD decruft - rm -rf profiles diff --git a/tests/fuzz/Readme.md b/tests/fuzz/Readme.md index 794f770090f..28e4bb2da3a 100644 --- a/tests/fuzz/Readme.md +++ b/tests/fuzz/Readme.md @@ -1,7 +1,27 @@ # Fuzz Tests -By default, every test in this directory will be run as a fuzz test for several minutes each during builds. To run all fuzz tests simply run `make fuzz` from the top `s2n` directory to compile s2n with the proper flags and run the fuzz tests. To run a specific subset of fuzz tests, simply set the FUZZ_TESTS variable as follows: - -> FUZZ_TESTS="test1 test2 test3" +By default, every test in this directory will be run as a fuzz test for several minutes each during builds. To run all fuzz tests, run the following commands from the top `s2n-tls` directory: +1. Compile with S2N_FUZZ_TEST option enabled +``` +cmake . -Bbuild \ +-DS2N_FUZZ_TEST=on +``` +2. Compile the project +``` +cmake --build ./build -- -j $(nproc) +``` +3. Run fuzz tests +``` +cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure" +``` + +To run a specific fuzz test, pass in the name of the test using the following command: +``` +cmake --build build/ --target test -- ARGS="-L fuzz -R --output-on-failure" +``` +For example, +``` +cmake --build build/ --target test -- ARGS="-L fuzz -R s2n_client_fuzz_test --output-on-failure" +``` #### Each Fuzz Test should conform to the following rules: 1. End in either `*_test.c` or `*_negative_test.c`. @@ -25,6 +45,8 @@ As the tests run, more detailed coverage reports are placed in the following dir Each test outputs an HTML file which displays line by line coverage statistics and a .txt report which gives per-function coverage statistics in human-readable ASCII. After all fuzz tests have ran, a matching pair of coverage reports is generated for the total coverage of S2N by the entire set of tests performed. +Currently, this option isn't enabled for cmake build. See [#4748](https://github.com/aws/s2n-tls/issues/4748). + ## Fuzz Test Directory Structure For a test with name `$TEST_NAME`, its files should be laid out with the following structure: @@ -40,6 +62,10 @@ For a test with name `$TEST_NAME`, its files should be laid out with the followi # Corpus A Corpus is a directory of "interesting" inputs that result in a good branch/code coverage. These inputs will be permuted in random ways and checked to see if this permutation results in greater branch coverage or in a failure (Segfault, Memory Leak, Buffer Overflow, Non-zero return code, etc). If the permutation results in greater branch coverage, then it will be added to the Corpus directory. If a Memory leak or a Crash is detected, that file will **not** be added to the corpus for that test, and will instead be written to the current directory (`s2n/tests/fuzz/crash-*` or `s2n/tests/fuzz/leak-*`). These files will be automatically deleted for any Negative Fuzz tests that are expected to crash or leak memory so as to not clutter the directory. +To continuously improve corpus inputs, we have a scheduled job that runs every day for approximately 8 hours. These tests begin with corpus files stored in an S3 bucket. At the end of each run, the existing corpus files are replaced with updated ones, potentially increasing branch coverage over time. This process allows for gradual and automated enhancement of the corpus. + +To enable this, two environment variables must be defined: `CORPUS_UPLOAD_LOC` and `ARTIFACT_UPLOAD_LOC`. `CORPUS_UPLOAD_LOC` specifies where corpus files are stored, while `ARTIFACT_UPLOAD_LOC`defines where output logs from fuzzing are saved, which can be used for debugging if a new bug is detected during fuzzing. + # LD_PRELOAD The `LD_PRELOAD` directory contains function overrides for each Fuzz test that will be used **instead** of the original functions defined elsewhere. These function overrides will only be used during fuzz tests, and will not effect the rest of the s2n codebase when not fuzzing. Using `LD_PRELOAD` instead of C Preprocessor `#ifdef`'s is preferable in the following ways: @@ -51,10 +77,3 @@ Each Fuzz test will have up to two `LD_PRELOAD` function override files used: 1. A test specific `${TEST_NAME}_overrides.c` file that contains overrides specific to that test. 2. `global_overrides.c` file that contains overrides that will be used in every fuzz test. - -# American Fuzzy Lop (AFL) - -To use AFL set the environment variable `AFL_FUZZ` to true, in addition to `FUZZ_TIMEOUT_SEC`. -The runFuzzTest.sh script will terminate afl when it reaches `FUZZ_TIMEOUT_SEC`. AFL reports will be created under `tests/fuzz/results/TEST_NAME/fuzzer_stats`. - -Note that afl runs as a single process. [Parallelization](https://github.com/google/AFL/blob/master/docs/parallel_fuzzing.txt) has not been scripted yet for this project. \ No newline at end of file diff --git a/tests/fuzz/runFuzzTest.sh b/tests/fuzz/runFuzzTest.sh index 58517ea302f..487b36901d2 100755 --- a/tests/fuzz/runFuzzTest.sh +++ b/tests/fuzz/runFuzzTest.sh @@ -33,8 +33,8 @@ ARTIFACT_UPLOAD_LOC=$4 MIN_TEST_PER_SEC="1000" MIN_FEATURES_COVERED="100" -# Failures for negative tests on AFL can be ignored. -if [[ $TEST_NAME == *_negative_test && "$AFL_FUZZ" != "true" ]]; +# Failures for negative tests can be ignored. +if [[ $TEST_NAME == *_negative_test ]]; then EXPECTED_TEST_FAILURE=1 else @@ -100,44 +100,6 @@ else cp -r ./corpus/${TEST_NAME}/. "${TEMP_CORPUS_DIR}" fi -# Run AFL instead of libfuzzer if AFL_FUZZ is set. Not compatible with fuzz coverage. -if [[ ${AFL_FUZZ} == "true" && ${FUZZ_COVERAGE} != "true" ]]; then - unset LD_PRELOAD - # See https://aflplus.plus/docs/env_variables/ - export AFL_NO_UI=true - export AFL_HARDEN=true - printf "Running AFL %-s %-40s for %5d sec... " "${FIPS_TEST_MSG}" ${TEST_NAME} ${FUZZ_TIMEOUT_SEC} - mkdir -p results/${TEST_NAME} - set +e - timeout ${FUZZ_TIMEOUT_SEC} ${LIBFUZZER_INSTALL_DIR}/afl-fuzz -i corpus/${TEST_NAME} -o results/${TEST_NAME} -m none ./${TEST_NAME} 2>&1> ./results/${TEST_NAME}/console_output.log - returncode=$? - # See the timeout man page for specifics - if [[ ${returncode} -ne 124 ]]; then - printf "\033[33;1mWARNING!\033[0m AFL exited with an unexpected return value: %8d" ${returncode} - fi - set -e - CRASH_COUNT=$(sed -n -e 's/^unique_crashes *: //p' ./results/${TEST_NAME}/fuzzer_stats) - TEST_COUNT=$(sed -n -e 's/^execs_done *: //p' ./results/${TEST_NAME}/fuzzer_stats) - FLOAT_TESTS_PER_SEC=$(sed -n -e 's/^execs_per_sec *: //p' ./results/${TEST_NAME}/fuzzer_stats) - TESTS_PER_SEC=$(echo "($FLOAT_TESTS_PER_SEC+.5)/1"|bc) - - if [[ ${TESTS_PER_SEC} -lt 10 ]]; then - printf "\033[33;1mWARNING!\033[0m %10d tests, only %6d tests per second; test is too slow.\n" ${TEST_COUNT} ${TESTS_PER_SEC} - fi - if [[ ${CRASH_COUNT} -gt 0 ]]; then - ACTUAL_TEST_FAILURE=1 - fi - if [[ ${ACTUAL_TEST_FAILURE} == ${EXPECTED_TEST_FAILURE} ]]; then - printf "\033[32;1mPASSED\033[0m %8d tests, %.1f test/sec\n" ${TEST_COUNT} ${TESTS_PER_SEC} - exit 0 - else - printf "\033[31;1mFAILED\033[0m %10d tests, %6d unique crashes\n" ${TEST_COUNT} ${CRASH_COUNT} - exit -1 - fi -else - printf "Running %-s %-40s for %5d sec with %2d threads... " "${FIPS_TEST_MSG}" ${TEST_NAME} ${FUZZ_TIMEOUT_SEC} ${NUM_CPU_THREADS} -fi - # Setup and clean profile structure if FUZZ_COVERAGE is enabled, otherwise run as normal if [[ "$FUZZ_COVERAGE" == "true" ]]; then mkdir -p "./profiles/${TEST_NAME}"