Merge pull request #654 from AFLplusplus/dev

fix LTO

GNUmakefile

@@ -307,7 +307,7 @@ all: test_x86 test_shm test_python ready $(PROGS) afl-as llvm gcc_plugin test_bu
 
 .PHONY: llvm
 llvm:
- -$(MAKE) -f GNUmakefile.llvm
+ -$(MAKE) -j -f GNUmakefile.llvm
  @test -e afl-cc || { echo "[-] Compiling afl-cc failed. You seem not to have a working compiler." ; exit 1; }
 
 .PHONY: gcc_plugin
@@ -579,7 +579,7 @@ deepclean: clean
 
 .PHONY: distrib
 distrib: all
- -$(MAKE) -f GNUmakefile.llvm
+ -$(MAKE) -j -f GNUmakefile.llvm
  -$(MAKE) -f GNUmakefile.gcc_plugin
  $(MAKE) -C utils/libdislocator
  $(MAKE) -C utils/libtokencap
@@ -602,7 +602,7 @@ binary-only: test_shm test_python ready $(PROGS)
 
 .PHONY: source-only
 source-only: all
- -$(MAKE) -f GNUmakefile.llvm
+ -$(MAKE) -j -f GNUmakefile.llvm
  -$(MAKE) -f GNUmakefile.gcc_plugin
  $(MAKE) -C utils/libdislocator
  $(MAKE) -C utils/libtokencap

README.md

@@ -31,18 +31,18 @@ With afl++ 3.0 we introduced changes that break some previous afl and afl++
 behaviours and defaults:
 
  * There are no llvm_mode and gcc_plugin subdirectories anymore and there is
- only one compiler: afl-cc. All previous compilers now symlink to this one
- compiler. All instrumentation source code is now in the `instrumentation/`
- folder.
+ only one compiler: afl-cc. All previous compilers now symlink to this.
+ All instrumentation source code is now in the `instrumentation/` folder.
  * The gcc_plugin was replaced with a new version submitted by AdaCore that
  supports more features. thank you!
  * qemu_mode got upgraded to QEMU 5.1, but to be able to build this a current
  ninja build tool version and python3 setuptools are required.
  qemu_mode also got new options like snapshotting, instrumenting specific
  shared libraries, etc. Additionally QEMU 5.1 supports more CPU targets so
  this is really worth it.
- * When instrumenting targets, afl-cc will not supersede optimizations. This
- allows to fuzz targets as same as they are built for debug or release.
+ * When instrumenting targets, afl-cc will not supersede optimizations anymore
+ if any were given. This allows to fuzz targets as same as they are built
+ for debug or release.
  * afl-fuzz:
  * if neither -M or -S is specified, `-S default` is assumed, so more
  fuzzers can easily be added later
@@ -88,7 +88,7 @@ behaviours and defaults:
  | Ngram prev_loc Coverage | | x(6) | | | |
  | Context Coverage | | x(6) | | | |
  | Auto Dictionary | | x(7) | | | |
- | Snapshot LKM Support | | x | x  | (x)(5) | |
+ | Snapshot LKM Support | | x(8) | x(8) | (x)(5) | |
 
  1. default for LLVM >= 9.0, env var for older version due an efficiency bug in llvm <= 8
  2. GCC creates non-performant code, hence it is disabled in gcc_plugin
@@ -97,6 +97,7 @@ behaviours and defaults:
  5. upcoming, development in the branch
  6. not compatible with LTO instrumentation and needs at least LLVM >= 4.1
  7. automatic in LTO mode with LLVM >= 11, an extra pass for all LLVM version that writes to a file to use with afl-fuzz' `-x`
+ 8. the snapshot LKM is currently unmaintained due to too many kernel changes coming too fast :-(
 
  Among others, the following features and patches have been integrated:
 
@@ -139,9 +140,6 @@ behaviours and defaults:
 
 ## Help wanted
 
-We were happy to be part of [Google Summer of Code 2020](https://summerofcode.withgoogle.com/organizations/5100744400699392/)
-and we will try to participate again in 2021!
-
 We have several ideas we would like to see in AFL++ to make it even better.
 However, we already work on so many things that we do not have the time for
 all the big ideas.
@@ -206,7 +204,7 @@ These build targets exist:
 afl++ binaries by passing the STATIC=1 argument to make:
 
 ```shell
-make all STATIC=1
+make STATIC=1
 ```
 
 These build options exist:
@@ -283,9 +281,9 @@ anything below 9 is not recommended.
  |
  v
  +--------------------------------+
- | if you want to instrument only | -> use GCC_PLUGIN mode (afl-gcc-fast/afl-g++-fast)
- | parts of the target | see [instrumentation/README.gcc_plugin.md](instrumentation/README.gcc_plugin.md) and
- +--------------------------------+ [instrumentation/README.instrument_list.md](instrumentation/README.instrument_list.md)
+ | gcc 5+ is available  | -> use GCC_PLUGIN mode (afl-gcc-fast/afl-g++-fast)
+ +--------------------------------+ see [instrumentation/README.gcc_plugin.md](instrumentation/README.gcc_plugin.md) and
+   [instrumentation/README.instrument_list.md](instrumentation/README.instrument_list.md)
  |
  | if not, or if you do not have a gcc with plugin support
  |
@@ -298,17 +296,17 @@ Clickable README links for the chosen compiler:
  * [LTO mode - afl-clang-lto](instrumentation/README.lto.md)
  * [LLVM mode - afl-clang-fast](instrumentation/README.llvm.md)
  * [GCC_PLUGIN mode - afl-gcc-fast](instrumentation/README.gcc_plugin.md)
- * GCC mode (afl-gcc) has no README as it has no own features
+ * GCC/CLANG mode (afl-gcc/afl-clang) have no README as they have no own features
 
 You can select the mode for the afl-cc compiler by:
- 1. passing --afl-MODE command line options to the compiler via CFLAGS/CXXFLAGS/CPPFLAGS
- 2. use a symlink to afl-cc: afl-gcc, afl-g++, afl-clang, afl-clang++,
+ 1. use a symlink to afl-cc: afl-gcc, afl-g++, afl-clang, afl-clang++,
  afl-clang-fast, afl-clang-fast++, afl-clang-lto, afl-clang-lto++,
- afl-gcc-fast, afl-g++-fast
- 3. using the environment variable AFL_CC_COMPILER with MODE
+ afl-gcc-fast, afl-g++-fast (recommended!)
+ 2. using the environment variable AFL_CC_COMPILER with MODE
+ 3. passing --afl-MODE command line options to the compiler via CFLAGS/CXXFLAGS/CPPFLAGS
 
 MODE can be one of: LTO (afl-clang-lto*), LLVM (afl-clang-fast*), GCC_PLUGIN
-(afl-g*-fast) or GCC (afl-gcc/afl-g++).
+(afl-g*-fast) or GCC (afl-gcc/afl-g++) or CLANG(afl-clang/afl-clang++).
 
 Because no afl specific command-line options are accepted (beside the
 --afl-MODE command), the compile-time tools make fairly broad use of environment
@@ -338,25 +336,21 @@ The following options are available when you instrument with LTO mode (afl-clang
  You can read more about this in [instrumentation/README.cmplog.md](instrumentation/README.cmplog.md)
 
 If you use LTO, LLVM or GCC_PLUGIN mode (afl-clang-fast/afl-clang-lto/afl-gcc-fast)
- you have the option to selectively only instrument parts of the target that you
+you have the option to selectively only instrument parts of the target that you
 are interested in:
 
  * To instrument only those parts of the target that you are interested in
  create a file with all the filenames of the source code that should be
  instrumented.
- For afl-clang-lto and afl-gcc-fast - or afl-clang-fast if either the clang
- version is below 7 or the CLASSIC instrumentation is used - just put one
+ For afl-clang-lto and afl-gcc-fast - or afl-clang-fast if a mode other than
+ DEFAULT/PCGUARD is used or you have llvm > 10.0.0 - just put one
  filename or function per line (no directory information necessary for
  filenames9, and either set `export AFL_LLVM_ALLOWLIST=allowlist.txt` **or**
  `export AFL_LLVM_DENYLIST=denylist.txt` - depending on if you want per
  default to instrument unless noted (DENYLIST) or not perform instrumentation
  unless requested (ALLOWLIST).
  **NOTE:** During optimization functions might be inlined and then would not match!
  See [instrumentation/README.instrument_list.md](instrumentation/README.instrument_list.md)
- For afl-clang-fast > 6.0 or if PCGUARD instrumentation is used then use the
- llvm sancov allow-list feature: [http://clang.llvm.org/docs/SanitizerCoverage.html](http://clang.llvm.org/docs/SanitizerCoverage.html)
- The llvm sancov format works with the allowlist/denylist feature of afl++
- however afl++'s format is more flexible.
 
 There are many more options and modes available however these are most of the
 time less effective. See:
@@ -696,7 +690,7 @@ Note that there are also a lot of tools out there that help fuzzing with afl++
 (some might be deprecated or unsupported):
 
 Minimization of test cases:
- * [afl-pytmin](https://github.com/ilsani/afl-pytmin) - a wrapper for afl-tmin that tries to speed up the process of the minimization of test case by using many CPU cores.
+ * [afl-pytmin](https://github.com/ilsani/afl-pytmin) - a wrapper for afl-tmin that tries to speed up the process of minimization of a single test case by using many CPU cores.
  * [afl-ddmin-mod](https://github.com/MarkusTeufelberger/afl-ddmin-mod) - a variation of afl-tmin based on the ddmin algorithm. 
  * [halfempty](https://github.com/googleprojectzero/halfempty) - is a fast utility for minimizing test cases by Tavis Ormandy based on parallelization. 
 
@@ -751,7 +745,7 @@ the speed compared to qemu_mode (but slower than persistent mode).
 ### Unicorn
 
 For non-Linux binaries you can use afl++'s unicorn mode which can emulate
-anything you want - for the price of speed and the user writing scripts.
+anything you want - for the price of speed and user written scripts.
 See [unicorn_mode](unicorn_mode/README.md).
 
 It can be easily built by:
@@ -763,16 +757,16 @@ cd unicorn_mode
 ### Shared libraries
 
 If the goal is to fuzz a dynamic library then there are two options available.
-For both you need to write a small hardness that loads and calls the library.
+For both you need to write a small harness that loads and calls the library.
 Faster is the frida solution: [utils/afl_frida/README.md](utils/afl_frida/README.md)
 
 Another, less precise and slower option is using ptrace with debugger interrupt
-instrumentation: [utils/afl_untracer/README.md](utils/afl_untracer/README.md)
+instrumentation: [utils/afl_untracer/README.md](utils/afl_untracer/README.md).
 
 ### More
 
 A more comprehensive description of these and other options can be found in
-[docs/binaryonly_fuzzing.md](docs/binaryonly_fuzzing.md)
+[docs/binaryonly_fuzzing.md](docs/binaryonly_fuzzing.md).
 
 ## Challenges of guided fuzzing
 

docs/Changelog.md

@@ -22,6 +22,9 @@ sending a mail to <[email protected]>.
  - added AFL_LLVM_INSTRUMENT option NATIVE for native clang pc-guard
  support (less performant than our own), GCC for old afl-gcc and
  CLANG for old afl-clang
+ - warn on any _AFL and __AFL env var
+ - LLVM mode is now compiled with -j4, unicorn with all cores. qemu was
+ already building with all cores, the gcc plugin needs only one.
  - added dummy Makefile to instrumentation/
 
 

include/alloc-inl.h

@@ -363,7 +363,8 @@ static inline void *DFL_ck_realloc(void *orig, u32 size) {
 
  if (orig) {
 
- memcpy((char *)ret + ALLOC_OFF_HEAD, (char *)orig + ALLOC_OFF_HEAD, MIN(size, old_size));
+ memcpy((char *)ret + ALLOC_OFF_HEAD, (char *)orig + ALLOC_OFF_HEAD,
+ MIN(size, old_size));
  memset((char *)orig + ALLOC_OFF_HEAD, 0xFF, old_size);
 
  ALLOC_C1((char *)orig + ALLOC_OFF_HEAD) = ALLOC_MAGIC_F;

src/afl-analyze.c

@@ -103,11 +103,11 @@ static u32 map_size = MAP_SIZE;
 /* Classify tuple counts. This is a slow & naive version, but good enough here.
  */
 
-#define TIMES4(x) x,x,x,x
-#define TIMES8(x) TIMES4(x),TIMES4(x)
-#define TIMES16(x) TIMES8(x),TIMES8(x)
-#define TIMES32(x) TIMES16(x),TIMES16(x)
-#define TIMES64(x) TIMES32(x),TIMES32(x)
+#define TIMES4(x) x, x, x, x
+#define TIMES8(x) TIMES4(x), TIMES4(x)
+#define TIMES16(x) TIMES8(x), TIMES8(x)
+#define TIMES32(x) TIMES16(x), TIMES16(x)
+#define TIMES64(x) TIMES32(x), TIMES32(x)
 static u8 count_class_lookup[256] = {
 
  [0] = 0,
@@ -121,6 +121,7 @@ static u8 count_class_lookup[256] = {
  [128] = TIMES64(128)
 
 };
+
 #undef TIMES64
 #undef TIMES32
 #undef TIMES16

src/afl-cc.c

@@ -1030,9 +1030,9 @@ int main(int argc, char **argv, char **envp) {
 
  compiler_mode = GCC;
 
- } else if (strncmp(callname, "afl-clang", 9) == 0 &&
+ } else if (strcmp(callname, "afl-clang") == 0 ||
 
- strstr(callname, "fast") == NULL) {
+ strcmp(callname, "afl-clang++") == 0) {
 
  compiler_mode = CLANG;
 
@@ -1076,13 +1076,13 @@ int main(int argc, char **argv, char **envp) {
 
  }
 
- if (strncmp(callname, "afl-clang", 9) == 0 &&
- strstr(callname, "fast") == NULL) {
+ if (strcmp(callname, "afl-clang") == 0 ||
+ strcmp(callname, "afl-clang++") == 0) {
 
  clang_mode = 1;
  compiler_mode = CLANG;
 
- if (strncmp(callname, "afl-clang++", 11) == 0) { plusplus_mode = 1; }
+ if (strcmp(callname, "afl-clang++") == 0) { plusplus_mode = 1; }
 
  }
 

src/afl-common.c

@@ -432,7 +432,9 @@ void check_environment_vars(char **envp) {
  char *env, *val;
  while ((env = envp[index++]) != NULL) {
 
- if (strncmp(env, "ALF_", 4) == 0) {
+ if (strncmp(env, "ALF_", 4) == 0 || strncmp(env, "_ALF", 4) == 0 ||
+ strncmp(env, "__ALF", 5) == 0 || strncmp(env, "_AFL", 4) == 0 ||
+ strncmp(env, "__AFL", 5) == 0) {
 
  WARNF("Potentially mistyped AFL environment variable: %s", env);
  issue_detected = 1;

src/afl-fuzz-bitmap.c

@@ -26,7 +26,7 @@
 #include "afl-fuzz.h"
 #include <limits.h>
 #if !defined NAME_MAX
-#define NAME_MAX _XOPEN_NAME_MAX
+ #define NAME_MAX _XOPEN_NAME_MAX
 #endif
 
 /* Write bitmap to file. The bitmap is useful mostly for the secret
@@ -143,12 +143,14 @@ u32 count_non_255_bytes(afl_state_t *afl, u8 *mem) {
  and replacing it with 0x80 or 0x01 depending on whether the tuple
  is hit or not. Called on every new crash or timeout, should be
  reasonably fast. */
-#define TIMES4(x) x,x,x,x
-#define TIMES8(x) TIMES4(x),TIMES4(x)
-#define TIMES16(x) TIMES8(x),TIMES8(x)
-#define TIMES32(x) TIMES16(x),TIMES16(x)
-#define TIMES64(x) TIMES32(x),TIMES32(x)
-#define TIMES255(x) TIMES64(x),TIMES64(x),TIMES64(x),TIMES32(x),TIMES16(x),TIMES8(x),TIMES4(x),x,x,x
+#define TIMES4(x) x, x, x, x
+#define TIMES8(x) TIMES4(x), TIMES4(x)
+#define TIMES16(x) TIMES8(x), TIMES8(x)
+#define TIMES32(x) TIMES16(x), TIMES16(x)
+#define TIMES64(x) TIMES32(x), TIMES32(x)
+#define TIMES255(x) \
+ TIMES64(x), TIMES64(x), TIMES64(x), TIMES32(x), TIMES16(x), TIMES8(x), \
+ TIMES4(x), x, x, x
 const u8 simplify_lookup[256] = {
 
  [0] = 1, [1] = TIMES255(128)
@@ -172,6 +174,7 @@ const u8 count_class_lookup8[256] = {
  [128] = TIMES64(128)
 
 };
+
 #undef TIMES255
 #undef TIMES64
 #undef TIMES32

src/afl-fuzz-run.c

@@ -29,7 +29,7 @@
 #include <signal.h>
 #include <limits.h>
 #if !defined NAME_MAX
-#define NAME_MAX _XOPEN_NAME_MAX
+ #define NAME_MAX _XOPEN_NAME_MAX
 #endif
 
 #include "cmplog.h"

src/afl-showmap.c

@@ -98,17 +98,23 @@ static sharedmem_t * shm_fuzz;
 /* Classify tuple counts. Instead of mapping to individual bits, as in
  afl-fuzz.c, we map to more user-friendly numbers between 1 and 8. */
 
-#define TIMES4(x) x,x,x,x
-#define TIMES8(x) TIMES4(x),TIMES4(x)
-#define TIMES16(x) TIMES8(x),TIMES8(x)
-#define TIMES32(x) TIMES16(x),TIMES16(x)
-#define TIMES64(x) TIMES32(x),TIMES32(x)
-#define TIMES96(x) TIMES64(x),TIMES32(x)
-#define TIMES128(x) TIMES64(x),TIMES64(x)
+#define TIMES4(x) x, x, x, x
+#define TIMES8(x) TIMES4(x), TIMES4(x)
+#define TIMES16(x) TIMES8(x), TIMES8(x)
+#define TIMES32(x) TIMES16(x), TIMES16(x)
+#define TIMES64(x) TIMES32(x), TIMES32(x)
+#define TIMES96(x) TIMES64(x), TIMES32(x)
+#define TIMES128(x) TIMES64(x), TIMES64(x)
 static const u8 count_class_human[256] = {
 
- [0] = 0, [1] = 1, [2] = 2, [3] = 3,
- [4] = TIMES4(4), [8] = TIMES8(5),[16] = TIMES16(6),[32] = TIMES96(7),
+ [0] = 0,
+ [1] = 1,
+ [2] = 2,
+ [3] = 3,
+ [4] = TIMES4(4),
+ [8] = TIMES8(5),
+ [16] = TIMES16(6),
+ [32] = TIMES96(7),
  [128] = TIMES128(8)
 
 };
@@ -126,6 +132,7 @@ static const u8 count_class_binary[256] = {
  [128] = TIMES64(128)
 
 };
+
 #undef TIMES128
 #undef TIMES96
 #undef TIMES64

src/afl-tmin.c

@@ -98,11 +98,11 @@ static sharedmem_t * shm_fuzz;
 /* Classify tuple counts. This is a slow & naive version, but good enough here.
  */
 
-#define TIMES4(x) x,x,x,x
-#define TIMES8(x) TIMES4(x),TIMES4(x)
-#define TIMES16(x) TIMES8(x),TIMES8(x)
-#define TIMES32(x) TIMES16(x),TIMES16(x)
-#define TIMES64(x) TIMES32(x),TIMES32(x)
+#define TIMES4(x) x, x, x, x
+#define TIMES8(x) TIMES4(x), TIMES4(x)
+#define TIMES16(x) TIMES8(x), TIMES8(x)
+#define TIMES32(x) TIMES16(x), TIMES16(x)
+#define TIMES64(x) TIMES32(x), TIMES32(x)
 static const u8 count_class_lookup[256] = {
 
  [0] = 0,
@@ -116,6 +116,7 @@ static const u8 count_class_lookup[256] = {
  [128] = TIMES64(128)
 
 };
+
 #undef TIMES64
 #undef TIMES32
 #undef TIMES16

unicorn_mode/UNICORNAFL_VERSION

@@ -1 +1 @@
-8cca4801
+768e6bb2