From ab6b4468fcd6cd634a053bbfa747d8582907d0c0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Fri, 9 Feb 2024 16:56:12 -0800 Subject: [PATCH 1/6] Fuzzer: Use a directory for important fuzz testcases --- scripts/fuzz_opt.py | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py index 251b23d033a..dc4f92155d4 100755 --- a/scripts/fuzz_opt.py +++ b/scripts/fuzz_opt.py @@ -200,24 +200,12 @@ def randomize_fuzz_settings(): def init_important_initial_contents(): - FIXED_IMPORTANT_INITIAL_CONTENTS = [ - # Perenially-important passes - os.path.join('lit', 'passes', 'optimize-instructions-mvp.wast'), - os.path.join('passes', 'optimize-instructions_fuzz-exec.wast'), - ] - MANUAL_RECENT_INITIAL_CONTENTS = [ - # Recently-added or modified passes. These can be added to and pruned - # frequently. - os.path.join('lit', 'passes', 'once-reduction.wast'), - os.path.join('passes', 'remove-unused-brs_enable-multivalue.wast'), - os.path.join('lit', 'passes', 'optimize-instructions-bulk-memory.wast'), - os.path.join('lit', 'passes', 'optimize-instructions-ignore-traps.wast'), - os.path.join('lit', 'passes', 'optimize-instructions-gc.wast'), - os.path.join('lit', 'passes', 'optimize-instructions-gc-iit.wast'), - os.path.join('lit', 'passes', 'optimize-instructions-call_ref.wast'), - os.path.join('lit', 'passes', 'inlining_splitting.wast'), - os.path.join('heap-types.wast'), - ] + # Always grab important fuzz testcases from the designated directory for + # that. We consider them fixed content to always use. + FIXED_IMPORTANT_INITIAL_CONTENTS = fuzz_cases + + # If auto_initial_contents is set we'll also grab all test files that are + # recent. RECENT_DAYS = 30 # Returns the list of test wast/wat files added or modified within the @@ -268,9 +256,6 @@ def is_git_repo(): if shared.options.auto_initial_contents: print(f'(automatically selected: within last {RECENT_DAYS} days):') recent_contents += auto_select_recent_initial_contents() - else: - print('(manually selected):') - recent_contents = MANUAL_RECENT_INITIAL_CONTENTS for test in recent_contents: print(' ' + test) print() @@ -335,7 +320,7 @@ def pick_initial_contents(): return # some of the time use initial contents that are known to be especially # important - if random.random() < 0.5: + if IMPORTANT_INITIAL_CONTENTS and random.random() < 0.5: test_name = random.choice(IMPORTANT_INITIAL_CONTENTS) else: test_name = random.choice(all_tests) @@ -1393,6 +1378,7 @@ def handle(self, wasm): test_suffixes = ['*.wasm', '*.wast', '*.wat'] + core_tests = shared.get_tests(shared.get_test_dir('.'), test_suffixes) passes_tests = shared.get_tests(shared.get_test_dir('passes'), test_suffixes) spec_tests = shared.get_tests(shared.get_test_dir('spec'), test_suffixes) @@ -1402,6 +1388,8 @@ def handle(self, wasm): lit_tests = shared.get_tests(shared.get_test_dir('lit'), test_suffixes, recursive=True) all_tests = core_tests + passes_tests + spec_tests + wasm2js_tests + lld_tests + unit_tests + lit_tests +fuzz_cases = shared.get_tests(shared.get_test_dir('fuzz'), test_suffixes, recursive=True) + # Do one test, given an input file for -ttf and some optimizations to run def test_one(random_input, given_wasm): From 92a00e7d3b4124c1a232dc85def1e674635ed305 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 12 Feb 2024 12:39:23 -0800 Subject: [PATCH 2/6] work --- scripts/fuzz_opt.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py index dc4f92155d4..613b808c683 100755 --- a/scripts/fuzz_opt.py +++ b/scripts/fuzz_opt.py @@ -200,9 +200,15 @@ def randomize_fuzz_settings(): def init_important_initial_contents(): - # Always grab important fuzz testcases from the designated directory for - # that. We consider them fixed content to always use. - FIXED_IMPORTANT_INITIAL_CONTENTS = fuzz_cases + # Always grab important fuzz testcases from the designated local directory + # for that, 'fuzz', if it exists. We consider them fixed content to always + # use. This is, you can easily add fuzz testcases to be handled with high + # importance by creating a directory ./fuzz (parallel to ./test etc.) and + # putting wasm files in it. + fuzz_dir = os.path.join(options.binaryen_root, 'fuzz') + if os.path.exists(fuzz_dir): + fuzz_cases = shared.get_tests(fuzz_dir, test_suffixes, recursive=True) + FIXED_IMPORTANT_INITIAL_CONTENTS = fuzz_cases # If auto_initial_contents is set we'll also grab all test files that are # recent. @@ -1388,8 +1394,6 @@ def handle(self, wasm): lit_tests = shared.get_tests(shared.get_test_dir('lit'), test_suffixes, recursive=True) all_tests = core_tests + passes_tests + spec_tests + wasm2js_tests + lld_tests + unit_tests + lit_tests -fuzz_cases = shared.get_tests(shared.get_test_dir('fuzz'), test_suffixes, recursive=True) - # Do one test, given an input file for -ttf and some optimizations to run def test_one(random_input, given_wasm): From b58cbc4037d4f62ed8edacaa737e1c86457bf483 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 12 Feb 2024 12:41:33 -0800 Subject: [PATCH 3/6] work --- scripts/fuzz_opt.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py index 613b808c683..cba88a25ce8 100755 --- a/scripts/fuzz_opt.py +++ b/scripts/fuzz_opt.py @@ -200,12 +200,14 @@ def randomize_fuzz_settings(): def init_important_initial_contents(): + FIXED_IMPORTANT_INITIAL_CONTENTS = [] + # Always grab important fuzz testcases from the designated local directory # for that, 'fuzz', if it exists. We consider them fixed content to always # use. This is, you can easily add fuzz testcases to be handled with high # importance by creating a directory ./fuzz (parallel to ./test etc.) and # putting wasm files in it. - fuzz_dir = os.path.join(options.binaryen_root, 'fuzz') + fuzz_dir = os.path.join(shared.options.binaryen_root, 'fuzz') if os.path.exists(fuzz_dir): fuzz_cases = shared.get_tests(fuzz_dir, test_suffixes, recursive=True) FIXED_IMPORTANT_INITIAL_CONTENTS = fuzz_cases @@ -252,7 +254,7 @@ def is_git_repo(): 'so not automatically selecting initial contents.') shared.options.auto_initial_contents = False - print('- Perenially-important initial contents:') + print('- Important provided initial contents:') for test in FIXED_IMPORTANT_INITIAL_CONTENTS: print(' ' + test) print() From 2a7da34399589f4b2ca9ad5a3744342a758f31d3 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 12 Feb 2024 12:43:45 -0800 Subject: [PATCH 4/6] work --- scripts/fuzz_opt.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py index cba88a25ce8..4956a2d48ec 100755 --- a/scripts/fuzz_opt.py +++ b/scripts/fuzz_opt.py @@ -10,6 +10,10 @@ That will run forever or until it finds a problem. +If you create a local directory 'fuzz' (under the binaryen root directory, +that is, parallel to 'test' etc.) and put wasm files in it then they are +treated as important content to fuzz with high frequency. + Setup: Some tools are optional, like emcc and wasm2c. The v8 shell (d8), however, is used in various sub-fuzzers and so it is mandatory. @@ -202,11 +206,7 @@ def randomize_fuzz_settings(): def init_important_initial_contents(): FIXED_IMPORTANT_INITIAL_CONTENTS = [] - # Always grab important fuzz testcases from the designated local directory - # for that, 'fuzz', if it exists. We consider them fixed content to always - # use. This is, you can easily add fuzz testcases to be handled with high - # importance by creating a directory ./fuzz (parallel to ./test etc.) and - # putting wasm files in it. + # If the fuzz dir exists, its contents are always important to us. fuzz_dir = os.path.join(shared.options.binaryen_root, 'fuzz') if os.path.exists(fuzz_dir): fuzz_cases = shared.get_tests(fuzz_dir, test_suffixes, recursive=True) From f2e33ec7c6fb4fa3b0169411cfda376e07024274 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 12 Feb 2024 13:23:51 -0800 Subject: [PATCH 5/6] add the dir --- scripts/fuzz_opt.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py index 4956a2d48ec..b514c5e7e7c 100755 --- a/scripts/fuzz_opt.py +++ b/scripts/fuzz_opt.py @@ -10,9 +10,9 @@ That will run forever or until it finds a problem. -If you create a local directory 'fuzz' (under the binaryen root directory, -that is, parallel to 'test' etc.) and put wasm files in it then they are -treated as important content to fuzz with high frequency. +You can put files in the local directory 'fuzz' (under the top level of the +binaryen repo) and the fuzzer will treat them as important content to fuzz +with high frequency. Setup: Some tools are optional, like emcc and wasm2c. The v8 shell (d8), however, is used in various sub-fuzzers and so it is mandatory. @@ -204,13 +204,10 @@ def randomize_fuzz_settings(): def init_important_initial_contents(): - FIXED_IMPORTANT_INITIAL_CONTENTS = [] - - # If the fuzz dir exists, its contents are always important to us. + # Fuzz dir contents are always important to us. fuzz_dir = os.path.join(shared.options.binaryen_root, 'fuzz') - if os.path.exists(fuzz_dir): - fuzz_cases = shared.get_tests(fuzz_dir, test_suffixes, recursive=True) - FIXED_IMPORTANT_INITIAL_CONTENTS = fuzz_cases + fuzz_cases = shared.get_tests(fuzz_dir, test_suffixes, recursive=True) + FIXED_IMPORTANT_INITIAL_CONTENTS = fuzz_cases # If auto_initial_contents is set we'll also grab all test files that are # recent. From 1693597c2fc1f6f9f51bcb77daf85d3f8b479805 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 12 Feb 2024 13:24:25 -0800 Subject: [PATCH 6/6] file --- fuzz/readme.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 fuzz/readme.txt diff --git a/fuzz/readme.txt b/fuzz/readme.txt new file mode 100644 index 00000000000..54df3bd12ad --- /dev/null +++ b/fuzz/readme.txt @@ -0,0 +1,3 @@ +The wasm contents of this directory (*.wasm, *.wast, *.wat files) are treated as +important contents by the fuzzer, which will test them with high frequency. This +is useful when you have some local files you want the fuzzer to focus on.