Add ClusterFuzz merge scripts Adds and uses merge scripts for generating/uploading ClusterFuzz corpora from the CI ClusterFuzz builder. Bug: 441327468, 385317083 Change-Id: I204e4754dd6bae83e42db49980aa3f464064135e Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/265716 Commit-Queue: Brian Sheedy <bsheedy@google.com> Reviewed-by: Antonio Maiorano <amaiorano@google.com> Reviewed-by: Ryan Harrison <rharrison@chromium.org> Reviewed-by: dan sinclair <dsinclair@chromium.org>
diff --git a/infra/specs/ci.json b/infra/specs/ci.json index 4c8dd29..4f0a5b1 100644 --- a/infra/specs/ci.json +++ b/infra/specs/ci.json
@@ -127,10 +127,18 @@ "args": [ "--adapter-vendor-id=0x1AE0", "--use-wire", - "--wire-trace-dir=${ISOLATED_OUTDIR}" + "--wire-trace-dir=${ISOLATED_OUTDIR}/clusterfuzz" ], "merge": { - "script": "//scripts/merge_scripts/true_noop_merge.py" + "args": [ + "--fuzzer-name", + "dawn_wire_server_and_frontend_fuzzer", + "--fuzzer-name", + "dawn_wire_server_and_vulkan_backend_fuzzer", + "--fuzzer-name", + "dawn_wire_server_and_d3d12_backend_fuzzer" + ], + "script": "//scripts/merge_scripts/generate_wire_trace_fuzz_corpora.py" }, "name": "dawn_wire_trace_end2end_sws_tests", "resultdb": { @@ -152,10 +160,18 @@ { "args": [ "--use-wire", - "--wire-trace-dir=${ISOLATED_OUTDIR}" + "--wire-trace-dir=${ISOLATED_OUTDIR}/clusterfuzz" ], "merge": { - "script": "//scripts/merge_scripts/true_noop_merge.py" + "args": [ + "--fuzzer-name", + "dawn_wire_server_and_frontend_fuzzer", + "--fuzzer-name", + "dawn_wire_server_and_vulkan_backend_fuzzer", + "--fuzzer-name", + "dawn_wire_server_and_d3d12_backend_fuzzer" + ], + "script": "//scripts/merge_scripts/generate_wire_trace_fuzz_corpora.py" }, "name": "dawn_wire_trace_unittests", "resultdb": { @@ -180,12 +196,16 @@ "args": [ "-generate", "-out", - "${ISOLATED_OUTDIR}", + "${ISOLATED_OUTDIR}/clusterfuzz", "-ir", "--append-cwd-as-build" ], "merge": { - "script": "//scripts/merge_scripts/true_noop_merge.py" + "args": [ + "--fuzzer-name", + "tint_ir_fuzzer" + ], + "script": "//scripts/merge_scripts/generate_tint_fuzz_corpora.py" }, "name": "tint_ir_fuzzer_corpus_generate_tests", "resultdb": { @@ -208,11 +228,15 @@ "args": [ "-generate", "-out", - "${ISOLATED_OUTDIR}", + "${ISOLATED_OUTDIR}/clusterfuzz", "--append-cwd-as-build" ], "merge": { - "script": "//scripts/merge_scripts/true_noop_merge.py" + "args": [ + "--fuzzer-name", + "tint_wgsl_fuzzer" + ], + "script": "//scripts/merge_scripts/generate_tint_fuzz_corpora.py" }, "name": "tint_wgsl_fuzzer_corpus_generate_tests", "resultdb": {
diff --git a/infra/specs/generate_test_spec_json.py b/infra/specs/generate_test_spec_json.py index 2c8fa04..d448259 100755 --- a/infra/specs/generate_test_spec_json.py +++ b/infra/specs/generate_test_spec_json.py
@@ -66,11 +66,43 @@ 'result_format': 'single', }, }, + 'tint_ir_merge': { + 'merge': { + 'script': '//scripts/merge_scripts/generate_tint_fuzz_corpora.py', + 'args': [ + '--fuzzer-name', + 'tint_ir_fuzzer', + ], + }, + }, + 'tint_wgsl_merge': { + 'merge': { + 'script': '//scripts/merge_scripts/generate_tint_fuzz_corpora.py', + 'args': [ + '--fuzzer-name', + 'tint_wgsl_fuzzer', + ], + }, + }, 'true_noop_merge': { 'merge': { 'script': '//scripts/merge_scripts/true_noop_merge.py', }, }, + 'wire_trace_merge': { + 'merge': { + 'script': + '//scripts/merge_scripts/generate_wire_trace_fuzz_corpora.py', + 'args': [ + '--fuzzer-name', + 'dawn_wire_server_and_frontend_fuzzer', + '--fuzzer-name', + 'dawn_wire_server_and_vulkan_backend_fuzzer', + '--fuzzer-name', + 'dawn_wire_server_and_d3d12_backend_fuzzer', + ], + }, + }, } MIXIN_FILEPATH = os.path.join(THIS_DIR, 'mixins.pyl')
diff --git a/infra/specs/mixins.pyl b/infra/specs/mixins.pyl index df998a4..44f177f 100644 --- a/infra/specs/mixins.pyl +++ b/infra/specs/mixins.pyl
@@ -62,7 +62,18 @@ 'result_adapter_single': {'resultdb': {'result_format': 'single'}}, 'swarming_containment_auto': { 'fail_if_unused': False, 'swarming': {'containment_type': 'AUTO'}}, + 'tint_ir_merge': { 'merge': { 'args': ['--fuzzer-name', 'tint_ir_fuzzer'], + 'script': '//scripts/merge_scripts/generate_tint_fuzz_corpora.py'}}, + 'tint_wgsl_merge': { 'merge': { 'args': ['--fuzzer-name', 'tint_wgsl_fuzzer'], + 'script': '//scripts/merge_scripts/generate_tint_fuzz_corpora.py'}}, 'true_noop_merge': { 'merge': { 'script': '//scripts/merge_scripts/true_noop_merge.py'}}, 'win10': {'swarming': {'dimensions': {'os': 'Windows-10-19045'}}}, + 'wire_trace_merge': { 'merge': { 'args': [ '--fuzzer-name', + 'dawn_wire_server_and_frontend_fuzzer', + '--fuzzer-name', + 'dawn_wire_server_and_vulkan_backend_fuzzer', + '--fuzzer-name', + 'dawn_wire_server_and_d3d12_backend_fuzzer'], + 'script': '//scripts/merge_scripts/generate_wire_trace_fuzz_corpora.py'}}, 'x86-64': { 'fail_if_unused': False, 'swarming': {'dimensions': {'cpu': 'x86-64'}}}}
diff --git a/infra/specs/test_suites.pyl b/infra/specs/test_suites.pyl index c752d3d..e2c0de2 100644 --- a/infra/specs/test_suites.pyl +++ b/infra/specs/test_suites.pyl
@@ -22,11 +22,11 @@ 'args': [ '--adapter-vendor-id=0x1AE0', '--use-wire', - '--wire-trace-dir=${ISOLATED_OUTDIR}', + '--wire-trace-dir=${ISOLATED_OUTDIR}/clusterfuzz', ], 'mixins': [ 'result_adapter_gtest_json', - 'true_noop_merge', + 'wire_trace_merge', ], 'test': 'dawn_end2end_tests', }, @@ -106,11 +106,11 @@ 'dawn_wire_trace_unittests': { 'args': [ '--use-wire', - '--wire-trace-dir=${ISOLATED_OUTDIR}', + '--wire-trace-dir=${ISOLATED_OUTDIR}/clusterfuzz', ], 'mixins': [ 'result_adapter_gtest_json', - 'true_noop_merge', + 'wire_trace_merge', ], 'test': 'dawn_unittests', }, @@ -157,12 +157,12 @@ 'args': [ '-generate', '-out', - '${ISOLATED_OUTDIR}', + '${ISOLATED_OUTDIR}/clusterfuzz', '--append-cwd-as-build', ], 'mixins': [ 'result_adapter_single', - 'true_noop_merge', + 'tint_wgsl_merge', ], 'test': 'fuzzer_corpus_tests', }, @@ -170,13 +170,13 @@ 'args': [ '-generate', '-out', - '${ISOLATED_OUTDIR}', + '${ISOLATED_OUTDIR}/clusterfuzz', '-ir', '--append-cwd-as-build', ], 'mixins': [ 'result_adapter_single', - 'true_noop_merge', + 'tint_ir_merge', ], 'test': 'fuzzer_corpus_tests', },
diff --git a/scripts/merge_scripts/fuzz_corpora_common.py b/scripts/merge_scripts/fuzz_corpora_common.py new file mode 100644 index 0000000..dc7381e --- /dev/null +++ b/scripts/merge_scripts/fuzz_corpora_common.py
@@ -0,0 +1,128 @@ +# Copyright 2025 The Dawn & Tint Authors +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Common code for ClusterFuzz corpora generation/uploading.""" + +import argparse +import hashlib +import os +import shutil +import subprocess +import sys + +DAWN_ROOT = os.path.realpath( + os.path.join(os.path.dirname(__file__), '..', '..')) + +BUCKET = 'clusterfuzz-corpus' +BUCKET_DIRECTORY = 'libfuzzer' + + +def upload_directory_to_gcs(local_directory: str, fuzzer_name: str, + clobber: bool) -> None: + """Uploads the contents of a directory to the ClusterFuzz GCS bucket. + + Args: + local_directory: The path to the local directory whose contents will + be uploaded. + fuzzer_name: The ClusterFuzz fuzzer name to upload the files under. + clobber: Whether to clobber identically named files in the GCS bucket. + """ + gsutil_path = os.path.join(DAWN_ROOT, 'third_party', 'depot_tools', + 'gsutil.py') + if not os.path.exists(gsutil_path): + raise RuntimeError(f'Unable to find gsutil.py at {gsutil_path}') + + cmd = [ + sys.executable, + '-u', # Unbuffered output. + gsutil_path, + '-m', # Multithreaded. + '-o', # Parallel upload. + 'GSUtil:parallel_composite_upload_threshold=50M', + 'cp', + '-r', # Recursive. + ] + if not clobber: + cmd.append('-n') + cmd.extend([ + os.path.join(local_directory, '*'), + f'gs://{BUCKET}/{BUCKET_DIRECTORY}/{fuzzer_name}', + ]) + p = subprocess.run(cmd, check=True) + + +def hash_trace_files(trace_files: list[str], output_directory: str) -> None: + """Creates copies of trace files with hash-based names. + + Args: + trace_files: A list of filepaths to trace files to process. + output_directory: A filepath to a directory that the copies will be + placed in. + """ + for tf in trace_files: + with open(tf, 'rb') as infile: + digest = hashlib.md5(infile.read()).hexdigest() + filename = os.path.join(output_directory, f'trace_{digest}') + shutil.copyfile(tf, filename) + + +def find_raw_trace_files(output_jsons: list[str], + subdirectory: str) -> list[str]: + """Finds all raw trace files produced by the test. + + Args: + output_jsons: A list of filepaths to the output.json files produced + by each shard. + subdirectory: The subdirectory of the isolated output directory that + is expected to contain the raw trace files. + + Returns: + A list of filepaths, one for each found trace file. + """ + trace_files = [] + for json_file in output_jsons: + isolated_outdir = os.path.dirname(json_file) + dirname = os.path.join(isolated_outdir, subdirectory) + for f in os.listdir(dirname): + trace_files.append(os.path.join(dirname, f)) + if not trace_files: + raise RuntimeError('Did not find any wire trace files') + return trace_files + + +def add_common_arguments(parser: argparse.ArgumentParser) -> None: + """Adds common fuzz corpora-related arguments to a parser. + + Args: + parser: The ArgumentParser to add arguments to. + """ + parser.add_argument( + '--fuzzer-name', + required=True, + dest='fuzzer_names', + action='append', + help=('A fuzzer name to upload files to. Can be specified multiple ' + 'times'))
diff --git a/scripts/merge_scripts/generate_fuzz_corpora.py b/scripts/merge_scripts/generate_fuzz_corpora.py deleted file mode 100644 index 99543ec..0000000 --- a/scripts/merge_scripts/generate_fuzz_corpora.py +++ /dev/null
@@ -1,159 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright 2025 The Dawn & Tint Authors -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Merge script to generate/upload fuzz corpora for use in ClusterFuzz. - -Note that this "merge" script does not actually merge any data - all Dawn -results are handled by ResultDB. -""" - -import argparse -import hashlib -import os -import shutil -import subprocess -import sys -import tempfile -from typing import List - -DAWN_ROOT = os.path.realpath( - os.path.join(os.path.dirname(__file__), '..', '..')) -TESTING_MERGE_SCRIPTS = os.path.join(DAWN_ROOT, 'testing', 'merge_scripts') -sys.path.insert(0, TESTING_MERGE_SCRIPTS) - -try: - import merge_api -except ImportError as e: - raise RuntimeError( - 'Unable to import merge_api - are you running in a Chromium checkout? ' - 'This merge script only supports standalone Dawn checkouts.') from e - -TRACE_SUBDIR = 'wire_traces' -FUZZER_NAMES = ( - 'dawn_wire_server_and_frontend_fuzzer', - 'dawn_wire_server_and_vulkan_backend_fuzzer', - 'dawn_wire_server_and_d3d12_backend_fuzzer', -) -BUCKET = 'clusterfuzz-corpus' -BUCKET_DIRECTORY = 'libfuzzer' - - -def upload_files(input_directory: str) -> None: - """Uploads all files in a directory to the ClusterFuzz bucket. - - One upload will be performed for each fuzzer in |FUZZER_NAMES|. - - Args: - input_directory: A filepath to a directory whose contents will be - uploaded. - """ - gsutil_path = os.path.join(DAWN_ROOT, 'third_party', 'depot_tools', - 'gsutil.py') - if not os.path.exists(gsutil_path): - raise RuntimeError(f'Unable to find gsutil.py at {gsutil_path}') - - for fn in FUZZER_NAMES: - # This is effectively the same command line that would be run by the - # older dawn/gn.py recipe for uploading the corpora when using the - # gsutil recipe module. - cmd = [ - sys.executable, - '-u', # Unbuffered output. - gsutil_path, - '-m', # Multithreaded. - '-o', # Parallel upload. - 'GSUtil:parallel_composite_upload_threshold=50M', - 'cp', - '-r', # Recursive. - '-n', # No clobber. - os.path.join(input_directory, '*'), - f'gs://{BUCKET}/{BUCKET_DIRECTORY}/{fn}', - ] - p = subprocess.run(cmd) - p.check_returncode() - - -def hash_trace_files(trace_files: List[str], output_directory: str) -> None: - """Creates copies of trace files with hash-based names. - - Args: - trace_files: A list of filepaths to trace files to process. - output_directory: A filepath to a directory that the copies will be - placed in. - """ - for tf in trace_files: - with open(tf, 'rb') as infile: - digest = hashlib.md5(infile.read()).hexdigest() - filename = os.path.join(output_directory, f'trace_{digest}') - shutil.copyfile(tf, filename) - - -def find_trace_files(output_jsons: List[str]) -> List[str]: - """Finds all wire trace files produced by the test. - - Args: - output_jsons: A list of filepaths to the output.json files produced - by each shard. - - Returns: - A list of filepaths, one for each found trace file. - """ - trace_files = [] - for json_file in output_jsons: - isolated_outdir = os.path.dirname(json_file) - dirname = os.path.join(isolated_outdir, TRACE_SUBDIR) - for f in os.listdir(dirname): - trace_files.append(os.path.join(dirname, f)) - if not trace_files: - raise RuntimeError('Did not find any wire trace files') - return trace_files - - -def generate_and_upload_fuzz_corpora(output_jsons: List[str]) -> None: - """Generates and uploads fuzz corpora to ClusterFuzz. - - One corpus will be generated for each fuzzer in |FUZZER_NAMES|. - - Args: - output_jsons: A list of filepaths to the output.json files produced - by each shard. - """ - trace_files = find_trace_files(output_jsons) - with tempfile.TemporaryDirectory() as tempdir: - hash_trace_files(trace_files, tempdir) - upload_files(tempdir) - - -def main(cmdline_args: List[str]) -> None: - parser = merge_api.ArgumentParser() - args = parser.parse_args(cmdline_args) - generate_and_upload_fuzz_corpora(args.jsons_to_merge) - - -if __name__ == '__main__': - main(sys.argv[1:])
diff --git a/scripts/merge_scripts/generate_tint_fuzz_corpora.py b/scripts/merge_scripts/generate_tint_fuzz_corpora.py new file mode 100755 index 0000000..106132c --- /dev/null +++ b/scripts/merge_scripts/generate_tint_fuzz_corpora.py
@@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# +# Copyright 2025 The Dawn & Tint Authors +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Merge script to upload Tint fuzz corpora for use in ClusterFuzz. + +Note that this "merge" script does not actually merge any data - all Dawn +results are handled by ResultDB. +""" + +import os +import tempfile +import shutil +import sys + +ISOLATED_OUT_SUBDIR = 'clusterfuzz' +DAWN_ROOT = os.path.realpath( + os.path.join(os.path.dirname(__file__), '..', '..')) +sys.path.insert(0, DAWN_ROOT) + +from scripts.merge_scripts import fuzz_corpora_common + +try: + from testing.merge_scripts import merge_api +except ImportError as e: + raise RuntimeError( + 'Unable to import merge_api - are you running in a Chromium checkout? ' + 'This merge script only supports standalone Dawn checkouts.') from e + + +def main() -> None: + parser = merge_api.ArgumentParser() + fuzz_corpora_common.add_common_arguments(parser) + args = parser.parse_args() + # Tint tests upload their files directly with clobbering. This is in + # contrast to the wire trace tests, which use hash-based file names and do + # not clobber. + trace_files = fuzz_corpora_common.find_raw_trace_files( + args.jsons_to_merge, ISOLATED_OUT_SUBDIR) + with tempfile.TemporaryDirectory() as tempdir: + for tf in trace_files: + shutil.copy(tf, tempdir) + for fn in args.fuzzer_names: + fuzz_corpora_common.upload_directory_to_gcs(tempdir, + fn, + clobber=False) + + +if __name__ == '__main__': + main()
diff --git a/scripts/merge_scripts/generate_wire_trace_fuzz_corpora.py b/scripts/merge_scripts/generate_wire_trace_fuzz_corpora.py new file mode 100755 index 0000000..89d59e2 --- /dev/null +++ b/scripts/merge_scripts/generate_wire_trace_fuzz_corpora.py
@@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# +# Copyright 2025 The Dawn & Tint Authors +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Merge script to upload wire trace fuzz corpora for use in ClusterFuzz. + +Note that this "merge" script does not actually merge any data - all Dawn +results are handled by ResultDB. +""" + +import os +import tempfile +import sys + +ISOLATED_OUT_SUBDIR = 'clusterfuzz' +DAWN_ROOT = os.path.realpath( + os.path.join(os.path.dirname(__file__), '..', '..')) +sys.path.insert(0, DAWN_ROOT) + +from scripts.merge_scripts import fuzz_corpora_common + +try: + from testing.merge_scripts import merge_api +except ImportError as e: + raise RuntimeError( + 'Unable to import merge_api - are you running in a Chromium checkout? ' + 'This merge script only supports standalone Dawn checkouts.') from e + + +def main() -> None: + parser = merge_api.ArgumentParser() + fuzz_corpora_common.add_common_arguments(parser) + args = parser.parse_args() + # Wire trace tests uniquely name their files based on hashes and upload + # Without clobbering. This is in contrast with Tint tests, which upload + # as-is without clobbering. + trace_files = fuzz_corpora_common.find_raw_trace_files( + args.jsons_to_merge, ISOLATED_OUT_SUBDIR) + with tempfile.TemporaryDirectory() as tempdir: + fuzz_corpora_common.hash_trace_files(trace_files, tempdir) + for fn in args.fuzzer_names: + fuzz_corpora_common.upload_directory_to_gcs(tempdir, + fn, + clobber=False) + + +if __name__ == '__main__': + main()
diff --git a/tools/src/cmd/fuzz/main.go b/tools/src/cmd/fuzz/main.go index 6e2e2ef..3c4bd8c 100644 --- a/tools/src/cmd/fuzz/main.go +++ b/tools/src/cmd/fuzz/main.go
@@ -180,6 +180,11 @@ } else { return err } + } else { + err := c.osWrapper.MkdirAll(c.out, os.ModePerm) + if err != nil { + return err + } } if !fileutils.IsDir(c.out, c.osWrapper) {