Support dawn.node on Swarming
Updates run_dawn_node_cts.py to have the underlying runner output
results to a temporary file, and if --isolated-script-test-output is
present, convert those results into a format that result_adapter can
ingest.
Ultimately, this means that the dawn_node_cts suite can now be run
on Swarming. The no_swarming mixin is removed as a result.
The format that the results are converted into is basically only
suitable for the current use of dawn_node_cts (ensuring that running via
Node is not completely broken). If testing via Node is ever expanded on
the bots, the conversion will need to be improved to a format that
supports multiple tests or native ResultDB integration will be needed.
In either case, the underlying runner's output format will also need to
be improved since it currently only gives a pass/fail per test without
any additional information.
Bug: 441328362
Change-Id: I131b276448059f63b00a578b81467e26d1c0f4c4
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/264115
Auto-Submit: Brian Sheedy <bsheedy@google.com>
Reviewed-by: Yuly Novikov <ynovikov@chromium.org>
Commit-Queue: Brian Sheedy <bsheedy@google.com>
diff --git a/infra/specs/ci.json b/infra/specs/ci.json
index 18175cc..3824b89 100644
--- a/infra/specs/ci.json
+++ b/infra/specs/ci.json
@@ -189,7 +189,23 @@
"SwiftShader",
"webgpu:api,operation,adapter,requestDevice:default:*"
],
+ "merge": {
+ "script": "//testing/merge_scripts/noop_merge.py"
+ },
"name": "dawn_node_sws_cts",
+ "resultdb": {
+ "result_format": "single"
+ },
+ "swarming": {
+ "containment_type": "AUTO",
+ "dimensions": {
+ "cpu": "x86-64",
+ "gpu": "none",
+ "os": "Ubuntu-22.04",
+ "pool": "chromium.tests.gpu"
+ },
+ "service_account": "chromium-tester@chops-service-accounts.iam.gserviceaccount.com"
+ },
"test": "dawn_node_cts",
"test_id_prefix": "ninja://scripts/dawn_node_cts:dawn_node_cts/"
}
@@ -323,7 +339,23 @@
"SwiftShader",
"webgpu:api,operation,adapter,requestDevice:default:*"
],
+ "merge": {
+ "script": "//testing/merge_scripts/noop_merge.py"
+ },
"name": "dawn_node_sws_cts",
+ "resultdb": {
+ "result_format": "single"
+ },
+ "swarming": {
+ "containment_type": "AUTO",
+ "dimensions": {
+ "cpu": "x86-64",
+ "gpu": "none",
+ "os": "Ubuntu-22.04",
+ "pool": "chromium.tests.gpu"
+ },
+ "service_account": "chromium-tester@chops-service-accounts.iam.gserviceaccount.com"
+ },
"test": "dawn_node_cts",
"test_id_prefix": "ninja://scripts/dawn_node_cts:dawn_node_cts/"
}
@@ -767,7 +799,23 @@
"SwiftShader",
"webgpu:api,operation,adapter,requestDevice:default:*"
],
+ "merge": {
+ "script": "//testing/merge_scripts/noop_merge.py"
+ },
"name": "dawn_node_sws_cts",
+ "resultdb": {
+ "result_format": "single"
+ },
+ "swarming": {
+ "containment_type": "AUTO",
+ "dimensions": {
+ "cpu": "x86-64",
+ "display_attached": "1",
+ "gpu": "8086:3e9b",
+ "os": "Mac-14.5"
+ },
+ "service_account": "chromium-tester@chops-service-accounts.iam.gserviceaccount.com"
+ },
"test": "dawn_node_cts",
"test_id_prefix": "ninja://scripts/dawn_node_cts:dawn_node_cts/"
}
@@ -901,7 +949,23 @@
"SwiftShader",
"webgpu:api,operation,adapter,requestDevice:default:*"
],
+ "merge": {
+ "script": "//testing/merge_scripts/noop_merge.py"
+ },
"name": "dawn_node_sws_cts",
+ "resultdb": {
+ "result_format": "single"
+ },
+ "swarming": {
+ "containment_type": "AUTO",
+ "dimensions": {
+ "cpu": "x86-64",
+ "display_attached": "1",
+ "gpu": "8086:3e9b",
+ "os": "Mac-14.5"
+ },
+ "service_account": "chromium-tester@chops-service-accounts.iam.gserviceaccount.com"
+ },
"test": "dawn_node_cts",
"test_id_prefix": "ninja://scripts/dawn_node_cts:dawn_node_cts/"
}
@@ -1050,7 +1114,23 @@
"SwiftShader",
"webgpu:api,operation,adapter,requestDevice:default:*"
],
+ "merge": {
+ "script": "//testing/merge_scripts/noop_merge.py"
+ },
"name": "dawn_node_sws_cts",
+ "resultdb": {
+ "result_format": "single"
+ },
+ "swarming": {
+ "containment_type": "AUTO",
+ "dimensions": {
+ "cpu": "x86-64",
+ "gpu": "none",
+ "os": "Windows-10-19045",
+ "pool": "chromium.tests.gpu"
+ },
+ "service_account": "chromium-tester@chops-service-accounts.iam.gserviceaccount.com"
+ },
"test": "dawn_node_cts",
"test_id_prefix": "ninja://scripts/dawn_node_cts:dawn_node_cts/"
}
@@ -1208,7 +1288,23 @@
"SwiftShader",
"webgpu:api,operation,adapter,requestDevice:default:*"
],
+ "merge": {
+ "script": "//testing/merge_scripts/noop_merge.py"
+ },
"name": "dawn_node_sws_cts",
+ "resultdb": {
+ "result_format": "single"
+ },
+ "swarming": {
+ "containment_type": "AUTO",
+ "dimensions": {
+ "cpu": "x86-64",
+ "gpu": "none",
+ "os": "Windows-10-19045",
+ "pool": "chromium.tests.gpu"
+ },
+ "service_account": "chromium-tester@chops-service-accounts.iam.gserviceaccount.com"
+ },
"test": "dawn_node_cts",
"test_id_prefix": "ninja://scripts/dawn_node_cts:dawn_node_cts/"
}
@@ -1366,7 +1462,23 @@
"SwiftShader",
"webgpu:api,operation,adapter,requestDevice:default:*"
],
+ "merge": {
+ "script": "//testing/merge_scripts/noop_merge.py"
+ },
"name": "dawn_node_sws_cts",
+ "resultdb": {
+ "result_format": "single"
+ },
+ "swarming": {
+ "containment_type": "AUTO",
+ "dimensions": {
+ "cpu": "x86-64",
+ "gpu": "none",
+ "os": "Windows-10-19045",
+ "pool": "chromium.tests.gpu"
+ },
+ "service_account": "chromium-tester@chops-service-accounts.iam.gserviceaccount.com"
+ },
"test": "dawn_node_cts",
"test_id_prefix": "ninja://scripts/dawn_node_cts:dawn_node_cts/"
}
diff --git a/infra/specs/generate_test_spec_json.py b/infra/specs/generate_test_spec_json.py
index 8b32b6e..c26ee29 100755
--- a/infra/specs/generate_test_spec_json.py
+++ b/infra/specs/generate_test_spec_json.py
@@ -61,6 +61,11 @@
'result_format': 'gtest_json',
},
},
+ 'result_adapter_single': {
+ 'resultdb': {
+ 'result_format': 'single',
+ },
+ },
}
MIXIN_FILEPATH = os.path.join(THIS_DIR, 'mixins.pyl')
diff --git a/infra/specs/mixins.pyl b/infra/specs/mixins.pyl
index 92c3593..44b2221 100644
--- a/infra/specs/mixins.pyl
+++ b/infra/specs/mixins.pyl
@@ -51,6 +51,7 @@
'swarming': {'dimensions': {'gpu': 'none'}}},
'no_swarming': {'swarming': {'can_use_on_swarming_builders': False}},
'result_adapter_gtest_json': {'resultdb': {'result_format': 'gtest_json'}},
+ 'result_adapter_single': {'resultdb': {'result_format': 'single'}},
'swarming_containment_auto': { 'fail_if_unused': False,
'swarming': {'containment_type': 'AUTO'}},
'win10': {'swarming': {'dimensions': {'os': 'Windows-10-19045'}}},
diff --git a/infra/specs/test_suites.pyl b/infra/specs/test_suites.pyl
index c3632e7..093487d 100644
--- a/infra/specs/test_suites.pyl
+++ b/infra/specs/test_suites.pyl
@@ -72,9 +72,17 @@
'webgpu:api,operation,adapter,requestDevice:default:*',
],
'mixins': [
- # TODO(crbug.com/441328362): Support running these tests on Swarming.
- 'no_swarming',
+ 'result_adapter_single',
],
+ 'merge': {
+ # This should be removed in favor of updating the recipe code to allow
+ # the merge script to be skipped entirely. noop is needed currently
+ # because standard_isolated_script_merge does not work properly
+ # without some additional Python dependencies. However, noop_merge
+ # locks us to a single shard, and we should not have any need for the
+ # outputs of merge scripts for Dawn.
+ 'script': '//testing/merge_scripts/noop_merge.py',
+ },
'test': 'dawn_node_cts',
},
},
diff --git a/scripts/dawn_node_cts/run_dawn_node_cts.py b/scripts/dawn_node_cts/run_dawn_node_cts.py
index 987ef40..9e255eb 100755
--- a/scripts/dawn_node_cts/run_dawn_node_cts.py
+++ b/scripts/dawn_node_cts/run_dawn_node_cts.py
@@ -31,10 +31,12 @@
import contextlib
import functools
import glob
+import json
import logging
import os
import subprocess
import sys
+import tempfile
import node_helpers
@@ -52,7 +54,8 @@
subprocess.run(cmd, check=True)
-def run_node_cts(output_directory: str, args_to_forward: list[str]) -> None:
+def run_node_cts(output_directory: str, args_to_forward: list[str],
+ output_filepath: str) -> None:
logging.info('Running CTS via node in %s', os.getcwd())
npx_wrapper = os.path.join(THIS_DIR, 'run_npx.py')
if sys.platform == 'win32':
@@ -63,12 +66,36 @@
'run-cts',
'-bin',
output_directory,
+ '-output',
+ output_filepath,
'-npx',
npx_wrapper,
] + args_to_forward
subprocess.run(cmd, check=True)
+def convert_results_for_resultdb_ingestion(
+ test_output_filepath: str, isolated_output_filepath: str) -> None:
+ # This is a very crude conversion, although the output of run-cts does not
+ # give us a lot to work with. If the information it provides is ever
+ # improved, this conversion can likely be substituted for native ResultDB
+ # integration instead of relying on result_adapter.
+ with open(test_output_filepath, encoding='utf-8') as infile:
+ test_results = json.load(infile)
+
+ converted_test_results = {
+ 'failures': [],
+ 'valid': True,
+ }
+
+ for r in test_results:
+ if r['Status'] != 'pass':
+ converted_test_results['failures'].append(r['TestCase'])
+
+ with open(isolated_output_filepath, 'w', encoding='utf-8') as outfile:
+ json.dump(converted_test_results, outfile)
+
+
def main() -> None:
logging.getLogger().setLevel(logging.INFO)
parser = argparse.ArgumentParser('Runs the WebGPU CTS via NodeJS')
@@ -89,7 +116,15 @@
with contextlib.chdir(CTS_DIR):
node_helpers.add_node_to_path()
install_npm_deps_in_current_dir()
- run_node_cts(args.output_directory, unknown_args)
+ output_fd, output_filepath = tempfile.mkstemp()
+ os.close(output_fd)
+ try:
+ run_node_cts(args.output_directory, unknown_args, output_filepath)
+ if args.isolated_script_test_output:
+ convert_results_for_resultdb_ingestion(
+ output_filepath, args.isolated_script_test_output)
+ finally:
+ os.remove(output_filepath)
if __name__ == '__main__':
