Output LiteRT-LM benchmark results to file Write the benchmark metrics output to a proto file instead of stdout, and include that file in the CAS outputs so that it can be pushed to a GCS bucket with a results merge script in a follow-up. For now the file can be manually inspected in the CAS Outputs on the Swarming Task Page. Bug: 549160087 Change-Id: I3cb865beb50c08e6de0e1be4f20590e971376089 Cq-Include-Trybots: luci.dawn.try:dawn-try-linux-x64-nvidia-gtx1660-rel Include-Ci-Only-Tests: ci:dawn-linux-x64-nvidia-gtx1660-rel|litert_lm_benchmark Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/333955 Commit-Queue: Natalie Chouinard <chouinard@google.com> Reviewed-by: dan sinclair <dsinclair@chromium.org>
diff --git a/scripts/run_litert_lm_benchmark.py b/scripts/run_litert_lm_benchmark.py index fead45d..8aeadba 100755 --- a/scripts/run_litert_lm_benchmark.py +++ b/scripts/run_litert_lm_benchmark.py
@@ -38,7 +38,7 @@ import subprocess -def run_litert_lm() -> subprocess.CompletedProcess: +def run_litert_lm(metric_proto_file_path: Path) -> subprocess.CompletedProcess: repo_root = Path(__file__).resolve().parent.parent binary_path = Path.cwd() / 'litert_lm_advanced_main' model_path = repo_root / 'third_party' / 'litert-lm' / 'data' / 'model.litertlm' @@ -48,11 +48,11 @@ '--benchmark', '--backend=gpu', f'--model_path={model_path}', + f'--metric_proto_file_path={metric_proto_file_path}', ] print(f"Executing: {' '.join(cmd)}") - # The .so files are isolated alongside the binary in the current build out directory, # so explicitly add them to the library path for execution on Linux bots. env = os.environ.copy() @@ -89,7 +89,15 @@ help='Currently unused, needed for bot support.') args = parser.parse_args() - proc = run_litert_lm() + # Determine output directory for results. + if args.isolated_script_test_output: + output_dir = Path(args.isolated_script_test_output).parent + else: + output_dir = Path.cwd() + + # Run the benchmark and generate metric results file. + metric_file = output_dir / 'litert_lm_metrics.pb' + proc = run_litert_lm(metric_file) if args.isolated_script_test_output: generate_results_for_resultdb_ingestion( not proc.returncode, args.isolated_script_test_output)