[tools][cts] Switch CTS roller to use BigQuery
Updates the CTS roller to use BigQuery to retrieve ResultDB data instead
of a ResultDB RPC.
Locally, this is reducing the runtime of "tools/run cts results" on an
already merged roll from ~22.5 minutes to ~3.5 minutes.
Bug: chromium:342446313
Change-Id: I3f190feefb1f7aaba06b5fe70abf9d75d32fe011
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/189762
Commit-Queue: Brian Sheedy <bsheedy@google.com>
Reviewed-by: Ben Clayton <bclayton@google.com>
Auto-Submit: Brian Sheedy <bsheedy@google.com>
diff --git a/go.mod b/go.mod
index d84291d..e6f538b 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,7 @@
go 1.18
require (
+ cloud.google.com/go/bigquery v1.52.0
cloud.google.com/go/storage v1.33.0
github.com/andygrunwald/go-gerrit v0.0.0-20230508072829-423d372345aa
github.com/ben-clayton/webidlparser v0.0.0-20210923100217-8ba896ded094
@@ -10,7 +11,7 @@
github.com/fatih/color v1.13.0
github.com/google/go-cmp v0.5.9
github.com/mattn/go-colorable v0.1.12
- github.com/mattn/go-isatty v0.0.16
+ github.com/mattn/go-isatty v0.0.17
github.com/mzohreva/gographviz v0.0.0-20180226085351-533f4a37d9c6
github.com/sergi/go-diff v1.3.1
github.com/shirou/gopsutil v3.21.11+incompatible
@@ -28,15 +29,21 @@
cloud.google.com/go/compute v1.20.1 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.0 // indirect
+ github.com/andybalholm/brotli v1.0.4 // indirect
+ github.com/apache/arrow/go/v12 v12.0.0 // indirect
+ github.com/apache/thrift v0.16.0 // indirect
github.com/chromedp/cdproto v0.0.0-20231011050154-1d073bb38998 // indirect
github.com/chromedp/sysutil v1.0.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.3.0 // indirect
+ github.com/goccy/go-json v0.9.11 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
+ github.com/golang/snappy v0.0.4 // indirect
+ github.com/google/flatbuffers v2.0.8+incompatible // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/google/uuid v1.3.0 // indirect
@@ -44,24 +51,32 @@
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/julienschmidt/httprouter v1.3.0 // indirect
+ github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/klauspost/compress v1.16.3 // indirect
+ github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/maruel/subcommands v1.1.1 // indirect
+ github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
+ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mzohreva/GoGraphviz v0.0.0-20180226085351-533f4a37d9c6 // indirect
+ github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
github.com/texttheater/golang-levenshtein v1.0.1 // indirect
github.com/tklauser/go-sysconf v0.3.10 // indirect
github.com/tklauser/numcpus v0.4.0 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
+ github.com/zeebo/xxh3 v1.0.2 // indirect
go.opencensus.io v0.24.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
+ golang.org/x/mod v0.10.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.14.0 // indirect
golang.org/x/term v0.14.0 // indirect
golang.org/x/text v0.13.0 // indirect
+ golang.org/x/tools v0.9.1 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.8-0.20221117013220-504804fb50de // indirect
google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 // indirect
diff --git a/go.sum b/go.sum
index cd3cc97..2e106ed 100644
--- a/go.sum
+++ b/go.sum
@@ -2,6 +2,8 @@
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk=
cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI=
+cloud.google.com/go/bigquery v1.52.0 h1:JKLNdxI0N+TIUWD6t9KN646X27N5dQWq9dZbbTWZ8hc=
+cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4=
cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg=
cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
@@ -11,9 +13,15 @@
cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M=
cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
+github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/andygrunwald/go-gerrit v0.0.0-20230508072829-423d372345aa h1:bGSzPoUh/2eduqGEk54TCoB4v81MVi6Hr3+fYQwFrBM=
github.com/andygrunwald/go-gerrit v0.0.0-20230508072829-423d372345aa/go.mod h1:SeP12EkHZxEVjuJ2HZET304NBtHGG2X6w2Gzd0QXAZw=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/apache/arrow/go/v12 v12.0.0 h1:xtZE63VWl7qLdB0JObIXvvhGjoVNrQ9ciIHG2OK5cmc=
+github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg=
+github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY=
+github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
github.com/ben-clayton/webidlparser v0.0.0-20210923100217-8ba896ded094 h1:CTVJdI6oUCRNucMEmoh3c2U88DesoPtefsxKhoZ1WuQ=
github.com/ben-clayton/webidlparser v0.0.0-20210923100217-8ba896ded094/go.mod h1:bV550SPlMos7UhMprxlm14XTBTpKHSUZ8Q4Id5qQuyw=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -52,11 +60,14 @@
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.3.0 h1:sbeU3Y4Qzlb+MOzIe6mQGf7QR4Hkv6ZD0qhGkBFL2O0=
github.com/gobwas/ws v1.3.0/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
+github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
+github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -74,6 +85,10 @@
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM=
+github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -104,8 +119,12 @@
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
+github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
@@ -129,6 +148,12 @@
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
+github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
+github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
+github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mzohreva/GoGraphviz v0.0.0-20180226085351-533f4a37d9c6 h1:yd4o0qJNQc2PBlymcRCUHM0ltxciTkPjUuVTj8cDbLA=
@@ -137,6 +162,8 @@
github.com/mzohreva/gographviz v0.0.0-20180226085351-533f4a37d9c6/go.mod h1:bBX+0ahzrDlJ3NtK345LYHaf+xDeNwttNkWPy+NL6wM=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
+github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
+github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -172,6 +199,8 @@
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
go.chromium.org/luci v0.0.0-20230311013728-313c8e2205bc h1:MQqGMDf3p8O6ESdGcDWhBAUWDB7oqJfpP4WEIIHIWY4=
go.chromium.org/luci v0.0.0-20230311013728-313c8e2205bc/go.mod h1:69M9nsLBv0vKXTuj42kKWHb9KEulBn2rGHCmGUIkpig=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
@@ -188,8 +217,11 @@
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
+golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -263,6 +295,8 @@
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
+golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/tools/src/cmd/cts/common/results.go b/tools/src/cmd/cts/common/results.go
index f0e71e8..c2b6240 100644
--- a/tools/src/cmd/cts/common/results.go
+++ b/tools/src/cmd/cts/common/results.go
@@ -29,6 +29,7 @@
import (
"context"
+ "errors"
"flag"
"fmt"
"log"
@@ -95,7 +96,7 @@
if err != nil {
return nil, err
}
- rdb, err := resultsdb.New(ctx, auth)
+ client, err := resultsdb.NewBigQueryClient(ctx, resultsdb.DefaultQueryProject)
if err != nil {
return nil, err
}
@@ -114,7 +115,7 @@
}
fmt.Printf("scanning for latest patchset of %v...\n", latest.Number)
var resultsByExecutionMode result.ResultsByExecutionMode
- resultsByExecutionMode, *ps, err = MostRecentResultsForChange(ctx, cfg, r.CacheDir, gerrit, bb, rdb, latest.Number)
+ resultsByExecutionMode, *ps, err = MostRecentResultsForChange(ctx, cfg, r.CacheDir, gerrit, bb, client, latest.Number)
if err != nil {
return nil, err
}
@@ -144,7 +145,7 @@
return nil, err
}
- resultsByExecutionMode, err := CacheResults(ctx, cfg, *ps, r.CacheDir, rdb, builds)
+ resultsByExecutionMode, err := CacheResults(ctx, cfg, *ps, r.CacheDir, client, builds)
if err != nil {
return nil, err
}
@@ -162,7 +163,7 @@
cfg Config,
ps gerrit.Patchset,
cacheDir string,
- rdb *resultsdb.ResultsDB,
+ client *resultsdb.BigQueryClient,
builds BuildsByName) (result.ResultsByExecutionMode, error) {
var cachePath string
@@ -177,7 +178,7 @@
}
log.Printf("fetching results from cl %v ps %v...", ps.Change, ps.Patchset)
- resultsByExecutionMode, err := GetRawResults(ctx, cfg, rdb, builds)
+ resultsByExecutionMode, err := GetRawResults(ctx, cfg, client, builds)
if err != nil {
return nil, err
}
@@ -201,10 +202,10 @@
func GetResults(
ctx context.Context,
cfg Config,
- rdb *resultsdb.ResultsDB,
+ client *resultsdb.BigQueryClient,
builds BuildsByName) (result.ResultsByExecutionMode, error) {
- resultsByExecutionMode, err := GetRawResults(ctx, cfg, rdb, builds)
+ resultsByExecutionMode, err := GetRawResults(ctx, cfg, client, builds)
if err != nil {
return nil, err
}
@@ -224,26 +225,26 @@
func GetRawResults(
ctx context.Context,
cfg Config,
- rdb *resultsdb.ResultsDB,
+ client *resultsdb.BigQueryClient,
builds BuildsByName) (result.ResultsByExecutionMode, error) {
fmt.Printf("fetching results from resultdb...")
lastPrintedDot := time.Now()
- toStatus := func(s rdbpb.TestStatus) result.Status {
+ toStatus := func(s string) result.Status {
switch s {
default:
return result.Unknown
- case rdbpb.TestStatus_PASS:
+ case rdbpb.TestStatus_PASS.String():
return result.Pass
- case rdbpb.TestStatus_FAIL:
+ case rdbpb.TestStatus_FAIL.String():
return result.Failure
- case rdbpb.TestStatus_CRASH:
+ case rdbpb.TestStatus_CRASH.String():
return result.Crash
- case rdbpb.TestStatus_ABORT:
+ case rdbpb.TestStatus_ABORT.String():
return result.Abort
- case rdbpb.TestStatus_SKIP:
+ case rdbpb.TestStatus_SKIP.String():
return result.Skip
}
}
@@ -252,36 +253,34 @@
for _, test := range cfg.Tests {
results := result.List{}
for _, prefix := range test.Prefixes {
- err := rdb.QueryTestResults(ctx, builds.ids(), prefix+".*", func(rpb *rdbpb.TestResult) error {
+
+ err := client.QueryTestResults(ctx, builds.ids(), prefix, func(r *resultsdb.QueryResult) error {
if time.Since(lastPrintedDot) > 5*time.Second {
lastPrintedDot = time.Now()
fmt.Printf(".")
}
- if !strings.HasPrefix(rpb.GetTestId(), prefix) {
- return nil
+ if !strings.HasPrefix(r.TestId, prefix) {
+ return errors.New(fmt.Sprintf("Test ID %s did not start with %s even though query should have filtered.", r.TestId, prefix))
}
- testName := rpb.GetTestId()[len(prefix):]
- status := toStatus(rpb.Status)
+ testName := r.TestId[len(prefix):]
+ status := toStatus(r.Status)
tags := result.NewTags()
-
- duration := rpb.GetDuration().AsDuration()
+ duration := time.Duration(r.Duration * float64(time.Second))
mayExonerate := false
- for _, sp := range rpb.Tags {
- if sp.Key == "typ_tag" {
- tags.Add(sp.Value)
- }
- if sp.Key == "javascript_duration" {
+ for _, tagPair := range r.Tags {
+ if tagPair.Key == "typ_tag" {
+ tags.Add(tagPair.Value)
+ } else if tagPair.Key == "javascript_duration" {
var err error
- if duration, err = time.ParseDuration(sp.Value); err != nil {
+ if duration, err = time.ParseDuration(tagPair.Value); err != nil {
return err
}
- }
- if sp.Key == "may_exonerate" {
+ } else if tagPair.Key == "may_exonerate" {
var err error
- if mayExonerate, err = strconv.ParseBool(sp.Value); err != nil {
+ if mayExonerate, err = strconv.ParseBool(tagPair.Value); err != nil {
return err
}
}
@@ -351,7 +350,7 @@
cacheDir string,
g *gerrit.Gerrit,
bb *buildbucket.Buildbucket,
- rdb *resultsdb.ResultsDB,
+ client *resultsdb.BigQueryClient,
change int) (result.ResultsByExecutionMode, gerrit.Patchset, error) {
ps, err := LatestPatchset(g, change)
@@ -369,7 +368,7 @@
return nil, gerrit.Patchset{}, err
}
- results, err := CacheResults(ctx, cfg, ps, cacheDir, rdb, builds)
+ results, err := CacheResults(ctx, cfg, ps, cacheDir, client, builds)
if err != nil {
return nil, gerrit.Patchset{}, err
}
diff --git a/tools/src/cmd/cts/roll/roll.go b/tools/src/cmd/cts/roll/roll.go
index 3e389cd..d0cf2ee 100644
--- a/tools/src/cmd/cts/roll/roll.go
+++ b/tools/src/cmd/cts/roll/roll.go
@@ -171,7 +171,7 @@
if err != nil {
return err
}
- rdb, err := resultsdb.New(ctx, auth)
+ client, err := resultsdb.NewBigQueryClient(ctx, resultsdb.DefaultQueryProject)
if err != nil {
return err
}
@@ -183,7 +183,7 @@
auth: auth,
bb: bb,
parentSwarmingRunID: c.flags.parentSwarmingRunID,
- rdb: rdb,
+ client: client,
git: git,
gerrit: gerrit,
gitiles: gitilesRepos{dawn: dawn},
@@ -202,7 +202,7 @@
auth auth.Options
bb *buildbucket.Buildbucket
parentSwarmingRunID string
- rdb *resultsdb.ResultsDB
+ client *resultsdb.BigQueryClient
git *git.Git
gerrit *gerrit.Gerrit
gitiles gitilesRepos
@@ -413,7 +413,7 @@
// Gather the build results
log.Println("gathering results...")
- psResultsByExecutionMode, err = common.CacheResults(ctx, r.cfg, ps, r.flags.cacheDir, r.rdb, builds)
+ psResultsByExecutionMode, err = common.CacheResults(ctx, r.cfg, ps, r.flags.cacheDir, r.client, builds)
if err != nil {
return err
}
diff --git a/tools/src/resultsdb/resultsdb.go b/tools/src/resultsdb/resultsdb.go
index 8d01b48..66a3b9b 100644
--- a/tools/src/resultsdb/resultsdb.go
+++ b/tools/src/resultsdb/resultsdb.go
@@ -31,78 +31,98 @@
import (
"context"
"fmt"
+ "strings"
+ "cloud.google.com/go/bigquery"
"dawn.googlesource.com/dawn/tools/src/buildbucket"
- "go.chromium.org/luci/auth"
- "go.chromium.org/luci/grpc/prpc"
- "go.chromium.org/luci/hardcoded/chromeinfra"
- rdbpb "go.chromium.org/luci/resultdb/proto/v1"
- "google.golang.org/protobuf/types/known/fieldmaskpb"
+ "google.golang.org/api/iterator"
)
-// ResultsDB is the client to communicate with ResultDB.
-type ResultsDB struct {
- client rdbpb.ResultDBClient
+// BigQueryClient is a wrapper around bigquery.Client so that we can define new
+// methods.
+type BigQueryClient struct {
+ client *bigquery.Client
}
-// New creates a client to communicate with ResultDB.
-func New(ctx context.Context, credentials auth.Options) (*ResultsDB, error) {
- http, err := auth.NewAuthenticator(ctx, auth.InteractiveLogin, credentials).Client()
+// QueryResult contains all of the data for a single test result from a ResultDB
+// BigQuery query.
+type QueryResult struct {
+ TestId string
+ Status string
+ Tags []struct {
+ Key string
+ Value string
+ }
+ Duration float64
+}
+
+// DefaultQueryProject is the default BigQuery project to use when running
+// queries.
+const DefaultQueryProject string = "chrome-unexpected-pass-data"
+
+// NewBigQueryClient creates a client for running BigQuery queries. The
+// intention is for this to be used for querying ResultDB tables, but there is
+// nothing ResultDB-specific about the resulting client.
+func NewBigQueryClient(ctx context.Context, project string) (*BigQueryClient, error) {
+ client, err := bigquery.NewClient(ctx, project)
if err != nil {
return nil, err
}
- client, err := rdbpb.NewResultDBPRPCClient(
- &prpc.Client{
- C: http,
- Host: chromeinfra.ResultDBHost,
- Options: prpc.DefaultOptions(),
- }), nil
+ // By default, results are retrieved in chunks as they're iterated over, but
+ // that results in slow performance. Enabling the Storage API allows us to get
+ // all results at once, resulting in a ~8-10x speed increase.
+ err = client.EnableStorageReadClient(ctx)
if err != nil {
return nil, err
}
-
- return &ResultsDB{client}, nil
+ return &BigQueryClient{client}, nil
}
-// QueryTestResults fetches the test results for the given builds.
-// f is called once per page of test variants.
-func (r *ResultsDB) QueryTestResults(
- ctx context.Context,
- builds []buildbucket.BuildID,
- filterRegex string,
- f func(*rdbpb.TestResult) error) error {
+// QueryTestResults fetches the test results for the given builds using
+// BigQuery.
+//
+// f is called once per result and is expected to handle any processing or
+// storage of results.
+func (bq *BigQueryClient) QueryTestResults(
+ ctx context.Context, builds []buildbucket.BuildID, testPrefix string, f func(*QueryResult) error) error {
+ // test_id gets renamed since the column names need to match the struct names
+ // unless we want to get results in a generic bigquery.Value slice and
+ // manually copy data over.
+ base_query := `
+ SELECT
+ test_id AS testid,
+ status,
+ tags,
+ duration
+ FROM ` + "`chrome-luci-data.chromium.gpu_try_test_results`" + ` tr
+ WHERE
+ exported.id IN UNNEST([%s])
+ AND STARTS_WITH(tr.test_id, "%v")`
- invocationNames := make([]string, len(builds))
- for i, id := range builds {
- invocationNames[i] = fmt.Sprintf("invocations/build-%v", id)
+ var buildIds []string
+ for _, id := range builds {
+ buildIds = append(buildIds, fmt.Sprintf(`"build-%v"`, id))
+ }
+ query := fmt.Sprintf(base_query, strings.Join(buildIds, ","), testPrefix)
+
+ q := bq.client.Query(query)
+ iter, err := q.Read(ctx)
+ if err != nil {
+ return err
}
- pageToken := ""
+ var row QueryResult
for {
- rsp, err := r.client.QueryTestResults(ctx, &rdbpb.QueryTestResultsRequest{
- Invocations: invocationNames,
- Predicate: &rdbpb.TestResultPredicate{
- TestIdRegexp: filterRegex,
- },
- ReadMask: &fieldmaskpb.FieldMask{Paths: []string{
- "test_id", "status", "tags", "duration",
- }},
- PageSize: 1000, // Maximum page size.
- PageToken: pageToken,
- })
+ err := iter.Next(&row)
+ if err == iterator.Done {
+ break
+ }
if err != nil {
return err
}
- for _, res := range rsp.TestResults {
- if err := f(res); err != nil {
- return err
- }
- }
-
- pageToken = rsp.GetNextPageToken()
- if pageToken == "" {
- break
+ if err := f(&row); err != nil {
+ return err
}
}
