[tools][cts] Switch CTS roller to use BigQuery Updates the CTS roller to use BigQuery to retrieve ResultDB data instead of a ResultDB RPC. Locally, this is reducing the runtime of "tools/run cts results" on an already merged roll from ~22.5 minutes to ~3.5 minutes. Bug: chromium:342446313 Change-Id: I3f190feefb1f7aaba06b5fe70abf9d75d32fe011 Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/189762 Commit-Queue: Brian Sheedy <bsheedy@google.com> Reviewed-by: Ben Clayton <bclayton@google.com> Auto-Submit: Brian Sheedy <bsheedy@google.com>
diff --git a/go.mod b/go.mod index d84291d..e6f538b 100644 --- a/go.mod +++ b/go.mod
@@ -3,6 +3,7 @@ go 1.18 require ( + cloud.google.com/go/bigquery v1.52.0 cloud.google.com/go/storage v1.33.0 github.com/andygrunwald/go-gerrit v0.0.0-20230508072829-423d372345aa github.com/ben-clayton/webidlparser v0.0.0-20210923100217-8ba896ded094 @@ -10,7 +11,7 @@ github.com/fatih/color v1.13.0 github.com/google/go-cmp v0.5.9 github.com/mattn/go-colorable v0.1.12 - github.com/mattn/go-isatty v0.0.16 + github.com/mattn/go-isatty v0.0.17 github.com/mzohreva/gographviz v0.0.0-20180226085351-533f4a37d9c6 github.com/sergi/go-diff v1.3.1 github.com/shirou/gopsutil v3.21.11+incompatible @@ -28,15 +29,21 @@ cloud.google.com/go/compute v1.20.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.0 // indirect + github.com/andybalholm/brotli v1.0.4 // indirect + github.com/apache/arrow/go/v12 v12.0.0 // indirect + github.com/apache/thrift v0.16.0 // indirect github.com/chromedp/cdproto v0.0.0-20231011050154-1d073bb38998 // indirect github.com/chromedp/sysutil v1.0.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.3.0 // indirect + github.com/goccy/go-json v0.9.11 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/flatbuffers v2.0.8+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.3.0 // indirect @@ -44,24 +51,32 @@ github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect + github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.16.3 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/maruel/subcommands v1.1.1 // indirect + github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect + github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mzohreva/GoGraphviz v0.0.0-20180226085351-533f4a37d9c6 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/texttheater/golang-levenshtein v1.0.1 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/crypto v0.14.0 // indirect + golang.org/x/mod v0.10.0 // indirect golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/term v0.14.0 // indirect golang.org/x/text v0.13.0 // indirect + golang.org/x/tools v0.9.1 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.8-0.20221117013220-504804fb50de // indirect google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 // indirect
diff --git a/go.sum b/go.sum index cd3cc97..2e106ed 100644 --- a/go.sum +++ b/go.sum
@@ -2,6 +2,8 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk= cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go/bigquery v1.52.0 h1:JKLNdxI0N+TIUWD6t9KN646X27N5dQWq9dZbbTWZ8hc= +cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg= cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= @@ -11,9 +13,15 @@ cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M= cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andygrunwald/go-gerrit v0.0.0-20230508072829-423d372345aa h1:bGSzPoUh/2eduqGEk54TCoB4v81MVi6Hr3+fYQwFrBM= github.com/andygrunwald/go-gerrit v0.0.0-20230508072829-423d372345aa/go.mod h1:SeP12EkHZxEVjuJ2HZET304NBtHGG2X6w2Gzd0QXAZw= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/arrow/go/v12 v12.0.0 h1:xtZE63VWl7qLdB0JObIXvvhGjoVNrQ9ciIHG2OK5cmc= +github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= +github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/ben-clayton/webidlparser v0.0.0-20210923100217-8ba896ded094 h1:CTVJdI6oUCRNucMEmoh3c2U88DesoPtefsxKhoZ1WuQ= github.com/ben-clayton/webidlparser v0.0.0-20210923100217-8ba896ded094/go.mod h1:bV550SPlMos7UhMprxlm14XTBTpKHSUZ8Q4Id5qQuyw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -52,11 +60,14 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.3.0 h1:sbeU3Y4Qzlb+MOzIe6mQGf7QR4Hkv6ZD0qhGkBFL2O0= github.com/gobwas/ws v1.3.0/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= +github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -74,6 +85,10 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= +github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -104,8 +119,12 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -129,6 +148,12 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mzohreva/GoGraphviz v0.0.0-20180226085351-533f4a37d9c6 h1:yd4o0qJNQc2PBlymcRCUHM0ltxciTkPjUuVTj8cDbLA= @@ -137,6 +162,8 @@ github.com/mzohreva/gographviz v0.0.0-20180226085351-533f4a37d9c6/go.mod h1:bBX+0ahzrDlJ3NtK345LYHaf+xDeNwttNkWPy+NL6wM= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -172,6 +199,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.chromium.org/luci v0.0.0-20230311013728-313c8e2205bc h1:MQqGMDf3p8O6ESdGcDWhBAUWDB7oqJfpP4WEIIHIWY4= go.chromium.org/luci v0.0.0-20230311013728-313c8e2205bc/go.mod h1:69M9nsLBv0vKXTuj42kKWHb9KEulBn2rGHCmGUIkpig= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= @@ -188,8 +217,11 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -263,6 +295,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/tools/src/cmd/cts/common/results.go b/tools/src/cmd/cts/common/results.go index f0e71e8..c2b6240 100644 --- a/tools/src/cmd/cts/common/results.go +++ b/tools/src/cmd/cts/common/results.go
@@ -29,6 +29,7 @@ import ( "context" + "errors" "flag" "fmt" "log" @@ -95,7 +96,7 @@ if err != nil { return nil, err } - rdb, err := resultsdb.New(ctx, auth) + client, err := resultsdb.NewBigQueryClient(ctx, resultsdb.DefaultQueryProject) if err != nil { return nil, err } @@ -114,7 +115,7 @@ } fmt.Printf("scanning for latest patchset of %v...\n", latest.Number) var resultsByExecutionMode result.ResultsByExecutionMode - resultsByExecutionMode, *ps, err = MostRecentResultsForChange(ctx, cfg, r.CacheDir, gerrit, bb, rdb, latest.Number) + resultsByExecutionMode, *ps, err = MostRecentResultsForChange(ctx, cfg, r.CacheDir, gerrit, bb, client, latest.Number) if err != nil { return nil, err } @@ -144,7 +145,7 @@ return nil, err } - resultsByExecutionMode, err := CacheResults(ctx, cfg, *ps, r.CacheDir, rdb, builds) + resultsByExecutionMode, err := CacheResults(ctx, cfg, *ps, r.CacheDir, client, builds) if err != nil { return nil, err } @@ -162,7 +163,7 @@ cfg Config, ps gerrit.Patchset, cacheDir string, - rdb *resultsdb.ResultsDB, + client *resultsdb.BigQueryClient, builds BuildsByName) (result.ResultsByExecutionMode, error) { var cachePath string @@ -177,7 +178,7 @@ } log.Printf("fetching results from cl %v ps %v...", ps.Change, ps.Patchset) - resultsByExecutionMode, err := GetRawResults(ctx, cfg, rdb, builds) + resultsByExecutionMode, err := GetRawResults(ctx, cfg, client, builds) if err != nil { return nil, err } @@ -201,10 +202,10 @@ func GetResults( ctx context.Context, cfg Config, - rdb *resultsdb.ResultsDB, + client *resultsdb.BigQueryClient, builds BuildsByName) (result.ResultsByExecutionMode, error) { - resultsByExecutionMode, err := GetRawResults(ctx, cfg, rdb, builds) + resultsByExecutionMode, err := GetRawResults(ctx, cfg, client, builds) if err != nil { return nil, err } @@ -224,26 +225,26 @@ func GetRawResults( ctx context.Context, cfg Config, - rdb *resultsdb.ResultsDB, + client *resultsdb.BigQueryClient, builds BuildsByName) (result.ResultsByExecutionMode, error) { fmt.Printf("fetching results from resultdb...") lastPrintedDot := time.Now() - toStatus := func(s rdbpb.TestStatus) result.Status { + toStatus := func(s string) result.Status { switch s { default: return result.Unknown - case rdbpb.TestStatus_PASS: + case rdbpb.TestStatus_PASS.String(): return result.Pass - case rdbpb.TestStatus_FAIL: + case rdbpb.TestStatus_FAIL.String(): return result.Failure - case rdbpb.TestStatus_CRASH: + case rdbpb.TestStatus_CRASH.String(): return result.Crash - case rdbpb.TestStatus_ABORT: + case rdbpb.TestStatus_ABORT.String(): return result.Abort - case rdbpb.TestStatus_SKIP: + case rdbpb.TestStatus_SKIP.String(): return result.Skip } } @@ -252,36 +253,34 @@ for _, test := range cfg.Tests { results := result.List{} for _, prefix := range test.Prefixes { - err := rdb.QueryTestResults(ctx, builds.ids(), prefix+".*", func(rpb *rdbpb.TestResult) error { + + err := client.QueryTestResults(ctx, builds.ids(), prefix, func(r *resultsdb.QueryResult) error { if time.Since(lastPrintedDot) > 5*time.Second { lastPrintedDot = time.Now() fmt.Printf(".") } - if !strings.HasPrefix(rpb.GetTestId(), prefix) { - return nil + if !strings.HasPrefix(r.TestId, prefix) { + return errors.New(fmt.Sprintf("Test ID %s did not start with %s even though query should have filtered.", r.TestId, prefix)) } - testName := rpb.GetTestId()[len(prefix):] - status := toStatus(rpb.Status) + testName := r.TestId[len(prefix):] + status := toStatus(r.Status) tags := result.NewTags() - - duration := rpb.GetDuration().AsDuration() + duration := time.Duration(r.Duration * float64(time.Second)) mayExonerate := false - for _, sp := range rpb.Tags { - if sp.Key == "typ_tag" { - tags.Add(sp.Value) - } - if sp.Key == "javascript_duration" { + for _, tagPair := range r.Tags { + if tagPair.Key == "typ_tag" { + tags.Add(tagPair.Value) + } else if tagPair.Key == "javascript_duration" { var err error - if duration, err = time.ParseDuration(sp.Value); err != nil { + if duration, err = time.ParseDuration(tagPair.Value); err != nil { return err } - } - if sp.Key == "may_exonerate" { + } else if tagPair.Key == "may_exonerate" { var err error - if mayExonerate, err = strconv.ParseBool(sp.Value); err != nil { + if mayExonerate, err = strconv.ParseBool(tagPair.Value); err != nil { return err } } @@ -351,7 +350,7 @@ cacheDir string, g *gerrit.Gerrit, bb *buildbucket.Buildbucket, - rdb *resultsdb.ResultsDB, + client *resultsdb.BigQueryClient, change int) (result.ResultsByExecutionMode, gerrit.Patchset, error) { ps, err := LatestPatchset(g, change) @@ -369,7 +368,7 @@ return nil, gerrit.Patchset{}, err } - results, err := CacheResults(ctx, cfg, ps, cacheDir, rdb, builds) + results, err := CacheResults(ctx, cfg, ps, cacheDir, client, builds) if err != nil { return nil, gerrit.Patchset{}, err }
diff --git a/tools/src/cmd/cts/roll/roll.go b/tools/src/cmd/cts/roll/roll.go index 3e389cd..d0cf2ee 100644 --- a/tools/src/cmd/cts/roll/roll.go +++ b/tools/src/cmd/cts/roll/roll.go
@@ -171,7 +171,7 @@ if err != nil { return err } - rdb, err := resultsdb.New(ctx, auth) + client, err := resultsdb.NewBigQueryClient(ctx, resultsdb.DefaultQueryProject) if err != nil { return err } @@ -183,7 +183,7 @@ auth: auth, bb: bb, parentSwarmingRunID: c.flags.parentSwarmingRunID, - rdb: rdb, + client: client, git: git, gerrit: gerrit, gitiles: gitilesRepos{dawn: dawn}, @@ -202,7 +202,7 @@ auth auth.Options bb *buildbucket.Buildbucket parentSwarmingRunID string - rdb *resultsdb.ResultsDB + client *resultsdb.BigQueryClient git *git.Git gerrit *gerrit.Gerrit gitiles gitilesRepos @@ -413,7 +413,7 @@ // Gather the build results log.Println("gathering results...") - psResultsByExecutionMode, err = common.CacheResults(ctx, r.cfg, ps, r.flags.cacheDir, r.rdb, builds) + psResultsByExecutionMode, err = common.CacheResults(ctx, r.cfg, ps, r.flags.cacheDir, r.client, builds) if err != nil { return err }
diff --git a/tools/src/resultsdb/resultsdb.go b/tools/src/resultsdb/resultsdb.go index 8d01b48..66a3b9b 100644 --- a/tools/src/resultsdb/resultsdb.go +++ b/tools/src/resultsdb/resultsdb.go
@@ -31,78 +31,98 @@ import ( "context" "fmt" + "strings" + "cloud.google.com/go/bigquery" "dawn.googlesource.com/dawn/tools/src/buildbucket" - "go.chromium.org/luci/auth" - "go.chromium.org/luci/grpc/prpc" - "go.chromium.org/luci/hardcoded/chromeinfra" - rdbpb "go.chromium.org/luci/resultdb/proto/v1" - "google.golang.org/protobuf/types/known/fieldmaskpb" + "google.golang.org/api/iterator" ) -// ResultsDB is the client to communicate with ResultDB. -type ResultsDB struct { - client rdbpb.ResultDBClient +// BigQueryClient is a wrapper around bigquery.Client so that we can define new +// methods. +type BigQueryClient struct { + client *bigquery.Client } -// New creates a client to communicate with ResultDB. -func New(ctx context.Context, credentials auth.Options) (*ResultsDB, error) { - http, err := auth.NewAuthenticator(ctx, auth.InteractiveLogin, credentials).Client() +// QueryResult contains all of the data for a single test result from a ResultDB +// BigQuery query. +type QueryResult struct { + TestId string + Status string + Tags []struct { + Key string + Value string + } + Duration float64 +} + +// DefaultQueryProject is the default BigQuery project to use when running +// queries. +const DefaultQueryProject string = "chrome-unexpected-pass-data" + +// NewBigQueryClient creates a client for running BigQuery queries. The +// intention is for this to be used for querying ResultDB tables, but there is +// nothing ResultDB-specific about the resulting client. +func NewBigQueryClient(ctx context.Context, project string) (*BigQueryClient, error) { + client, err := bigquery.NewClient(ctx, project) if err != nil { return nil, err } - client, err := rdbpb.NewResultDBPRPCClient( - &prpc.Client{ - C: http, - Host: chromeinfra.ResultDBHost, - Options: prpc.DefaultOptions(), - }), nil + // By default, results are retrieved in chunks as they're iterated over, but + // that results in slow performance. Enabling the Storage API allows us to get + // all results at once, resulting in a ~8-10x speed increase. + err = client.EnableStorageReadClient(ctx) if err != nil { return nil, err } - - return &ResultsDB{client}, nil + return &BigQueryClient{client}, nil } -// QueryTestResults fetches the test results for the given builds. -// f is called once per page of test variants. -func (r *ResultsDB) QueryTestResults( - ctx context.Context, - builds []buildbucket.BuildID, - filterRegex string, - f func(*rdbpb.TestResult) error) error { +// QueryTestResults fetches the test results for the given builds using +// BigQuery. +// +// f is called once per result and is expected to handle any processing or +// storage of results. +func (bq *BigQueryClient) QueryTestResults( + ctx context.Context, builds []buildbucket.BuildID, testPrefix string, f func(*QueryResult) error) error { + // test_id gets renamed since the column names need to match the struct names + // unless we want to get results in a generic bigquery.Value slice and + // manually copy data over. + base_query := ` + SELECT + test_id AS testid, + status, + tags, + duration + FROM ` + "`chrome-luci-data.chromium.gpu_try_test_results`" + ` tr + WHERE + exported.id IN UNNEST([%s]) + AND STARTS_WITH(tr.test_id, "%v")` - invocationNames := make([]string, len(builds)) - for i, id := range builds { - invocationNames[i] = fmt.Sprintf("invocations/build-%v", id) + var buildIds []string + for _, id := range builds { + buildIds = append(buildIds, fmt.Sprintf(`"build-%v"`, id)) + } + query := fmt.Sprintf(base_query, strings.Join(buildIds, ","), testPrefix) + + q := bq.client.Query(query) + iter, err := q.Read(ctx) + if err != nil { + return err } - pageToken := "" + var row QueryResult for { - rsp, err := r.client.QueryTestResults(ctx, &rdbpb.QueryTestResultsRequest{ - Invocations: invocationNames, - Predicate: &rdbpb.TestResultPredicate{ - TestIdRegexp: filterRegex, - }, - ReadMask: &fieldmaskpb.FieldMask{Paths: []string{ - "test_id", "status", "tags", "duration", - }}, - PageSize: 1000, // Maximum page size. - PageToken: pageToken, - }) + err := iter.Next(&row) + if err == iterator.Done { + break + } if err != nil { return err } - for _, res := range rsp.TestResults { - if err := f(res); err != nil { - return err - } - } - - pageToken = rsp.GetNextPageToken() - if pageToken == "" { - break + if err := f(&row); err != nil { + return err } }