[tools][cts] Export results to Google Sheets each roll
go/webgpu-cts-stats
Change-Id: Ie2770e0fe19a77ecf6f8fd8b79c593c39e36eff7
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/173520
Kokoro: Kokoro <noreply+kokoro@google.com>
Commit-Queue: Ben Clayton <bclayton@google.com>
Reviewed-by: Austin Eng <enga@chromium.org>
diff --git a/tools/src/auth/auth.go b/tools/src/auth/auth.go
index 7f6bac7..9bf4009 100644
--- a/tools/src/auth/auth.go
+++ b/tools/src/auth/auth.go
@@ -35,11 +35,12 @@
// DefaultAuthOptions returns the default authentication options for use by
// command line arguments.
-func DefaultAuthOptions() auth.Options {
+func DefaultAuthOptions(additionalScopes ...string) auth.Options {
def := chromeinfra.DefaultAuthOptions()
def.SecretsDir = fileutils.ExpandHome("~/.config/dawn-cts")
def.Scopes = append(def.Scopes,
"https://www.googleapis.com/auth/gerritcodereview",
auth.OAuthScopeEmail)
+ def.Scopes = append(def.Scopes, additionalScopes...)
return def
}
diff --git a/tools/src/cmd/cts/common/export.go b/tools/src/cmd/cts/common/export.go
new file mode 100644
index 0000000..0c2123f
--- /dev/null
+++ b/tools/src/cmd/cts/common/export.go
@@ -0,0 +1,230 @@
+// Copyright 2024 The Dawn & Tint Authors
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+// list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package common
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "os/exec"
+ "strings"
+ "time"
+
+ "dawn.googlesource.com/dawn/tools/src/container"
+ "dawn.googlesource.com/dawn/tools/src/cts/result"
+ "go.chromium.org/luci/auth"
+ "google.golang.org/api/option"
+ "google.golang.org/api/sheets/v4"
+)
+
+// Export appends the results binned by status and number of unimplemented tests to a Google Sheets document.
+func Export(ctx context.Context,
+ credentials auth.Options,
+ sheetsID, ctsDir, node, npmPath string,
+ resultsByExecutionMode result.ResultsByExecutionMode) error {
+ // Create the sheets service client
+ http, err := auth.NewAuthenticator(ctx, auth.InteractiveLogin, credentials).Client()
+ if err != nil {
+ return err
+ }
+
+ s, err := sheets.NewService(ctx, option.WithHTTPClient(http))
+ if err != nil {
+ return fmt.Errorf("unable to create sheets client: %w", err)
+ }
+
+ // Get the CTS statistics spreadsheet
+ spreadsheet, err := s.Spreadsheets.Get(sheetsID).Do()
+ if err != nil {
+ return fmt.Errorf("failed to get spreadsheet: %w", err)
+ }
+
+ // Grab the CTS revision to count the number of unimplemented tests
+ numUnimplemented, err := CountUnimplementedTests(ctx, ctsDir, node, npmPath)
+ if err != nil {
+ return fmt.Errorf("failed to obtain number of unimplemented tests: %w", err)
+ }
+
+ // Generate a new set of counts of test by status
+ for mode, results := range resultsByExecutionMode {
+ name := string(mode) + "-data"
+ // Scan the sheets of the spreadsheet (tabs at the bottom) for the sheet named `name`.
+ var dataSheet *sheets.Sheet
+ for _, sheet := range spreadsheet.Sheets {
+ if strings.ToLower(sheet.Properties.Title) == name {
+ dataSheet = sheet
+ break
+ }
+ }
+ if dataSheet == nil {
+ return fmt.Errorf("failed to find sheet '%v'", name)
+ }
+
+ // Fetch the table column names
+ columns, err := fetchRow[string](s, spreadsheet, dataSheet, 0)
+
+ counts := container.NewMap[string, int]()
+ for _, r := range results {
+ counts[string(r.Status)] = counts[string(r.Status)] + 1
+ }
+
+ { // Report statuses that have no column
+ missingColumns := container.NewSet(counts.Keys()...)
+ missingColumns.RemoveAll(container.NewSet(columns...))
+ for _, status := range missingColumns.List() {
+ log.Println("no column for status", status)
+ }
+ }
+
+ // Generate new cell data based on the table column names
+ data := []any{}
+ for _, column := range columns {
+ switch strings.ToLower(column) {
+ case "date":
+ data = append(data, time.Now().UTC().Format("2006-01-02"))
+ case "unimplemented":
+ data = append(data, numUnimplemented)
+ default:
+ count, ok := counts[column]
+ if !ok {
+ log.Println("no results with status", column)
+ }
+ data = append(data, count)
+ }
+ }
+
+ // Insert a blank row under the column header row
+ if err := insertBlankRows(s, spreadsheet, dataSheet, 1, 1); err != nil {
+ return err
+ }
+
+ // Add a new row to the spreadsheet
+ _, err = s.Spreadsheets.Values.BatchUpdate(spreadsheet.SpreadsheetId,
+ &sheets.BatchUpdateValuesRequest{
+ ValueInputOption: "RAW",
+ Data: []*sheets.ValueRange{{
+ Range: rowRange(1, dataSheet),
+ Values: [][]any{data},
+ }},
+ }).Do()
+ if err != nil {
+ return fmt.Errorf("failed to update spreadsheet: %v\n\ndata: %+v", err, data)
+ }
+ }
+
+ return nil
+}
+
+// rowRange returns a sheets range ("name!Ai:i") for the entire row with the
+// given index.
+func rowRange(index int, sheet *sheets.Sheet) string {
+ return fmt.Sprintf("%v!A%v:%v", sheet.Properties.Title, index+1, index+1)
+}
+
+// columnRange returns a sheets range ("name!i1:i") for the entire column with
+// the given index.
+func columnRange(index int, sheet *sheets.Sheet) string {
+ col := 'A' + index
+ if index > 25 {
+ panic("UNIMPLEMENTED")
+ }
+ return fmt.Sprintf("%v!%c1:%c", sheet.Properties.Title, col, col)
+}
+
+// fetchRow returns all the values in the given sheet's row.
+func fetchRow[T any](srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, row int) ([]T, error) {
+ rng := rowRange(row, sheet)
+ data, err := srv.Spreadsheets.Values.Get(spreadsheet.SpreadsheetId, rng).Do()
+ if err != nil {
+ return nil, fmt.Errorf("Couldn't fetch %v: %w", rng, err)
+ }
+ out := make([]T, len(data.Values[0]))
+ for column, v := range data.Values[0] {
+ val, ok := v.(T)
+ if !ok {
+ return nil, fmt.Errorf("cell at %v:%v was type %T, but expected type %T", row, column, v, val)
+ }
+ out[column] = val
+ }
+ return out, nil
+}
+
+// insertBlankRows inserts blank rows into the given sheet.
+func insertBlankRows(srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, aboveRow, count int) error {
+ req := sheets.BatchUpdateSpreadsheetRequest{
+ Requests: []*sheets.Request{{
+ InsertRange: &sheets.InsertRangeRequest{
+ Range: &sheets.GridRange{
+ SheetId: sheet.Properties.SheetId,
+ StartRowIndex: int64(aboveRow),
+ EndRowIndex: int64(aboveRow + count),
+ },
+ ShiftDimension: "ROWS",
+ }},
+ },
+ }
+ if _, err := srv.Spreadsheets.BatchUpdate(spreadsheet.SpreadsheetId, &req).Do(); err != nil {
+ return fmt.Errorf("BatchUpdate failed: %v", err)
+ }
+ return nil
+}
+
+// CountUnimplementedTests returns the number of unimplemented tests in the CTS.
+// It does this by running the cmdline.ts CTS command with '--list-unimplemented webgpu:*'.
+func CountUnimplementedTests(ctx context.Context, ctsDir, node, npmPath string) (int, error) {
+ if err := InstallCTSDeps(ctx, ctsDir, npmPath); err != nil {
+ return 0, err
+ }
+
+ // Run 'src/common/runtime/cmdline.ts' to obtain the full test list
+ cmd := exec.CommandContext(ctx, node,
+ "-e", "require('./src/common/tools/setup-ts-in-node.js');require('./src/common/runtime/cmdline.ts');",
+ "--", // Start of arguments
+ // src/common/runtime/helper/sys.ts expects 'node file.js <args>'
+ // and slices away the first two arguments. When running with '-e', args
+ // start at 1, so just inject a placeholder argument.
+ "placeholder-arg",
+ "--list-unimplemented",
+ "webgpu:*",
+ )
+ cmd.Dir = ctsDir
+
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ return 0, fmt.Errorf("failed to gather unimplemented tests: %w", err)
+ }
+
+ lines := strings.Split(string(out), "\n")
+ count := 0
+ for _, line := range lines {
+ if strings.TrimSpace(line) != "" {
+ count++
+ }
+ }
+ return count, nil
+}
diff --git a/tools/src/cmd/cts/export/export.go b/tools/src/cmd/cts/export/export.go
index 7a1f181..c7319ac 100644
--- a/tools/src/cmd/cts/export/export.go
+++ b/tools/src/cmd/cts/export/export.go
@@ -31,23 +31,17 @@
"context"
"flag"
"fmt"
- "io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
- "strings"
- "time"
"dawn.googlesource.com/dawn/tools/src/auth"
"dawn.googlesource.com/dawn/tools/src/cmd/cts/common"
- "dawn.googlesource.com/dawn/tools/src/cts/result"
"dawn.googlesource.com/dawn/tools/src/fileutils"
"dawn.googlesource.com/dawn/tools/src/git"
"dawn.googlesource.com/dawn/tools/src/gitiles"
"go.chromium.org/luci/auth/client/authcli"
- "golang.org/x/oauth2"
- "golang.org/x/oauth2/google"
"google.golang.org/api/sheets/v4"
)
@@ -57,8 +51,10 @@
type cmd struct {
flags struct {
- auth authcli.Flags
- results common.ResultSource
+ auth authcli.Flags
+ results common.ResultSource
+ npmPath string
+ nodePath string
}
}
@@ -71,8 +67,11 @@
}
func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) {
- c.flags.auth.Register(flag.CommandLine, auth.DefaultAuthOptions())
+ c.flags.auth.Register(flag.CommandLine, auth.DefaultAuthOptions(sheets.SpreadsheetsScope))
c.flags.results.RegisterFlags(cfg)
+ npmPath, _ := exec.LookPath("npm")
+ flag.StringVar(&c.flags.npmPath, "npm", npmPath, "path to npm")
+ flag.StringVar(&c.flags.nodePath, "node", fileutils.NodePath(), "path to node")
return nil, nil
}
@@ -83,48 +82,6 @@
return fmt.Errorf("failed to obtain authentication options: %w", err)
}
- // Load the credentials used for accessing the sheets document
- authdir := fileutils.ExpandHome(os.ExpandEnv(auth.SecretsDir))
- credentialsPath := filepath.Join(authdir, "credentials.json")
- b, err := ioutil.ReadFile(credentialsPath)
- if err != nil {
- return fmt.Errorf("unable to read credentials file '%v'\n"+
- "Obtain this file from: https://console.developers.google.com/apis/credentials\n%w",
- credentialsPath, err)
- }
- credentials, err := google.CredentialsFromJSON(ctx, b, "https://www.googleapis.com/auth/spreadsheets")
- if err != nil {
- return fmt.Errorf("unable to parse client secret file to config: %w", err)
- }
-
- // Create the sheets service client
- s, err := sheets.New(oauth2.NewClient(ctx, credentials.TokenSource))
- if err != nil {
- return fmt.Errorf("unable to create sheets client: %w", err)
- }
-
- // Get the CTS statistics spreadsheet
- spreadsheet, err := s.Spreadsheets.Get(cfg.Sheets.ID).Do()
- if err != nil {
- return fmt.Errorf("failed to get spreadsheet: %w", err)
- }
-
- // Scan the sheets of the spreadsheet (tabs at the bottom) for the 'data'
- // sheet.
- var dataSheet *sheets.Sheet
- for _, sheet := range spreadsheet.Sheets {
- if strings.ToLower(sheet.Properties.Title) == "data" {
- dataSheet = sheet
- break
- }
- }
- if dataSheet == nil {
- return fmt.Errorf("failed to find data sheet")
- }
-
- // Fetch the table column names
- columns, err := fetchRow[string](s, spreadsheet, dataSheet, 0)
-
// Grab the resultsByExecutionMode
resultsByExecutionMode, err := c.flags.results.GetResults(ctx, cfg, auth)
if err != nil {
@@ -133,6 +90,8 @@
if len(resultsByExecutionMode) == 0 {
return fmt.Errorf("no results found")
}
+
+ // Note: GetResults() will update the Patchset to the latest roll, if not explicitly set.
ps := c.flags.results.Patchset
// Find the CTS revision
@@ -149,185 +108,38 @@
return fmt.Errorf("failed to find CTS hash in deps: %w", err)
}
- // Grab the CTS revision to count the number of unimplemented tests
- numUnimplemented, err := countUnimplementedTests(cfg, ctsHash)
- if err != nil {
- return fmt.Errorf("failed to obtain number of unimplemented tests: %w", err)
- }
+ log.Printf("checking out cts @ '%v'...", ctsHash)
- // Generate a new set of counts of test by status
- log.Printf("exporting results from cl %v ps %v...", ps.Change, ps.Patchset)
- for _, results := range resultsByExecutionMode {
- counts := map[result.Status]int{}
- for _, r := range results {
- counts[r.Status] = counts[r.Status] + 1
- }
-
- // Generate new cell data based on the table column names
- data := []any{}
- for _, column := range columns {
- switch strings.ToLower(column) {
- case "date":
- data = append(data, time.Now().UTC().Format("2006-01-02"))
- case "change":
- data = append(data, ps.Change)
- case "unimplemented":
- data = append(data, numUnimplemented)
- default:
- count, ok := counts[result.Status(column)]
- if !ok {
- log.Println("no results with status", column)
- }
- data = append(data, count)
- }
- }
-
- // Insert a blank row under the column header row
- if err := insertBlankRows(s, spreadsheet, dataSheet, 1, 1); err != nil {
- return err
- }
-
- // Add a new row to the spreadsheet
- _, err = s.Spreadsheets.Values.BatchUpdate(spreadsheet.SpreadsheetId,
- &sheets.BatchUpdateValuesRequest{
- ValueInputOption: "RAW",
- Data: []*sheets.ValueRange{{
- Range: rowRange(1, dataSheet),
- Values: [][]any{data},
- }},
- }).Do()
- if err != nil {
- return fmt.Errorf("failed to update spreadsheet: %v", err)
- }
- }
-
- return nil
-}
-
-// rowRange returns a sheets range ("name!Ai:i") for the entire row with the
-// given index.
-func rowRange(index int, sheet *sheets.Sheet) string {
- return fmt.Sprintf("%v!A%v:%v", sheet.Properties.Title, index+1, index+1)
-}
-
-// columnRange returns a sheets range ("name!i1:i") for the entire column with
-// the given index.
-func columnRange(index int, sheet *sheets.Sheet) string {
- col := 'A' + index
- if index > 25 {
- panic("UNIMPLEMENTED")
- }
- return fmt.Sprintf("%v!%c1:%c", sheet.Properties.Title, col, col)
-}
-
-// fetchRow returns all the values in the given sheet's row.
-func fetchRow[T any](srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, row int) ([]T, error) {
- rng := rowRange(row, sheet)
- data, err := srv.Spreadsheets.Values.Get(spreadsheet.SpreadsheetId, rng).Do()
- if err != nil {
- return nil, fmt.Errorf("Couldn't fetch %v: %w", rng, err)
- }
- out := make([]T, len(data.Values[0]))
- for column, v := range data.Values[0] {
- val, ok := v.(T)
- if !ok {
- return nil, fmt.Errorf("cell at %v:%v was type %T, but expected type %T", row, column, v, val)
- }
- out[column] = val
- }
- return out, nil
-}
-
-// insertBlankRows inserts blank rows into the given sheet.
-func insertBlankRows(srv *sheets.Service, spreadsheet *sheets.Spreadsheet, sheet *sheets.Sheet, aboveRow, count int) error {
- req := sheets.BatchUpdateSpreadsheetRequest{
- Requests: []*sheets.Request{{
- InsertRange: &sheets.InsertRangeRequest{
- Range: &sheets.GridRange{
- SheetId: sheet.Properties.SheetId,
- StartRowIndex: int64(aboveRow),
- EndRowIndex: int64(aboveRow + count),
- },
- ShiftDimension: "ROWS",
- }},
- },
- }
- if _, err := srv.Spreadsheets.BatchUpdate(spreadsheet.SpreadsheetId, &req).Do(); err != nil {
- return fmt.Errorf("BatchUpdate failed: %v", err)
- }
- return nil
-}
-
-// countUnimplementedTests checks out the WebGPU CTS at ctsHash, builds the node
-// command line tool, and runs it with '--list-unimplemented webgpu:*' to count
-// the total number of unimplemented tests, which is returned.
-func countUnimplementedTests(cfg common.Config, ctsHash string) (int, error) {
tmpDir, err := os.MkdirTemp("", "dawn-cts-export")
if err != nil {
- return 0, err
+ return err
}
defer os.RemoveAll(tmpDir)
- dir := filepath.Join(tmpDir, "cts")
+ ctsDir := filepath.Join(tmpDir, "cts")
gitExe, err := exec.LookPath("git")
if err != nil {
- return 0, fmt.Errorf("failed to find git on PATH: %w", err)
+ return fmt.Errorf("failed to find git on PATH: %w", err)
}
git, err := git.New(gitExe)
if err != nil {
- return 0, err
+ return err
}
- log.Printf("cloning cts to '%v'...", dir)
- repo, err := git.Clone(dir, cfg.Git.CTS.HttpsURL(), nil)
+ log.Printf("cloning cts to '%v'...", ctsDir)
+ repo, err := git.Clone(ctsDir, cfg.Git.CTS.HttpsURL(), nil)
if err != nil {
- return 0, fmt.Errorf("failed to clone cts: %v", err)
+ return fmt.Errorf("failed to clone cts: %v", err)
}
- log.Printf("checking out cts @ '%v'...", ctsHash)
if _, err := repo.Fetch(ctsHash, nil); err != nil {
- return 0, fmt.Errorf("failed to fetch cts: %v", err)
+ return fmt.Errorf("failed to fetch cts: %v", err)
}
if err := repo.Checkout(ctsHash, nil); err != nil {
- return 0, fmt.Errorf("failed to clone cts: %v", err)
+ return fmt.Errorf("failed to clone cts: %v", err)
}
- {
- npm, err := exec.LookPath("npm")
- if err != nil {
- return 0, fmt.Errorf("failed to find npm on PATH: %w", err)
- }
- cmd := exec.Command(npm, "ci")
- cmd.Dir = dir
- if out, err := cmd.CombinedOutput(); err != nil {
- return 0, fmt.Errorf("failed to run npm ci: %w\n%v", err, string(out))
- }
- }
- {
- npx, err := exec.LookPath("npx")
- if err != nil {
- return 0, fmt.Errorf("failed to find npx on PATH: %w", err)
- }
- cmd := exec.Command(npx, "grunt", "run:build-out-node")
- cmd.Dir = dir
- if out, err := cmd.CombinedOutput(); err != nil {
- return 0, fmt.Errorf("failed to build CTS typescript: %w\n%v", err, string(out))
- }
- }
- {
- sh, err := exec.LookPath("node")
- if err != nil {
- return 0, fmt.Errorf("failed to find sh on PATH: %w", err)
- }
- cmd := exec.Command(sh, "./tools/run_node", "--list-unimplemented", "webgpu:*")
- cmd.Dir = dir
- out, err := cmd.CombinedOutput()
- if err != nil {
- return 0, fmt.Errorf("failed to gather unimplemented tests: %w", err)
- }
- lines := strings.Split(string(out), "\n")
- return len(lines), nil
- }
+ return common.Export(ctx, auth, cfg.Sheets.ID, ctsDir, c.flags.nodePath, c.flags.npmPath, resultsByExecutionMode)
}
diff --git a/tools/src/cmd/cts/roll/roll.go b/tools/src/cmd/cts/roll/roll.go
index f1e0734..7b77bc4 100644
--- a/tools/src/cmd/cts/roll/roll.go
+++ b/tools/src/cmd/cts/roll/roll.go
@@ -60,6 +60,7 @@
"dawn.googlesource.com/dawn/tools/src/resultsdb"
"go.chromium.org/luci/auth"
"go.chromium.org/luci/auth/client/authcli"
+ "google.golang.org/api/sheets/v4"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
@@ -106,7 +107,7 @@
func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, error) {
gitPath, _ := exec.LookPath("git")
npmPath, _ := exec.LookPath("npm")
- c.flags.auth.Register(flag.CommandLine, commonAuth.DefaultAuthOptions())
+ c.flags.auth.Register(flag.CommandLine, commonAuth.DefaultAuthOptions(sheets.SpreadsheetsScope))
flag.StringVar(&c.flags.gitPath, "git", gitPath, "path to git")
flag.StringVar(&c.flags.npmPath, "npm", npmPath, "path to npm")
flag.StringVar(&c.flags.nodePath, "node", fileutils.NodePath(), "path to node")
@@ -368,6 +369,18 @@
return fmt.Errorf("failed to update change '%v': %v", changeID, err)
}
+ var psResultsByExecutionMode result.ResultsByExecutionMode
+
+ defer func() {
+ // Export the results to the Google Sheets whether the roll succeeded or failed.
+ if psResultsByExecutionMode != nil {
+ log.Println("exporting results...")
+ if err := common.Export(ctx, r.auth, r.cfg.Sheets.ID, r.ctsDir, r.flags.nodePath, r.flags.npmPath, psResultsByExecutionMode); err != nil {
+ log.Println("failed to update results spreadsheet: ", err)
+ }
+ }
+ }()
+
// Begin main roll loop
for attempt := 0; ; attempt++ {
// Kick builds
@@ -395,7 +408,7 @@
// Gather the build results
log.Println("gathering results...")
- psResultsByExecutionMode, err := common.CacheResults(ctx, r.cfg, ps, r.flags.cacheDir, r.rdb, builds)
+ psResultsByExecutionMode, err = common.CacheResults(ctx, r.cfg, ps, r.flags.cacheDir, r.rdb, builds)
if err != nil {
return err
}
