tint: Skip e2e test validation for known-good
Cache the hash of the test output, and only re-validate if the output changes.
This speeds up incremental end-to-end testing.
Change-Id: I44bc1eff62320df1e80524e8481d1c012fb67f42
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/109761
Commit-Queue: Ben Clayton <bclayton@google.com>
Reviewed-by: James Price <jrprice@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Auto-Submit: Ben Clayton <bclayton@google.com>
diff --git a/.gitignore b/.gitignore
index 15b0736..bb0834e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -39,6 +39,9 @@
/tools/memory
/out
+# Tint test validation cache file
+/test/tint/validation.cache
+
# Modified from https://www.gitignore.io/api/vim,macos,linux,emacs,windows,sublimetext,visualstudio,visualstudiocode,intellij
### Emacs ###
diff --git a/src/tint/cmd/main.cc b/src/tint/cmd/main.cc
index 2b0268d..d3f9161 100644
--- a/src/tint/cmd/main.cc
+++ b/src/tint/cmd/main.cc
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <charconv>
#include <cstdio>
#include <fstream>
#include <iostream>
@@ -62,6 +63,11 @@
exit(1);
}
+/// Prints the given hash value in a format string that the end-to-end test runner can parse.
+void PrintHash(uint32_t hash) {
+ std::cout << "<<HASH: 0x" << std::hex << hash << ">>" << std::endl;
+}
+
enum class Format {
kNone = -1,
kSpirv,
@@ -82,9 +88,12 @@
bool parse_only = false;
bool disable_workgroup_init = false;
bool validate = false;
+ bool print_hash = false;
bool demangle = false;
bool dump_inspector_bindings = false;
+ std::unordered_set<uint32_t> skip_hash;
+
Format format = Format::kNone;
bool emit_single_entry_point = false;
@@ -122,8 +131,7 @@
-o <name> -- Output file name. Use "-" for standard output
--transform <name list> -- Runs transforms, name list is comma separated
Available transforms:
-${transforms}
- --parse-only -- Stop after parsing the input
+${transforms} --parse-only -- Stop after parsing the input
--disable-workgroup-init -- Disable workgroup memory zero initialization.
--demangle -- Preserve original source names. Demangle them.
Affects AST dumping, and text-based output languages.
@@ -135,6 +143,9 @@
default to binding 0 of the largest used group plus 1,
or group 0 if no resource bound.
--validate -- Validates the generated shader with all available validators
+ --skip-hash <hash list> -- Skips validation if the hash of the output is equal to any
+ of the hash codes in the comma separated list of hashes
+ --print-hash -- Emit the hash of the output program
--fxc -- Path to FXC dll, used to validate HLSL output.
When specified, automatically enables HLSL validation with FXC
--dxc -- Path to DXC executable, used to validate HLSL output.
@@ -431,6 +442,24 @@
opts->dump_inspector_bindings = true;
} else if (arg == "--validate") {
opts->validate = true;
+ } else if (arg == "--skip-hash") {
+ ++i;
+ if (i >= args.size()) {
+ std::cerr << "Missing hash value for " << arg << std::endl;
+ return false;
+ }
+ for (auto hash : split_on_comma(args[i])) {
+ uint32_t value = 0;
+ int base = 10;
+ if (hash.size() > 2 && hash[0] == '0' && (hash[1] == 'x' || hash[1] == 'X')) {
+ hash = hash.substr(2);
+ base = 16;
+ }
+ std::from_chars(hash.data(), hash.data() + hash.size(), value, base);
+ opts->skip_hash.emplace(value);
+ }
+ } else if (arg == "--print-hash") {
+ opts->print_hash = true;
} else if (arg == "--fxc") {
++i;
if (i >= args.size()) {
@@ -682,7 +711,12 @@
}
}
- if (options.validate) {
+ const auto hash = tint::utils::CRC32(result.spirv.data(), result.spirv.size());
+ if (options.print_hash) {
+ PrintHash(hash);
+ }
+
+ if (options.validate && options.skip_hash.count(hash) == 0) {
// Use Vulkan 1.1, since this is what Tint, internally, uses.
spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_1);
tools.SetMessageConsumer(
@@ -722,7 +756,12 @@
return false;
}
- if (options.validate) {
+ const auto hash = tint::utils::CRC32(result.wgsl.data(), result.wgsl.size());
+ if (options.print_hash) {
+ PrintHash(hash);
+ }
+
+ if (options.validate && options.skip_hash.count(hash) == 0) {
// Attempt to re-parse the output program with Tint's WGSL reader.
auto source = std::make_unique<tint::Source::File>(options.input_filename, result.wgsl);
auto reparsed_program = tint::reader::wgsl::Parse(source.get());
@@ -772,7 +811,12 @@
return false;
}
- if (options.validate) {
+ const auto hash = tint::utils::CRC32(result.msl.c_str());
+ if (options.print_hash) {
+ PrintHash(hash);
+ }
+
+ if (options.validate && options.skip_hash.count(hash) == 0) {
tint::val::Result res;
#ifdef TINT_ENABLE_MSL_VALIDATION_USING_METAL_API
res = tint::val::MslUsingMetalAPI(result.msl);
@@ -828,11 +872,17 @@
return false;
}
+ const auto hash = tint::utils::CRC32(result.hlsl.c_str());
+ if (options.print_hash) {
+ PrintHash(hash);
+ }
+
// If --fxc or --dxc was passed, then we must explicitly find and validate with that respective
// compiler.
const bool must_validate_dxc = !options.dxc_path.empty();
const bool must_validate_fxc = !options.fxc_path.empty();
- if (options.validate || must_validate_dxc || must_validate_fxc) {
+ if ((options.validate || must_validate_dxc || must_validate_fxc) &&
+ (options.skip_hash.count(hash) == 0)) {
tint::val::Result dxc_res;
bool dxc_found = false;
if (options.validate || must_validate_dxc) {
@@ -959,7 +1009,12 @@
return false;
}
- if (options.validate) {
+ const auto hash = tint::utils::CRC32(result.glsl.c_str());
+ if (options.print_hash) {
+ PrintHash(hash);
+ }
+
+ if (options.validate && options.skip_hash.count(hash) == 0) {
for (auto entry_pt : result.entry_points) {
EShLanguage lang = pipeline_stage_to_esh_language(entry_pt.second);
glslang::TShader shader(lang);
diff --git a/src/tint/utils/crc32.h b/src/tint/utils/crc32.h
index 5123612..c9697b7 100644
--- a/src/tint/utils/crc32.h
+++ b/src/tint/utils/crc32.h
@@ -16,56 +16,62 @@
#define SRC_TINT_UTILS_CRC32_H_
#include <stdint.h>
+#include <cstddef>
namespace tint::utils {
-/// @returns the CRC32 of the string `s`.
+constexpr uint32_t kCRC32LUT[] = {
+ 0, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
+ 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
+ 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
+ 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
+ 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
+ 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
+ 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
+ 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
+ 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
+ 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
+ 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
+ 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
+ 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
+ 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
+ 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
+ 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
+ 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
+ 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
+ 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
+ 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
+ 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
+ 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
+ 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
+ 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
+ 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
+ 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
+ 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
+ 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
+ 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
+ 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
+ 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
+ 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d};
+
+/// @returns the CRC32 of the string @p s.
/// @note this function can be used to calculate the CRC32 of a string literal
/// at compile time.
/// @see https://en.wikipedia.org/wiki/Cyclic_redundancy_check#CRC-32_algorithm
constexpr uint32_t CRC32(const char* s) {
- constexpr uint32_t kLUT[] = {
- 0, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535,
- 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd,
- 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d,
- 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,
- 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4,
- 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c,
- 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac,
- 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
- 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab,
- 0xb6662d3d, 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f,
- 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb,
- 0x086d3d2d, 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
- 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea,
- 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, 0x4db26158, 0x3ab551ce,
- 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a,
- 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
- 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409,
- 0xce61e49f, 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81,
- 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739,
- 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
- 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, 0x8708a3d2, 0x1e01f268,
- 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0,
- 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8,
- 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
- 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef,
- 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703,
- 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7,
- 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a,
- 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae,
- 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
- 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, 0x88085ae6,
- 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
- 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d,
- 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5,
- 0x47b2cf7f, 0x30b5ffe9, 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605,
- 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
- 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d};
-
uint32_t crc = 0xffffffff;
for (auto* p = s; *p != '\0'; ++p) {
- crc = (crc >> 8) ^ kLUT[static_cast<uint8_t>(crc) ^ static_cast<uint8_t>(*p)];
+ crc = (crc >> 8) ^ kCRC32LUT[static_cast<uint8_t>(crc) ^ static_cast<uint8_t>(*p)];
+ }
+ return crc ^ 0xffffffff;
+}
+
+/// @returns the CRC32 of the data at @p ptr of size @p size.
+inline uint32_t CRC32(const void* ptr, size_t size) {
+ auto* p = static_cast<const uint8_t*>(ptr);
+ uint32_t crc = 0xffffffff;
+ while (size--) {
+ crc = (crc >> 8) ^ kCRC32LUT[static_cast<uint8_t>(crc) ^ *p++];
}
return crc ^ 0xffffffff;
}
diff --git a/tools/src/cmd/test-runner/main.go b/tools/src/cmd/test-runner/main.go
index e8ea9dd..f83fec1 100644
--- a/tools/src/cmd/test-runner/main.go
+++ b/tools/src/cmd/test-runner/main.go
@@ -17,6 +17,8 @@
import (
"context"
+ "crypto/sha256"
+ "encoding/json"
"flag"
"fmt"
"io/ioutil"
@@ -190,6 +192,8 @@
defaultMSLExe = "metal.exe"
}
+ toolchainHash := sha256.New()
+
// If explicit verification compilers have been specified, check they exist.
// Otherwise, look on PATH for them, but don't error if they cannot be found.
for _, tool := range []struct {
@@ -222,9 +226,18 @@
}
color.Unset()
fmt.Println()
+
+ toolchainHash.Write([]byte(tool.name))
+ if s, err := os.Stat(*tool.path); err == nil {
+ toolchainHash.Write([]byte(s.ModTime().String()))
+ toolchainHash.Write([]byte(fmt.Sprint(s.Size())))
+ }
}
fmt.Println()
+ validationCache := loadValidationCache(fmt.Sprintf("%x", toolchainHash.Sum(nil)))
+ defer saveValidationCache(validationCache)
+
// Build the list of results.
// These hold the chans used to report the job results.
results := make([]map[outputFormat]chan status, len(files))
@@ -247,6 +260,7 @@
xcrunPath: xcrunPath,
generateExpected: generateExpected,
generateSkip: generateSkip,
+ validationCache: validationCache,
}
for cpu := 0; cpu < numCPU; cpu++ {
go func() {
@@ -326,6 +340,8 @@
printFormatsHeader()
printHorizontalLine()
+ newKnownGood := knownGoodHashes{}
+
for i, file := range files {
results := results[i]
@@ -343,6 +359,11 @@
for _, format := range formats {
columnWidth := formatWidth(format)
result := <-results[format]
+
+ // Update the known-good hashes
+ newKnownGood[fileAndFormat{file, format}] = result.passHashes
+
+ // Update stats
stats := statsByFmt[format]
stats.numTests++
stats.timeTaken += result.timeTaken
@@ -351,6 +372,7 @@
file: file, format: format, err: err,
})
}
+
switch result.code {
case pass:
green.Fprintf(row, alignCenter("PASS", columnWidth))
@@ -375,6 +397,17 @@
}
}
+ // Update the validation cache known-good hashes.
+ // This has to be done after all the results have been collected to avoid
+ // concurrent access on the map.
+ for ff, hashes := range newKnownGood {
+ if len(newKnownGood) > 0 {
+ validationCache.knownGood[ff] = hashes
+ } else {
+ delete(validationCache.knownGood, ff)
+ }
+ }
+
printHorizontalLine()
printFormatsHeader()
printHorizontalLine()
@@ -497,9 +530,10 @@
)
type status struct {
- code statusCode
- err error
- timeTaken time.Duration
+ code statusCode
+ err error
+ timeTaken time.Duration
+ passHashes []string
}
type job struct {
@@ -517,6 +551,7 @@
xcrunPath string
generateExpected bool
generateSkip bool
+ validationCache validationCache
}
func (j job) run(cfg runConfig) {
@@ -558,6 +593,14 @@
args := []string{
file,
"--format", strings.Split(string(j.format), "-")[0], // 'hlsl-fxc' -> 'hlsl', etc.
+ "--print-hash",
+ }
+
+ // Append any skip-hashes, if they're found.
+ if j.format != "wgsl" { // Don't skip 'wgsl' as this 'toolchain' is ever changing.
+ if skipHashes := cfg.validationCache.knownGood[fileAndFormat{file, j.format}]; len(skipHashes) > 0 {
+ args = append(args, "--skip-hash", strings.Join(skipHashes, ","))
+ }
}
// Can we validate?
@@ -597,6 +640,7 @@
timeTaken := time.Since(start)
out = strings.ReplaceAll(out, "\r\n", "\n")
+ out, hashes := extractValidationHashes(out)
matched := expected == "" || expected == out
canEmitPassExpectationFile := true
@@ -627,7 +671,7 @@
switch {
case ok && matched:
// Test passed
- return status{code: pass, timeTaken: timeTaken}
+ return status{code: pass, timeTaken: timeTaken, passHashes: hashes}
// --- Below this point the test has failed ---
@@ -676,6 +720,24 @@
}()
}
+var reValidationHash = regexp.MustCompile(`<<HASH: ([^>]*)>>\n`)
+
+// Parses and returns the validation hashes emitted by tint, or an empty string
+// if the hash wasn't found, along with the input string with the validation
+// hashes removed.
+func extractValidationHashes(in string) (out string, hashes []string) {
+ matches := reValidationHash.FindAllStringSubmatch(in, -1)
+ if matches == nil {
+ return in, nil
+ }
+ out = in
+ for _, match := range matches {
+ out = strings.ReplaceAll(out, match[0], "")
+ hashes = append(hashes, match[1])
+ }
+ return out, hashes
+}
+
// indent returns the string 's' indented with 'n' whitespace characters
func indent(s string, n int) string {
tab := strings.Repeat(" ", n)
@@ -803,3 +865,109 @@
}
return sb.String()
}
+
+// fileAndFormat is a pair of test file path and output format.
+type fileAndFormat struct {
+ file string
+ format outputFormat
+}
+
+// Used to optimize end-to-end testing of tint
+type validationCache struct {
+ // A hash of all the validation toolchains in use.
+ toolchainHash string
+ // A map of fileAndFormat to known-good (validated) output hashes.
+ knownGood knownGoodHashes
+}
+
+// A map of fileAndFormat to known-good (validated) output hashes.
+type knownGoodHashes map[fileAndFormat][]string
+
+// The serialized form of a known-good validation.cache file
+type ValidationCacheFile struct {
+ ToolchainHash string
+ KnownGood []ValidationCacheFileKnownGood
+}
+
+type ValidationCacheFileKnownGood struct {
+ File string
+ Format outputFormat
+ Hashes []string
+}
+
+func validationCachePath() string {
+ return filepath.Join(fileutils.DawnRoot(), "test", "tint", "validation.cache")
+}
+
+// loadValidationCache attempts to load the validation cache.
+// Returns an empty cache if the file could not be loaded, or if toolchains have changed.
+func loadValidationCache(toolchainHash string) validationCache {
+ out := validationCache{
+ toolchainHash: toolchainHash,
+ knownGood: knownGoodHashes{},
+ }
+
+ file, err := os.Open(validationCachePath())
+ if err != nil {
+ return out
+ }
+ defer file.Close()
+
+ content := ValidationCacheFile{}
+ if err := json.NewDecoder(file).Decode(&content); err != nil {
+ return out
+ }
+
+ if content.ToolchainHash != toolchainHash {
+ color.Set(color.FgYellow)
+ fmt.Println("Toolchains have changed - clearing validation cache")
+ color.Unset()
+ return out
+ }
+
+ for _, knownGood := range content.KnownGood {
+ out.knownGood[fileAndFormat{knownGood.File, knownGood.Format}] = knownGood.Hashes
+ }
+
+ return out
+}
+
+// saveValidationCache saves the validation cache file.
+func saveValidationCache(vc validationCache) error {
+ out := ValidationCacheFile{
+ ToolchainHash: vc.toolchainHash,
+ KnownGood: make([]ValidationCacheFileKnownGood, 0, len(vc.knownGood)),
+ }
+
+ for ff, hashes := range vc.knownGood {
+ out.KnownGood = append(out.KnownGood, ValidationCacheFileKnownGood{
+ File: ff.file,
+ Format: ff.format,
+ Hashes: hashes,
+ })
+ }
+
+ sort.Slice(out.KnownGood, func(i, j int) bool {
+ switch {
+ case out.KnownGood[i].File < out.KnownGood[j].File:
+ return true
+ case out.KnownGood[i].File > out.KnownGood[j].File:
+ return false
+ case out.KnownGood[i].Format < out.KnownGood[j].Format:
+ return true
+ case out.KnownGood[i].Format > out.KnownGood[j].Format:
+ return false
+ }
+ return false
+ })
+
+ file, err := os.Create(validationCachePath())
+ if err != nil {
+ return fmt.Errorf("failed to save the validation cache file: %w", err)
+ }
+ defer file.Close()
+
+ enc := json.NewEncoder(file)
+ enc.SetIndent("", " ")
+ return enc.Encode(&out)
+}
