Delete the remove_stale_autogen_files mechanism.
Previously when moving around directories for generated files, Dawn ran
into an issue where stale files where #included instead of the new ones,
causing compilation failures. To get around this a
remove_stale_autogen_files mechanism was added that scans the gen/
directory for files not in an allow-list of directories.
This mechanism is now causing problems for bringing up Dawn standalone
tests on Android as these test also generate files in Dawn's gen/
directories, and their files get deleted by remove_stale_autogen_files.
We are not foresseing any additional shuffling of directories and it's
safe to expect that all stale files have been removed from CI builder
caches at this time. So remove_stale_autogen_files can go. This is what
this CL does.
Fixed: dawn:1543
Change-Id: I7dbf1eae6c55b7659f3837b6d4a565052001ce57
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/103040
Reviewed-by: Austin Eng <enga@chromium.org>
Commit-Queue: Corentin Wallez <cwallez@chromium.org>
diff --git a/generator/BUILD.gn b/generator/BUILD.gn
deleted file mode 100644
index a1d954a..0000000
--- a/generator/BUILD.gn
+++ /dev/null
@@ -1,63 +0,0 @@
-# Copyright 2019 The Dawn Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import("../scripts/dawn_overrides_with_defaults.gni")
-import("dawn_generator.gni")
-
-# The list of directories in which to check for stale autogenerated files.
-# It should include the list of all directories in which we ever generated
-# files but we can't just put dawn_gen_root because there are more than
-# autogenerated sources there.
-_stale_dirs = [
- "dawn",
- "dawn/native",
- "dawn/wire",
- "mock",
- "src",
-]
-
-_allowed_output_dirs_file =
- "${dawn_gen_root}/removed_stale_autogen_files.allowed_output_dirs"
-write_file(_allowed_output_dirs_file, dawn_allowed_gen_output_dirs)
-
-_stale_dirs_file = "${dawn_gen_root}/removed_stale_autogen_files.stale_dirs"
-write_file(_stale_dirs_file, _stale_dirs)
-
-_stamp_file = "${dawn_gen_root}/removed_stale_autogen_files.stamp"
-
-# An action that removes autogenerated files that aren't in allowed directories
-# see dawn_generator.gni for more details.
-action("remove_stale_autogen_files") {
- script = "remove_files.py"
- args = [
- "--root-dir",
- rebase_path(dawn_gen_root, root_build_dir),
- "--allowed-output-dirs-file",
- rebase_path(_allowed_output_dirs_file, root_build_dir),
- "--stale-dirs-file",
- rebase_path(_stale_dirs_file, root_build_dir),
- "--stamp",
- rebase_path(_stamp_file, root_build_dir),
- ]
-
- # Have the "list of file" inputs as a dependency so that the action reruns
- # as soon as they change.
- inputs = [
- _allowed_output_dirs_file,
- _stale_dirs_file,
- ]
-
- # Output a stamp file so we don't re-run this action on every build.
- outputs = [ _stamp_file ]
-}
diff --git a/generator/dawn_generator.gni b/generator/dawn_generator.gni
index 28c5301..91e4cf7 100644
--- a/generator/dawn_generator.gni
+++ b/generator/dawn_generator.gni
@@ -15,41 +15,6 @@
import("../scripts/dawn_overrides_with_defaults.gni")
import("generator_lib.gni")
-# Dawn used to put autogenerated files in a lot of different places. When we
-# started to move them around, some compilation issues arised because some
-# stale include files stayed in the build directory and were picked up.
-# To counter this, now Dawn does the following:
-#
-# 1. The generated output file directory structure has to match the structure
-# of the source tree, starting at dawn_gen_root (gen/ or
-# gen/third_party/dawn depending on where we are).
-# 2. include and dawn_gen_root/include has to match the structure of
-# the source tree too.
-# 3. Dawn files must use include relative to src/ or include such as
-# "dawn/dawn.h" or "dawn/native/backend/BackendStuff.h".
-#
-# The allowed list below ensure 1). Include directory rules for Dawn ensure 3)
-# and 2) is something we need to enforce in code review.
-#
-# However GN's toolchains automatically add some include directories for us
-# which breaks 3) slightly. To avoid stale headers in for example
-# dawn_gen_root/src/dawn/dawn/ to be picked up (instead of
-# dawn_gen_root/src/dawn), we have a special action that removes files in
-# disallowed gen directories.
-
-dawn_allowed_gen_output_dirs = [
- "src/dawn/",
- "src/dawn/common/",
- "src/dawn/native/",
- "src/dawn/native/opengl/",
- "src/dawn/wire/client/",
- "src/dawn/wire/server/",
- "src/dawn/wire/",
- "include/dawn/",
- "emscripten-bits/",
- "webgpu-headers/",
-]
-
# Template to help invoking Dawn code generators based on generator_lib
#
# dawn_generator("my_target_gen") {
@@ -84,12 +49,7 @@
# Force Dawn's autogenerated file structure to mirror exactly the source
# tree but start at ${dawn_gen_root} instead of ${dawn_root}
- allowed_output_dirs = dawn_allowed_gen_output_dirs
custom_gen_dir = dawn_gen_root
-
- # Make sure that we delete stale autogenerated file in directories that are
- # no longer used by code generation to avoid include conflicts.
- deps = [ "${dawn_root}/generator:remove_stale_autogen_files" ]
}
}
diff --git a/generator/generator_lib.gni b/generator/generator_lib.gni
index 8b9e04c..b7a5c1c 100644
--- a/generator/generator_lib.gni
+++ b/generator/generator_lib.gni
@@ -37,11 +37,6 @@
#
# jinja2_path: Optional Jinja2 installation path.
#
-# allowed_output_dirs: Optional list of directories that are the only
-# directories in which files of `outputs` are allowed to be (and not
-# in children directories). Generation will fail if an output isn't
-# in a directory in the list.
-#
# root_dir: Optional root source dir for Python dependencies
# computation. Defaults to "${generator_lib_dir}/..". Any dependency
# outside of this directory is considered a system file and will be
@@ -118,19 +113,6 @@
rebase_path(_expected_outputs_file, root_build_dir),
]
- # Check that all of the outputs are in a directory that's allowed. This is
- # useful to keep the list of directories in sink with other parts of the
- # build.
- if (defined(invoker.allowed_output_dirs)) {
- _allowed_output_dirs_file = "${_gen_dir}/${target_name}.allowed_output_dirs"
- write_file(_allowed_output_dirs_file, invoker.allowed_output_dirs)
-
- _generator_args += [
- "--allowed-output-dirs-file",
- rebase_path(_allowed_output_dirs_file, root_build_dir),
- ]
- }
-
# The code generator invocation that will write the JSON tarball, check the
# outputs are what's expected and write a depfile for Ninja.
action(_json_tarball_target) {
diff --git a/generator/generator_lib.py b/generator/generator_lib.py
index 11b3ed2..dbff07e 100644
--- a/generator/generator_lib.py
+++ b/generator/generator_lib.py
@@ -259,12 +259,6 @@
help=('Optional source root directory for Python dependency '
'computations'))
parser.add_argument(
- '--allowed-output-dirs-file',
- default=None,
- type=str,
- help=("File containing a list of allowed directories where files "
- "can be output."))
- parser.add_argument(
'--print-cmake-dependencies',
default=False,
action="store_true",
@@ -326,32 +320,6 @@
outputs = _do_renders(renders, args.template_dir)
- # The caller wants to assert that the outputs are only in specific
- # directories.
- if args.allowed_output_dirs_file != None:
- with open(args.allowed_output_dirs_file) as f:
- allowed_dirs = set([line.strip() for line in f.readlines()])
-
- for directory in allowed_dirs:
- if not directory.endswith('/'):
- print('Allowed directory entry "{}" doesn\'t '
- 'end with /'.format(directory))
- return 1
-
- def check_in_subdirectory(path, directory):
- return path.startswith(
- directory) and not '/' in path[len(directory):]
-
- for render in renders:
- if not any(
- check_in_subdirectory(render.output, directory)
- for directory in allowed_dirs):
- print('Output file "{}" is not in the allowed directory '
- 'list below:'.format(render.output))
- for directory in sorted(allowed_dirs):
- print(' "{}"'.format(directory))
- return 1
-
# Output the JSON tarball
if args.output_json_tarball != None:
json_root = {}
diff --git a/generator/remove_files.py b/generator/remove_files.py
deleted file mode 100644
index 6ddf463..0000000
--- a/generator/remove_files.py
+++ /dev/null
@@ -1,91 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2019 The Dawn Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse, glob, os, sys
-
-
-def check_in_subdirectory(path, directory):
- return path.startswith(directory) and not '/' in path[len(directory):]
-
-
-def check_is_allowed(path, allowed_dirs):
- return any(
- check_in_subdirectory(path, directory) for directory in allowed_dirs)
-
-
-def get_all_files_in_dir(find_directory):
- result = []
- for (directory, _, files) in os.walk(find_directory):
- result += [os.path.join(directory, filename) for filename in files]
- return result
-
-
-def run():
- # Parse command line arguments
- parser = argparse.ArgumentParser(
- description="Removes stale autogenerated files from gen/ directories.")
- parser.add_argument(
- '--root-dir',
- type=str,
- help='The root directory, all other paths in files are relative to it.'
- )
- parser.add_argument(
- '--allowed-output-dirs-file',
- type=str,
- help='The file containing a list of allowed directories')
- parser.add_argument(
- '--stale-dirs-file',
- type=str,
- help=
- 'The file containing a list of directories to check for stale files')
- parser.add_argument('--stamp',
- type=str,
- help='A stamp written once this script completes')
- args = parser.parse_args()
-
- root_dir = args.root_dir
- stamp_file = args.stamp
-
- # Load the list of allowed and stale directories
- with open(args.allowed_output_dirs_file) as f:
- allowed_dirs = set(
- [os.path.join(root_dir, line.strip()) for line in f.readlines()])
-
- for directory in allowed_dirs:
- if not directory.endswith('/'):
- print('Allowed directory entry "{}" doesn\'t end with /'.format(
- directory))
- return 1
-
- with open(args.stale_dirs_file) as f:
- stale_dirs = set([line.strip() for line in f.readlines()])
-
- # Remove all files in stale dirs that aren't in the allowed dirs.
- for stale_dir in stale_dirs:
- stale_dir = os.path.join(root_dir, stale_dir)
-
- for candidate in get_all_files_in_dir(stale_dir):
- if not check_is_allowed(candidate, allowed_dirs):
- os.remove(candidate)
-
- # Finished! Write the stamp file so ninja knows to not run this again.
- with open(stamp_file, "w") as f:
- f.write("")
-
- return 0
-
-
-if __name__ == "__main__":
- sys.exit(run())
diff --git a/src/tint/fuzzers/BUILD.gn b/src/tint/fuzzers/BUILD.gn
index 25e475e..d0c72a7 100644
--- a/src/tint/fuzzers/BUILD.gn
+++ b/src/tint/fuzzers/BUILD.gn
@@ -33,8 +33,6 @@
rebase_path(fuzzer_corpus_wgsl_dir, root_build_dir),
]
outputs = [ fuzzer_corpus_wgsl_stamp ]
-
- deps = [ "${dawn_root}/generator:remove_stale_autogen_files" ]
}
tint_fuzzer_common_libfuzzer_options = [
diff --git a/src/tint/fuzzers/tint_ast_fuzzer/BUILD.gn b/src/tint/fuzzers/tint_ast_fuzzer/BUILD.gn
index 58a36ea..416128c 100644
--- a/src/tint/fuzzers/tint_ast_fuzzer/BUILD.gn
+++ b/src/tint/fuzzers/tint_ast_fuzzer/BUILD.gn
@@ -23,7 +23,6 @@
sources = [ "protobufs/tint_ast_fuzzer.proto" ]
generate_python = false
use_protobuf_full = true
- deps = [ "${dawn_root}/generator:remove_stale_autogen_files" ]
}
source_set("tint_ast_fuzzer") {
