generator_lib.py: Add support for importing templates in templates
Previously importing template with {% include %} or {% from %} would
cause issue with the integration with the build system because
generator_lib.py wouldn't know which additional template files to add as
dependencies to the generator task.
Change get_file_renders into get_outputs that return both a list of
FileRenders and a list of templates that will be imported. Use that new
list both in the dependency information of the build systems, and as an
allow list in the PreprocessingLoader (so we don't forget to add new
imported templates in the future).
Fixed: 352690884
Change-Id: I9d63a209d8c82405dec16c8f5baf8b1538cf2760
Reviewed-on: https://dawn-review.googlesource.com/c/dawn/+/198254
Reviewed-by: Austin Eng <enga@chromium.org>
Commit-Queue: Corentin Wallez <cwallez@chromium.org>
Reviewed-by: Loko Kung <lokokung@google.com>
diff --git a/generator/dawn_gpu_info_generator.py b/generator/dawn_gpu_info_generator.py
index 2036cb9..4d196c7 100644
--- a/generator/dawn_gpu_info_generator.py
+++ b/generator/dawn_gpu_info_generator.py
@@ -29,7 +29,7 @@
import json, os, sys
from collections import namedtuple
-from generator_lib import Generator, run_generator, FileRender
+from generator_lib import Generator, run_generator, FileRender, GeneratorOutput
def parse_mask(mask):
@@ -223,18 +223,19 @@
def get_dependencies(self, args):
return [os.path.abspath(args.gpu_info_json)]
- def get_file_renders(self, args):
+ def get_outputs(self, args):
with open(args.gpu_info_json) as f:
loaded_json = json.loads(f.read())
params = parse_json(loaded_json)
- return [
+ renders = [
FileRender("dawn/common/GPUInfo.h",
"src/dawn/common/GPUInfo_autogen.h", [params]),
FileRender("dawn/common/GPUInfo.cpp",
"src/dawn/common/GPUInfo_autogen.cpp", [params]),
]
+ return GeneratorOutput(renders=renders, imported_templates=[])
if __name__ == "__main__":
diff --git a/generator/dawn_json_generator.py b/generator/dawn_json_generator.py
index 59c8273..d6dce98 100644
--- a/generator/dawn_json_generator.py
+++ b/generator/dawn_json_generator.py
@@ -29,7 +29,7 @@
import json, os, sys
from collections import namedtuple, defaultdict
-from generator_lib import Generator, run_generator, FileRender
+from generator_lib import Generator, run_generator, FileRender, GeneratorOutput
############################################################
# OBJECT MODEL
@@ -1217,7 +1217,7 @@
'Comma-separated subset of targets to output. Available targets: '
+ ', '.join(allowed_targets))
- def get_file_renders(self, args):
+ def get_outputs(self, args):
with open(args.dawn_json) as f:
loaded_json = json.loads(f.read())
@@ -1239,6 +1239,7 @@
lpm_json = json.loads(f.read())
renders = []
+ imported_templates = []
params_dawn = parse_json(
loaded_json,
@@ -1256,6 +1257,7 @@
api = metadata.api.lower()
prefix = metadata.proc_table_prefix.lower()
if 'headers' in targets:
+ imported_templates.append('BSD_LICENSE')
renders.append(
FileRender('api.h', 'include/dawn/' + api + '.h',
[RENDER_PARAMS_BASE, params_all]))
@@ -1317,6 +1319,7 @@
loaded_json,
enabled_tags=['compat', 'upstream', 'native'],
disabled_tags=['dawn'])
+ imported_templates.append('BSD_LICENSE')
renders.append(
FileRender('api.h', 'webgpu-headers/' + api + '.h',
[RENDER_PARAMS_BASE, params_upstream]))
@@ -1326,6 +1329,7 @@
params_emscripten = parse_json(
loaded_json, enabled_tags=['compat', 'emscripten'])
# system/include/webgpu
+ imported_templates.append('BSD_LICENSE')
renders.append(
FileRender('api.h', 'src/emdawnwebgpu/include/webgpu/webgpu.h',
[RENDER_PARAMS_BASE, params_emscripten]))
@@ -1586,6 +1590,11 @@
params_kotlin = compute_kotlin_params(loaded_json, kotlin_json)
kt_file_path = params_kotlin['kotlin_package'].replace('.', '/')
jni_name = params_kotlin['jni_name']
+
+ imported_templates += [
+ "art/api_kotlin_types.kt",
+ ]
+
by_category = params_kotlin['by_category']
for structure in by_category['structure']:
renders.append(
@@ -1637,6 +1646,11 @@
if "jni" in targets:
params_kotlin = compute_kotlin_params(loaded_json, kotlin_json)
+
+ imported_templates += [
+ "art/api_jni_types.kt",
+ ]
+
renders.append(
FileRender('art/structures.h', 'cpp/structures.h',
[RENDER_PARAMS_BASE, params_kotlin]))
@@ -1647,7 +1661,8 @@
FileRender('art/methods.cpp', 'cpp/methods.cpp',
[RENDER_PARAMS_BASE, params_kotlin]))
- return renders
+ return GeneratorOutput(renders=renders,
+ imported_templates=imported_templates)
def get_dependencies(self, args):
deps = [os.path.abspath(args.dawn_json)]
diff --git a/generator/dawn_version_generator.py b/generator/dawn_version_generator.py
index 8898c8c..7e2bc8c 100644
--- a/generator/dawn_version_generator.py
+++ b/generator/dawn_version_generator.py
@@ -28,7 +28,7 @@
import os, subprocess, sys, shutil
-from generator_lib import Generator, run_generator, FileRender
+from generator_lib import Generator, run_generator, FileRender, GeneratorOutput
def get_git():
# Will find git, git.exe, git.bat...
@@ -147,13 +147,14 @@
return []
return []
- def get_file_renders(self, args):
+ def get_outputs(self, args):
params = compute_params(args)
- return [
+ renders = [
FileRender("dawn/common/Version.h",
"src/dawn/common/Version_autogen.h", [params]),
]
+ return GeneratorOutput(renders=renders, imported_templates=[])
if __name__ == "__main__":
diff --git a/generator/generator_lib.py b/generator/generator_lib.py
index e8338b2..0f58a52 100644
--- a/generator/generator_lib.py
+++ b/generator/generator_lib.py
@@ -66,6 +66,16 @@
#
FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts'])
+# A GeneratorOutput represent everything an invocation of the generator will
+# produce.
+#
+# renders: an iterable of FileRenders.
+#
+# imported_templates: paths to additional templates that will be imported.
+# Trying to import with {% from %} will enforce that the file is listed
+# to ensure the dependency information produced is correct.
+GeneratorOutput = namedtuple('GeneratorOutput',
+ ['renders', 'imported_templates'])
# The interface that must be implemented by generators.
class Generator:
@@ -77,7 +87,7 @@
"""Add generator-specific argparse arguments."""
pass
- def get_file_renders(self, args):
+ def get_outputs(self, args):
"""Return the list of FileRender objects to process."""
return []
@@ -120,13 +130,21 @@
# A custom Jinja2 template loader that removes the extra indentation
# of the template blocks so that the output is correctly indented
class _PreprocessingLoader(jinja2.BaseLoader):
- def __init__(self, path):
+
+ def __init__(self, path, allow_list):
self.path = path
+ self.allow_list = set(allow_list)
+
+ # Check that all the listed templates exist.
+ for template in self.allow_list:
+ if not os.path.exists(os.path.join(self.path, template)):
+ raise jinja2.TemplateNotFound(template)
def get_source(self, environment, template):
- path = os.path.join(self.path, template)
- if not os.path.exists(path):
+ if not template in self.allow_list:
raise jinja2.TemplateNotFound(template)
+
+ path = os.path.join(self.path, template)
mtime = os.path.getmtime(path)
with open(path) as f:
source = self.preprocess(f.read())
@@ -177,8 +195,11 @@
_FileOutput = namedtuple('FileOutput', ['name', 'content'])
-def _do_renders(renders, template_dir):
- loader = _PreprocessingLoader(template_dir)
+def _do_renders(output, template_dir):
+ template_allow_list = [render.template for render in output.renders
+ ] + list(output.imported_templates)
+ loader = _PreprocessingLoader(template_dir, template_allow_list)
+
env = jinja2.Environment(
extensions=['jinja2.ext.do', 'jinja2.ext.loopcontrols'],
loader=loader,
@@ -202,7 +223,7 @@
}
outputs = []
- for render in renders:
+ for render in output.renders:
params = {}
params.update(base_params)
for param_dict in render.params_dicts:
@@ -316,14 +337,18 @@
args = parser.parse_args()
- renders = generator.get_file_renders(args)
+ output = generator.get_outputs(args)
# Output a list of all dependencies for CMake or the tarball for GN/Ninja.
if args.depfile != None or args.print_cmake_dependencies:
dependencies = generator.get_dependencies(args)
dependencies += [
args.template_dir + os.path.sep + render.template
- for render in renders
+ for render in output.renders
+ ]
+ dependencies += [
+ args.template_dir + os.path.sep + template
+ for template in output.imported_templates
]
dependencies += _compute_python_dependencies(args.root_dir)
@@ -342,7 +367,7 @@
with open(args.expected_outputs_file) as f:
expected = set([line.strip() for line in f.readlines()])
- actual = {render.output for render in renders}
+ actual = {render.output for render in output.renders}
if actual != expected:
print("Wrong expected outputs, caller expected:\n " +
@@ -355,16 +380,16 @@
sys.stdout.write(
_cmake_path_list([
os.path.join(args.output_dir, render.output)
- for render in renders
+ for render in output.renders
]))
return 0
- outputs = _do_renders(renders, args.template_dir)
+ render_outputs = _do_renders(output, args.template_dir)
# Output the JSON tarball
if args.output_json_tarball != None:
json_root = {}
- for output in outputs:
+ for output in render_outputs:
json_root[output.name] = output.content
with open(args.output_json_tarball, 'w') as f:
@@ -372,7 +397,7 @@
# Output the files directly.
if args.output_dir != None:
- for output in outputs:
+ for output in render_outputs:
output_path = os.path.join(args.output_dir, output.name)
directory = os.path.dirname(output_path)
diff --git a/generator/opengl_loader_generator.py b/generator/opengl_loader_generator.py
index 3865ffb..671e006 100644
--- a/generator/opengl_loader_generator.py
+++ b/generator/opengl_loader_generator.py
@@ -30,7 +30,7 @@
from collections import namedtuple
import xml.etree.ElementTree as etree
-from generator_lib import Generator, run_generator, FileRender
+from generator_lib import Generator, run_generator, FileRender, GeneratorOutput
class ProcName:
@@ -261,7 +261,7 @@
'The JSON file that defines the OpenGL and GLES extensions to use.'
)
- def get_file_renders(self, args):
+ def get_outputs(self, args):
supported_extensions = []
with open(args.supported_extensions) as f:
supported_extensions_json = json.loads(f.read())
@@ -271,7 +271,7 @@
params = compute_params(
etree.parse(args.gl_xml).getroot(), supported_extensions)
- return [
+ renders = [
FileRender(
'opengl/OpenGLFunctionsBase.cpp',
'src/dawn/native/opengl/OpenGLFunctionsBase_autogen.cpp',
@@ -283,6 +283,7 @@
'src/dawn/native/opengl/opengl_platform_autogen.h',
[params]),
]
+ return GeneratorOutput(renders=renders, imported_templates=[])
def get_dependencies(self, args):
return [
