blob: dbff07e27bfe20c32c85bf4aa3e21611cf58c804 [file] [log] [blame]
Corentin Wallez59382b72020-04-17 20:43:07 +00001#!/usr/bin/env python3
Corentin Wallez0c38e922019-06-07 08:59:17 +00002# Copyright 2019 The Dawn Authors
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000015"""Module to create generators that render multiple Jinja2 templates for GN.
16
17A helper module that can be used to create generator scripts (clients)
18that expand one or more Jinja2 templates, without outputs usable from
19GN and Ninja build-based systems. See generator_lib.gni as well.
20
21Clients should create a Generator sub-class, then call run_generator()
22with a proper derived class instance.
23
24Clients specify a list of FileRender operations, each one of them will
25output a file into a temporary output directory through Jinja2 expansion.
26All temporary output files are then grouped and written to into a single JSON
27file, that acts as a convenient single GN output target. Use extract_json.py
28to extract the output files from the JSON tarball in another GN action.
29
30--depfile can be used to specify an output Ninja dependency file for the
31JSON tarball, to ensure it is regenerated any time one of its dependencies
32changes.
33
34Finally, --expected-output-files can be used to check the list of generated
35output files.
36"""
37
Corentin Wallez0c38e922019-06-07 08:59:17 +000038import argparse, json, os, re, sys
39from collections import namedtuple
40
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000041# A FileRender represents a single Jinja2 template render operation:
42#
43# template: Jinja2 template name, relative to --template-dir path.
44#
45# output: Output file path, relative to temporary output directory.
46#
47# params_dicts: iterable of (name:string -> value:string) dictionaries.
48# All of them will be merged before being sent as Jinja2 template
49# expansion parameters.
50#
51# Example:
52# FileRender('api.c', 'src/project_api.c', [{'PROJECT_VERSION': '1.0.0'}])
53#
54FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts'])
55
Kai Ninomiya01aeca22020-07-15 19:51:17 +000056
Corentin Wallez0c38e922019-06-07 08:59:17 +000057# The interface that must be implemented by generators.
58class Generator:
59 def get_description(self):
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000060 """Return generator description for --help."""
Corentin Wallez0c38e922019-06-07 08:59:17 +000061 return ""
62
63 def add_commandline_arguments(self, parser):
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000064 """Add generator-specific argparse arguments."""
Corentin Wallez0c38e922019-06-07 08:59:17 +000065 pass
66
67 def get_file_renders(self, args):
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000068 """Return the list of FileRender objects to process."""
Corentin Wallez0c38e922019-06-07 08:59:17 +000069 return []
70
71 def get_dependencies(self, args):
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000072 """Return a list of extra input dependencies."""
Corentin Wallez0c38e922019-06-07 08:59:17 +000073 return []
74
Kai Ninomiya01aeca22020-07-15 19:51:17 +000075
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000076# Allow custom Jinja2 installation path through an additional python
77# path from the arguments if present. This isn't done through the regular
78# argparse because PreprocessingLoader uses jinja2 in the global scope before
79# "main" gets to run.
80#
81# NOTE: If this argument appears several times, this only uses the first
82# value, while argparse would typically keep the last one!
83kJinja2Path = '--jinja2-path'
Corentin Wallez45f91852019-09-18 00:59:40 +000084try:
85 jinja2_path_argv_index = sys.argv.index(kJinja2Path)
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000086 # Add parent path for the import to succeed.
87 path = os.path.join(sys.argv[jinja2_path_argv_index + 1], os.pardir)
Corentin Wallez0c38e922019-06-07 08:59:17 +000088 sys.path.insert(1, path)
Corentin Wallez45f91852019-09-18 00:59:40 +000089except ValueError:
90 # --jinja2-path isn't passed, ignore the exception and just import Jinja2
91 # assuming it already is in the Python PATH.
92 pass
Corentin Wallez0c38e922019-06-07 08:59:17 +000093
94import jinja2
95
Kai Ninomiya01aeca22020-07-15 19:51:17 +000096
Corentin Wallez0c38e922019-06-07 08:59:17 +000097# A custom Jinja2 template loader that removes the extra indentation
98# of the template blocks so that the output is correctly indented
Corentin Wallezdf69f242019-06-13 10:22:32 +000099class _PreprocessingLoader(jinja2.BaseLoader):
Corentin Wallez0c38e922019-06-07 08:59:17 +0000100 def __init__(self, path):
101 self.path = path
102
103 def get_source(self, environment, template):
104 path = os.path.join(self.path, template)
105 if not os.path.exists(path):
106 raise jinja2.TemplateNotFound(template)
107 mtime = os.path.getmtime(path)
108 with open(path) as f:
109 source = self.preprocess(f.read())
110 return source, path, lambda: mtime == os.path.getmtime(path)
111
Corentin Wallez8f938712019-07-08 19:20:22 +0000112 blockstart = re.compile('{%-?\s*(if|elif|else|for|block|macro)[^}]*%}')
113 blockend = re.compile('{%-?\s*(end(if|for|block|macro)|elif|else)[^}]*%}')
Corentin Wallez0c38e922019-06-07 08:59:17 +0000114
115 def preprocess(self, source):
116 lines = source.split('\n')
117
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000118 # Compute the current indentation level of the template blocks and
119 # remove their indentation
Corentin Wallez0c38e922019-06-07 08:59:17 +0000120 result = []
121 indentation_level = 0
122
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000123 # Filter lines that are pure comments. line_comment_prefix is not
124 # enough because it removes the comment but doesn't completely remove
125 # the line, resulting in more verbose output.
Corentin Wallez1bf31672020-01-15 15:39:12 +0000126 lines = filter(lambda line: not line.strip().startswith('//*'), lines)
127
128 # Remove indentation templates have for the Jinja control flow.
Corentin Wallez0c38e922019-06-07 08:59:17 +0000129 for line in lines:
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000130 # The capture in the regex adds one element per block start or end,
131 # so we divide by two. There is also an extra line chunk
132 # corresponding to the line end, so we subtract it.
Corentin Wallez0c38e922019-06-07 08:59:17 +0000133 numends = (len(self.blockend.split(line)) - 1) // 2
134 indentation_level -= numends
135
136 result.append(self.remove_indentation(line, indentation_level))
137
138 numstarts = (len(self.blockstart.split(line)) - 1) // 2
139 indentation_level += numstarts
140
141 return '\n'.join(result) + '\n'
142
143 def remove_indentation(self, line, n):
144 for _ in range(n):
145 if line.startswith(' '):
146 line = line[4:]
147 elif line.startswith('\t'):
148 line = line[1:]
149 else:
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000150 assert line.strip() == ''
Corentin Wallez0c38e922019-06-07 08:59:17 +0000151 return line
152
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000153
Corentin Wallez0c38e922019-06-07 08:59:17 +0000154_FileOutput = namedtuple('FileOutput', ['name', 'content'])
155
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000156
Corentin Wallez0c38e922019-06-07 08:59:17 +0000157def _do_renders(renders, template_dir):
Corentin Wallezdf69f242019-06-13 10:22:32 +0000158 loader = _PreprocessingLoader(template_dir)
Loko Kung4d835252022-03-19 00:21:48 +0000159 env = jinja2.Environment(extensions=['jinja2.ext.do'],
160 loader=loader,
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000161 lstrip_blocks=True,
162 trim_blocks=True,
163 line_comment_prefix='//*')
Corentin Wallez0c38e922019-06-07 08:59:17 +0000164
Corentin Wallez031fbbb2019-06-11 18:03:05 +0000165 def do_assert(expr):
166 assert expr
167 return ''
168
169 def debug(text):
170 print(text)
171
172 base_params = {
173 'enumerate': enumerate,
174 'format': format,
175 'len': len,
176 'debug': debug,
177 'assert': do_assert,
178 }
179
Corentin Wallez0c38e922019-06-07 08:59:17 +0000180 outputs = []
181 for render in renders:
182 params = {}
Corentin Wallez031fbbb2019-06-11 18:03:05 +0000183 params.update(base_params)
Corentin Wallez0c38e922019-06-07 08:59:17 +0000184 for param_dict in render.params_dicts:
185 params.update(param_dict)
186 content = env.get_template(render.template).render(**params)
187 outputs.append(_FileOutput(render.output, content))
188
189 return outputs
190
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000191
Corentin Wallez0c38e922019-06-07 08:59:17 +0000192# Compute the list of imported, non-system Python modules.
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +0000193# It assumes that any path outside of the root directory is system.
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000194def _compute_python_dependencies(root_dir=None):
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +0000195 if not root_dir:
196 # Assume this script is under generator/ by default.
197 root_dir = os.path.join(os.path.dirname(__file__), os.pardir)
198 root_dir = os.path.abspath(root_dir)
Corentin Wallez0c38e922019-06-07 08:59:17 +0000199
200 module_paths = (module.__file__ for module in sys.modules.values()
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000201 if module and hasattr(module, '__file__'))
Corentin Wallez0c38e922019-06-07 08:59:17 +0000202
203 paths = set()
204 for path in module_paths:
dan sinclaire8022ea2020-10-20 14:46:10 +0000205 # Builtin/namespaced modules may return None for the file path.
206 if not path:
207 continue
208
Corentin Wallez0c38e922019-06-07 08:59:17 +0000209 path = os.path.abspath(path)
210
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +0000211 if not path.startswith(root_dir):
Corentin Wallez0c38e922019-06-07 08:59:17 +0000212 continue
213
214 if (path.endswith('.pyc')
215 or (path.endswith('c') and not os.path.splitext(path)[1])):
216 path = path[:-1]
217
218 paths.add(path)
219
220 return paths
221
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000222
Corentin Wallez0c38e922019-06-07 08:59:17 +0000223def run_generator(generator):
224 parser = argparse.ArgumentParser(
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000225 description=generator.get_description(),
226 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
Corentin Wallez0c38e922019-06-07 08:59:17 +0000227 )
228
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000229 generator.add_commandline_arguments(parser)
230 parser.add_argument('--template-dir',
231 default='templates',
232 type=str,
233 help='Directory with template files.')
234 parser.add_argument(
235 kJinja2Path,
236 default=None,
237 type=str,
238 help='Additional python path to set before loading Jinja2')
239 parser.add_argument(
240 '--output-json-tarball',
241 default=None,
242 type=str,
243 help=('Name of the "JSON tarball" to create (tar is too annoying '
244 'to use in python).'))
245 parser.add_argument(
246 '--depfile',
247 default=None,
248 type=str,
249 help='Name of the Ninja depfile to create for the JSON tarball')
250 parser.add_argument(
251 '--expected-outputs-file',
252 default=None,
253 type=str,
254 help="File to compare outputs with and fail if it doesn't match")
255 parser.add_argument(
256 '--root-dir',
257 default=None,
258 type=str,
259 help=('Optional source root directory for Python dependency '
260 'computations'))
261 parser.add_argument(
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000262 '--print-cmake-dependencies',
263 default=False,
264 action="store_true",
265 help=("Prints a semi-colon separated list of dependencies to "
266 "stdout and exits."))
267 parser.add_argument(
268 '--print-cmake-outputs',
269 default=False,
270 action="store_true",
271 help=("Prints a semi-colon separated list of outputs to "
272 "stdout and exits."))
273 parser.add_argument('--output-dir',
274 default=None,
275 type=str,
276 help='Directory where to output generate files.')
Corentin Wallez0c38e922019-06-07 08:59:17 +0000277
278 args = parser.parse_args()
279
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000280 renders = generator.get_file_renders(args)
Corentin Wallez0c38e922019-06-07 08:59:17 +0000281
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000282 # Output a list of all dependencies for CMake or the tarball for GN/Ninja.
283 if args.depfile != None or args.print_cmake_dependencies:
284 dependencies = generator.get_dependencies(args)
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000285 dependencies += [
286 args.template_dir + os.path.sep + render.template
287 for render in renders
288 ]
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000289 dependencies += _compute_python_dependencies(args.root_dir)
290
291 if args.depfile != None:
292 with open(args.depfile, 'w') as f:
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000293 f.write(args.output_json_tarball + ": " +
294 " ".join(dependencies))
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000295
296 if args.print_cmake_dependencies:
297 sys.stdout.write(";".join(dependencies))
298 return 0
299
Corentin Wallez0c38e922019-06-07 08:59:17 +0000300 # The caller wants to assert that the outputs are what it expects.
301 # Load the file and compare with our renders.
302 if args.expected_outputs_file != None:
303 with open(args.expected_outputs_file) as f:
304 expected = set([line.strip() for line in f.readlines()])
305
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +0000306 actual = {render.output for render in renders}
Corentin Wallez0c38e922019-06-07 08:59:17 +0000307
308 if actual != expected:
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000309 print("Wrong expected outputs, caller expected:\n " +
310 repr(sorted(expected)))
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +0000311 print("Actual output:\n " + repr(sorted(actual)))
Corentin Wallez0c38e922019-06-07 08:59:17 +0000312 return 1
313
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000314 # Print the list of all the outputs for cmake.
315 if args.print_cmake_outputs:
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000316 sys.stdout.write(";".join([
317 os.path.join(args.output_dir, render.output) for render in renders
318 ]))
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000319 return 0
320
Corentin Wallez0c38e922019-06-07 08:59:17 +0000321 outputs = _do_renders(renders, args.template_dir)
322
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000323 # Output the JSON tarball
Corentin Wallez0c38e922019-06-07 08:59:17 +0000324 if args.output_json_tarball != None:
325 json_root = {}
326 for output in outputs:
327 json_root[output.name] = output.content
328
329 with open(args.output_json_tarball, 'w') as f:
330 f.write(json.dumps(json_root))
331
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000332 # Output the files directly.
333 if args.output_dir != None:
334 for output in outputs:
335 output_path = os.path.join(args.output_dir, output.name)
Corentin Wallez0c38e922019-06-07 08:59:17 +0000336
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000337 directory = os.path.dirname(output_path)
338 if not os.path.exists(directory):
339 os.makedirs(directory)
340
341 with open(output_path, 'w') as outfile:
342 outfile.write(output.content)