blob: dbbd6dfac6da746ccb5c315fcaba24bc6c3113c6 [file] [log] [blame]
Corentin Wallez59382b72020-04-17 20:43:07 +00001#!/usr/bin/env python3
Corentin Wallez0c38e922019-06-07 08:59:17 +00002# Copyright 2019 The Dawn Authors
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000015"""Module to create generators that render multiple Jinja2 templates for GN.
16
17A helper module that can be used to create generator scripts (clients)
18that expand one or more Jinja2 templates, without outputs usable from
19GN and Ninja build-based systems. See generator_lib.gni as well.
20
21Clients should create a Generator sub-class, then call run_generator()
22with a proper derived class instance.
23
24Clients specify a list of FileRender operations, each one of them will
25output a file into a temporary output directory through Jinja2 expansion.
26All temporary output files are then grouped and written to into a single JSON
27file, that acts as a convenient single GN output target. Use extract_json.py
28to extract the output files from the JSON tarball in another GN action.
29
30--depfile can be used to specify an output Ninja dependency file for the
31JSON tarball, to ensure it is regenerated any time one of its dependencies
32changes.
33
34Finally, --expected-output-files can be used to check the list of generated
35output files.
36"""
37
Corentin Wallez0c38e922019-06-07 08:59:17 +000038import argparse, json, os, re, sys
39from collections import namedtuple
40
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000041# A FileRender represents a single Jinja2 template render operation:
42#
43# template: Jinja2 template name, relative to --template-dir path.
44#
45# output: Output file path, relative to temporary output directory.
46#
47# params_dicts: iterable of (name:string -> value:string) dictionaries.
48# All of them will be merged before being sent as Jinja2 template
49# expansion parameters.
50#
51# Example:
52# FileRender('api.c', 'src/project_api.c', [{'PROJECT_VERSION': '1.0.0'}])
53#
54FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts'])
55
Kai Ninomiya01aeca22020-07-15 19:51:17 +000056
Corentin Wallez0c38e922019-06-07 08:59:17 +000057# The interface that must be implemented by generators.
58class Generator:
59 def get_description(self):
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000060 """Return generator description for --help."""
Corentin Wallez0c38e922019-06-07 08:59:17 +000061 return ""
62
63 def add_commandline_arguments(self, parser):
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000064 """Add generator-specific argparse arguments."""
Corentin Wallez0c38e922019-06-07 08:59:17 +000065 pass
66
67 def get_file_renders(self, args):
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000068 """Return the list of FileRender objects to process."""
Corentin Wallez0c38e922019-06-07 08:59:17 +000069 return []
70
71 def get_dependencies(self, args):
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000072 """Return a list of extra input dependencies."""
Corentin Wallez0c38e922019-06-07 08:59:17 +000073 return []
74
Kai Ninomiya01aeca22020-07-15 19:51:17 +000075
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000076# Allow custom Jinja2 installation path through an additional python
77# path from the arguments if present. This isn't done through the regular
78# argparse because PreprocessingLoader uses jinja2 in the global scope before
79# "main" gets to run.
80#
81# NOTE: If this argument appears several times, this only uses the first
82# value, while argparse would typically keep the last one!
83kJinja2Path = '--jinja2-path'
Corentin Wallez45f91852019-09-18 00:59:40 +000084try:
85 jinja2_path_argv_index = sys.argv.index(kJinja2Path)
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +000086 # Add parent path for the import to succeed.
87 path = os.path.join(sys.argv[jinja2_path_argv_index + 1], os.pardir)
Corentin Wallez0c38e922019-06-07 08:59:17 +000088 sys.path.insert(1, path)
Corentin Wallez45f91852019-09-18 00:59:40 +000089except ValueError:
90 # --jinja2-path isn't passed, ignore the exception and just import Jinja2
91 # assuming it already is in the Python PATH.
92 pass
dan sinclair6b67a902023-04-07 07:52:36 +000093kMarkupSafePath = '--markupsafe-path'
94try:
95 markupsafe_path_argv_index = sys.argv.index(kMarkupSafePath)
96 # Add parent path for the import to succeed.
97 path = os.path.join(sys.argv[markupsafe_path_argv_index + 1], os.pardir)
98 sys.path.insert(1, path)
99except ValueError:
100 # --markupsafe-path isn't passed, ignore the exception and just import
101 # assuming it already is in the Python PATH.
102 pass
Corentin Wallez0c38e922019-06-07 08:59:17 +0000103
104import jinja2
105
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000106
Corentin Wallez0c38e922019-06-07 08:59:17 +0000107# A custom Jinja2 template loader that removes the extra indentation
108# of the template blocks so that the output is correctly indented
Corentin Wallezdf69f242019-06-13 10:22:32 +0000109class _PreprocessingLoader(jinja2.BaseLoader):
Corentin Wallez0c38e922019-06-07 08:59:17 +0000110 def __init__(self, path):
111 self.path = path
112
113 def get_source(self, environment, template):
114 path = os.path.join(self.path, template)
115 if not os.path.exists(path):
116 raise jinja2.TemplateNotFound(template)
117 mtime = os.path.getmtime(path)
118 with open(path) as f:
119 source = self.preprocess(f.read())
120 return source, path, lambda: mtime == os.path.getmtime(path)
121
Corentin Wallez8f938712019-07-08 19:20:22 +0000122 blockstart = re.compile('{%-?\s*(if|elif|else|for|block|macro)[^}]*%}')
123 blockend = re.compile('{%-?\s*(end(if|for|block|macro)|elif|else)[^}]*%}')
Corentin Wallez0c38e922019-06-07 08:59:17 +0000124
125 def preprocess(self, source):
126 lines = source.split('\n')
127
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000128 # Compute the current indentation level of the template blocks and
129 # remove their indentation
Corentin Wallez0c38e922019-06-07 08:59:17 +0000130 result = []
131 indentation_level = 0
132
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000133 # Filter lines that are pure comments. line_comment_prefix is not
134 # enough because it removes the comment but doesn't completely remove
135 # the line, resulting in more verbose output.
Corentin Wallez1bf31672020-01-15 15:39:12 +0000136 lines = filter(lambda line: not line.strip().startswith('//*'), lines)
137
138 # Remove indentation templates have for the Jinja control flow.
Corentin Wallez0c38e922019-06-07 08:59:17 +0000139 for line in lines:
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000140 # The capture in the regex adds one element per block start or end,
141 # so we divide by two. There is also an extra line chunk
142 # corresponding to the line end, so we subtract it.
Corentin Wallez0c38e922019-06-07 08:59:17 +0000143 numends = (len(self.blockend.split(line)) - 1) // 2
144 indentation_level -= numends
145
146 result.append(self.remove_indentation(line, indentation_level))
147
148 numstarts = (len(self.blockstart.split(line)) - 1) // 2
149 indentation_level += numstarts
150
151 return '\n'.join(result) + '\n'
152
153 def remove_indentation(self, line, n):
154 for _ in range(n):
155 if line.startswith(' '):
156 line = line[4:]
157 elif line.startswith('\t'):
158 line = line[1:]
159 else:
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000160 assert line.strip() == ''
Corentin Wallez0c38e922019-06-07 08:59:17 +0000161 return line
162
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000163
Corentin Wallez0c38e922019-06-07 08:59:17 +0000164_FileOutput = namedtuple('FileOutput', ['name', 'content'])
165
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000166
Corentin Wallez0c38e922019-06-07 08:59:17 +0000167def _do_renders(renders, template_dir):
Corentin Wallezdf69f242019-06-13 10:22:32 +0000168 loader = _PreprocessingLoader(template_dir)
Loko Kung7c8dfbc2023-06-09 23:55:39 +0000169 env = jinja2.Environment(
170 extensions=['jinja2.ext.do', 'jinja2.ext.loopcontrols'],
171 loader=loader,
172 lstrip_blocks=True,
173 trim_blocks=True,
174 line_comment_prefix='//*')
Corentin Wallez0c38e922019-06-07 08:59:17 +0000175
Corentin Wallez031fbbb2019-06-11 18:03:05 +0000176 def do_assert(expr):
177 assert expr
178 return ''
179
180 def debug(text):
181 print(text)
182
183 base_params = {
184 'enumerate': enumerate,
185 'format': format,
186 'len': len,
187 'debug': debug,
188 'assert': do_assert,
189 }
190
Corentin Wallez0c38e922019-06-07 08:59:17 +0000191 outputs = []
192 for render in renders:
193 params = {}
Corentin Wallez031fbbb2019-06-11 18:03:05 +0000194 params.update(base_params)
Corentin Wallez0c38e922019-06-07 08:59:17 +0000195 for param_dict in render.params_dicts:
196 params.update(param_dict)
197 content = env.get_template(render.template).render(**params)
198 outputs.append(_FileOutput(render.output, content))
199
200 return outputs
201
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000202
Corentin Wallez0c38e922019-06-07 08:59:17 +0000203# Compute the list of imported, non-system Python modules.
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +0000204# It assumes that any path outside of the root directory is system.
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000205def _compute_python_dependencies(root_dir=None):
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +0000206 if not root_dir:
207 # Assume this script is under generator/ by default.
208 root_dir = os.path.join(os.path.dirname(__file__), os.pardir)
209 root_dir = os.path.abspath(root_dir)
Corentin Wallez0c38e922019-06-07 08:59:17 +0000210
211 module_paths = (module.__file__ for module in sys.modules.values()
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000212 if module and hasattr(module, '__file__'))
Corentin Wallez0c38e922019-06-07 08:59:17 +0000213
214 paths = set()
215 for path in module_paths:
dan sinclaire8022ea2020-10-20 14:46:10 +0000216 # Builtin/namespaced modules may return None for the file path.
217 if not path:
218 continue
219
Corentin Wallez0c38e922019-06-07 08:59:17 +0000220 path = os.path.abspath(path)
221
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +0000222 if not path.startswith(root_dir):
Corentin Wallez0c38e922019-06-07 08:59:17 +0000223 continue
224
225 if (path.endswith('.pyc')
226 or (path.endswith('c') and not os.path.splitext(path)[1])):
227 path = path[:-1]
228
229 paths.add(path)
230
231 return paths
232
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000233
Corentin Wallez0c38e922019-06-07 08:59:17 +0000234def run_generator(generator):
235 parser = argparse.ArgumentParser(
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000236 description=generator.get_description(),
237 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
Corentin Wallez0c38e922019-06-07 08:59:17 +0000238 )
239
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000240 generator.add_commandline_arguments(parser)
241 parser.add_argument('--template-dir',
242 default='templates',
243 type=str,
244 help='Directory with template files.')
245 parser.add_argument(
246 kJinja2Path,
247 default=None,
248 type=str,
249 help='Additional python path to set before loading Jinja2')
250 parser.add_argument(
dan sinclair6b67a902023-04-07 07:52:36 +0000251 kMarkupSafePath,
252 default=None,
253 type=str,
254 help='Additional python path to set before loading MarkupSafe')
255 parser.add_argument(
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000256 '--output-json-tarball',
257 default=None,
258 type=str,
259 help=('Name of the "JSON tarball" to create (tar is too annoying '
260 'to use in python).'))
261 parser.add_argument(
262 '--depfile',
263 default=None,
264 type=str,
265 help='Name of the Ninja depfile to create for the JSON tarball')
266 parser.add_argument(
267 '--expected-outputs-file',
268 default=None,
269 type=str,
270 help="File to compare outputs with and fail if it doesn't match")
271 parser.add_argument(
272 '--root-dir',
273 default=None,
274 type=str,
275 help=('Optional source root directory for Python dependency '
276 'computations'))
277 parser.add_argument(
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000278 '--print-cmake-dependencies',
279 default=False,
280 action="store_true",
281 help=("Prints a semi-colon separated list of dependencies to "
282 "stdout and exits."))
283 parser.add_argument(
284 '--print-cmake-outputs',
285 default=False,
286 action="store_true",
287 help=("Prints a semi-colon separated list of outputs to "
288 "stdout and exits."))
289 parser.add_argument('--output-dir',
290 default=None,
291 type=str,
292 help='Directory where to output generate files.')
Corentin Wallez0c38e922019-06-07 08:59:17 +0000293
294 args = parser.parse_args()
295
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000296 renders = generator.get_file_renders(args)
Corentin Wallez0c38e922019-06-07 08:59:17 +0000297
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000298 # Output a list of all dependencies for CMake or the tarball for GN/Ninja.
299 if args.depfile != None or args.print_cmake_dependencies:
300 dependencies = generator.get_dependencies(args)
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000301 dependencies += [
302 args.template_dir + os.path.sep + render.template
303 for render in renders
304 ]
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000305 dependencies += _compute_python_dependencies(args.root_dir)
306
307 if args.depfile != None:
308 with open(args.depfile, 'w') as f:
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000309 f.write(args.output_json_tarball + ": " +
310 " ".join(dependencies))
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000311
312 if args.print_cmake_dependencies:
313 sys.stdout.write(";".join(dependencies))
314 return 0
315
Corentin Wallez0c38e922019-06-07 08:59:17 +0000316 # The caller wants to assert that the outputs are what it expects.
317 # Load the file and compare with our renders.
318 if args.expected_outputs_file != None:
319 with open(args.expected_outputs_file) as f:
320 expected = set([line.strip() for line in f.readlines()])
321
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +0000322 actual = {render.output for render in renders}
Corentin Wallez0c38e922019-06-07 08:59:17 +0000323
324 if actual != expected:
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000325 print("Wrong expected outputs, caller expected:\n " +
326 repr(sorted(expected)))
David 'Digit' Turner5dee3e82019-06-24 14:31:06 +0000327 print("Actual output:\n " + repr(sorted(actual)))
Corentin Wallez0c38e922019-06-07 08:59:17 +0000328 return 1
329
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000330 # Print the list of all the outputs for cmake.
331 if args.print_cmake_outputs:
Kai Ninomiya01aeca22020-07-15 19:51:17 +0000332 sys.stdout.write(";".join([
333 os.path.join(args.output_dir, render.output) for render in renders
334 ]))
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000335 return 0
336
Corentin Wallez0c38e922019-06-07 08:59:17 +0000337 outputs = _do_renders(renders, args.template_dir)
338
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000339 # Output the JSON tarball
Corentin Wallez0c38e922019-06-07 08:59:17 +0000340 if args.output_json_tarball != None:
341 json_root = {}
342 for output in outputs:
343 json_root[output.name] = output.content
344
345 with open(args.output_json_tarball, 'w') as f:
346 f.write(json.dumps(json_root))
347
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000348 # Output the files directly.
349 if args.output_dir != None:
350 for output in outputs:
351 output_path = os.path.join(args.output_dir, output.name)
Corentin Wallez0c38e922019-06-07 08:59:17 +0000352
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000353 directory = os.path.dirname(output_path)
Corentin Wallez20a6ca02022-12-19 11:14:54 +0000354 os.makedirs(directory, exist_ok=True)
Corentin Wallez7fe6efb2020-02-05 17:16:05 +0000355
356 with open(output_path, 'w') as outfile:
357 outfile.write(output.content)