| #!/usr/bin/env python3 | 
 | # Copyright 2019 The Dawn & Tint Authors | 
 | # | 
 | # Redistribution and use in source and binary forms, with or without | 
 | # modification, are permitted provided that the following conditions are met: | 
 | # | 
 | # 1. Redistributions of source code must retain the above copyright notice, this | 
 | #    list of conditions and the following disclaimer. | 
 | # | 
 | # 2. Redistributions in binary form must reproduce the above copyright notice, | 
 | #    this list of conditions and the following disclaimer in the documentation | 
 | #    and/or other materials provided with the distribution. | 
 | # | 
 | # 3. Neither the name of the copyright holder nor the names of its | 
 | #    contributors may be used to endorse or promote products derived from | 
 | #    this software without specific prior written permission. | 
 | # | 
 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | 
 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | 
 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE | 
 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE | 
 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | 
 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR | 
 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER | 
 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | 
 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 
 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 
 | """Module to create generators that render multiple Jinja2 templates for GN. | 
 |  | 
 | A helper module that can be used to create generator scripts (clients) | 
 | that expand one or more Jinja2 templates, without outputs usable from | 
 | GN and Ninja build-based systems. See generator_lib.gni as well. | 
 |  | 
 | Clients should create a Generator sub-class, then call run_generator() | 
 | with a proper derived class instance. | 
 |  | 
 | Clients specify a list of FileRender operations, each one of them will | 
 | output a file into a temporary output directory through Jinja2 expansion. | 
 | All temporary output files are then grouped and written to into a single JSON | 
 | file, that acts as a convenient single GN output target. Use extract_json.py | 
 | to extract the output files from the JSON tarball in another GN action. | 
 |  | 
 | --depfile can be used to specify an output Ninja dependency file for the | 
 | JSON tarball, to ensure it is regenerated any time one of its dependencies | 
 | changes. | 
 |  | 
 | Finally, --expected-output-files can be used to check the list of generated | 
 | output files. | 
 | """ | 
 |  | 
 | import argparse, json, os, re, sys | 
 | from collections import namedtuple | 
 |  | 
 | # A FileRender represents a single Jinja2 template render operation: | 
 | # | 
 | #   template: Jinja2 template name, relative to --template-dir path. | 
 | # | 
 | #   output: Output file path, relative to temporary output directory. | 
 | # | 
 | #   params_dicts: iterable of (name:string -> value:string) dictionaries. | 
 | #       All of them will be merged before being sent as Jinja2 template | 
 | #       expansion parameters. | 
 | # | 
 | # Example: | 
 | #   FileRender('api.c', 'src/project_api.c', [{'PROJECT_VERSION': '1.0.0'}]) | 
 | # | 
 | FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts']) | 
 |  | 
 | # A GeneratorOutput represent everything an invocation of the generator will | 
 | # produce. | 
 | # | 
 | #   renders: an iterable of FileRenders. | 
 | # | 
 | #   imported_templates: paths to additional templates that will be imported. | 
 | #       Trying to import with {% from %} will enforce that the file is listed | 
 | #       to ensure the dependency information produced is correct. | 
 | GeneratorOutput = namedtuple('GeneratorOutput', | 
 |                              ['renders', 'imported_templates']) | 
 |  | 
 | # The interface that must be implemented by generators. | 
 | class Generator: | 
 |     def get_description(self): | 
 |         """Return generator description for --help.""" | 
 |         return "" | 
 |  | 
 |     def add_commandline_arguments(self, parser): | 
 |         """Add generator-specific argparse arguments.""" | 
 |         pass | 
 |  | 
 |     def get_outputs(self, args): | 
 |         """Return the list of FileRender objects to process.""" | 
 |         return [] | 
 |  | 
 |     def get_dependencies(self, args): | 
 |         """Return a list of extra input dependencies.""" | 
 |         return [] | 
 |  | 
 |  | 
 | # Allow custom Jinja2 installation path through an additional python | 
 | # path from the arguments if present. This isn't done through the regular | 
 | # argparse because PreprocessingLoader uses jinja2 in the global scope before | 
 | # "main" gets to run. | 
 | # | 
 | # NOTE: If this argument appears several times, this only uses the first | 
 | #       value, while argparse would typically keep the last one! | 
 | kJinja2Path = '--jinja2-path' | 
 | try: | 
 |     jinja2_path_argv_index = sys.argv.index(kJinja2Path) | 
 |     # Add parent path for the import to succeed. | 
 |     path = os.path.join(sys.argv[jinja2_path_argv_index + 1], os.pardir) | 
 |     sys.path.insert(1, path) | 
 | except ValueError: | 
 |     # --jinja2-path isn't passed, ignore the exception and just import Jinja2 | 
 |     # assuming it already is in the Python PATH. | 
 |     pass | 
 | kMarkupSafePath = '--markupsafe-path' | 
 | try: | 
 |     markupsafe_path_argv_index = sys.argv.index(kMarkupSafePath) | 
 |     # Add parent path for the import to succeed. | 
 |     path = os.path.join(sys.argv[markupsafe_path_argv_index + 1], os.pardir) | 
 |     sys.path.insert(1, path) | 
 | except ValueError: | 
 |     # --markupsafe-path isn't passed, ignore the exception and just import | 
 |     # assuming it already is in the Python PATH. | 
 |     pass | 
 |  | 
 | import jinja2 | 
 |  | 
 |  | 
 | # A custom Jinja2 template loader that removes the extra indentation | 
 | # of the template blocks so that the output is correctly indented | 
 | class _PreprocessingLoader(jinja2.BaseLoader): | 
 |  | 
 |     def __init__(self, path, allow_list): | 
 |         self.path = path | 
 |         self.allow_list = set(allow_list) | 
 |  | 
 |         # Check that all the listed templates exist. | 
 |         for template in self.allow_list: | 
 |             if not os.path.exists(os.path.join(self.path, template)): | 
 |                 raise jinja2.TemplateNotFound(template) | 
 |  | 
 |     def get_source(self, environment, template): | 
 |         if not template in self.allow_list: | 
 |             raise jinja2.TemplateNotFound(template) | 
 |  | 
 |         path = os.path.join(self.path, template) | 
 |         mtime = os.path.getmtime(path) | 
 |         with open(path) as f: | 
 |             source = self.preprocess(f.read()) | 
 |         return source, path, lambda: mtime == os.path.getmtime(path) | 
 |  | 
 |     blockstart = re.compile(r'{%-?\s*(if|elif|else|for|block|macro)[^}]*%}') | 
 |     blockend = re.compile(r'{%-?\s*(end(if|for|block|macro)|elif|else)[^}]*%}') | 
 |  | 
 |     def preprocess(self, source): | 
 |         lines = source.split('\n') | 
 |  | 
 |         # Compute the current indentation level of the template blocks and | 
 |         # remove their indentation | 
 |         result = [] | 
 |         indentation_level = 0 | 
 |  | 
 |         # Filter lines that are pure comments. line_comment_prefix is not | 
 |         # enough because it removes the comment but doesn't completely remove | 
 |         # the line, resulting in more verbose output. | 
 |         lines = filter(lambda line: not line.strip().startswith('//*'), lines) | 
 |  | 
 |         # Remove indentation templates have for the Jinja control flow. | 
 |         for line in lines: | 
 |             # The capture in the regex adds one element per block start or end, | 
 |             # so we divide by two. There is also an extra line chunk | 
 |             # corresponding to the line end, so we subtract it. | 
 |             numends = (len(self.blockend.split(line)) - 1) // 2 | 
 |             indentation_level -= numends | 
 |  | 
 |             result.append(self.remove_indentation(line, indentation_level)) | 
 |  | 
 |             numstarts = (len(self.blockstart.split(line)) - 1) // 2 | 
 |             indentation_level += numstarts | 
 |  | 
 |         return '\n'.join(result) + '\n' | 
 |  | 
 |     def remove_indentation(self, line, n): | 
 |         for _ in range(n): | 
 |             if line.startswith(' '): | 
 |                 line = line[4:] | 
 |             elif line.startswith('\t'): | 
 |                 line = line[1:] | 
 |             else: | 
 |                 assert line.strip() == '' | 
 |         return line | 
 |  | 
 |  | 
 | _FileOutput = namedtuple('FileOutput', ['name', 'content']) | 
 |  | 
 |  | 
 | def _do_renders(output, template_dir): | 
 |     template_allow_list = [render.template for render in output.renders | 
 |                            ] + list(output.imported_templates) | 
 |     loader = _PreprocessingLoader(template_dir, template_allow_list) | 
 |  | 
 |     env = jinja2.Environment( | 
 |         extensions=['jinja2.ext.do', 'jinja2.ext.loopcontrols'], | 
 |         loader=loader, | 
 |         lstrip_blocks=True, | 
 |         trim_blocks=True, | 
 |         line_comment_prefix='//*') | 
 |  | 
 |     def do_assert(expr, message=''): | 
 |         assert expr, message | 
 |         return '' | 
 |  | 
 |     def debug(text): | 
 |         print(text) | 
 |  | 
 |     base_params = { | 
 |         'enumerate': enumerate, | 
 |         'format': format, | 
 |         'len': len, | 
 |         'debug': debug, | 
 |         'assert': do_assert, | 
 |     } | 
 |  | 
 |     outputs = [] | 
 |     for render in output.renders: | 
 |         params = {} | 
 |         params.update(base_params) | 
 |         for param_dict in render.params_dicts: | 
 |             params.update(param_dict) | 
 |         content = env.get_template(render.template).render(**params) | 
 |         outputs.append(_FileOutput(render.output, content)) | 
 |  | 
 |     return outputs | 
 |  | 
 |  | 
 | # Compute the list of imported, non-system Python modules. | 
 | # It assumes that any path outside of the root directory is system. | 
 | def _compute_python_dependencies(root_dir=None): | 
 |     if not root_dir: | 
 |         # Assume this script is under generator/ by default. | 
 |         root_dir = os.path.join(os.path.dirname(__file__), os.pardir) | 
 |     root_dir = os.path.abspath(root_dir) | 
 |  | 
 |     module_paths = (module.__file__ for module in sys.modules.values() | 
 |                     if module and hasattr(module, '__file__')) | 
 |  | 
 |     paths = set() | 
 |     for path in module_paths: | 
 |         # Builtin/namespaced modules may return None for the file path. | 
 |         if not path: | 
 |             continue | 
 |  | 
 |         path = os.path.abspath(path) | 
 |  | 
 |         if not path.startswith(root_dir): | 
 |             continue | 
 |  | 
 |         if (path.endswith('.pyc') | 
 |                 or (path.endswith('c') and not os.path.splitext(path)[1])): | 
 |             path = path[:-1] | 
 |  | 
 |         paths.add(path) | 
 |  | 
 |     return sorted(paths) | 
 |  | 
 |  | 
 | # Computes the string representing a cmake list of paths. | 
 | def _cmake_path_list(paths): | 
 |     if os.name == "nt": | 
 |         # On Windows CMake still expects paths to be separated by forward | 
 |         # slashes | 
 |         return (";".join(paths)).replace("\\", "/") | 
 |     else: | 
 |         return ";".join(paths) | 
 |  | 
 |  | 
 | def run_generator(generator): | 
 |     parser = argparse.ArgumentParser( | 
 |         description=generator.get_description(), | 
 |         formatter_class=argparse.ArgumentDefaultsHelpFormatter, | 
 |     ) | 
 |  | 
 |     generator.add_commandline_arguments(parser) | 
 |     parser.add_argument('--template-dir', | 
 |                         default='templates', | 
 |                         type=str, | 
 |                         help='Directory with template files.') | 
 |     parser.add_argument( | 
 |         kJinja2Path, | 
 |         default=None, | 
 |         type=str, | 
 |         help='Additional python path to set before loading Jinja2') | 
 |     parser.add_argument( | 
 |         kMarkupSafePath, | 
 |         default=None, | 
 |         type=str, | 
 |         help='Additional python path to set before loading MarkupSafe') | 
 |     parser.add_argument( | 
 |         '--output-json-tarball', | 
 |         default=None, | 
 |         type=str, | 
 |         help=('Name of the "JSON tarball" to create (tar is too annoying ' | 
 |               'to use in python).')) | 
 |     parser.add_argument( | 
 |         '--depfile', | 
 |         default=None, | 
 |         type=str, | 
 |         help='Name of the Ninja depfile to create for the JSON tarball') | 
 |     parser.add_argument( | 
 |         '--expected-outputs-file', | 
 |         default=None, | 
 |         type=str, | 
 |         help="File to compare outputs with and fail if it doesn't match") | 
 |     parser.add_argument( | 
 |         '--root-dir', | 
 |         default=None, | 
 |         type=str, | 
 |         help=('Optional source root directory for Python dependency ' | 
 |               'computations')) | 
 |     parser.add_argument( | 
 |         '--print-cmake-dependencies', | 
 |         default=False, | 
 |         action="store_true", | 
 |         help=("Prints a semi-colon separated list of dependencies to " | 
 |               "stdout and exits.")) | 
 |     parser.add_argument( | 
 |         '--print-cmake-outputs', | 
 |         default=False, | 
 |         action="store_true", | 
 |         help=("Prints a semi-colon separated list of outputs to " | 
 |               "stdout and exits.")) | 
 |     parser.add_argument('--output-dir', | 
 |                         default=None, | 
 |                         type=str, | 
 |                         help='Directory where to output generate files.') | 
 |  | 
 |     args = parser.parse_args() | 
 |  | 
 |     output = generator.get_outputs(args) | 
 |  | 
 |     # Output a list of all dependencies for CMake or the tarball for GN/Ninja. | 
 |     if args.depfile != None or args.print_cmake_dependencies: | 
 |         dependencies = generator.get_dependencies(args) | 
 |         dependencies += [ | 
 |             args.template_dir + os.path.sep + render.template | 
 |             for render in output.renders | 
 |         ] | 
 |         dependencies += [ | 
 |             args.template_dir + os.path.sep + template | 
 |             for template in output.imported_templates | 
 |         ] | 
 |         dependencies += _compute_python_dependencies(args.root_dir) | 
 |  | 
 |         if args.depfile != None: | 
 |             with open(args.depfile, 'w') as f: | 
 |                 f.write(args.output_json_tarball + ": " + | 
 |                         " ".join(dependencies)) | 
 |  | 
 |         if args.print_cmake_dependencies: | 
 |             sys.stdout.write(_cmake_path_list(dependencies)) | 
 |             return 0 | 
 |  | 
 |     # The caller wants to assert that the outputs are what it expects. | 
 |     # Load the file and compare with our renders. | 
 |     if args.expected_outputs_file != None: | 
 |         with open(args.expected_outputs_file) as f: | 
 |             expected = set([line.strip() for line in f.readlines()]) | 
 |  | 
 |         actual = {render.output for render in output.renders} | 
 |  | 
 |         if actual != expected: | 
 |             print("Wrong expected outputs, caller expected:\n    " + | 
 |                   repr(sorted(expected))) | 
 |             print("Actual output:\n    " + repr(sorted(actual))) | 
 |             return 1 | 
 |  | 
 |     # Print the list of all the outputs for cmake. | 
 |     if args.print_cmake_outputs: | 
 |         sys.stdout.write( | 
 |             _cmake_path_list([ | 
 |                 os.path.join(args.output_dir, render.output) | 
 |                 for render in output.renders | 
 |             ])) | 
 |         return 0 | 
 |  | 
 |     render_outputs = _do_renders(output, args.template_dir) | 
 |  | 
 |     # Output the JSON tarball | 
 |     if args.output_json_tarball != None: | 
 |         json_root = {} | 
 |         for output in render_outputs: | 
 |             json_root[output.name] = output.content | 
 |  | 
 |         with open(args.output_json_tarball, 'w') as f: | 
 |             f.write(json.dumps(json_root)) | 
 |  | 
 |     # Output the files directly. | 
 |     if args.output_dir != None: | 
 |         for output in render_outputs: | 
 |             output_path = os.path.join(args.output_dir, output.name) | 
 |  | 
 |             directory = os.path.dirname(output_path) | 
 |             os.makedirs(directory, exist_ok=True) | 
 |  | 
 |             with open(output_path, 'w') as outfile: | 
 |                 outfile.write(output.content) |