Elie Michel | 9ae8ed2 | 2023-05-12 20:19:41 +0000 | [diff] [blame] | 1 | # Copyright 2023 The Dawn & Tint Authors |
| 2 | # |
Austin Eng | cc2516a | 2023-10-17 20:57:54 +0000 | [diff] [blame] | 3 | # Redistribution and use in source and binary forms, with or without |
| 4 | # modification, are permitted provided that the following conditions are met: |
Elie Michel | 9ae8ed2 | 2023-05-12 20:19:41 +0000 | [diff] [blame] | 5 | # |
Austin Eng | cc2516a | 2023-10-17 20:57:54 +0000 | [diff] [blame] | 6 | # 1. Redistributions of source code must retain the above copyright notice, this |
| 7 | # list of conditions and the following disclaimer. |
Elie Michel | 9ae8ed2 | 2023-05-12 20:19:41 +0000 | [diff] [blame] | 8 | # |
Austin Eng | cc2516a | 2023-10-17 20:57:54 +0000 | [diff] [blame] | 9 | # 2. Redistributions in binary form must reproduce the above copyright notice, |
| 10 | # this list of conditions and the following disclaimer in the documentation |
| 11 | # and/or other materials provided with the distribution. |
| 12 | # |
| 13 | # 3. Neither the name of the copyright holder nor the names of its |
| 14 | # contributors may be used to endorse or promote products derived from |
| 15 | # this software without specific prior written permission. |
| 16 | # |
| 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
| 18 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| 19 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| 20 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE |
| 21 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
| 23 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
| 24 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
| 25 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 26 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
Elie Michel | 9ae8ed2 | 2023-05-12 20:19:41 +0000 | [diff] [blame] | 27 | """ |
| 28 | Helper script to download Dawn's source dependencies without the need to |
| 29 | install depot_tools by manually. This script implements a subset of |
| 30 | `gclient sync`. |
| 31 | |
| 32 | This helps embedders, for example through CMake, get all the sources with |
| 33 | a single add_subdirectory call (or FetchContent) instead of more complex setups |
| 34 | |
| 35 | Note that this script executes blindly the content of DEPS file, run it only on |
| 36 | a project that you trust not to contain malicious DEPS files. |
| 37 | """ |
| 38 | |
| 39 | import os |
| 40 | import sys |
| 41 | import subprocess |
| 42 | import argparse |
| 43 | from pathlib import Path |
| 44 | |
| 45 | parser = argparse.ArgumentParser( |
| 46 | prog='fetch_dawn_dependencies', |
| 47 | description=__doc__, |
| 48 | ) |
| 49 | |
| 50 | parser.add_argument('-d', |
| 51 | '--directory', |
| 52 | type=str, |
| 53 | default="", |
| 54 | help=""" |
| 55 | Working directory, in which we read and apply DEPS files recusively. If not |
| 56 | specified, the current working directory is used. |
| 57 | """) |
| 58 | |
| 59 | parser.add_argument('-g', |
| 60 | '--git', |
| 61 | type=str, |
| 62 | default="git", |
| 63 | help=""" |
| 64 | Path to the git command used to. By default, git is retrieved from the PATH. |
| 65 | You may also use this option to specify extra argument for all calls to git. |
| 66 | """) |
| 67 | |
| 68 | parser.add_argument('-s', |
| 69 | '--shallow', |
| 70 | action='store_true', |
| 71 | default=True, |
| 72 | help=""" |
| 73 | Clone repositories without commit history (only getting data for the |
| 74 | requested commit). |
| 75 | NB: The git server hosting the dependencies must have turned on the |
| 76 | `uploadpack.allowReachableSHA1InWant` option. |
| 77 | NB2: git submodules may not work as expected (but they are not used by Dawn |
| 78 | dependencies). |
| 79 | """) |
| 80 | parser.add_argument('-ns', |
| 81 | '--no-shallow', |
| 82 | action='store_false', |
| 83 | dest='shallow', |
| 84 | help="Deactivate shallow cloning.") |
| 85 | |
| 86 | parser.add_argument('-t', |
| 87 | '--use-test-deps', |
| 88 | action='store_true', |
| 89 | default=False, |
| 90 | help=""" |
| 91 | Fetch dependencies needed for testing |
| 92 | """) |
| 93 | |
| 94 | |
| 95 | def main(args): |
| 96 | # The dependencies that we need to pull from the DEPS files. |
| 97 | # Dependencies of dependencies are prefixed by their ancestors. |
| 98 | required_submodules = [ |
Corentin Wallez | 8c70529 | 2023-10-23 20:15:19 +0000 | [diff] [blame] | 99 | 'third_party/abseil-cpp', |
| 100 | 'third_party/glfw', |
| 101 | 'third_party/jinja2', |
| 102 | 'third_party/khronos/EGL-Registry', |
| 103 | 'third_party/khronos/OpenGL-Registry', |
Christophe Dehais | 01e013c | 2024-06-05 12:25:17 +0000 | [diff] [blame] | 104 | 'third_party/libprotobuf-mutator/src', |
| 105 | 'third_party/protobuf', |
Corentin Wallez | 8c70529 | 2023-10-23 20:15:19 +0000 | [diff] [blame] | 106 | 'third_party/markupsafe', |
Yuly Novikov | 143523a | 2024-05-23 15:59:58 +0000 | [diff] [blame] | 107 | 'third_party/glslang/src', |
| 108 | 'third_party/spirv-headers/src', |
| 109 | 'third_party/spirv-tools/src', |
| 110 | 'third_party/vulkan-headers/src', |
| 111 | 'third_party/vulkan-loader/src', |
| 112 | 'third_party/vulkan-utility-libraries/src', |
Elie Michel | 9ae8ed2 | 2023-05-12 20:19:41 +0000 | [diff] [blame] | 113 | ] |
| 114 | |
| 115 | if args.use_test_deps: |
| 116 | required_submodules += [ |
| 117 | 'third_party/googletest', |
| 118 | ] |
| 119 | |
| 120 | root_dir = Path(args.directory).resolve() |
| 121 | |
| 122 | process_dir(args, root_dir, required_submodules) |
| 123 | |
| 124 | |
| 125 | def process_dir(args, dir_path, required_submodules): |
| 126 | """ |
| 127 | Install dependencies for the provided directory by processing the DEPS file |
| 128 | that it contains (if it exists). |
| 129 | Recursively install dependencies in sub-directories that are created by |
| 130 | cloning dependencies. |
| 131 | """ |
| 132 | deps_path = dir_path / 'DEPS' |
| 133 | if not deps_path.is_file(): |
| 134 | return |
| 135 | |
| 136 | log(f"Listing dependencies from {dir_path}") |
| 137 | DEPS = open(deps_path).read() |
| 138 | |
| 139 | ldict = {} |
| 140 | exec(DEPS, globals(), ldict) |
| 141 | deps = ldict.get('deps') |
| 142 | variables = ldict.get('vars', {}) |
| 143 | |
| 144 | if deps is None: |
Christophe Dehais | 01e013c | 2024-06-05 12:25:17 +0000 | [diff] [blame] | 145 | log(f"WARNING: DEPS file '{deps_path}' does not define a 'deps' variable" |
Elie Michel | 9ae8ed2 | 2023-05-12 20:19:41 +0000 | [diff] [blame] | 146 | ) |
Christophe Dehais | 01e013c | 2024-06-05 12:25:17 +0000 | [diff] [blame] | 147 | return |
Elie Michel | 9ae8ed2 | 2023-05-12 20:19:41 +0000 | [diff] [blame] | 148 | |
| 149 | for submodule in required_submodules: |
| 150 | if submodule not in deps: |
| 151 | continue |
| 152 | submodule_path = dir_path / Path(submodule) |
| 153 | |
| 154 | raw_url = deps[submodule]['url'] |
| 155 | git_url, git_tag = raw_url.format(**variables).rsplit('@', 1) |
| 156 | |
| 157 | # Run git from within the submodule's path (don't use for clone) |
| 158 | git = lambda *x: subprocess.run([args.git, '-C', submodule_path, *x], |
| 159 | capture_output=True) |
| 160 | |
| 161 | log(f"Fetching dependency '{submodule}'") |
Sergei Kachkov | 6275ada | 2023-09-01 20:56:26 +0000 | [diff] [blame] | 162 | if (submodule_path / ".git").is_dir(): |
Elie Michel | 9ae8ed2 | 2023-05-12 20:19:41 +0000 | [diff] [blame] | 163 | # The module was already cloned, but we may need to update it |
| 164 | proc = git('rev-parse', 'HEAD') |
| 165 | need_update = proc.stdout.decode().strip() != git_tag |
| 166 | |
| 167 | if need_update: |
| 168 | # The module was already cloned, but we may need to update it |
| 169 | proc = git('cat-file', '-t', git_tag) |
| 170 | git_tag_exists = proc.returncode == 0 |
| 171 | |
| 172 | if not git_tag_exists: |
| 173 | log(f"Updating '{submodule_path}' from '{git_url}'") |
| 174 | if args.shallow: |
| 175 | git('fetch', 'origin', git_tag, '--depth', '1') |
| 176 | else: |
| 177 | git('fetch', 'origin') |
| 178 | |
| 179 | log(f"Checking out tag '{git_tag}'") |
| 180 | git('checkout', git_tag) |
| 181 | |
| 182 | else: |
Sergei Kachkov | 6275ada | 2023-09-01 20:56:26 +0000 | [diff] [blame] | 183 | if args.shallow: |
| 184 | log(f"Shallow cloning '{git_url}' at '{git_tag}' into '{submodule_path}'" |
| 185 | ) |
| 186 | shallow_clone(git, git_url, git_tag) |
| 187 | else: |
| 188 | log(f"Cloning '{git_url}' into '{submodule_path}'") |
| 189 | subprocess.run([ |
| 190 | args.git, |
| 191 | 'clone', |
| 192 | '--recurse-submodules', |
| 193 | git_url, |
| 194 | submodule_path, |
| 195 | ], |
| 196 | capture_output=True) |
| 197 | |
| 198 | log(f"Checking out tag '{git_tag}'") |
| 199 | git('checkout', git_tag) |
Elie Michel | 9ae8ed2 | 2023-05-12 20:19:41 +0000 | [diff] [blame] | 200 | |
| 201 | # Recursive call |
| 202 | required_subsubmodules = [ |
| 203 | m[len(submodule) + 1:] for m in required_submodules |
| 204 | if m.startswith(submodule + "/") |
| 205 | ] |
| 206 | process_dir(args, submodule_path, required_subsubmodules) |
| 207 | |
| 208 | |
Sergei Kachkov | 6275ada | 2023-09-01 20:56:26 +0000 | [diff] [blame] | 209 | def shallow_clone(git, git_url, git_tag): |
Elie Michel | 9ae8ed2 | 2023-05-12 20:19:41 +0000 | [diff] [blame] | 210 | """ |
| 211 | Fetching only 1 commit is not exposed in the git clone API, so we decompose |
| 212 | it manually in git init, git fetch, git reset. |
| 213 | """ |
Elie Michel | 9ae8ed2 | 2023-05-12 20:19:41 +0000 | [diff] [blame] | 214 | git('init') |
Elie Michel | 9cbc657 | 2023-07-10 12:02:47 +0000 | [diff] [blame] | 215 | git('fetch', git_url, git_tag, '--depth', '1') |
Elie Michel | 9ae8ed2 | 2023-05-12 20:19:41 +0000 | [diff] [blame] | 216 | |
| 217 | |
| 218 | def log(msg): |
| 219 | """Just makes it look good in the CMake log flow.""" |
| 220 | print(f"-- -- {msg}") |
| 221 | |
| 222 | |
| 223 | class Var: |
| 224 | """ |
| 225 | Mock Var class, that the content of DEPS files assume to exist when they |
| 226 | are exec-ed. |
| 227 | """ |
| 228 | def __init__(self, name): |
| 229 | self.name = name |
| 230 | |
| 231 | def __add__(self, text): |
| 232 | return self.name + text |
| 233 | |
| 234 | def __radd__(self, text): |
| 235 | return text + self.name |
| 236 | |
| 237 | |
| 238 | if __name__ == "__main__": |
| 239 | main(parser.parse_args()) |