blob: 3c6d29af0e269a25e009da6cc2a83a323310e440 [file] [log] [blame]
Elie Michel9ae8ed22023-05-12 20:19:41 +00001# Copyright 2023 The Dawn & Tint Authors
2#
Austin Engcc2516a2023-10-17 20:57:54 +00003# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are met:
Elie Michel9ae8ed22023-05-12 20:19:41 +00005#
Austin Engcc2516a2023-10-17 20:57:54 +00006# 1. Redistributions of source code must retain the above copyright notice, this
7# list of conditions and the following disclaimer.
Elie Michel9ae8ed22023-05-12 20:19:41 +00008#
Austin Engcc2516a2023-10-17 20:57:54 +00009# 2. Redistributions in binary form must reproduce the above copyright notice,
10# this list of conditions and the following disclaimer in the documentation
11# and/or other materials provided with the distribution.
12#
13# 3. Neither the name of the copyright holder nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
21# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
23# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
24# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
25# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Elie Michel9ae8ed22023-05-12 20:19:41 +000027"""
28Helper script to download Dawn's source dependencies without the need to
29install depot_tools by manually. This script implements a subset of
30`gclient sync`.
31
32This helps embedders, for example through CMake, get all the sources with
33a single add_subdirectory call (or FetchContent) instead of more complex setups
34
35Note that this script executes blindly the content of DEPS file, run it only on
36a project that you trust not to contain malicious DEPS files.
37"""
38
39import os
40import sys
41import subprocess
42import argparse
43from pathlib import Path
44
45parser = argparse.ArgumentParser(
46 prog='fetch_dawn_dependencies',
47 description=__doc__,
48)
49
50parser.add_argument('-d',
51 '--directory',
52 type=str,
53 default="",
54 help="""
55 Working directory, in which we read and apply DEPS files recusively. If not
56 specified, the current working directory is used.
57 """)
58
59parser.add_argument('-g',
60 '--git',
61 type=str,
62 default="git",
63 help="""
64 Path to the git command used to. By default, git is retrieved from the PATH.
65 You may also use this option to specify extra argument for all calls to git.
66 """)
67
68parser.add_argument('-s',
69 '--shallow',
70 action='store_true',
71 default=True,
72 help="""
73 Clone repositories without commit history (only getting data for the
74 requested commit).
75 NB: The git server hosting the dependencies must have turned on the
76 `uploadpack.allowReachableSHA1InWant` option.
77 NB2: git submodules may not work as expected (but they are not used by Dawn
78 dependencies).
79 """)
80parser.add_argument('-ns',
81 '--no-shallow',
82 action='store_false',
83 dest='shallow',
84 help="Deactivate shallow cloning.")
85
86parser.add_argument('-t',
87 '--use-test-deps',
88 action='store_true',
89 default=False,
90 help="""
91 Fetch dependencies needed for testing
92 """)
93
94
95def main(args):
96 # The dependencies that we need to pull from the DEPS files.
97 # Dependencies of dependencies are prefixed by their ancestors.
98 required_submodules = [
Corentin Wallez8c705292023-10-23 20:15:19 +000099 'third_party/abseil-cpp',
100 'third_party/glfw',
101 'third_party/jinja2',
102 'third_party/khronos/EGL-Registry',
103 'third_party/khronos/OpenGL-Registry',
Christophe Dehais01e013c2024-06-05 12:25:17 +0000104 'third_party/libprotobuf-mutator/src',
105 'third_party/protobuf',
Corentin Wallez8c705292023-10-23 20:15:19 +0000106 'third_party/markupsafe',
Yuly Novikov143523a2024-05-23 15:59:58 +0000107 'third_party/glslang/src',
108 'third_party/spirv-headers/src',
109 'third_party/spirv-tools/src',
110 'third_party/vulkan-headers/src',
111 'third_party/vulkan-loader/src',
112 'third_party/vulkan-utility-libraries/src',
Elie Michel9ae8ed22023-05-12 20:19:41 +0000113 ]
114
115 if args.use_test_deps:
116 required_submodules += [
117 'third_party/googletest',
118 ]
119
120 root_dir = Path(args.directory).resolve()
121
122 process_dir(args, root_dir, required_submodules)
123
124
125def process_dir(args, dir_path, required_submodules):
126 """
127 Install dependencies for the provided directory by processing the DEPS file
128 that it contains (if it exists).
129 Recursively install dependencies in sub-directories that are created by
130 cloning dependencies.
131 """
132 deps_path = dir_path / 'DEPS'
133 if not deps_path.is_file():
134 return
135
136 log(f"Listing dependencies from {dir_path}")
137 DEPS = open(deps_path).read()
138
139 ldict = {}
140 exec(DEPS, globals(), ldict)
141 deps = ldict.get('deps')
142 variables = ldict.get('vars', {})
143
144 if deps is None:
Christophe Dehais01e013c2024-06-05 12:25:17 +0000145 log(f"WARNING: DEPS file '{deps_path}' does not define a 'deps' variable"
Elie Michel9ae8ed22023-05-12 20:19:41 +0000146 )
Christophe Dehais01e013c2024-06-05 12:25:17 +0000147 return
Elie Michel9ae8ed22023-05-12 20:19:41 +0000148
149 for submodule in required_submodules:
150 if submodule not in deps:
151 continue
152 submodule_path = dir_path / Path(submodule)
153
154 raw_url = deps[submodule]['url']
155 git_url, git_tag = raw_url.format(**variables).rsplit('@', 1)
156
157 # Run git from within the submodule's path (don't use for clone)
158 git = lambda *x: subprocess.run([args.git, '-C', submodule_path, *x],
159 capture_output=True)
160
161 log(f"Fetching dependency '{submodule}'")
Sergei Kachkov6275ada2023-09-01 20:56:26 +0000162 if (submodule_path / ".git").is_dir():
Elie Michel9ae8ed22023-05-12 20:19:41 +0000163 # The module was already cloned, but we may need to update it
164 proc = git('rev-parse', 'HEAD')
165 need_update = proc.stdout.decode().strip() != git_tag
166
167 if need_update:
168 # The module was already cloned, but we may need to update it
169 proc = git('cat-file', '-t', git_tag)
170 git_tag_exists = proc.returncode == 0
171
172 if not git_tag_exists:
173 log(f"Updating '{submodule_path}' from '{git_url}'")
174 if args.shallow:
175 git('fetch', 'origin', git_tag, '--depth', '1')
176 else:
177 git('fetch', 'origin')
178
179 log(f"Checking out tag '{git_tag}'")
180 git('checkout', git_tag)
181
182 else:
Sergei Kachkov6275ada2023-09-01 20:56:26 +0000183 if args.shallow:
184 log(f"Shallow cloning '{git_url}' at '{git_tag}' into '{submodule_path}'"
185 )
186 shallow_clone(git, git_url, git_tag)
187 else:
188 log(f"Cloning '{git_url}' into '{submodule_path}'")
189 subprocess.run([
190 args.git,
191 'clone',
192 '--recurse-submodules',
193 git_url,
194 submodule_path,
195 ],
196 capture_output=True)
197
198 log(f"Checking out tag '{git_tag}'")
199 git('checkout', git_tag)
Elie Michel9ae8ed22023-05-12 20:19:41 +0000200
201 # Recursive call
202 required_subsubmodules = [
203 m[len(submodule) + 1:] for m in required_submodules
204 if m.startswith(submodule + "/")
205 ]
206 process_dir(args, submodule_path, required_subsubmodules)
207
208
Sergei Kachkov6275ada2023-09-01 20:56:26 +0000209def shallow_clone(git, git_url, git_tag):
Elie Michel9ae8ed22023-05-12 20:19:41 +0000210 """
211 Fetching only 1 commit is not exposed in the git clone API, so we decompose
212 it manually in git init, git fetch, git reset.
213 """
Elie Michel9ae8ed22023-05-12 20:19:41 +0000214 git('init')
Elie Michel9cbc6572023-07-10 12:02:47 +0000215 git('fetch', git_url, git_tag, '--depth', '1')
Elie Michel9ae8ed22023-05-12 20:19:41 +0000216
217
218def log(msg):
219 """Just makes it look good in the CMake log flow."""
220 print(f"-- -- {msg}")
221
222
223class Var:
224 """
225 Mock Var class, that the content of DEPS files assume to exist when they
226 are exec-ed.
227 """
228 def __init__(self, name):
229 self.name = name
230
231 def __add__(self, text):
232 return self.name + text
233
234 def __radd__(self, text):
235 return text + self.name
236
237
238if __name__ == "__main__":
239 main(parser.parse_args())