1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
#!/usr/bin/env python3
"""Regenerate third-party/rust/BUCK, with every git dependency as an archive.
just buckify # after editing third-party/rust/Cargo.toml or a fixup
reindeer emits `git_fetch` for a dependency taken from a git repository, and
the prelude runs that rule locally and only locally: it shells out to git, so
the action is tagged `LocalRequired` and a `--remote-only` build dies on it
with "Incompatible executor preferences" before it compiles anything. Seven
targets in this graph came from git, so remote execution was never actually
whole — a local worker had to be there to clone.
A git host serves the same commit as a tarball, which `http_archive` fetches
the way every crates.io dependency here is already fetched: by URL and digest,
no git anywhere, and nothing tied to the machine that started the build. So
this runs reindeer and then rewrites those targets into that shape.
The digests are recorded in git-archives.json beside the generated file rather
than taken fresh each run, because that is the whole point of a digest: GitHub
builds these tarballs on demand, and if one ever comes back different, the
build should say so loudly instead of quietly carrying on with other bytes.
The `.git` suffix goes with the rule. A fetched archive's output directory is
named after its target, and every crate_root beneath one of these points at
`<name-without-.git>/…`, which is what git_fetch happened to produce.
"""
import hashlib
import json
import pathlib
import re
import subprocess
import sys
import urllib.request
HERE = pathlib.Path(__file__).resolve().parent
THIRD_PARTY = HERE.parent / "third-party" / "rust"
BUCK = THIRD_PARTY / "BUCK"
LOCK = THIRD_PARTY / "git-archives.json"
GIT_FETCH = re.compile(
r"^git_fetch\(\n(?P<body>(?: .*\n)*?)\)\n", re.MULTILINE
)
FIELD = re.compile(r'^ (\w+) = (.*?),$', re.MULTILINE | re.DOTALL)
def field(body, name):
m = re.search(rf'^ {name} = "([^"]*)",$', body, re.MULTILINE)
return m.group(1) if m else None
def sub_targets(body):
m = re.search(r"^ sub_targets = \[\n(.*?)^ \],$", body, re.MULTILINE | re.DOTALL)
return re.findall(r'"([^"]+)"', m.group(1)) if m else []
def digest(url, key, lock):
if key in lock:
return lock[key]
print(f" fetching {url}", file=sys.stderr)
with urllib.request.urlopen(url) as response:
lock[key] = hashlib.sha256(response.read()).hexdigest()
return lock[key]
def archive(body, lock):
name = field(body, "name")
repo = field(body, "repo").rstrip("/")
rev = field(body, "rev")
# What the tarball unpacks into: the repository's own name, and the commit.
prefix = f"{repo.rsplit('/', 1)[1]}-{rev}"
url = f"{repo}/archive/{rev}.tar.gz"
lines = [
"http_archive(",
f' name = "{name.removesuffix(".git")}",',
f' sha256 = "{digest(url, f"{repo}@{rev}", lock)}",',
f' strip_prefix = "{prefix}",',
f' urls = ["{url}"],',
]
if subs := sub_targets(body):
lines.append(" sub_targets = [")
lines += [f' "{s}",' for s in subs]
lines.append(" ],")
lines += [" visibility = [],", ")"]
return name, "\n".join(lines) + "\n"
def main():
subprocess.run(
["reindeer", "--third-party-dir", str(THIRD_PARTY), "buckify"], check=True
)
lock = json.loads(LOCK.read_text()) if LOCK.exists() else {}
text = BUCK.read_text()
names = []
def replace(match):
name, block = archive(match.group("body"), lock)
names.append(name)
return block
text = GIT_FETCH.sub(replace, text)
if not names:
print("no git_fetch targets — nothing to rewrite")
# Every reference to one of them loses the suffix too: srcs, sub_target
# labels, and the paths a crate_root is written against.
for name in names:
text = text.replace(name, name.removesuffix(".git"))
BUCK.write_text(text)
LOCK.write_text(json.dumps(dict(sorted(lock.items())), indent=2) + "\n")
for name in names:
print(f" {name} -> http_archive")
if __name__ == "__main__":
main()
|