| Fetch the git dependencies as archives, so nothing has to run here 807f681 nandi 18d ago | 1 | #!/usr/bin/env python3 |
| 2 | """Regenerate third-party/rust/BUCK, with every git dependency as an archive. |
| 3 | |
| 4 | just buckify # after editing third-party/rust/Cargo.toml or a fixup |
| 5 | |
| 6 | reindeer emits `git_fetch` for a dependency taken from a git repository, and |
| 7 | the prelude runs that rule locally and only locally: it shells out to git, so |
| 8 | the action is tagged `LocalRequired` and a `--remote-only` build dies on it |
| 9 | with "Incompatible executor preferences" before it compiles anything. Seven |
| 10 | targets in this graph came from git, so remote execution was never actually |
| 11 | whole — a local worker had to be there to clone. |
| 12 | |
| 13 | A git host serves the same commit as a tarball, which `http_archive` fetches |
| 14 | the way every crates.io dependency here is already fetched: by URL and digest, |
| 15 | no git anywhere, and nothing tied to the machine that started the build. So |
| 16 | this runs reindeer and then rewrites those targets into that shape. |
| 17 | |
| 18 | The digests are recorded in git-archives.json beside the generated file rather |
| 19 | than taken fresh each run, because that is the whole point of a digest: GitHub |
| 20 | builds these tarballs on demand, and if one ever comes back different, the |
| 21 | build should say so loudly instead of quietly carrying on with other bytes. |
| 22 | |
| 23 | The `.git` suffix goes with the rule. A fetched archive's output directory is |
| 24 | named after its target, and every crate_root beneath one of these points at |
| 25 | `<name-without-.git>/…`, which is what git_fetch happened to produce. |
| 26 | """ |
| 27 | |
| 28 | import hashlib |
| 29 | import json |
| 30 | import pathlib |
| 31 | import re |
| 32 | import subprocess |
| 33 | import sys |
| 34 | import urllib.request |
| 35 | |
| 36 | HERE = pathlib.Path(__file__).resolve().parent |
| 37 | THIRD_PARTY = HERE.parent / "third-party" / "rust" |
| 38 | BUCK = THIRD_PARTY / "BUCK" |
| 39 | LOCK = THIRD_PARTY / "git-archives.json" |
| 40 | |
| 41 | GIT_FETCH = re.compile( |
| 42 | r"^git_fetch\(\n(?P<body>(?: .*\n)*?)\)\n", re.MULTILINE |
| 43 | ) |
| 44 | FIELD = re.compile(r'^ (\w+) = (.*?),$', re.MULTILINE | re.DOTALL) |
| 45 | |
| 46 | |
| 47 | def field(body, name): |
| 48 | m = re.search(rf'^ {name} = "([^"]*)",$', body, re.MULTILINE) |
| 49 | return m.group(1) if m else None |
| 50 | |
| 51 | |
| 52 | def sub_targets(body): |
| 53 | m = re.search(r"^ sub_targets = \[\n(.*?)^ \],$", body, re.MULTILINE | re.DOTALL) |
| 54 | return re.findall(r'"([^"]+)"', m.group(1)) if m else [] |
| 55 | |
| 56 | |
| 57 | def digest(url, key, lock): |
| 58 | if key in lock: |
| 59 | return lock[key] |
| 60 | print(f" fetching {url}", file=sys.stderr) |
| 61 | with urllib.request.urlopen(url) as response: |
| 62 | lock[key] = hashlib.sha256(response.read()).hexdigest() |
| 63 | return lock[key] |
| 64 | |
| 65 | |
| 66 | def archive(body, lock): |
| 67 | name = field(body, "name") |
| 68 | repo = field(body, "repo").rstrip("/") |
| 69 | rev = field(body, "rev") |
| 70 | # What the tarball unpacks into: the repository's own name, and the commit. |
| 71 | prefix = f"{repo.rsplit('/', 1)[1]}-{rev}" |
| 72 | url = f"{repo}/archive/{rev}.tar.gz" |
| 73 | lines = [ |
| 74 | "http_archive(", |
| 75 | f' name = "{name.removesuffix(".git")}",', |
| 76 | f' sha256 = "{digest(url, f"{repo}@{rev}", lock)}",', |
| 77 | f' strip_prefix = "{prefix}",', |
| 78 | f' urls = ["{url}"],', |
| 79 | ] |
| 80 | if subs := sub_targets(body): |
| 81 | lines.append(" sub_targets = [") |
| 82 | lines += [f' "{s}",' for s in subs] |
| 83 | lines.append(" ],") |
| 84 | lines += [" visibility = [],", ")"] |
| 85 | return name, "\n".join(lines) + "\n" |
| 86 | |
| 87 | |
| 88 | def main(): |
| 89 | subprocess.run( |
| 90 | ["reindeer", "--third-party-dir", str(THIRD_PARTY), "buckify"], check=True |
| 91 | ) |
| 92 | lock = json.loads(LOCK.read_text()) if LOCK.exists() else {} |
| 93 | text = BUCK.read_text() |
| 94 | |
| 95 | names = [] |
| 96 | def replace(match): |
| 97 | name, block = archive(match.group("body"), lock) |
| 98 | names.append(name) |
| 99 | return block |
| 100 | text = GIT_FETCH.sub(replace, text) |
| 101 | |
| 102 | if not names: |
| 103 | print("no git_fetch targets — nothing to rewrite") |
| 104 | # Every reference to one of them loses the suffix too: srcs, sub_target |
| 105 | # labels, and the paths a crate_root is written against. |
| 106 | for name in names: |
| 107 | text = text.replace(name, name.removesuffix(".git")) |
| 108 | |
| 109 | BUCK.write_text(text) |
| 110 | LOCK.write_text(json.dumps(dict(sorted(lock.items())), indent=2) + "\n") |
| 111 | for name in names: |
| 112 | print(f" {name} -> http_archive") |
| 113 | |
| 114 | |
| 115 | if __name__ == "__main__": |
| 116 | main() |