1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
|
package main
import (
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
)
// readReleaseScript returns the release builder, so its rules can be asserted
// without running it: running it cross-compiles five binaries, which is not a
// unit test. (Running the tagging script, on the other hand, is done below,
// against a throwaway clone.)
func readReleaseScript(t *testing.T) string {
t.Helper()
script, err := os.ReadFile("02-build-releases.sh")
if err != nil {
t.Fatalf("cannot read the release script: %v", err)
}
return string(script)
}
func TestTheReleaseScriptStampsTheBinariesItShips(t *testing.T) {
// Without -ldflags on the cross-compile, every downloaded binary reports
// "devel" while the release page names a version. The host binary would
// still be right, so nothing but this notices.
script := readReleaseScript(t)
build := commandContaining(t, script, "GOARCH=")
if !strings.Contains(build, "-ldflags") {
t.Errorf("the cross-compile does not stamp a version:\n%s", build)
}
}
func TestTheReleaseScriptTakesTheStampFromTheMakefile(t *testing.T) {
// Repeating the -X paths in the script is how the host binary and the
// downloads would come to disagree about which package holds the version.
script := readReleaseScript(t)
if !strings.Contains(script, "make --no-print-directory ldflags") {
t.Error("the script does not read the linker flags from the Makefile")
}
if strings.Contains(script, "version.stamp=") {
t.Error("the script spells out the -X path, which the Makefile already owns")
}
}
func TestTheReleaseScriptStampsTheTagItIsReleasing(t *testing.T) {
// The release *is* ${TAG}, so that is what the binaries say. Letting the
// Makefile's default stand would stamp `git describe`, which answers a
// different question — where HEAD is — and disagrees the moment anyone
// commits after tagging.
script := readReleaseScript(t)
flags := commandContaining(t, script, "ldflags")
if !strings.Contains(flags, `VERSION="${TAG}"`) {
t.Errorf("the stamp does not come from TAG:\n%s", flags)
}
if build := commandContaining(t, script, "make build"); !strings.Contains(build, `VERSION="${TAG}"`) {
t.Errorf("the host build carries a different version from the assets:\n%s", build)
}
}
func TestTheReleaseScriptDoesNotParseTheVersionOutOfProse(t *testing.T) {
// `-version` is written for a person and has changed shape once already;
// awk '{print $NF}' on it read a timestamp and failed a release.
script := readReleaseScript(t)
if strings.Contains(script, "$NF") {
t.Error("the script reads a field out of the -version line, which is prose")
}
}
func TestTheMakefileHandsOutTheFlagsThatStampABuild(t *testing.T) {
// The contract the release script depends on: `make ldflags` prints flags
// that actually put *the Makefile's own version* into a binary.
//
// It is checked against `make version` rather than against "not devel",
// because a checkout with no tags — a fresh clone, or a repository that has
// never had a release — correctly reports devel, and a test that called
// that a failure would be testing the tags rather than the flags.
version, err := exec.Command("make", "--no-print-directory", "version").Output()
if err != nil {
t.Fatalf("make version: %v", err)
}
// internal/version drops the leading v of a tag, so the comparison has to
// as well: `make version` says v0.2.1 and the binary says 0.2.1.
number := strings.TrimPrefix(strings.Fields(strings.TrimSpace(string(version)))[0], "v")
flags, err := exec.Command("make", "--no-print-directory", "ldflags").Output()
if err != nil {
t.Fatalf("make ldflags: %v", err)
}
binary := filepath.Join(t.TempDir(), "turbo-golo")
build := exec.Command("go", "build", "-trimpath", "-ldflags", strings.TrimSpace(string(flags)), "-o", binary, ".")
build.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS="+runtime.GOOS, "GOARCH="+runtime.GOARCH)
if out, err := build.CombinedOutput(); err != nil {
t.Fatalf("building with those flags failed: %v\n%s", err, out)
}
reported, err := exec.Command(binary, "-version").Output()
if err != nil {
t.Fatalf("the stamped binary does not run: %v", err)
}
if !strings.Contains(string(reported), number) {
t.Errorf("-version printed %q, want it to carry the Makefile's version %q", reported, number)
}
if strings.Contains(string(reported), "unknown") {
t.Errorf("-version printed %q, so nothing reached the linker at all", reported)
}
}
func TestMakeLdflagsTakesTheVersionItIsGiven(t *testing.T) {
// The release script overrides VERSION with the tag it is releasing, and
// everything downstream rests on that override reaching the linker.
flags, err := exec.Command("make", "--no-print-directory", "ldflags", "VERSION=v9.9.9").Output()
if err != nil {
t.Fatalf("make ldflags: %v", err)
}
binary := filepath.Join(t.TempDir(), "turbo-golo")
build := exec.Command("go", "build", "-trimpath", "-ldflags", strings.TrimSpace(string(flags)), "-o", binary, ".")
build.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS="+runtime.GOOS, "GOARCH="+runtime.GOARCH)
if out, err := build.CombinedOutput(); err != nil {
t.Fatalf("building with those flags failed: %v\n%s", err, out)
}
reported, err := exec.Command(binary, "-version").Output()
if err != nil {
t.Fatalf("the stamped binary does not run: %v", err)
}
if !strings.Contains(string(reported), "9.9.9") {
t.Errorf("-version printed %q, so VERSION=v9.9.9 never reached the linker", reported)
}
}
// commandContaining returns the first shell command of a script holding a
// fragment, with backslash continuations joined: a command's flags are often
// on the line after the one that names it, and a test about the command should
// not depend on where it happens to wrap.
func commandContaining(t *testing.T, script, fragment string) string {
t.Helper()
joined := strings.ReplaceAll(script, "\\\n", " ")
for _, line := range strings.Split(joined, "\n") {
if strings.Contains(line, fragment) {
return strings.TrimSpace(line)
}
}
t.Fatalf("no command in the script contains %q", fragment)
return ""
}
// readTagScript returns the tagging script, whose failure modes are what the
// release builder is left to notice when they are not caught here.
func readTagScript(t *testing.T) string {
t.Helper()
script, err := os.ReadFile("01-release.tag.sh")
if err != nil {
t.Fatalf("cannot read the tagging script: %v", err)
}
return string(script)
}
func TestTheTagScriptStopsOnTheFirstFailure(t *testing.T) {
// Without this, `git tag` refusing a tag that already existed was skipped
// in silence and the `git push` after it pushed the OLD tag, cutting a
// release from a commit nobody meant.
if !strings.Contains(readTagScript(t), "set -euo pipefail") {
t.Error("the tagging script does not stop on a failing step")
}
}
func TestTheTagScriptRefusesATagThatAlreadyExists(t *testing.T) {
script := readTagScript(t)
for _, want := range []string{
"git rev-parse -q --verify", // taken locally
"git ls-remote --tags origin", // taken on the remote, after a local delete
} {
if !strings.Contains(script, want) {
t.Errorf("the tagging script never checks %q", want)
}
}
}
func TestTheTagScriptSurvivesHavingNothingToCommit(t *testing.T) {
// Under `set -e` a plain `git commit` with a clean tree ends the release,
// which is wrong: the work being already committed is the normal case on a
// second run.
script := readTagScript(t)
if !strings.Contains(script, "git diff --cached --quiet") {
t.Error("the tagging script commits without checking there is anything to commit")
}
}
func TestTheTagScriptTagsOnlyAfterThePushSucceeded(t *testing.T) {
// A tag left behind pointing at a commit the remote has never seen is the
// state that needs a force push to escape.
script := readTagScript(t)
push := strings.Index(script, `git push origin "$(git rev-parse`)
tag := strings.Index(script, `git tag -a "${TAG}"`)
if push < 0 || tag < 0 {
t.Fatal("the tagging script no longer pushes and tags")
}
if tag < push {
t.Error("the script tags before pushing, so a rejected push leaves a stray tag")
}
}
// skipInsideARelease stops a test that runs the tagging script from running
// while the tagging script is running it.
//
// The script sets this before `make check`, and `make check` runs this suite.
// Without the guard the two call each other forever — which is not a test-only
// hazard: a real release would recurse in exactly the same way. The Release
// workflow sets it too, for the same reason.
func skipInsideARelease(t *testing.T) {
t.Helper()
if os.Getenv("TURBO_GOLO_RELEASING") != "" {
t.Skip("running inside a release; not starting another one")
}
}
func TestTheTagScriptRunsTheSuiteBeforePublishing(t *testing.T) {
// A version people will download, and the proxy will cache, is the wrong
// place to find out the suite was red.
if !strings.Contains(readTagScript(t), "make --no-print-directory check") {
t.Error("the tagging script publishes without running make check")
}
}
func TestTheTagScriptRefusesAReplaceDirective(t *testing.T) {
// The proxy serves go.mod as written, so `go install …@TAG` on a module
// carrying a replace looks for turbo-core in a directory that does not
// exist on the installer's machine.
if !strings.Contains(readTagScript(t), "replace") {
t.Error("the tagging script does not check go.mod for a replace directive")
}
}
func TestThisModuleHasNoReplaceDirective(t *testing.T) {
// The check above only helps if it is true today as well.
data, err := os.ReadFile("go.mod")
if err != nil {
t.Fatalf("reading go.mod: %v", err)
}
for _, line := range strings.Split(string(data), "\n") {
if strings.HasPrefix(strings.TrimSpace(line), "replace ") {
t.Errorf("go.mod carries %q; a published module must not", line)
}
}
}
func TestTheReleaseToolingNeedsNoPersonalToken(t *testing.T) {
// The Release workflow publishes with the job's own GITHUB_TOKEN, which is
// the only credential Rickub's release API accepts. A script still reading
// a token file is a credential that cannot work and has to be kept
// somewhere all the same — and a 02 or 04 left in the tree is a second
// pipeline somebody will run by mistake.
for _, script := range []string{"01-release.tag.sh", "02-build-releases.sh"} {
data, err := os.ReadFile(script)
if err != nil {
t.Fatalf("reading %s: %v", script, err)
}
for _, secret := range []string{"token.env", "${TOKEN}"} {
if strings.Contains(string(data), secret) {
t.Errorf("%s still reads %s", script, secret)
}
}
}
for _, gone := range []string{"02-release.publish.sh", "04-release.upload-binaries.sh"} {
if _, err := os.Stat(gone); err == nil {
t.Errorf("%s is still there; the workflow publishes and attaches the binaries now", gone)
}
}
}
func TestTheTagScriptTagsAndPushesForReal(t *testing.T) {
// The whole flow, in a throwaway clone with its own bare remote, so no tag
// is ever created in the real repository. Reading the script is not the
// same as running it: every guard above was added because one of them was
// wrong once.
skipInsideARelease(t)
if _, err := exec.LookPath("git"); err != nil {
t.Skip("git is not available")
}
remote, clone := throwawayClone(t)
out, err := runAllowingFailure(t, clone, "./01-release.tag.sh")
if err != nil {
t.Fatalf("the tagging script failed:\n%s", out)
}
if !strings.Contains(out, "published") {
t.Errorf("the script did not report publishing:\n%s", out)
}
tags, _ := runAllowingFailure(t, remote, "git", "tag")
if !strings.Contains(tags, "v0.0.1-test") {
t.Errorf("the remote has tags %q, want v0.0.1-test", strings.TrimSpace(tags))
}
}
func TestTheTagScriptRefusesATagItAlreadyPublished(t *testing.T) {
// Moving a published version is not an option: the proxy caches what it
// fetched, and the release page already carries binaries with that number.
skipInsideARelease(t)
if _, err := exec.LookPath("git"); err != nil {
t.Skip("git is not available")
}
_, clone := throwawayClone(t)
if out, err := runAllowingFailure(t, clone, "./01-release.tag.sh"); err != nil {
t.Fatalf("the first release failed:\n%s", out)
}
out, err := runAllowingFailure(t, clone, "./01-release.tag.sh")
if err == nil {
t.Fatalf("the script published the same tag twice:\n%s", out)
}
if !strings.Contains(out, "already exists") {
t.Errorf("the refusal does not say the tag is taken:\n%s", out)
}
}
// throwawayClone sets up a bare remote and a clone of it holding a copy of
// this module and a release.env naming a test version, and returns both paths.
func throwawayClone(t *testing.T) (remote, clone string) {
t.Helper()
root := t.TempDir()
remote = filepath.Join(root, "remote.git")
clone = filepath.Join(root, "clone")
runOrFail(t, root, "git", "init", "--bare", "--initial-branch=main", remote)
runOrFail(t, root, "git", "clone", remote, clone)
copyModuleInto(t, clone)
runOrFail(t, clone, "git", "config", "user.email", "test@example.test")
runOrFail(t, clone, "git", "config", "user.name", "Release Test")
writeTestFile(t, filepath.Join(clone, "release.env"), "TAG=v0.0.1-test\nABOUT=\"a throwaway release\"\n")
return remote, clone
}
// copyModuleInto copies the module's source into a directory, so the script can
// be run against a real checkout without touching this one.
//
// .git is left out because the target has its own; *.env because a test writes
// its own release.env — copying this checkout's would release whatever version
// happens to be in it; go.work because it would point the copy at a turbo-core
// checkout that is not what a release builds against; and the build outputs
// (bin, release, kits) and the demo project because they are hundreds of
// megabytes the script never reads.
//
// The copying is done here rather than by shelling out to cp, which on a
// network-backed working copy has been seen to write the right number of bytes
// and the wrong ones: every file in the copy came out NUL-filled.
func copyModuleInto(t *testing.T, target string) {
t.Helper()
entries, err := os.ReadDir(".")
if err != nil {
t.Fatalf("reading the module: %v", err)
}
for _, entry := range entries {
if leftOutOfTheCopy(entry.Name()) {
continue
}
copyTree(t, entry.Name(), filepath.Join(target, entry.Name()))
}
}
// leftOutOfTheCopy reports whether a top-level entry stays out of a throwaway
// copy of the module.
func leftOutOfTheCopy(name string) bool {
switch name {
case ".git", "bin", "release", "kits", "demo", "demos", "go.work", "go.work.sum":
return true
}
return strings.HasSuffix(name, ".env")
}
// copyTree copies a file or a directory to a new path.
//
// Anything that is neither a regular file nor a directory is skipped: the tool
// directories beside the source hold symlinks into caches that do not exist in
// a temporary copy, and the release scripts have no use for them.
func copyTree(t *testing.T, from, to string) {
t.Helper()
err := filepath.WalkDir(from, func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
relative, err := filepath.Rel(from, path)
if err != nil {
return err
}
destination := filepath.Join(to, relative)
if entry.IsDir() {
return os.MkdirAll(destination, 0o755)
}
if !entry.Type().IsRegular() {
return nil
}
info, err := entry.Info()
if err != nil {
return err
}
data, err := os.ReadFile(path)
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil {
return err
}
// The mode carries the execute bit, without which the scripts these
// tests exist to run cannot be run.
return os.WriteFile(destination, data, info.Mode().Perm())
})
if err != nil {
t.Fatalf("copying %s: %v", from, err)
}
}
// runOrFail executes a command in a directory, failing the test if it does not
// succeed.
func runOrFail(t *testing.T, dir string, name string, args ...string) {
t.Helper()
if out, err := runAllowingFailure(t, dir, name, args...); err != nil {
t.Fatalf("%s %v: %v\n%s", name, args, err, out)
}
}
// runAllowingFailure executes a command and returns its combined output along
// with whether it succeeded.
//
// GOWORK is switched off for the child: a go.work beside this checkout points
// at a turbo-core working tree, and a release is built against the published
// module, which is what a clean clone would see.
func runAllowingFailure(t *testing.T, dir string, name string, args ...string) (string, error) {
t.Helper()
command := exec.Command(name, args...)
command.Dir = dir
command.Env = append(os.Environ(), "GOWORK=off")
out, err := command.CombinedOutput()
return string(out), err
}
// writeTestFile creates a file, failing the test if it cannot.
func writeTestFile(t *testing.T, path, contents string) {
t.Helper()
if err := os.WriteFile(path, []byte(contents), 0o644); err != nil {
t.Fatalf("writing %s: %v", path, err)
}
}
func TestTheBuildScriptTakesTheTagFromTheCommandLine(t *testing.T) {
// release.env is git-ignored, so the workflow has none: it passes the tag
// it was started by. A script that only reads the file builds nothing in
// CI, or builds whatever version the file last named.
script := readReleaseScript(t)
if !strings.Contains(script, `TAG="${1:-${TAG:-}}"`) {
t.Error("the build script does not take the tag from its first argument")
}
if !strings.Contains(script, `[ -f release.env ]`) {
t.Error("the build script requires release.env, which CI does not have")
}
}
func TestTheBuildScriptRefusesATagThatIsNotAVersion(t *testing.T) {
// The proxy will not serve a tag it cannot read as a version, so a typo
// here builds perfectly and then fails at every `go install`.
//
// The refusal comes before anything is built or written, so the script
// alone is enough: it is run from an empty directory with no release.env.
dir := t.TempDir()
script, err := os.ReadFile("02-build-releases.sh")
if err != nil {
t.Fatalf("reading the build script: %v", err)
}
writeTestFile(t, filepath.Join(dir, "02-build-releases.sh"), string(script))
out, err := runAllowingFailure(t, dir, "bash", "./02-build-releases.sh", "v0.o.0")
if err == nil {
t.Fatalf("the script accepted a tag that is not a version:\n%s", out)
}
if !strings.Contains(out, "v1.2.3") {
t.Errorf("the refusal does not say what a tag should look like:\n%s", out)
}
}
func TestTheBuildScriptDoesNotHandOffToAnUploadScript(t *testing.T) {
// 04 attached the binaries to a release page a personal token had created.
// The workflow does both now; a script still pointing at 04 sends the
// reader to run something that is not there.
if strings.Contains(readReleaseScript(t), "04-release") {
t.Error("the build script still hands off to 04-release.upload-binaries.sh")
}
}
// readWorkflow returns the release workflow's text.
func readWorkflow(t *testing.T) string {
t.Helper()
data, err := os.ReadFile(filepath.Join(".github", "workflows", "release.yml"))
if err != nil {
t.Fatalf("reading the release workflow: %v", err)
}
return string(data)
}
func TestTheWorkflowPublishesOnATagPush(t *testing.T) {
// The tag push is the trigger: ./01-release.tag.sh ends by pushing one,
// and nothing else starts a release.
workflow := readWorkflow(t)
for _, want := range []string{"push:", "tags:", `- "v*"`} {
if !strings.Contains(workflow, want) {
t.Errorf("the workflow never declares %q", want)
}
}
// A workflow with the default read-only token cannot create a release, and
// fails at its last step after doing all the work.
if !strings.Contains(workflow, "contents: write") {
t.Error("the workflow does not ask for contents: write")
}
}
func TestTheWorkflowBuildsWithTheSameScriptAPersonRuns(t *testing.T) {
// A CI job that builds its own way is a second pipeline nobody tests, and
// the local one is then only ever exercised by accident.
if !strings.Contains(readWorkflow(t), "./02-build-releases.sh") {
t.Error("the workflow does not build the release with ./02-build-releases.sh")
}
}
func TestTheWorkflowAttachesWhatWasBuilt(t *testing.T) {
// Publishing a release page with no files attached is a silent half-job:
// the page exists and the downloads are not there.
workflow := readWorkflow(t)
for _, want := range []string{"turbo-golo-*", "SHA256SUMS", "fail_on_unmatched_files: true"} {
if !strings.Contains(workflow, want) {
t.Errorf("the workflow never mentions %q", want)
}
}
}
func TestTheWorkflowLinksToTheDocumentationAtThatTag(t *testing.T) {
// A release page is not inside the repository tree, so a relative path
// from it 404s — and a link to the branch would rot as the branch moves.
workflow := readWorkflow(t)
if !strings.Contains(workflow, "blob/${GITHUB_REF_NAME}") {
t.Error("the release notes do not link into the repository at the released tag")
}
if !strings.Contains(workflow, "/docs/en/README.md") {
t.Error("the release notes do not link to the documentation")
}
}
func TestTheWorkflowNeedsNoPersonalToken(t *testing.T) {
// The release API behind Rickub's /gh shim accepts the job's own
// GITHUB_TOKEN and refuses a personal one, so a secret referenced here is
// a credential that cannot work and still has to be kept somewhere.
if strings.Contains(readWorkflow(t), "secrets.") {
t.Error("the workflow reads a secret; the job's own token is the only credential the release API takes")
}
}
func TestTheWorkflowDoesNotStartAReleaseInsideItself(t *testing.T) {
// The suite it runs includes tests that run ./01-release.tag.sh against a
// throwaway clone. Locally the script exports this before calling make;
// in CI nothing calls the script, so the job has to set it itself.
if !strings.Contains(readWorkflow(t), "TURBO_GOLO_RELEASING") {
t.Error("the workflow runs the suite without TURBO_GOLO_RELEASING set")
}
}
|