package main import ( "os" "os/exec" "path/filepath" "runtime" "strings" "testing" ) // readReleaseScript returns the release builder, so its rules can be asserted // without running it: running it cross-compiles five binaries, which is not a // unit test. (Running the tagging script, on the other hand, is done below, // against a throwaway clone.) func readReleaseScript(t *testing.T) string { t.Helper() script, err := os.ReadFile("02-build-releases.sh") if err != nil { t.Fatalf("cannot read the release script: %v", err) } return string(script) } func TestTheReleaseScriptStampsTheBinariesItShips(t *testing.T) { // Without -ldflags on the cross-compile, every downloaded binary reports // "devel" while the release page names a version. The host binary would // still be right, so nothing but this notices. script := readReleaseScript(t) build := commandContaining(t, script, "GOARCH=") if !strings.Contains(build, "-ldflags") { t.Errorf("the cross-compile does not stamp a version:\n%s", build) } } func TestTheReleaseScriptTakesTheStampFromTheMakefile(t *testing.T) { // Repeating the -X paths in the script is how the host binary and the // downloads would come to disagree about which package holds the version. script := readReleaseScript(t) if !strings.Contains(script, "make --no-print-directory ldflags") { t.Error("the script does not read the linker flags from the Makefile") } if strings.Contains(script, "version.stamp=") { t.Error("the script spells out the -X path, which the Makefile already owns") } } func TestTheReleaseScriptStampsTheTagItIsReleasing(t *testing.T) { // The release *is* ${TAG}, so that is what the binaries say. Letting the // Makefile's default stand would stamp `git describe`, which answers a // different question — where HEAD is — and disagrees the moment anyone // commits after tagging. script := readReleaseScript(t) flags := commandContaining(t, script, "ldflags") if !strings.Contains(flags, `VERSION="${TAG}"`) { t.Errorf("the stamp does not come from TAG:\n%s", flags) } if build := commandContaining(t, script, "make build"); !strings.Contains(build, `VERSION="${TAG}"`) { t.Errorf("the host build carries a different version from the assets:\n%s", build) } } func TestTheReleaseScriptDoesNotParseTheVersionOutOfProse(t *testing.T) { // `-version` is written for a person and has changed shape once already; // awk '{print $NF}' on it read a timestamp and failed a release. script := readReleaseScript(t) if strings.Contains(script, "$NF") { t.Error("the script reads a field out of the -version line, which is prose") } } func TestTheMakefileHandsOutTheFlagsThatStampABuild(t *testing.T) { // The contract the release script depends on: `make ldflags` prints flags // that actually put *the Makefile's own version* into a binary. // // It is checked against `make version` rather than against "not devel", // because a checkout with no tags — a fresh clone, or a repository that has // never had a release — correctly reports devel, and a test that called // that a failure would be testing the tags rather than the flags. version, err := exec.Command("make", "--no-print-directory", "version").Output() if err != nil { t.Fatalf("make version: %v", err) } // internal/version drops the leading v of a tag, so the comparison has to // as well: `make version` says v0.2.1 and the binary says 0.2.1. number := strings.TrimPrefix(strings.Fields(strings.TrimSpace(string(version)))[0], "v") flags, err := exec.Command("make", "--no-print-directory", "ldflags").Output() if err != nil { t.Fatalf("make ldflags: %v", err) } binary := filepath.Join(t.TempDir(), "turbo-python") build := exec.Command("go", "build", "-trimpath", "-ldflags", strings.TrimSpace(string(flags)), "-o", binary, ".") build.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS="+runtime.GOOS, "GOARCH="+runtime.GOARCH) if out, err := build.CombinedOutput(); err != nil { t.Fatalf("building with those flags failed: %v\n%s", err, out) } reported, err := exec.Command(binary, "-version").Output() if err != nil { t.Fatalf("the stamped binary does not run: %v", err) } if !strings.Contains(string(reported), number) { t.Errorf("-version printed %q, want it to carry the Makefile's version %q", reported, number) } if strings.Contains(string(reported), "unknown") { t.Errorf("-version printed %q, so nothing reached the linker at all", reported) } } func TestMakeLdflagsTakesTheVersionItIsGiven(t *testing.T) { // The release script overrides VERSION with the tag it is releasing, and // everything downstream rests on that override reaching the linker. flags, err := exec.Command("make", "--no-print-directory", "ldflags", "VERSION=v9.9.9").Output() if err != nil { t.Fatalf("make ldflags: %v", err) } binary := filepath.Join(t.TempDir(), "turbo-python") build := exec.Command("go", "build", "-trimpath", "-ldflags", strings.TrimSpace(string(flags)), "-o", binary, ".") build.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS="+runtime.GOOS, "GOARCH="+runtime.GOARCH) if out, err := build.CombinedOutput(); err != nil { t.Fatalf("building with those flags failed: %v\n%s", err, out) } reported, err := exec.Command(binary, "-version").Output() if err != nil { t.Fatalf("the stamped binary does not run: %v", err) } if !strings.Contains(string(reported), "9.9.9") { t.Errorf("-version printed %q, so VERSION=v9.9.9 never reached the linker", reported) } } // commandContaining returns the first shell command of a script holding a // fragment, with backslash continuations joined: a command's flags are often // on the line after the one that names it, and a test about the command should // not depend on where it happens to wrap. func commandContaining(t *testing.T, script, fragment string) string { t.Helper() joined := strings.ReplaceAll(script, "\\\n", " ") for _, line := range strings.Split(joined, "\n") { if strings.Contains(line, fragment) { return strings.TrimSpace(line) } } t.Fatalf("no command in the script contains %q", fragment) return "" } // readTagScript returns the tagging script, whose failure modes are what the // release builder is left to notice when they are not caught here. func readTagScript(t *testing.T) string { t.Helper() script, err := os.ReadFile("01-release.tag.sh") if err != nil { t.Fatalf("cannot read the tagging script: %v", err) } return string(script) } func TestTheTagScriptStopsOnTheFirstFailure(t *testing.T) { // Without this, `git tag` refusing a tag that already existed was skipped // in silence and the `git push` after it pushed the OLD tag, cutting a // release from a commit nobody meant. if !strings.Contains(readTagScript(t), "set -euo pipefail") { t.Error("the tagging script does not stop on a failing step") } } func TestTheTagScriptRefusesATagThatAlreadyExists(t *testing.T) { script := readTagScript(t) for _, want := range []string{ "git rev-parse -q --verify", // taken locally "git ls-remote --tags origin", // taken on the remote, after a local delete } { if !strings.Contains(script, want) { t.Errorf("the tagging script never checks %q", want) } } } func TestTheTagScriptSurvivesHavingNothingToCommit(t *testing.T) { // Under `set -e` a plain `git commit` with a clean tree ends the release, // which is wrong: the work being already committed is the normal case on a // second run. script := readTagScript(t) if !strings.Contains(script, "git diff --cached --quiet") { t.Error("the tagging script commits without checking there is anything to commit") } } func TestTheTagScriptTagsOnlyAfterThePushSucceeded(t *testing.T) { // A tag left behind pointing at a commit the remote has never seen is the // state that needs a force push to escape. script := readTagScript(t) push := strings.Index(script, `git push origin "$(git rev-parse`) tag := strings.Index(script, `git tag -a "${TAG}"`) if push < 0 || tag < 0 { t.Fatal("the tagging script no longer pushes and tags") } if tag < push { t.Error("the script tags before pushing, so a rejected push leaves a stray tag") } } // skipInsideARelease stops a test that runs the tagging script from running // while the tagging script is running it. // // The script sets this before `make check`, and `make check` runs this suite. // Without the guard the two call each other forever — which is not a test-only // hazard: a real release would recurse in exactly the same way. The Release // workflow sets it too, for the same reason. func skipInsideARelease(t *testing.T) { t.Helper() if os.Getenv("TURBO_PYTHON_RELEASING") != "" { t.Skip("running inside a release; not starting another one") } } func TestTheTagScriptRunsTheSuiteBeforePublishing(t *testing.T) { // A version people will download, and the proxy will cache, is the wrong // place to find out the suite was red. if !strings.Contains(readTagScript(t), "make --no-print-directory check") { t.Error("the tagging script publishes without running make check") } } func TestTheTagScriptRefusesAReplaceDirective(t *testing.T) { // The proxy serves go.mod as written, so `go install …@TAG` on a module // carrying a replace looks for turbo-core in a directory that does not // exist on the installer's machine. if !strings.Contains(readTagScript(t), "replace") { t.Error("the tagging script does not check go.mod for a replace directive") } } func TestThisModuleHasNoReplaceDirective(t *testing.T) { // The check above only helps if it is true today as well. data, err := os.ReadFile("go.mod") if err != nil { t.Fatalf("reading go.mod: %v", err) } for _, line := range strings.Split(string(data), "\n") { if strings.HasPrefix(strings.TrimSpace(line), "replace ") { t.Errorf("go.mod carries %q; a published module must not", line) } } } func TestTheReleaseToolingNeedsNoPersonalToken(t *testing.T) { // The Release workflow publishes with the job's own GITHUB_TOKEN, which is // the only credential Rickub's release API accepts. A script still reading // a token file is a credential that cannot work and has to be kept // somewhere all the same — and a 02 or 04 left in the tree is a second // pipeline somebody will run by mistake. for _, script := range []string{"01-release.tag.sh", "02-build-releases.sh"} { data, err := os.ReadFile(script) if err != nil { t.Fatalf("reading %s: %v", script, err) } for _, secret := range []string{"token.env", "${TOKEN}"} { if strings.Contains(string(data), secret) { t.Errorf("%s still reads %s", script, secret) } } } for _, gone := range []string{"02-release.publish.sh", "04-release.upload-binaries.sh"} { if _, err := os.Stat(gone); err == nil { t.Errorf("%s is still there; the workflow publishes and attaches the binaries now", gone) } } } func TestTheTagScriptTagsAndPushesForReal(t *testing.T) { // The whole flow, in a throwaway clone with its own bare remote, so no tag // is ever created in the real repository. Reading the script is not the // same as running it: every guard above was added because one of them was // wrong once. skipInsideARelease(t) if _, err := exec.LookPath("git"); err != nil { t.Skip("git is not available") } remote, clone := throwawayClone(t) out, err := runAllowingFailure(t, clone, "./01-release.tag.sh") if err != nil { t.Fatalf("the tagging script failed:\n%s", out) } if !strings.Contains(out, "published") { t.Errorf("the script did not report publishing:\n%s", out) } tags, _ := runAllowingFailure(t, remote, "git", "tag") if !strings.Contains(tags, "v0.0.1-test") { t.Errorf("the remote has tags %q, want v0.0.1-test", strings.TrimSpace(tags)) } } func TestTheTagScriptRefusesATagItAlreadyPublished(t *testing.T) { // Moving a published version is not an option: the proxy caches what it // fetched, and the release page already carries binaries with that number. skipInsideARelease(t) if _, err := exec.LookPath("git"); err != nil { t.Skip("git is not available") } _, clone := throwawayClone(t) if out, err := runAllowingFailure(t, clone, "./01-release.tag.sh"); err != nil { t.Fatalf("the first release failed:\n%s", out) } out, err := runAllowingFailure(t, clone, "./01-release.tag.sh") if err == nil { t.Fatalf("the script published the same tag twice:\n%s", out) } if !strings.Contains(out, "already exists") { t.Errorf("the refusal does not say the tag is taken:\n%s", out) } } // throwawayClone sets up a bare remote and a clone of it holding a copy of // this module and a release.env naming a test version, and returns both paths. func throwawayClone(t *testing.T) (remote, clone string) { t.Helper() root := t.TempDir() remote = filepath.Join(root, "remote.git") clone = filepath.Join(root, "clone") runOrFail(t, root, "git", "init", "--bare", "--initial-branch=main", remote) runOrFail(t, root, "git", "clone", remote, clone) copyModuleInto(t, clone) runOrFail(t, clone, "git", "config", "user.email", "test@example.test") runOrFail(t, clone, "git", "config", "user.name", "Release Test") writeTestFile(t, filepath.Join(clone, "release.env"), "TAG=v0.0.1-test\nABOUT=\"a throwaway release\"\n") return remote, clone } // copyModuleInto copies the module's source into a directory, so the script can // be run against a real checkout without touching this one. // // .git is left out because the target has its own; *.env because a test writes // its own release.env — copying this checkout's would release whatever version // happens to be in it; go.work because it would point the copy at a turbo-core // checkout that is not what a release builds against; and the build outputs // (bin, release, kits) and the demo project because they are hundreds of // megabytes the script never reads. // // The copying is done here rather than by shelling out to cp, which on a // network-backed working copy has been seen to write the right number of bytes // and the wrong ones: every file in the copy came out NUL-filled. func copyModuleInto(t *testing.T, target string) { t.Helper() entries, err := os.ReadDir(".") if err != nil { t.Fatalf("reading the module: %v", err) } for _, entry := range entries { if leftOutOfTheCopy(entry.Name()) { continue } copyTree(t, entry.Name(), filepath.Join(target, entry.Name())) } } // leftOutOfTheCopy reports whether a top-level entry stays out of a throwaway // copy of the module. func leftOutOfTheCopy(name string) bool { switch name { case ".git", "bin", "release", "kits", "demo", "demos", "go.work", "go.work.sum": return true } return strings.HasSuffix(name, ".env") } // copyTree copies a file or a directory to a new path. // // Anything that is neither a regular file nor a directory is skipped: the tool // directories beside the source hold symlinks into caches that do not exist in // a temporary copy, and the release scripts have no use for them. func copyTree(t *testing.T, from, to string) { t.Helper() err := filepath.WalkDir(from, func(path string, entry os.DirEntry, err error) error { if err != nil { return err } relative, err := filepath.Rel(from, path) if err != nil { return err } destination := filepath.Join(to, relative) if entry.IsDir() { return os.MkdirAll(destination, 0o755) } if !entry.Type().IsRegular() { return nil } info, err := entry.Info() if err != nil { return err } data, err := os.ReadFile(path) if err != nil { return err } if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { return err } // The mode carries the execute bit, without which the scripts these // tests exist to run cannot be run. return os.WriteFile(destination, data, info.Mode().Perm()) }) if err != nil { t.Fatalf("copying %s: %v", from, err) } } // runOrFail executes a command in a directory, failing the test if it does not // succeed. func runOrFail(t *testing.T, dir string, name string, args ...string) { t.Helper() if out, err := runAllowingFailure(t, dir, name, args...); err != nil { t.Fatalf("%s %v: %v\n%s", name, args, err, out) } } // runAllowingFailure executes a command and returns its combined output along // with whether it succeeded. // // GOWORK is switched off for the child: a go.work beside this checkout points // at a turbo-core working tree, and a release is built against the published // module, which is what a clean clone would see. func runAllowingFailure(t *testing.T, dir string, name string, args ...string) (string, error) { t.Helper() command := exec.Command(name, args...) command.Dir = dir command.Env = append(os.Environ(), "GOWORK=off") out, err := command.CombinedOutput() return string(out), err } // writeTestFile creates a file, failing the test if it cannot. func writeTestFile(t *testing.T, path, contents string) { t.Helper() if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { t.Fatalf("writing %s: %v", path, err) } } func TestTheBuildScriptTakesTheTagFromTheCommandLine(t *testing.T) { // release.env is git-ignored, so the workflow has none: it passes the tag // it was started by. A script that only reads the file builds nothing in // CI, or builds whatever version the file last named. script := readReleaseScript(t) if !strings.Contains(script, `TAG="${1:-${TAG:-}}"`) { t.Error("the build script does not take the tag from its first argument") } if !strings.Contains(script, `[ -f release.env ]`) { t.Error("the build script requires release.env, which CI does not have") } } func TestTheBuildScriptRefusesATagThatIsNotAVersion(t *testing.T) { // The proxy will not serve a tag it cannot read as a version, so a typo // here builds perfectly and then fails at every `go install`. // // The refusal comes before anything is built or written, so the script // alone is enough: it is run from an empty directory with no release.env. dir := t.TempDir() script, err := os.ReadFile("02-build-releases.sh") if err != nil { t.Fatalf("reading the build script: %v", err) } writeTestFile(t, filepath.Join(dir, "02-build-releases.sh"), string(script)) out, err := runAllowingFailure(t, dir, "bash", "./02-build-releases.sh", "v0.o.0") if err == nil { t.Fatalf("the script accepted a tag that is not a version:\n%s", out) } if !strings.Contains(out, "v1.2.3") { t.Errorf("the refusal does not say what a tag should look like:\n%s", out) } } func TestTheBuildScriptDoesNotHandOffToAnUploadScript(t *testing.T) { // 04 attached the binaries to a release page a personal token had created. // The workflow does both now; a script still pointing at 04 sends the // reader to run something that is not there. if strings.Contains(readReleaseScript(t), "04-release") { t.Error("the build script still hands off to 04-release.upload-binaries.sh") } } // readWorkflow returns the release workflow's text. func readWorkflow(t *testing.T) string { t.Helper() data, err := os.ReadFile(filepath.Join(".github", "workflows", "release.yml")) if err != nil { t.Fatalf("reading the release workflow: %v", err) } return string(data) } func TestTheWorkflowPublishesOnATagPush(t *testing.T) { // The tag push is the trigger: ./01-release.tag.sh ends by pushing one, // and nothing else starts a release. workflow := readWorkflow(t) for _, want := range []string{"push:", "tags:", `- "v*"`} { if !strings.Contains(workflow, want) { t.Errorf("the workflow never declares %q", want) } } // A workflow with the default read-only token cannot create a release, and // fails at its last step after doing all the work. if !strings.Contains(workflow, "contents: write") { t.Error("the workflow does not ask for contents: write") } } func TestTheWorkflowBuildsWithTheSameScriptAPersonRuns(t *testing.T) { // A CI job that builds its own way is a second pipeline nobody tests, and // the local one is then only ever exercised by accident. if !strings.Contains(readWorkflow(t), "./02-build-releases.sh") { t.Error("the workflow does not build the release with ./02-build-releases.sh") } } func TestTheWorkflowAttachesWhatWasBuilt(t *testing.T) { // Publishing a release page with no files attached is a silent half-job: // the page exists and the downloads are not there. workflow := readWorkflow(t) for _, want := range []string{"turbo-python-*", "SHA256SUMS", "fail_on_unmatched_files: true"} { if !strings.Contains(workflow, want) { t.Errorf("the workflow never mentions %q", want) } } } func TestTheWorkflowLinksToTheDocumentationAtThatTag(t *testing.T) { // A release page is not inside the repository tree, so a relative path // from it 404s — and a link to the branch would rot as the branch moves. workflow := readWorkflow(t) if !strings.Contains(workflow, "blob/${GITHUB_REF_NAME}") { t.Error("the release notes do not link into the repository at the released tag") } if !strings.Contains(workflow, "/docs/en/README.md") { t.Error("the release notes do not link to the documentation") } } func TestTheWorkflowNeedsNoPersonalToken(t *testing.T) { // The release API behind Rickub's /gh shim accepts the job's own // GITHUB_TOKEN and refuses a personal one, so a secret referenced here is // a credential that cannot work and still has to be kept somewhere. if strings.Contains(readWorkflow(t), "secrets.") { t.Error("the workflow reads a secret; the job's own token is the only credential the release API takes") } } func TestTheWorkflowDoesNotStartAReleaseInsideItself(t *testing.T) { // The suite it runs includes tests that run ./01-release.tag.sh against a // throwaway clone. Locally the script exports this before calling make; // in CI nothing calls the script, so the job has to set it itself. if !strings.Contains(readWorkflow(t), "TURBO_PYTHON_RELEASING") { t.Error("the workflow runs the suite without TURBO_PYTHON_RELEASING set") } }