ci: retain and rehearse deployment rollbackUnverified
5f1ff8c parent: 30f04fc modified
.github/workflows/deploy-bunny.yml +8 -0 | @@ -172,6 +172,14 @@ jobs: | ||
| 172 | 172 | esac |
| 173 | 173 | done < <(find var/config-site -type f -print0) |
| 174 | 174 | |
| 175 | + - name: Retain rollback bundle | |
| 176 | + uses: actions/upload-artifact@v4 | |
| 177 | + with: | |
| 178 | + name: bunny-rollback-${{ github.run_id }} | |
| 179 | + path: var/deploy-backup/ | |
| 180 | + if-no-files-found: error | |
| 181 | + retention-days: 30 | |
| 182 | + | |
| 175 | 183 | - name: Upload static site to Bunny Storage |
| 176 | 184 | id: upload |
| 177 | 185 | shell: bash |
| @@ -172,6 +172,14 @@ jobs: | |||
| 172 | esac | 172 | esac |
| 173 | done < <(find var/config-site -type f -print0) | 173 | done < <(find var/config-site -type f -print0) |
| 174 | 174 | ||
| 175 | + - name: Retain rollback bundle | ||
| 176 | + uses: actions/upload-artifact@v4 | ||
| 177 | + with: | ||
| 178 | + name: bunny-rollback-${{ github.run_id }} | ||
| 179 | + path: var/deploy-backup/ | ||
| 180 | + if-no-files-found: error | ||
| 181 | + retention-days: 30 | ||
| 182 | + | ||
| 175 | - name: Upload static site to Bunny Storage | 183 | - name: Upload static site to Bunny Storage |
| 176 | id: upload | 184 | id: upload |
| 177 | shell: bash | 185 | shell: bash |
added
.github/workflows/rollback-bunny.yml +160 -0 | new file mode 100644 | ||
| @@ -0,0 +1,160 @@ | ||
| 1 | +name: Roll back Bunny configuration site | |
| 2 | + | |
| 3 | +on: | |
| 4 | + workflow_dispatch: | |
| 5 | + inputs: | |
| 6 | + deployment_run_id: | |
| 7 | + description: Successful deployment run whose pre-deployment bundle must be restored | |
| 8 | + required: true | |
| 9 | + type: string | |
| 10 | + | |
| 11 | +permissions: | |
| 12 | + actions: read | |
| 13 | + contents: read | |
| 14 | + | |
| 15 | +concurrency: | |
| 16 | + group: deploy-bunny-production | |
| 17 | + cancel-in-progress: false | |
| 18 | + | |
| 19 | +jobs: | |
| 20 | + rollback: | |
| 21 | + name: Restore retained mutable files | |
| 22 | + runs-on: ubuntu-latest | |
| 23 | + environment: production | |
| 24 | + steps: | |
| 25 | + - name: Validate deployment run | |
| 26 | + id: source | |
| 27 | + env: | |
| 28 | + DEPLOYMENT_RUN_ID: ${{ inputs.deployment_run_id }} | |
| 29 | + GH_TOKEN: ${{ github.token }} | |
| 30 | + GH_REPO: ${{ github.repository }} | |
| 31 | + run: | | |
| 32 | + set -euo pipefail | |
| 33 | + [[ "$DEPLOYMENT_RUN_ID" =~ ^[1-9][0-9]*$ ]] | |
| 34 | + conclusion="$(gh api "repos/$GH_REPO/actions/runs/$DEPLOYMENT_RUN_ID" --jq .conclusion)" | |
| 35 | + test "$conclusion" = success | |
| 36 | + | |
| 37 | + - name: Download retained rollback bundle | |
| 38 | + env: | |
| 39 | + DEPLOYMENT_RUN_ID: ${{ inputs.deployment_run_id }} | |
| 40 | + GH_TOKEN: ${{ github.token }} | |
| 41 | + GH_REPO: ${{ github.repository }} | |
| 42 | + run: | | |
| 43 | + gh run download "$DEPLOYMENT_RUN_ID" \ | |
| 44 | + --repo "$GH_REPO" \ | |
| 45 | + --name "bunny-rollback-$DEPLOYMENT_RUN_ID" \ | |
| 46 | + --dir var/deploy-backup | |
| 47 | + test -d var/deploy-backup/files | |
| 48 | + test -f var/deploy-backup/missing | |
| 49 | + | |
| 50 | + - name: Restore previous mutable files | |
| 51 | + env: | |
| 52 | + BUNNY_STORAGE_ENDPOINT: ${{ vars.BUNNY_STORAGE_ENDPOINT }} | |
| 53 | + BUNNY_STORAGE_PASSWORD: ${{ secrets.BUNNY_STORAGE_PASSWORD }} | |
| 54 | + run: | | |
| 55 | + set -euo pipefail | |
| 56 | + test -n "$BUNNY_STORAGE_ENDPOINT" | |
| 57 | + test -n "$BUNNY_STORAGE_PASSWORD" | |
| 58 | + endpoint="${BUNNY_STORAGE_ENDPOINT%/}" | |
| 59 | + case "$endpoint" in | |
| 60 | + https://*) ;; | |
| 61 | + *) echo "BUNNY_STORAGE_ENDPOINT must use HTTPS" >&2; exit 1 ;; | |
| 62 | + esac | |
| 63 | + | |
| 64 | + restore_file() { | |
| 65 | + local file="$1" | |
| 66 | + local relative_path="${file#var/deploy-backup/files/}" | |
| 67 | + local mime_type | |
| 68 | + case "$relative_path" in | |
| 69 | + *[!A-Za-z0-9._/-]* | /* | *../*) | |
| 70 | + echo "Refusing unsafe rollback path: $relative_path" >&2 | |
| 71 | + exit 1 | |
| 72 | + ;; | |
| 73 | + esac | |
| 74 | + mime_type="$(file --brief --mime-type "$file")" | |
| 75 | + echo "Restoring $relative_path" | |
| 76 | + curl --fail --silent --show-error \ | |
| 77 | + --retry 3 --retry-all-errors \ | |
| 78 | + --request PUT \ | |
| 79 | + --header "AccessKey: $BUNNY_STORAGE_PASSWORD" \ | |
| 80 | + --header "Content-Type: $mime_type" \ | |
| 81 | + --upload-file "$file" \ | |
| 82 | + "$endpoint/$relative_path" | |
| 83 | + } | |
| 84 | + export endpoint BUNNY_STORAGE_PASSWORD | |
| 85 | + export -f restore_file | |
| 86 | + find var/deploy-backup/files -type f -print0 \ | |
| 87 | + | xargs -0 --no-run-if-empty --max-args=1 --max-procs=16 \ | |
| 88 | + bash -c 'restore_file "$1"' _ | |
| 89 | + | |
| 90 | + while IFS= read -r -d '' relative_path; do | |
| 91 | + case "$relative_path" in | |
| 92 | + *[!A-Za-z0-9._/-]* | /* | *../*) | |
| 93 | + echo "Refusing unsafe rollback deletion: $relative_path" >&2 | |
| 94 | + exit 1 | |
| 95 | + ;; | |
| 96 | + esac | |
| 97 | + status="$(curl --silent --show-error \ | |
| 98 | + --request DELETE \ | |
| 99 | + --header "AccessKey: $BUNNY_STORAGE_PASSWORD" \ | |
| 100 | + --output /dev/null \ | |
| 101 | + --write-out '%{http_code}' \ | |
| 102 | + "$endpoint/$relative_path")" | |
| 103 | + case "$status" in | |
| 104 | + 200 | 201 | 204 | 404) ;; | |
| 105 | + *) | |
| 106 | + echo "Could not remove $relative_path during rollback: HTTP $status" >&2 | |
| 107 | + exit 1 | |
| 108 | + ;; | |
| 109 | + esac | |
| 110 | + done < var/deploy-backup/missing | |
| 111 | + | |
| 112 | + - name: Purge Bunny CDN cache | |
| 113 | + env: | |
| 114 | + BUNNY_API_KEY: ${{ secrets.BUNNY_API_KEY }} | |
| 115 | + BUNNY_PULL_ZONE_ID: ${{ vars.BUNNY_PULL_ZONE_ID }} | |
| 116 | + run: | | |
| 117 | + set -euo pipefail | |
| 118 | + test -n "$BUNNY_API_KEY" | |
| 119 | + test -n "$BUNNY_PULL_ZONE_ID" | |
| 120 | + curl --fail --silent --show-error \ | |
| 121 | + --retry 3 --retry-all-errors \ | |
| 122 | + --request POST \ | |
| 123 | + --header "AccessKey: $BUNNY_API_KEY" \ | |
| 124 | + "https://api.bunny.net/pullzone/$BUNNY_PULL_ZONE_ID/purgeCache" | |
| 125 | + | |
| 126 | + - name: Verify restored mutable files | |
| 127 | + env: | |
| 128 | + CONFIG_SITE_ORIGIN: ${{ vars.CONFIG_SITE_ORIGIN }} | |
| 129 | + run: | | |
| 130 | + set -euo pipefail | |
| 131 | + test -n "$CONFIG_SITE_ORIGIN" | |
| 132 | + origin="${CONFIG_SITE_ORIGIN%/}" | |
| 133 | + | |
| 134 | + for attempt in 1 2 3; do | |
| 135 | + failed=0 | |
| 136 | + while IFS= read -r -d '' file; do | |
| 137 | + relative_path="${file#var/deploy-backup/files/}" | |
| 138 | + if ! curl --fail --silent --show-error \ | |
| 139 | + --header "Cache-Control: no-cache" \ | |
| 140 | + "$origin/$relative_path" \ | |
| 141 | + | cmp --silent "$file" -; then | |
| 142 | + echo "Restored bytes differ for $relative_path" >&2 | |
| 143 | + failed=1 | |
| 144 | + fi | |
| 145 | + done < <(find var/deploy-backup/files -type f -print0) | |
| 146 | + while IFS= read -r -d '' relative_path; do | |
| 147 | + status="$(curl --silent --show-error \ | |
| 148 | + --header "Cache-Control: no-cache" \ | |
| 149 | + --output /dev/null \ | |
| 150 | + --write-out '%{http_code}' \ | |
| 151 | + "$origin/$relative_path")" | |
| 152 | + if [[ "$status" != 404 ]]; then | |
| 153 | + echo "New mutable file remains after rollback: $relative_path (HTTP $status)" >&2 | |
| 154 | + failed=1 | |
| 155 | + fi | |
| 156 | + done < var/deploy-backup/missing | |
| 157 | + if [[ "$failed" = 0 ]]; then exit 0; fi | |
| 158 | + if [[ "$attempt" != 3 ]]; then sleep 10; fi | |
| 159 | + done | |
| 160 | + exit 1 | |
| new file mode 100644 | |||
| @@ -0,0 +1,160 @@ | |||
| 1 | +name: Roll back Bunny configuration site | ||
| 2 | + | ||
| 3 | +on: | ||
| 4 | + workflow_dispatch: | ||
| 5 | + inputs: | ||
| 6 | + deployment_run_id: | ||
| 7 | + description: Successful deployment run whose pre-deployment bundle must be restored | ||
| 8 | + required: true | ||
| 9 | + type: string | ||
| 10 | + | ||
| 11 | +permissions: | ||
| 12 | + actions: read | ||
| 13 | + contents: read | ||
| 14 | + | ||
| 15 | +concurrency: | ||
| 16 | + group: deploy-bunny-production | ||
| 17 | + cancel-in-progress: false | ||
| 18 | + | ||
| 19 | +jobs: | ||
| 20 | + rollback: | ||
| 21 | + name: Restore retained mutable files | ||
| 22 | + runs-on: ubuntu-latest | ||
| 23 | + environment: production | ||
| 24 | + steps: | ||
| 25 | + - name: Validate deployment run | ||
| 26 | + id: source | ||
| 27 | + env: | ||
| 28 | + DEPLOYMENT_RUN_ID: ${{ inputs.deployment_run_id }} | ||
| 29 | + GH_TOKEN: ${{ github.token }} | ||
| 30 | + GH_REPO: ${{ github.repository }} | ||
| 31 | + run: | | ||
| 32 | + set -euo pipefail | ||
| 33 | + [[ "$DEPLOYMENT_RUN_ID" =~ ^[1-9][0-9]*$ ]] | ||
| 34 | + conclusion="$(gh api "repos/$GH_REPO/actions/runs/$DEPLOYMENT_RUN_ID" --jq .conclusion)" | ||
| 35 | + test "$conclusion" = success | ||
| 36 | + | ||
| 37 | + - name: Download retained rollback bundle | ||
| 38 | + env: | ||
| 39 | + DEPLOYMENT_RUN_ID: ${{ inputs.deployment_run_id }} | ||
| 40 | + GH_TOKEN: ${{ github.token }} | ||
| 41 | + GH_REPO: ${{ github.repository }} | ||
| 42 | + run: | | ||
| 43 | + gh run download "$DEPLOYMENT_RUN_ID" \ | ||
| 44 | + --repo "$GH_REPO" \ | ||
| 45 | + --name "bunny-rollback-$DEPLOYMENT_RUN_ID" \ | ||
| 46 | + --dir var/deploy-backup | ||
| 47 | + test -d var/deploy-backup/files | ||
| 48 | + test -f var/deploy-backup/missing | ||
| 49 | + | ||
| 50 | + - name: Restore previous mutable files | ||
| 51 | + env: | ||
| 52 | + BUNNY_STORAGE_ENDPOINT: ${{ vars.BUNNY_STORAGE_ENDPOINT }} | ||
| 53 | + BUNNY_STORAGE_PASSWORD: ${{ secrets.BUNNY_STORAGE_PASSWORD }} | ||
| 54 | + run: | | ||
| 55 | + set -euo pipefail | ||
| 56 | + test -n "$BUNNY_STORAGE_ENDPOINT" | ||
| 57 | + test -n "$BUNNY_STORAGE_PASSWORD" | ||
| 58 | + endpoint="${BUNNY_STORAGE_ENDPOINT%/}" | ||
| 59 | + case "$endpoint" in | ||
| 60 | + https://*) ;; | ||
| 61 | + *) echo "BUNNY_STORAGE_ENDPOINT must use HTTPS" >&2; exit 1 ;; | ||
| 62 | + esac | ||
| 63 | + | ||
| 64 | + restore_file() { | ||
| 65 | + local file="$1" | ||
| 66 | + local relative_path="${file#var/deploy-backup/files/}" | ||
| 67 | + local mime_type | ||
| 68 | + case "$relative_path" in | ||
| 69 | + *[!A-Za-z0-9._/-]* | /* | *../*) | ||
| 70 | + echo "Refusing unsafe rollback path: $relative_path" >&2 | ||
| 71 | + exit 1 | ||
| 72 | + ;; | ||
| 73 | + esac | ||
| 74 | + mime_type="$(file --brief --mime-type "$file")" | ||
| 75 | + echo "Restoring $relative_path" | ||
| 76 | + curl --fail --silent --show-error \ | ||
| 77 | + --retry 3 --retry-all-errors \ | ||
| 78 | + --request PUT \ | ||
| 79 | + --header "AccessKey: $BUNNY_STORAGE_PASSWORD" \ | ||
| 80 | + --header "Content-Type: $mime_type" \ | ||
| 81 | + --upload-file "$file" \ | ||
| 82 | + "$endpoint/$relative_path" | ||
| 83 | + } | ||
| 84 | + export endpoint BUNNY_STORAGE_PASSWORD | ||
| 85 | + export -f restore_file | ||
| 86 | + find var/deploy-backup/files -type f -print0 \ | ||
| 87 | + | xargs -0 --no-run-if-empty --max-args=1 --max-procs=16 \ | ||
| 88 | + bash -c 'restore_file "$1"' _ | ||
| 89 | + | ||
| 90 | + while IFS= read -r -d '' relative_path; do | ||
| 91 | + case "$relative_path" in | ||
| 92 | + *[!A-Za-z0-9._/-]* | /* | *../*) | ||
| 93 | + echo "Refusing unsafe rollback deletion: $relative_path" >&2 | ||
| 94 | + exit 1 | ||
| 95 | + ;; | ||
| 96 | + esac | ||
| 97 | + status="$(curl --silent --show-error \ | ||
| 98 | + --request DELETE \ | ||
| 99 | + --header "AccessKey: $BUNNY_STORAGE_PASSWORD" \ | ||
| 100 | + --output /dev/null \ | ||
| 101 | + --write-out '%{http_code}' \ | ||
| 102 | + "$endpoint/$relative_path")" | ||
| 103 | + case "$status" in | ||
| 104 | + 200 | 201 | 204 | 404) ;; | ||
| 105 | + *) | ||
| 106 | + echo "Could not remove $relative_path during rollback: HTTP $status" >&2 | ||
| 107 | + exit 1 | ||
| 108 | + ;; | ||
| 109 | + esac | ||
| 110 | + done < var/deploy-backup/missing | ||
| 111 | + | ||
| 112 | + - name: Purge Bunny CDN cache | ||
| 113 | + env: | ||
| 114 | + BUNNY_API_KEY: ${{ secrets.BUNNY_API_KEY }} | ||
| 115 | + BUNNY_PULL_ZONE_ID: ${{ vars.BUNNY_PULL_ZONE_ID }} | ||
| 116 | + run: | | ||
| 117 | + set -euo pipefail | ||
| 118 | + test -n "$BUNNY_API_KEY" | ||
| 119 | + test -n "$BUNNY_PULL_ZONE_ID" | ||
| 120 | + curl --fail --silent --show-error \ | ||
| 121 | + --retry 3 --retry-all-errors \ | ||
| 122 | + --request POST \ | ||
| 123 | + --header "AccessKey: $BUNNY_API_KEY" \ | ||
| 124 | + "https://api.bunny.net/pullzone/$BUNNY_PULL_ZONE_ID/purgeCache" | ||
| 125 | + | ||
| 126 | + - name: Verify restored mutable files | ||
| 127 | + env: | ||
| 128 | + CONFIG_SITE_ORIGIN: ${{ vars.CONFIG_SITE_ORIGIN }} | ||
| 129 | + run: | | ||
| 130 | + set -euo pipefail | ||
| 131 | + test -n "$CONFIG_SITE_ORIGIN" | ||
| 132 | + origin="${CONFIG_SITE_ORIGIN%/}" | ||
| 133 | + | ||
| 134 | + for attempt in 1 2 3; do | ||
| 135 | + failed=0 | ||
| 136 | + while IFS= read -r -d '' file; do | ||
| 137 | + relative_path="${file#var/deploy-backup/files/}" | ||
| 138 | + if ! curl --fail --silent --show-error \ | ||
| 139 | + --header "Cache-Control: no-cache" \ | ||
| 140 | + "$origin/$relative_path" \ | ||
| 141 | + | cmp --silent "$file" -; then | ||
| 142 | + echo "Restored bytes differ for $relative_path" >&2 | ||
| 143 | + failed=1 | ||
| 144 | + fi | ||
| 145 | + done < <(find var/deploy-backup/files -type f -print0) | ||
| 146 | + while IFS= read -r -d '' relative_path; do | ||
| 147 | + status="$(curl --silent --show-error \ | ||
| 148 | + --header "Cache-Control: no-cache" \ | ||
| 149 | + --output /dev/null \ | ||
| 150 | + --write-out '%{http_code}' \ | ||
| 151 | + "$origin/$relative_path")" | ||
| 152 | + if [[ "$status" != 404 ]]; then | ||
| 153 | + echo "New mutable file remains after rollback: $relative_path (HTTP $status)" >&2 | ||
| 154 | + failed=1 | ||
| 155 | + fi | ||
| 156 | + done < var/deploy-backup/missing | ||
| 157 | + if [[ "$failed" = 0 ]]; then exit 0; fi | ||
| 158 | + if [[ "$attempt" != 3 ]]; then sleep 10; fi | ||
| 159 | + done | ||
| 160 | + exit 1 | ||
modified
packages/config-page/test/config-site-build.test.ts +9 -0 | @@ -131,6 +131,10 @@ test("Bunny releases deploy production tags while preview tags stop after qualit | ||
| 131 | 131 | new URL("../../../.github/workflows/deploy-bunny.yml", import.meta.url), |
| 132 | 132 | "utf8", |
| 133 | 133 | ); |
| 134 | + const rollbackWorkflow = readFileSync( | |
| 135 | + new URL("../../../.github/workflows/rollback-bunny.yml", import.meta.url), | |
| 136 | + "utf8", | |
| 137 | + ); | |
| 134 | 138 | assert.doesNotMatch(workflow, /branches:\s*\[develop\]/u); |
| 135 | 139 | assert.match(workflow, /tags:\s*\["v\*", "pre-v\*"\]/u); |
| 136 | 140 | assert.match(workflow, /startsWith\(github\.ref_name, 'v'\)/u); |
| @@ -155,7 +159,12 @@ test("Bunny releases deploy production tags while preview tags stop after qualit | ||
| 155 | 159 | /if \[\[ -f var\/config-site\/catalog\/manifest\.json \]\]; then\s+upload_file/u, |
| 156 | 160 | ); |
| 157 | 161 | assert.match(workflow, /Back up mutable production files[\s\S]+id: backup/u); |
| 162 | + assert.match(workflow, /name: bunny-rollback-\$\{\{ github\.run_id \}\}[\s\S]+retention-days: 30/u); | |
| 158 | 163 | assert.match(workflow, /Verify served static site[\s\S]+npm run verify:config-site/u); |
| 159 | 164 | assert.match(workflow, /failure\(\) && steps\.backup\.outcome == 'success'/u); |
| 160 | 165 | assert.match(workflow, /Restoring \$relative_path[\s\S]+Removing newly introduced mutable file/u); |
| 166 | + assert.match(rollbackWorkflow, /deployment_run_id:[\s\S]+type: string/u); | |
| 167 | + assert.match(rollbackWorkflow, /gh run download "\$DEPLOYMENT_RUN_ID"[\s\S]+bunny-rollback-\$DEPLOYMENT_RUN_ID/u); | |
| 168 | + assert.match(rollbackWorkflow, /conclusion="\$\(gh api[\s\S]+test "\$conclusion" = success/u); | |
| 169 | + assert.match(rollbackWorkflow, /Restore previous mutable files[\s\S]+Purge Bunny CDN cache[\s\S]+Verify restored mutable files/u); | |
| 161 | 170 | }); |
| @@ -131,6 +131,10 @@ test("Bunny releases deploy production tags while preview tags stop after qualit | |||
| 131 | new URL("../../../.github/workflows/deploy-bunny.yml", import.meta.url), | 131 | new URL("../../../.github/workflows/deploy-bunny.yml", import.meta.url), |
| 132 | "utf8", | 132 | "utf8", |
| 133 | ); | 133 | ); |
| 134 | + const rollbackWorkflow = readFileSync( | ||
| 135 | + new URL("../../../.github/workflows/rollback-bunny.yml", import.meta.url), | ||
| 136 | + "utf8", | ||
| 137 | + ); | ||
| 134 | assert.doesNotMatch(workflow, /branches:\s*\[develop\]/u); | 138 | assert.doesNotMatch(workflow, /branches:\s*\[develop\]/u); |
| 135 | assert.match(workflow, /tags:\s*\["v\*", "pre-v\*"\]/u); | 139 | assert.match(workflow, /tags:\s*\["v\*", "pre-v\*"\]/u); |
| 136 | assert.match(workflow, /startsWith\(github\.ref_name, 'v'\)/u); | 140 | assert.match(workflow, /startsWith\(github\.ref_name, 'v'\)/u); |
| @@ -155,7 +159,12 @@ test("Bunny releases deploy production tags while preview tags stop after qualit | |||
| 155 | /if \[\[ -f var\/config-site\/catalog\/manifest\.json \]\]; then\s+upload_file/u, | 159 | /if \[\[ -f var\/config-site\/catalog\/manifest\.json \]\]; then\s+upload_file/u, |
| 156 | ); | 160 | ); |
| 157 | assert.match(workflow, /Back up mutable production files[\s\S]+id: backup/u); | 161 | assert.match(workflow, /Back up mutable production files[\s\S]+id: backup/u); |
| 162 | + assert.match(workflow, /name: bunny-rollback-\$\{\{ github\.run_id \}\}[\s\S]+retention-days: 30/u); | ||
| 158 | assert.match(workflow, /Verify served static site[\s\S]+npm run verify:config-site/u); | 163 | assert.match(workflow, /Verify served static site[\s\S]+npm run verify:config-site/u); |
| 159 | assert.match(workflow, /failure\(\) && steps\.backup\.outcome == 'success'/u); | 164 | assert.match(workflow, /failure\(\) && steps\.backup\.outcome == 'success'/u); |
| 160 | assert.match(workflow, /Restoring \$relative_path[\s\S]+Removing newly introduced mutable file/u); | 165 | assert.match(workflow, /Restoring \$relative_path[\s\S]+Removing newly introduced mutable file/u); |
| 166 | + assert.match(rollbackWorkflow, /deployment_run_id:[\s\S]+type: string/u); | ||
| 167 | + assert.match(rollbackWorkflow, /gh run download "\$DEPLOYMENT_RUN_ID"[\s\S]+bunny-rollback-\$DEPLOYMENT_RUN_ID/u); | ||
| 168 | + assert.match(rollbackWorkflow, /conclusion="\$\(gh api[\s\S]+test "\$conclusion" = success/u); | ||
| 169 | + assert.match(rollbackWorkflow, /Restore previous mutable files[\s\S]+Purge Bunny CDN cache[\s\S]+Verify restored mutable files/u); | ||
| 161 | }); | 170 | }); |