ci: verify deploys and restore failuresUnverified
f224ada parent: 9dc8049 modified
.github/workflows/deploy-bunny.yml +112 -5 | @@ -62,20 +62,62 @@ jobs: | ||
| 62 | 62 | - run: npm ci |
| 63 | 63 | |
| 64 | 64 | - name: Refresh production catalog |
| 65 | - if: ${{ github.event_name == 'workflow_dispatch' && inputs.refresh_catalog }} | |
| 66 | 65 | env: |
| 67 | 66 | IDFM_DATASET_TOKEN: ${{ secrets.IDFM_DATASET_TOKEN }} |
| 68 | 67 | run: npm run catalog:refresh |
| 69 | 68 | |
| 70 | 69 | - name: Build static configuration site |
| 71 | - if: ${{ github.event_name == 'workflow_dispatch' && inputs.refresh_catalog }} | |
| 72 | 70 | run: npm run build:config-site |
| 73 | 71 | |
| 74 | - - name: Build configuration page | |
| 75 | - if: ${{ github.event_name != 'workflow_dispatch' || inputs.refresh_catalog != true }} | |
| 76 | - run: npm run build:config-page | |
| 72 | + - name: Back up mutable production files | |
| 73 | + id: backup | |
| 74 | + shell: bash | |
| 75 | + env: | |
| 76 | + BUNNY_STORAGE_ENDPOINT: ${{ vars.BUNNY_STORAGE_ENDPOINT }} | |
| 77 | + BUNNY_STORAGE_PASSWORD: ${{ secrets.BUNNY_STORAGE_PASSWORD }} | |
| 78 | + run: | | |
| 79 | + set -euo pipefail | |
| 80 | + | |
| 81 | + test -n "$BUNNY_STORAGE_ENDPOINT" | |
| 82 | + test -n "$BUNNY_STORAGE_PASSWORD" | |
| 83 | + endpoint="${BUNNY_STORAGE_ENDPOINT%/}" | |
| 84 | + mkdir -p var/deploy-backup/files | |
| 85 | + : > var/deploy-backup/missing | |
| 86 | + | |
| 87 | + while IFS= read -r -d '' file; do | |
| 88 | + relative_path="${file#var/config-site/}" | |
| 89 | + if [[ "$relative_path" =~ ^catalog/[a-f0-9]{64}/ ]]; then | |
| 90 | + continue | |
| 91 | + fi | |
| 92 | + case "$relative_path" in | |
| 93 | + *[!A-Za-z0-9._/-]* | /* | *../*) | |
| 94 | + echo "Refusing unsafe backup path: $relative_path" >&2 | |
| 95 | + exit 1 | |
| 96 | + ;; | |
| 97 | + esac | |
| 98 | + | |
| 99 | + target="var/deploy-backup/files/$relative_path" | |
| 100 | + mkdir -p "$(dirname "$target")" | |
| 101 | + status="$(curl --silent --show-error \ | |
| 102 | + --output "$target" \ | |
| 103 | + --write-out '%{http_code}' \ | |
| 104 | + --header "AccessKey: $BUNNY_STORAGE_PASSWORD" \ | |
| 105 | + "$endpoint/$relative_path")" | |
| 106 | + case "$status" in | |
| 107 | + 200) ;; | |
| 108 | + 404) | |
| 109 | + rm -f "$target" | |
| 110 | + printf '%s\0' "$relative_path" >> var/deploy-backup/missing | |
| 111 | + ;; | |
| 112 | + *) | |
| 113 | + echo "Could not back up $relative_path: HTTP $status" >&2 | |
| 114 | + exit 1 | |
| 115 | + ;; | |
| 116 | + esac | |
| 117 | + done < <(find var/config-site -type f -print0) | |
| 77 | 118 | |
| 78 | 119 | - name: Upload static site to Bunny Storage |
| 120 | + id: upload | |
| 79 | 121 | shell: bash |
| 80 | 122 | env: |
| 81 | 123 | BUNNY_STORAGE_ENDPOINT: ${{ vars.BUNNY_STORAGE_ENDPOINT }} |
| @@ -137,6 +179,71 @@ jobs: | ||
| 137 | 179 | --header "AccessKey: $BUNNY_API_KEY" \ |
| 138 | 180 | "https://api.bunny.net/pullzone/$BUNNY_PULL_ZONE_ID/purgeCache" |
| 139 | 181 | |
| 182 | + - name: Verify served static site | |
| 183 | + env: | |
| 184 | + CONFIG_SITE_ORIGIN: ${{ vars.CONFIG_SITE_ORIGIN }} | |
| 185 | + run: | | |
| 186 | + for attempt in 1 2 3; do | |
| 187 | + if npm run verify:config-site; then | |
| 188 | + exit 0 | |
| 189 | + fi | |
| 190 | + if [[ "$attempt" != 3 ]]; then sleep 10; fi | |
| 191 | + done | |
| 192 | + exit 1 | |
| 193 | + | |
| 194 | + - name: Restore previous mutable files after failure | |
| 195 | + if: ${{ failure() && steps.backup.outcome == 'success' }} | |
| 196 | + shell: bash | |
| 197 | + env: | |
| 198 | + BUNNY_STORAGE_ENDPOINT: ${{ vars.BUNNY_STORAGE_ENDPOINT }} | |
| 199 | + BUNNY_STORAGE_PASSWORD: ${{ secrets.BUNNY_STORAGE_PASSWORD }} | |
| 200 | + BUNNY_API_KEY: ${{ secrets.BUNNY_API_KEY }} | |
| 201 | + BUNNY_PULL_ZONE_ID: ${{ vars.BUNNY_PULL_ZONE_ID }} | |
| 202 | + run: | | |
| 203 | + set -euo pipefail | |
| 204 | + | |
| 205 | + endpoint="${BUNNY_STORAGE_ENDPOINT%/}" | |
| 206 | + restore_file() { | |
| 207 | + local file="$1" | |
| 208 | + local relative_path="${file#var/deploy-backup/files/}" | |
| 209 | + local mime_type | |
| 210 | + mime_type="$(file --brief --mime-type "$file")" | |
| 211 | + echo "Restoring $relative_path" | |
| 212 | + curl --fail --silent --show-error \ | |
| 213 | + --request PUT \ | |
| 214 | + --header "AccessKey: $BUNNY_STORAGE_PASSWORD" \ | |
| 215 | + --header "Content-Type: $mime_type" \ | |
| 216 | + --upload-file "$file" \ | |
| 217 | + "$endpoint/$relative_path" | |
| 218 | + } | |
| 219 | + export endpoint BUNNY_STORAGE_PASSWORD | |
| 220 | + export -f restore_file | |
| 221 | + find var/deploy-backup/files -type f -print0 \ | |
| 222 | + | xargs -0 --no-run-if-empty --max-args=1 --max-procs=16 \ | |
| 223 | + bash -c 'restore_file "$1"' _ | |
| 224 | + | |
| 225 | + while IFS= read -r -d '' relative_path; do | |
| 226 | + echo "Removing newly introduced mutable file $relative_path" | |
| 227 | + status="$(curl --silent --show-error \ | |
| 228 | + --request DELETE \ | |
| 229 | + --header "AccessKey: $BUNNY_STORAGE_PASSWORD" \ | |
| 230 | + --output /dev/null \ | |
| 231 | + --write-out '%{http_code}' \ | |
| 232 | + "$endpoint/$relative_path")" | |
| 233 | + case "$status" in | |
| 234 | + 200 | 201 | 204 | 404) ;; | |
| 235 | + *) | |
| 236 | + echo "Could not remove $relative_path during rollback: HTTP $status" >&2 | |
| 237 | + exit 1 | |
| 238 | + ;; | |
| 239 | + esac | |
| 240 | + done < var/deploy-backup/missing | |
| 241 | + | |
| 242 | + curl --fail --silent --show-error \ | |
| 243 | + --request POST \ | |
| 244 | + --header "AccessKey: $BUNNY_API_KEY" \ | |
| 245 | + "https://api.bunny.net/pullzone/$BUNNY_PULL_ZONE_ID/purgeCache" | |
| 246 | + | |
| 140 | 247 | publish: |
| 141 | 248 | name: Publish GitHub release notes |
| 142 | 249 | needs: [test, deploy] |
| @@ -62,20 +62,62 @@ jobs: | |||
| 62 | - run: npm ci | 62 | - run: npm ci |
| 63 | 63 | ||
| 64 | - name: Refresh production catalog | 64 | - name: Refresh production catalog |
| 65 | - if: ${{ github.event_name == 'workflow_dispatch' && inputs.refresh_catalog }} | ||
| 66 | env: | 65 | env: |
| 67 | IDFM_DATASET_TOKEN: ${{ secrets.IDFM_DATASET_TOKEN }} | 66 | IDFM_DATASET_TOKEN: ${{ secrets.IDFM_DATASET_TOKEN }} |
| 68 | run: npm run catalog:refresh | 67 | run: npm run catalog:refresh |
| 69 | 68 | ||
| 70 | - name: Build static configuration site | 69 | - name: Build static configuration site |
| 71 | - if: ${{ github.event_name == 'workflow_dispatch' && inputs.refresh_catalog }} | ||
| 72 | run: npm run build:config-site | 70 | run: npm run build:config-site |
| 73 | 71 | ||
| 74 | - - name: Build configuration page | 72 | + - name: Back up mutable production files |
| 75 | - if: ${{ github.event_name != 'workflow_dispatch' || inputs.refresh_catalog != true }} | 73 | + id: backup |
| 76 | - run: npm run build:config-page | 74 | + shell: bash |
| 75 | + env: | ||
| 76 | + BUNNY_STORAGE_ENDPOINT: ${{ vars.BUNNY_STORAGE_ENDPOINT }} | ||
| 77 | + BUNNY_STORAGE_PASSWORD: ${{ secrets.BUNNY_STORAGE_PASSWORD }} | ||
| 78 | + run: | | ||
| 79 | + set -euo pipefail | ||
| 80 | + | ||
| 81 | + test -n "$BUNNY_STORAGE_ENDPOINT" | ||
| 82 | + test -n "$BUNNY_STORAGE_PASSWORD" | ||
| 83 | + endpoint="${BUNNY_STORAGE_ENDPOINT%/}" | ||
| 84 | + mkdir -p var/deploy-backup/files | ||
| 85 | + : > var/deploy-backup/missing | ||
| 86 | + | ||
| 87 | + while IFS= read -r -d '' file; do | ||
| 88 | + relative_path="${file#var/config-site/}" | ||
| 89 | + if [[ "$relative_path" =~ ^catalog/[a-f0-9]{64}/ ]]; then | ||
| 90 | + continue | ||
| 91 | + fi | ||
| 92 | + case "$relative_path" in | ||
| 93 | + *[!A-Za-z0-9._/-]* | /* | *../*) | ||
| 94 | + echo "Refusing unsafe backup path: $relative_path" >&2 | ||
| 95 | + exit 1 | ||
| 96 | + ;; | ||
| 97 | + esac | ||
| 98 | + | ||
| 99 | + target="var/deploy-backup/files/$relative_path" | ||
| 100 | + mkdir -p "$(dirname "$target")" | ||
| 101 | + status="$(curl --silent --show-error \ | ||
| 102 | + --output "$target" \ | ||
| 103 | + --write-out '%{http_code}' \ | ||
| 104 | + --header "AccessKey: $BUNNY_STORAGE_PASSWORD" \ | ||
| 105 | + "$endpoint/$relative_path")" | ||
| 106 | + case "$status" in | ||
| 107 | + 200) ;; | ||
| 108 | + 404) | ||
| 109 | + rm -f "$target" | ||
| 110 | + printf '%s\0' "$relative_path" >> var/deploy-backup/missing | ||
| 111 | + ;; | ||
| 112 | + *) | ||
| 113 | + echo "Could not back up $relative_path: HTTP $status" >&2 | ||
| 114 | + exit 1 | ||
| 115 | + ;; | ||
| 116 | + esac | ||
| 117 | + done < <(find var/config-site -type f -print0) | ||
| 77 | 118 | ||
| 78 | - name: Upload static site to Bunny Storage | 119 | - name: Upload static site to Bunny Storage |
| 120 | + id: upload | ||
| 79 | shell: bash | 121 | shell: bash |
| 80 | env: | 122 | env: |
| 81 | BUNNY_STORAGE_ENDPOINT: ${{ vars.BUNNY_STORAGE_ENDPOINT }} | 123 | BUNNY_STORAGE_ENDPOINT: ${{ vars.BUNNY_STORAGE_ENDPOINT }} |
| @@ -137,6 +179,71 @@ jobs: | |||
| 137 | --header "AccessKey: $BUNNY_API_KEY" \ | 179 | --header "AccessKey: $BUNNY_API_KEY" \ |
| 138 | "https://api.bunny.net/pullzone/$BUNNY_PULL_ZONE_ID/purgeCache" | 180 | "https://api.bunny.net/pullzone/$BUNNY_PULL_ZONE_ID/purgeCache" |
| 139 | 181 | ||
| 182 | + - name: Verify served static site | ||
| 183 | + env: | ||
| 184 | + CONFIG_SITE_ORIGIN: ${{ vars.CONFIG_SITE_ORIGIN }} | ||
| 185 | + run: | | ||
| 186 | + for attempt in 1 2 3; do | ||
| 187 | + if npm run verify:config-site; then | ||
| 188 | + exit 0 | ||
| 189 | + fi | ||
| 190 | + if [[ "$attempt" != 3 ]]; then sleep 10; fi | ||
| 191 | + done | ||
| 192 | + exit 1 | ||
| 193 | + | ||
| 194 | + - name: Restore previous mutable files after failure | ||
| 195 | + if: ${{ failure() && steps.backup.outcome == 'success' }} | ||
| 196 | + shell: bash | ||
| 197 | + env: | ||
| 198 | + BUNNY_STORAGE_ENDPOINT: ${{ vars.BUNNY_STORAGE_ENDPOINT }} | ||
| 199 | + BUNNY_STORAGE_PASSWORD: ${{ secrets.BUNNY_STORAGE_PASSWORD }} | ||
| 200 | + BUNNY_API_KEY: ${{ secrets.BUNNY_API_KEY }} | ||
| 201 | + BUNNY_PULL_ZONE_ID: ${{ vars.BUNNY_PULL_ZONE_ID }} | ||
| 202 | + run: | | ||
| 203 | + set -euo pipefail | ||
| 204 | + | ||
| 205 | + endpoint="${BUNNY_STORAGE_ENDPOINT%/}" | ||
| 206 | + restore_file() { | ||
| 207 | + local file="$1" | ||
| 208 | + local relative_path="${file#var/deploy-backup/files/}" | ||
| 209 | + local mime_type | ||
| 210 | + mime_type="$(file --brief --mime-type "$file")" | ||
| 211 | + echo "Restoring $relative_path" | ||
| 212 | + curl --fail --silent --show-error \ | ||
| 213 | + --request PUT \ | ||
| 214 | + --header "AccessKey: $BUNNY_STORAGE_PASSWORD" \ | ||
| 215 | + --header "Content-Type: $mime_type" \ | ||
| 216 | + --upload-file "$file" \ | ||
| 217 | + "$endpoint/$relative_path" | ||
| 218 | + } | ||
| 219 | + export endpoint BUNNY_STORAGE_PASSWORD | ||
| 220 | + export -f restore_file | ||
| 221 | + find var/deploy-backup/files -type f -print0 \ | ||
| 222 | + | xargs -0 --no-run-if-empty --max-args=1 --max-procs=16 \ | ||
| 223 | + bash -c 'restore_file "$1"' _ | ||
| 224 | + | ||
| 225 | + while IFS= read -r -d '' relative_path; do | ||
| 226 | + echo "Removing newly introduced mutable file $relative_path" | ||
| 227 | + status="$(curl --silent --show-error \ | ||
| 228 | + --request DELETE \ | ||
| 229 | + --header "AccessKey: $BUNNY_STORAGE_PASSWORD" \ | ||
| 230 | + --output /dev/null \ | ||
| 231 | + --write-out '%{http_code}' \ | ||
| 232 | + "$endpoint/$relative_path")" | ||
| 233 | + case "$status" in | ||
| 234 | + 200 | 201 | 204 | 404) ;; | ||
| 235 | + *) | ||
| 236 | + echo "Could not remove $relative_path during rollback: HTTP $status" >&2 | ||
| 237 | + exit 1 | ||
| 238 | + ;; | ||
| 239 | + esac | ||
| 240 | + done < var/deploy-backup/missing | ||
| 241 | + | ||
| 242 | + curl --fail --silent --show-error \ | ||
| 243 | + --request POST \ | ||
| 244 | + --header "AccessKey: $BUNNY_API_KEY" \ | ||
| 245 | + "https://api.bunny.net/pullzone/$BUNNY_PULL_ZONE_ID/purgeCache" | ||
| 246 | + | ||
| 140 | publish: | 247 | publish: |
| 141 | name: Publish GitHub release notes | 248 | name: Publish GitHub release notes |
| 142 | needs: [test, deploy] | 249 | needs: [test, deploy] |
modified
packages/config-page/test/config-site-build.test.ts +7 -2 | @@ -143,10 +143,15 @@ test("Bunny releases deploy production tags while preview tags stop after qualit | ||
| 143 | 143 | assert.doesNotMatch(workflow, /--generate-notes/u); |
| 144 | 144 | assert.match(workflow, /github\.event_name == 'workflow_dispatch' && inputs\.refresh_catalog/u); |
| 145 | 145 | assert.match(workflow, /refresh_catalog:\s+description:[\s\S]+type: boolean/u); |
| 146 | - assert.match(workflow, /inputs\.refresh_catalog != true[\s\S]+npm run build:config-page/u); | |
| 147 | - assert.match(workflow, /inputs\.refresh_catalog \}\}[\s\S]+npm run catalog:refresh/u); | |
| 146 | + assert.match(workflow, /Refresh production catalog[\s\S]+npm run catalog:refresh/u); | |
| 147 | + assert.match(workflow, /Build static configuration site[\s\S]+npm run build:config-site/u); | |
| 148 | + assert.doesNotMatch(workflow, /npm run build:config-page/u); | |
| 148 | 149 | assert.match( |
| 149 | 150 | workflow, |
| 150 | 151 | /if \[\[ -f var\/config-site\/catalog\/manifest\.json \]\]; then\s+upload_file/u, |
| 151 | 152 | ); |
| 153 | + assert.match(workflow, /Back up mutable production files[\s\S]+id: backup/u); | |
| 154 | + assert.match(workflow, /Verify served static site[\s\S]+npm run verify:config-site/u); | |
| 155 | + assert.match(workflow, /failure\(\) && steps\.backup\.outcome == 'success'/u); | |
| 156 | + assert.match(workflow, /Restoring \$relative_path[\s\S]+Removing newly introduced mutable file/u); | |
| 152 | 157 | }); |
| @@ -143,10 +143,15 @@ test("Bunny releases deploy production tags while preview tags stop after qualit | |||
| 143 | assert.doesNotMatch(workflow, /--generate-notes/u); | 143 | assert.doesNotMatch(workflow, /--generate-notes/u); |
| 144 | assert.match(workflow, /github\.event_name == 'workflow_dispatch' && inputs\.refresh_catalog/u); | 144 | assert.match(workflow, /github\.event_name == 'workflow_dispatch' && inputs\.refresh_catalog/u); |
| 145 | assert.match(workflow, /refresh_catalog:\s+description:[\s\S]+type: boolean/u); | 145 | assert.match(workflow, /refresh_catalog:\s+description:[\s\S]+type: boolean/u); |
| 146 | - assert.match(workflow, /inputs\.refresh_catalog != true[\s\S]+npm run build:config-page/u); | 146 | + assert.match(workflow, /Refresh production catalog[\s\S]+npm run catalog:refresh/u); |
| 147 | - assert.match(workflow, /inputs\.refresh_catalog \}\}[\s\S]+npm run catalog:refresh/u); | 147 | + assert.match(workflow, /Build static configuration site[\s\S]+npm run build:config-site/u); |
| 148 | + assert.doesNotMatch(workflow, /npm run build:config-page/u); | ||
| 148 | assert.match( | 149 | assert.match( |
| 149 | workflow, | 150 | workflow, |
| 150 | /if \[\[ -f var\/config-site\/catalog\/manifest\.json \]\]; then\s+upload_file/u, | 151 | /if \[\[ -f var\/config-site\/catalog\/manifest\.json \]\]; then\s+upload_file/u, |
| 151 | ); | 152 | ); |
| 153 | + assert.match(workflow, /Back up mutable production files[\s\S]+id: backup/u); | ||
| 154 | + assert.match(workflow, /Verify served static site[\s\S]+npm run verify:config-site/u); | ||
| 155 | + assert.match(workflow, /failure\(\) && steps\.backup\.outcome == 'success'/u); | ||
| 156 | + assert.match(workflow, /Restoring \$relative_path[\s\S]+Removing newly introduced mutable file/u); | ||
| 152 | }); | 157 | }); |