[ci] Allow release corrections before PyPI publication

.github/workflows/release.yml

@@ -8,7 +8,7 @@ concurrency:
   group: release-${{ github.ref }}
   cancel-in-progress: false
 jobs:
-  validate:
+  check-release-version-eligibility:
     if: github.repository == 'nhairs/python-json-logger'
     runs-on: ubuntu-latest
     steps:
@@ -20,15 +20,19 @@ jobs:
         with:
           python-version: '3.12'
       - run: python -m pip install packaging
-      - run: python scripts/release.py prepare
-  ci:
-    needs: validate
+      - name: Check new-release eligibility before lint and tests
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          python scripts/release.py prepare-release-metadata
+          python scripts/release.py check-release-version-eligibility
+  lint-and-test:
+    needs: check-release-version-eligibility
     uses: ./.github/workflows/test-suite.yml
-  build:
-    permissions:
-      contents: read
-      actions: read
-    needs: [validate, ci]
+  build-and-verify-release-bundle:
+    outputs:
+      bundle-id: ${{ steps.bundle.outputs.artifact-id }}
+    needs: [check-release-version-eligibility, lint-and-test]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -39,40 +43,24 @@ jobs:
         with:
           python-version: '3.12'
       - run: python -m pip install build packaging twine
-      - name: Recover retained artifacts before considering a build
-        id: recover
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: python scripts/release.py recover
-      - name: Refuse to rebuild an existing release
-        if: steps.recover.outputs.recovered != 'true'
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          python scripts/release.py prepare
-          python scripts/release.py guard
-      - if: steps.recover.outputs.recovered != 'true'
-        run: |
+      - run: |
+          python scripts/release.py prepare-release-metadata
           python -m build --outdir bundle/dist
-          python scripts/release.py validate
+          python scripts/release.py validate-distributions
           python -m twine check bundle/dist/*
           python -m venv "$RUNNER_TEMP/release-install"
           "$RUNNER_TEMP/release-install/bin/python" -m pip install --no-deps bundle/dist/*.whl
           "$RUNNER_TEMP/release-install/bin/python" -I scripts/release_smoke.py
-      - name: Verify bundle identity and checksums
-        run: python scripts/release.py verify
-      # Re-retain official release recovery for downstream jobs. An available
-      # original artifact is never replaced; overwrite only clears expired entries.
+          python scripts/release.py verify-release-bundle
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        if: steps.recover.outputs.retained != 'true'
+        id: bundle
         with:
-          name: release-bundle
-          overwrite: true
+          name: release-bundle-${{ github.run_attempt }}
           path: bundle/
           if-no-files-found: error
           retention-days: 90
-  draft:
-    needs: build
+  stage-github-draft:
+    needs: build-and-verify-release-bundle
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -86,62 +74,112 @@ jobs:
       - run: python -m pip install packaging
       - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
-          name: release-bundle
+          artifact-ids: ${{ needs.build-and-verify-release-bundle.outputs.bundle-id }}
+          merge-multiple: true
           path: bundle
-      - run: python scripts/release.py stage
+      - run: python scripts/release.py stage-github-draft
         env:
           GH_TOKEN: ${{ github.token }}
-  select:
-    needs: draft
-    runs-on: ubuntu-latest
+  publish-to-pypi:
     outputs:
-      missing: ${{ steps.files.outputs.missing }}
+      uploaded: ${{ steps.upload.outcome }}
+    needs: [build-and-verify-release-bundle, stage-github-draft]
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/python-json-logger
+    permissions:
+      contents: read
+      id-token: write
     steps:
+      # Only the reviewed release helper is needed; preflight uses no third-party packages.
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
+          ref: ${{ github.sha }}
           persist-credentials: false
+          sparse-checkout: scripts/release.py
+          sparse-checkout-cone-mode: false
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: '3.12'
-      - run: python -m pip install packaging
       - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
-          name: release-bundle
+          artifact-ids: ${{ needs.build-and-verify-release-bundle.outputs.bundle-id }}
+          merge-multiple: true
           path: bundle
-      - name: Verify retained bytes and select missing PyPI files
-        id: files
-        run: python scripts/release.py select
+      - name: Recheck approved source, draft, bytes and PyPI absence
+        id: preflight
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: python -I scripts/release.py check-pypi-publication-readiness
+      - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
+        id: upload
+        with:
+          packages-dir: bundle/dist/
+      - name: Retain publication evidence even after failure
+        if: always()
+        env:
+          BUNDLE_ID: ${{ needs.build-and-verify-release-bundle.outputs.bundle-id }}
+          PREFLIGHT: ${{ steps.preflight.outcome }}
+          UPLOAD: ${{ steps.upload.outcome }}
+        shell: bash
+        run: |
+          mkdir -p evidence/attestations
+          cp bundle/release.json bundle/SHA256SUMS.json evidence/ 2>/dev/null || true
+          find bundle/dist -maxdepth 1 -name '*.publish.attestation' -exec cp {} evidence/attestations/ \; 2>/dev/null || true
+          count=$(find evidence/attestations -type f | wc -l)
+          if [ "$count" -gt 0 ]; then
+            (cd evidence && sha256sum attestations/* > attestation-checksums.txt)
+          fi
+          jq -n --arg run "$GITHUB_RUN_ID" --arg attempt "$GITHUB_RUN_ATTEMPT" \
+            --arg bundle "$BUNDLE_ID" --arg preflight "$PREFLIGHT" --arg upload "$UPLOAD" \
+            --argjson attestations "$count" \
+            '{run_id:$run, run_attempt:$attempt, bundle_artifact_id:$bundle,
+              preflight:$preflight, upload:$upload, attestation_count:$attestations,
+              attestation_status:(if $attestations == 0 then "none generated or available" else "retained" end)}' \
+            > evidence/publication.json
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        if: steps.files.outputs.missing == 'true'
+        if: always()
         with:
-          name: pypi-upload-${{ github.run_attempt }}
-          path: upload/
+          # A failed-job rerun may reuse a successful build's earlier bundle ID.
+          name: publication-evidence-${{ github.run_attempt }}
+          path: evidence/
           if-no-files-found: error
           retention-days: 90
-  publish:
-    needs: select
+  verify-and-publish-github-release:
+    if: ${{ !cancelled() && needs.publish-to-pypi.outputs.uploaded == 'success' }}
+    needs: [build-and-verify-release-bundle, publish-to-pypi]
     runs-on: ubuntu-latest
-    environment:
-      name: pypi
-      url: https://pypi.org/p/python-json-logger
     permissions:
       contents: write
-      id-token: write
     steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: '3.12'
+      - run: python -m pip install packaging
       - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        if: needs.select.outputs.missing == 'true'
         with:
-          name: pypi-upload-${{ github.run_attempt }}
-          path: upload
-      - name: Publish the verified GitHub release before uploading to PyPI
+          artifact-ids: ${{ needs.build-and-verify-release-bundle.outputs.bundle-id }}
+          merge-multiple: true
+          path: bundle
+      - name: Verify complete PyPI publication and publish the original GitHub draft
+        id: finalize
         env:
           GH_TOKEN: ${{ github.token }}
-        run: |
-          is_draft=$(gh release view "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft)
-          if [ "$is_draft" = true ]; then
-            gh release edit "$GITHUB_REF_NAME" --draft=false --repo "$GITHUB_REPOSITORY"
-          fi
-      - uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
-        if: needs.select.outputs.missing == 'true'
+        run: python scripts/release.py verify-and-publish-github-release
+      - name: Record finalization outcome
+        if: always()
+        env:
+          BUNDLE_ID: ${{ needs.build-and-verify-release-bundle.outputs.bundle-id }}
+          FINALIZE: ${{ steps.finalize.outcome }}
+        run: python scripts/release.py record-github-finalization-outcome
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        if: always()
         with:
-          packages-dir: upload/
+          name: finalization-evidence-${{ github.run_attempt }}
+          path: evidence/
+          if-no-files-found: error
+          retention-days: 90

docs/releasing.md

@@ -1,7 +1,8 @@
 # Release checklist
 
-The `release.yml` workflow builds distributions and stages a GitHub draft.
-After `pypi` approval, it publishes GitHub first, then uploads those bytes to PyPI.
+The `release.yml` workflow checks eligibility, runs CI, builds distributions, and
+stages a GitHub draft. After `pypi` approval, it uploads to PyPI and verifies both
+files before publishing GitHub.
 
 ## Prepare
 
@@ -21,40 +22,112 @@ git tag v4.3.0.rc1
 git push origin v4.3.0.rc1
 ```
 
+Before lint or tests, eligibility rejects any PyPI record for the target version,
+including partial or yanked releases. Only HTTP 404 means absent; service and
+network errors stop the run. A public GitHub release also blocks a new attempt
+and requires owner cleanup. Existing drafts are allowed.
+
 ## Approve
 
 - [ ] Check lint and the Linux, macOS, and Windows test results.
 - [ ] Review the GitHub draft: title, full changelog, and prerelease flag.
 - [ ] Check the wheel, sdist, `SHA256SUMS.json`, and `release.json` assets.
 - [ ] Approve the `pypi` deployment.
-- [ ] Confirm GitHub is published and PyPI has both files.
-
-File selection runs with read-only permissions before approval. The approved job
-publishes GitHub and uses OIDC to upload to PyPI. It does not check out or run
-repository scripts. If PyPI fails, GitHub remains published with the original files.
-
-## Recover
-
-- [ ] Use **Re-run all jobs** in the original run for any publishing failure.
-  This repeats file selection, reuses the original distributions, and requests approval.
-  **Re-run failed jobs** alone cannot refresh the upload selection.
-- [ ] Check that existing filenames and hashes match. Identical files are reused;
-  missing files are uploaded. Conflicts stop the run.
-- [ ] Leave tags and existing assets unchanged. Do not overwrite a conflicting release.
-- [ ] Check PyPI again after the retry succeeds.
-
-The `release-bundle` artifact retains distributions, checksums, notes, and commit
-metadata for 90 days. If it is missing or expired, the workflow retrieves the exact
-files from the official GitHub release and verifies their metadata and checksums.
-It never rebuilds a partially published release. Published GitHub assets are never
-modified; only matching releases can be reused.
-
-If neither retained artifacts nor matching official assets are available:
-
-- [ ] Stop automatic recovery.
-- [ ] Recover the original distributions and manifest from the draft or a verified archive.
-- [ ] Verify filenames, SHA-256 digests, tag, and commit before an owner-reviewed recovery.
-- [ ] If original bytes cannot be recovered, prepare a new version.
+- [ ] Confirm PyPI has both files and GitHub is published.
+
+Every build attempt creates fresh distributions. Staging replaces all assets on a
+same-commit draft, or deletes and recreates a draft for a different commit. Cleanup
+happens after checks and build succeed. Public releases are never replaced.
+
+After approval, publication readiness repeats the same version-eligibility policy:
+the live tag must still match, GitHub must remain unpublished, and PyPI must have
+no version record. It also verifies the built bundle's identity and checksums and
+compares every draft asset against that bundle. The early check allows no draft
+or an obsolete draft; readiness requires the exact staged build. PyPI absence is
+checked last, after the asset comparison.
+
+The approved job uses OIDC to upload the complete distribution set. It checks out
+only `scripts/release.py` at the run's commit without persisting checkout credentials.
+The shared Python checks run with isolation enabled and use only the standard
+library. A separate finalization job verifies both PyPI filenames and hashes before
+publishing the original draft.
+Keep the Trusted Publisher repository, workflow filename `release.yml`, and
+`pypi` environment unchanged.
+
+## Recover before any PyPI upload
+
+- [ ] Check PyPI to confirm no file for the version was accepted.
+- [ ] Cancel an obsolete run waiting for approval before starting its replacement.
+- [ ] Fix and merge the workflow or source change, then retarget the unpublished
+  tag under the repository's protection rules, or rerun the original run for a
+  transient failure with unchanged source.
+- [ ] Review the fresh build and draft, then approve the new deployment.
+
+Runs are serialized per tag without automatic cancellation; different tags run
+independently. The owner cancels stale waiting runs manually. If approved instead,
+a stale run fails its recheck when the tag moved or another run published PyPI.
+Avoid moving tags during an active upload. Rechecks cannot lock a tag against an
+owner changing it after the check.
+
+## Recover after a PyPI upload
+
+Any accepted file freezes the version's source and distribution bytes. Normal
+attempts stop rather than rebuild, select missing files, or use `skip-existing`.
+Deleting a PyPI version does not guarantee its filenames can be uploaded again.
+
+If the PyPI step succeeded but GitHub finalization failed:
+
+1. Select **Re-run failed jobs** on the original run. This reruns finalization
+   with the original build artifact ID without uploading to PyPI again.
+2. Confirm finalization verifies the complete PyPI hashes and original draft assets.
+3. Confirm the GitHub release is published.
+
+Do not select **Re-run all jobs** after publication: eligibility rejects the
+existing version. If PyPI accepted files but the upload step failed, or an upload
+was partial, stop automatic recovery. Recover the original distributions and
+manifest from the draft or retained bundle for an owner-reviewed manual recovery.
+This workflow does not automate partial uploads. If the original bytes cannot be
+recovered, prepare a new version.
+
+## Retained artifacts
+
+| Artifact | Contents |
+| --- | --- |
+| `release-bundle-N` | Original wheel and sdist, `SHA256SUMS.json`, full notes, and `release.json` with tag, version, commit, build run, and attempt. |
+| `finalization-evidence-N` | Original bundle artifact ID and GitHub finalization outcome. |
+| `publication-evidence-N` | Available `.publish.attestation` files, manifest, release metadata, bundle artifact ID, publication attempt, and step outcomes. |
+
+`N` is the job's workflow attempt. Artifacts are retained for 90 days and scoped
+to the workflow run. Downstream jobs use the exact build artifact ID, including
+when a failed job reruns with a newer attempt number. The old `pypi-upload-N`
+subset artifact is no longer needed because normal publication uploads both files.
+
+Evidence retention runs even after failure. If signing never started, the evidence
+records that no attestations are available. Distributions remain in the referenced
+bundle and GitHub draft. No credentials, OIDC tokens, or private keys are archived.
+
+Saved attestations bind the CI identity to exact distribution hashes; they are
+not upload credentials. PyPI requires Trusted Publishing authentication to accept
+these attestations. Ordinary local API-token uploads cannot replay that identity.
+The publishing action generates fresh attestations and rejects pre-existing
+attestation files when generation is enabled. Preserve evidence for investigation
+and future recovery rather than assuming it can be submitted by a local upload.
+
+## Release helper commands
+
+The workflow uses these `scripts/release.py` commands. Each command performs one
+release operation; job names describe the surrounding pipeline stage.
+
+| Command | Operation |
+| --- | --- |
+| `prepare-release-metadata` | Check tag/version, ancestry, changelog and release date; write metadata and notes. |
+| `check-release-version-eligibility` | Require PyPI absence and the current source tag; reject public GitHub releases. |
+| `validate-distributions` | Validate wheel/sdist metadata and write distribution checksums. |
+| `verify-release-bundle` | Check bundle tag, commit, workflow run, filenames and distribution hashes. |
+| `stage-github-draft` | Recheck eligibility and stage the complete draft asset set. |
+| `check-pypi-publication-readiness` | Recheck the approved bundle, live tag, draft assets and PyPI absence. |
+| `verify-and-publish-github-release` | Verify complete PyPI hashes and original draft assets, then publish GitHub. |
+| `record-github-finalization-outcome` | Record the finalization result and original bundle reference. |
 
 ## Validate workflow changes