[ci] Handle PyPI metadata delays and clarify release eligibility

docs/releasing.md

@@ -39,18 +39,25 @@ Every build attempt creates fresh distributions. Staging replaces all assets on
 same-commit draft, or deletes and recreates a draft for a different commit. Cleanup
 happens after checks and build succeed. Public releases are never replaced.
 
-After approval, publication readiness repeats the same version-eligibility policy:
-the live tag must still match, GitHub must remain unpublished, and PyPI must have
-no version record. It also verifies the built bundle's identity and checksums and
-compares every draft asset against that bundle. The early check allows no draft
-or an obsolete draft; readiness requires the exact staged build. PyPI absence is
-checked last, after the asset comparison.
+After approval, publication readiness verifies the built bundle's identity and
+checksums, then repeats the version-eligibility policy. PyPI absence is the first
+eligibility check: an existing version stops the run before tag or GitHub checks,
+without suggesting cleanup of a completed release. Then the live tag must still
+match and GitHub must remain unpublished.
+
+Readiness also compares every draft asset against the approved bundle. The early
+check allows no draft or an obsolete draft; readiness requires the exact staged
+build. Eligibility is rechecked after asset comparison, immediately before upload.
 
 The approved job uses OIDC to upload the complete distribution set. It checks out
 only `scripts/release.py` at the run's commit without persisting checkout credentials.
 The shared Python checks run with isolation enabled and use only the standard
 library. A separate finalization job verifies both PyPI filenames and hashes before
-publishing the original draft. The approved job needs `contents: write` because
+publishing the original draft. It waits up to five minutes for absent or incomplete
+PyPI JSON metadata to expose both uploaded files. Any unexpected filename or
+conflicting hash stops finalization immediately. API/network failures also stop
+the job; they are not treated as missing metadata.
+The approved job needs `contents: write` because
 GitHub requires push access to list draft releases and inspect their assets.
 Keep the Trusted Publisher repository, workflow filename `release.yml`, and
 `pypi` environment unchanged.
@@ -76,7 +83,9 @@ Any accepted file freezes the version's source and distribution bytes. Normal
 attempts stop rather than rebuild, select missing files, or use `skip-existing`.
 Deleting a PyPI version does not guarantee its filenames can be uploaded again.
 
-If the PyPI step succeeded but GitHub finalization failed:
+After a successful PyPI step and failed GitHub finalization, retry only finalization.
+If PyPI metadata is still incomplete after the bounded wait, the error reports
+the expected and observed files. Wait for metadata to settle before retrying.
 
 1. Select **Re-run failed jobs** on the original run. This reruns finalization
    with the original build artifact ID without uploading to PyPI again.

scripts/release.py

@@ -10,6 +10,7 @@
 import re
 import subprocess
 import tarfile
+import time
 import tomllib
 from typing import Any
 import urllib.error
@@ -200,16 +201,19 @@ def live_tag(api: GitHub) -> None:
 def require_current_commit(kind: str, current: str, expected: str) -> None:
     """Reject approvals for moved tags or tags pointing at non-commits."""
     if kind != "commit" or current != expected:
-        raise ValueError("Release tag moved; cancel this stale run")
+        raise ValueError(
+            f"Release tag no longer points to this run's commit: expected commit {expected}, "
+            f"found {kind} {current}. Cancel this stale run; use the run for the current tag."
+        )
     return None
 
 
 def eligible(bundle: Path, api: GitHub) -> None:
     """Reject any existing target PyPI version before expensive checks."""
     info = json.loads((bundle / "release.json").read_text())
+    require_unpublished(info["version"], pypi_files(info["version"]))
     live_tag(api)
     draft_action(api.find_release(info["tag"]), info["commit"])
-    require_unpublished(pypi_files(info["version"]))
     return None
 
 
@@ -257,11 +261,11 @@ def stage(bundle: Path, api: GitHub) -> None:
     return None
 
 
-def pypi_files(version: str) -> dict[str, str] | None:
+def pypi_files(version: str, timeout: float = 30) -> dict[str, str] | None:
     """Read published hashes; only HTTP 404 means that a version is absent."""
     try:
         with urllib.request.urlopen(
-            f"https://pypi.org/pypi/python-json-logger/{version}/json", timeout=30
+            f"https://pypi.org/pypi/python-json-logger/{version}/json", timeout=timeout
         ) as response:
             return {
                 item["filename"]: item["digests"]["sha256"] for item in json.load(response)["urls"]
@@ -272,18 +276,48 @@ def pypi_files(version: str) -> dict[str, str] | None:
         raise
 
 
-def require_unpublished(existing: dict[str, str] | None) -> None:
+def require_unpublished(version: str, existing: dict[str, str] | None) -> None:
     """Even an empty or yanked version record blocks ordinary publishing."""
     if existing is not None:
-        raise ValueError("PyPI version already exists; recover original bytes manually")
+        raise ValueError(
+            f"PyPI version {version} already exists; refusing a new release. "
+            "Do not rebuild or reupload this version."
+        )
     return None
 
 
-def require_published(manifest: dict[str, str], existing: dict[str, str] | None) -> None:
-    """Finalize only an exact, complete publication of the approved bundle."""
-    if existing != manifest:
-        raise ValueError("PyPI files do not match the complete release manifest")
-    return None
+def publication_complete(manifest: dict[str, str], existing: dict[str, str] | None) -> bool:
+    """Wait for missing metadata, but reject any conflicting public file immediately."""
+    if existing is None:
+        return False
+    if any(manifest.get(name) != digest for name, digest in existing.items()):
+        raise ValueError(
+            f"PyPI files conflict with the release manifest: expected {manifest}, observed {existing}"
+        )
+    return existing == manifest
+
+
+def wait_for_publication(version: str, manifest: dict[str, str]) -> None:
+    """Allow up to five minutes for newly uploaded files to appear in the JSON API."""
+    deadline = time.monotonic() + 300
+    existing = None
+    while (remaining := deadline - time.monotonic()) > 0:
+        existing = pypi_files(version, timeout=min(30, remaining))
+        if publication_complete(manifest, existing):
+            return None
+        remaining = deadline - time.monotonic()
+        if remaining > 0:
+            print(
+                f"Waiting for PyPI metadata ({remaining:.0f}s remaining): "
+                f"expected {sorted(manifest)}, observed {existing}",
+                flush=True,
+            )
+            time.sleep(min(10, remaining))
+    raise ValueError(
+        "Timed out waiting for complete PyPI metadata: "
+        f"expected {manifest}, observed {existing}. "
+        "Re-run failed finalization with the original bundle; do not rebuild or reupload."
+    )
 
 
 def verify_bundle(bundle: Path) -> dict[str, str]:
@@ -315,8 +349,17 @@ def verify_release(bundle: Path, api: GitHub, manifest: dict[str, str]) -> dict:
     """Verify exact GitHub release source, metadata and assets against the bundle."""
     info = json.loads((bundle / "release.json").read_text())
     release = api.find_release(info["tag"])
-    if release is None or release["target_commitish"] != info["commit"]:
-        raise ValueError("Missing or changed release draft")
+    if release is None:
+        raise ValueError(
+            f"Cannot find GitHub release for {info['tag']}. "
+            "The release may be absent, or the token may lack push access to view drafts."
+        )
+    if release["target_commitish"] != info["commit"]:
+        raise ValueError(
+            f"GitHub release source mismatch for {info['tag']}: "
+            f"bundle commit {info['commit']}, release target {release['target_commitish']}. "
+            "Use the run and bundle for the current release source."
+        )
     # Compare downloaded assets byte for byte, including attempt identity.
     assets = api.request(f'/releases/{release["id"]}/assets?per_page=100')
     expected = set(manifest) | {"release.json", "SHA256SUMS.json"}
@@ -346,10 +389,10 @@ def verify_release(bundle: Path, api: GitHub, manifest: dict[str, str]) -> dict:
 def preflight(bundle: Path, api: GitHub) -> None:
     """Recheck approved bytes and identities immediately before the PyPI upload."""
     manifest = verify_bundle(bundle)
+    # PyPI publication is the first eligibility gate, including for superseded runs.
+    eligible(bundle, api)
     verify_release(bundle, api, manifest)
-    # Repeat the same eligibility policy after approval, checking PyPI last.
-    # Early eligibility allows no draft or an obsolete draft; readiness also
-    # requires the exact approved source, metadata and distribution bytes above.
+    # Repeat after potentially slow asset downloads, immediately before upload.
     eligible(bundle, api)
     return None
 
@@ -359,7 +402,9 @@ def finalize(bundle: Path, api: GitHub) -> None:
     manifest = verify_bundle(bundle)
     live_tag(api)
     info = json.loads((bundle / "release.json").read_text())
-    require_published(manifest, pypi_files(info["version"]))
+    wait_for_publication(info["version"], manifest)
+    # Metadata visibility may take minutes; do not rely on the earlier tag check.
+    live_tag(api)
     release = verify_release(bundle, api, manifest)
     if release["draft"]:
         api.request(f'/releases/{release["id"]}', {"draft": False}, method="PATCH")

tests/test_release.py

@@ -100,27 +100,36 @@ def test_postrelease_preserves_original_date() -> None:
 
 @pytest.mark.parametrize("existing", [{}, {"wheel": "aaa"}, {"wheel": "aaa", "sdist": "bbb"}])
 def test_any_pypi_record_blocks_new_release(existing: dict[str, str]) -> None:
-    with pytest.raises(ValueError, match="already exists"):
-        release.require_unpublished(existing)
+    with pytest.raises(ValueError, match=r"PyPI version 4\.3\.0\.dev3 already exists"):
+        release.require_unpublished("4.3.0.dev3", existing)
     return None
 
 
 def test_absent_version_is_eligible() -> None:
-    release.require_unpublished(None)
+    release.require_unpublished("4.3.0.dev3", None)
+    return None
+
+
+@pytest.mark.parametrize("existing", [None, {}, {"wheel": "aaa"}])
+def test_finalization_waits_for_missing_metadata(existing: dict[str, str] | None) -> None:
+    assert not release.publication_complete({"wheel": "aaa", "sdist": "bbb"}, existing)
     return None
 
 
 @pytest.mark.parametrize(
-    "existing", [None, {}, {"wheel": "aaa"}, {"wheel": "wrong", "sdist": "bbb"}]
+    "existing",
+    [{"wheel": "wrong"}, {"wheel": "wrong", "sdist": "bbb"}, {"extra": "ccc"}],
 )
-def test_finalize_requires_complete_matching_publication(existing: dict[str, str] | None) -> None:
-    with pytest.raises(ValueError, match="complete release manifest"):
-        release.require_published({"wheel": "aaa", "sdist": "bbb"}, existing)
+def test_finalization_rejects_conflicting_publication(existing: dict[str, str]) -> None:
+    with pytest.raises(ValueError, match="conflict.*expected.*observed"):
+        release.publication_complete({"wheel": "aaa", "sdist": "bbb"}, existing)
     return None
 
 
 def test_complete_publication_can_finalize() -> None:
-    release.require_published({"wheel": "aaa", "sdist": "bbb"}, {"wheel": "aaa", "sdist": "bbb"})
+    assert release.publication_complete(
+        {"wheel": "aaa", "sdist": "bbb"}, {"wheel": "aaa", "sdist": "bbb"}
+    )
     return None
 
 
@@ -139,8 +148,10 @@ def test_public_release_requires_owner_cleanup() -> None:
 
 @pytest.mark.parametrize("kind,current", [("commit", "old"), ("tree", "approved")])
 def test_stale_run_cannot_publish(kind: str, current: str) -> None:
-    with pytest.raises(ValueError, match="stale run"):
+    with pytest.raises(ValueError, match="stale run") as error:
         release.require_current_commit(kind, current, "approved")
+    assert "expected commit approved" in str(error.value)
+    assert f"found {kind} {current}" in str(error.value)
     return None