build(release): upload release artifacts and checksums to GitHub release

.github/workflows/create_archive_and_notes.sh

@@ -31,6 +31,8 @@ git checkout "$TAG"
 # A prefix is added to better match the GitHub generated archives.
 PREFIX="rules_python-${TAG}"
 ARCHIVE="rules_python-$TAG.tar.gz"
+# TODO(https://github.com/bazel-contrib/rules_python/issues/4250): Add
+# artifacts_manifest.json to the release archive.
 git archive --format=tar "--prefix=${PREFIX}/" "$TAG" | gzip > "$ARCHIVE"
 SHA=$(shasum -a 256 "$ARCHIVE" | awk '{print $1}')
 

.github/workflows/release_build_tools.yaml → .github/workflows/release_build_artifacts.yaml

@@ -3,10 +3,7 @@
 # see dev/release_artifacts/build.sh. Linux x86_64 runners cross-compile the
 # files for other Linux CPUs. macOS and Windows files need those OSes' SDKs, so
 # they're built on those OSes.
-#
-# For now, this only verifies that the artifacts build; they aren't attached to
-# releases yet.
-name: "Release: Build Tools"
+name: "Release: Build Artifacts"
 
 on:
   workflow_call:
@@ -23,7 +20,7 @@ on:
         type: string
 
 permissions:
-  contents: read
+  contents: write
 
 defaults:
   run:
@@ -33,6 +30,8 @@ jobs:
   build:
     name: Build ${{ matrix.triple }}
     runs-on: ${{ matrix.runner }}
+    permissions:
+      contents: read
     # Builds take a few minutes; cancel hung ones instead of holding a runner
     # for the default 6 hours.
     timeout-minutes: 30
@@ -97,12 +96,70 @@ jobs:
         if: steps.build.outputs.files != ''
         run: dev/release_artifacts/verify.sh "$TRIPLE"
 
-      # Stores the files with this workflow run so they can be downloaded from
-      # the run's page. Nothing is added to the GitHub release.
+      # Stores the files with this workflow run so the publish job (and users
+      # viewing the run) can download them.
       - name: Upload artifacts
         if: steps.build.outputs.files != ''
         uses: actions/upload-artifact@v7
         with:
-          name: release-tools-${{ matrix.triple }}
+          name: release-artifacts-${{ matrix.triple }}
           path: ${{ steps.build.outputs.files }}
           if-no-files-found: error
+
+  publish:
+    name: Publish artifacts
+    needs: build
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+    env:
+      REF: ${{ inputs.ref || github.ref_name }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          ref: ${{ inputs.ref }}
+          persist-credentials: false
+
+      - name: Download built artifacts
+        uses: actions/download-artifact@v8
+        with:
+          pattern: release-artifacts-*
+          path: release-artifacts
+          merge-multiple: true
+
+      # The hashFiles(...) checks below are for backwards compatibility with
+      # patch releases of older release branches that predate these scripts and
+      # artifacts (and while artifact build failures are non-fatal). They can be
+      # removed once older release branches are no longer supported.
+      - name: Generate SHA256SUMS
+        if: hashFiles('release-artifacts/*') != ''
+        working-directory: release-artifacts
+        run: sha256sum * > ../SHA256SUMS
+
+      - name: Generate artifacts manifest
+        if: >-
+          hashFiles('dev/release_artifacts/generate_artifacts_manifest.py') != '' &&
+          hashFiles('release-artifacts/*') != ''
+        run: |
+          dev/release_artifacts/generate_artifacts_manifest.py \
+            "$REF" release-artifacts artifacts_manifest.json
+
+      - name: Upload artifacts to GitHub release
+        if: hashFiles('release-artifacts/*') != ''
+        uses: softprops/action-gh-release@v3
+        with:
+          tag_name: ${{ inputs.ref || github.ref_name }}
+          draft: true
+          fail_on_unmatched_files: true
+          files: |
+            release-artifacts/*
+            SHA256SUMS
+
+      - name: Upload artifacts manifest
+        if: hashFiles('artifacts_manifest.json') != ''
+        uses: actions/upload-artifact@v7
+        with:
+          name: release-artifacts-manifest
+          path: artifacts_manifest.json
+          if-no-files-found: error

.github/workflows/release_publish.yaml

@@ -46,19 +46,32 @@ on:
         default: true
 
 jobs:
-  build_tools:
-    # For now, this only verifies that the tools build. They aren't attached
-    # to the release, so no other job waits on this one. A failed build only
-    # adds a warning to this run. The run fails if a build hangs past the job
-    # timeout, the built files fail their checks, or the ref's dev/platforms
-    # lacks a job's platform.
-    name: Build tools
-    uses: ./.github/workflows/release_build_tools.yaml
+  create_release:
+    name: Create draft release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create draft release
+        uses: softprops/action-gh-release@v3
+        with:
+          draft: true
+          prerelease: ${{ contains( (inputs.tag_name || github.ref), '-rc') }}
+          tag_name: ${{ inputs.tag_name || github.ref_name }}
+
+  build_artifacts:
+    # Builds release artifacts and uploads them to the draft release. A failed
+    # build only adds a warning to this run, since releases don't require the
+    # artifacts yet. The run fails if a build hangs past the job timeout, the
+    # built files fail their checks, or the ref's dev/platforms lacks a job's
+    # platform.
+    name: Build artifacts
+    needs: create_release
+    uses: ./.github/workflows/release_build_artifacts.yaml
     with:
       ref: ${{ inputs.tag_name || github.ref_name }}
 
   release:
     name: Release
+    needs: build_artifacts
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -73,6 +86,7 @@ jobs:
           # Use GH feature to populate the changelog automatically
           generate_release_notes: true
           body_path: release_notes.txt
+          draft: false
           prerelease: ${{ contains( (inputs.tag_name || github.ref), '-rc') }}
           fail_on_unmatched_files: true
           files: rules_python-*.tar.gz

RELEASING.md

@@ -67,11 +67,12 @@ you can set the `publish_to_pypi` input to `false`:
 gh workflow run release_publish.yaml --ref <TAG> -f publish_to_pypi=false
 ```
 
-### Prebuilt tools
+### Release artifacts
 
 The release workflow also builds `//dev/release_artifacts:artifacts_for_release`
-for each platform to verify it builds. For now, the files aren't attached to
-the release, so a failed "Build tools" job doesn't block it.
+for each platform and uploads the built artifacts and `SHA256SUMS` to the GitHub
+release. For now, build failures only emit a warning rather than failing the
+release.
 
 Presubmit builds it for every platform too: natively on each CI host, and
 through `//dev/release_artifacts:cross_compiled_artifacts` for the others.

dev/release_artifacts/BUILD.bazel

@@ -115,3 +115,7 @@ bzl_library(
         "@rules_cc//cc:find_cc_toolchain_bzl",
     ],
 )
+
+exports_files([
+    "generate_artifacts_manifest.py",
+])

dev/release_artifacts/generate_artifacts_manifest.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""Generates an artifacts_manifest.json file for built release artifacts."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import sys
+from pathlib import Path
+from typing import TypedDict
+
+
+class ArtifactEntry(TypedDict):
+    exec_compatible_with: list[str]
+    filename: str
+    sha256: str
+    urls: list[str]
+
+
+_CPU_TO_CONSTRAINT = {
+    "aarch64": "@platforms//cpu:aarch64",
+    "armv7": "@platforms//cpu:armv7",
+    "i686": "@platforms//cpu:x86_32",
+    "powerpc64le": "@platforms//cpu:ppc64le",
+    "riscv64gc": "@platforms//cpu:riscv64",
+    "x86_64": "@platforms//cpu:x86_64",
+}
+
+_FILENAME_RE = re.compile(r"^(.+)-((?:" + "|".join(_CPU_TO_CONSTRAINT) + r")-.+)$")
+
+
+def _cpu_constraint(triple: str) -> str:
+    cpu = triple.split("-", 1)[0]
+    if cpu in _CPU_TO_CONSTRAINT:
+        return _CPU_TO_CONSTRAINT[cpu]
+    raise ValueError(f"Unsupported CPU in triple: {triple}")
+
+
+def _os_constraint(triple: str) -> str:
+    if triple.endswith("-apple-darwin"):
+        return "@platforms//os:macos"
+    if "-pc-windows-" in triple:
+        return "@platforms//os:windows"
+    if "-linux-" in triple:
+        return "@platforms//os:linux"
+    raise ValueError(f"Unsupported OS in triple: {triple}")
+
+
+def generate_manifest(
+    tag: str,
+    artifacts_dir: Path,
+    repo: str,
+) -> dict[str, dict[str, list[ArtifactEntry]]]:
+    artifacts: dict[str, list[ArtifactEntry]] = {}
+    for file_path in sorted(artifacts_dir.iterdir()):
+        filename = file_path.name
+        base = filename.removesuffix(".exe")
+        match = _FILENAME_RE.match(base)
+        if not match:
+            raise ValueError(
+                f"Cannot parse artifact and triple from filename: {filename}"
+            )
+        artifact, triple = match.group(1), match.group(2)
+
+        cpu = _cpu_constraint(triple)
+        os_constraint = _os_constraint(triple)
+        sha256 = hashlib.sha256(file_path.read_bytes()).hexdigest()
+        url = f"https://github.com/{repo}/releases/download/{tag}/{filename}"
+
+        artifacts.setdefault(artifact, []).append(
+            {
+                "exec_compatible_with": [cpu, os_constraint],
+                "filename": filename,
+                "sha256": sha256,
+                "urls": [url],
+            }
+        )
+
+    for entries in artifacts.values():
+        entries.sort(key=lambda entry: entry["filename"])
+
+    return {"artifacts": dict(sorted(artifacts.items()))}
+
+
+def _main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("tag", help="Release tag name (e.g. 2.4.0).")
+    parser.add_argument(
+        "artifacts_dir",
+        type=Path,
+        help="Directory containing built <artifact>-<triple>[.exe] files.",
+    )
+    parser.add_argument(
+        "output_file",
+        type=Path,
+        help="Path to the artifacts_manifest.json file to write.",
+    )
+    parser.add_argument(
+        "repo",
+        nargs="?",
+        default=os.environ.get("GITHUB_REPOSITORY") or "bazel-contrib/rules_python",
+        help="GitHub owner/repository name.",
+    )
+    args = parser.parse_args(argv)
+    try:
+        manifest = generate_manifest(args.tag, args.artifacts_dir, args.repo)
+    except ValueError as err:
+        print(f"::error::{err}")
+        return 1
+    args.output_file.parent.mkdir(parents=True, exist_ok=True)
+    args.output_file.write_text(
+        json.dumps(manifest, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(_main())