Skip to content

Commit ffbdac0

Browse files
committed
ci: Use package YAMLs as the package registry
Signed-off-by: Ludovic Henry <git@ludovic.dev>
1 parent 05f6508 commit ffbdac0

11 files changed

Lines changed: 57 additions & 1563 deletions

File tree

‎.github/workflows/nightly.yml‎

Lines changed: 6 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -100,12 +100,12 @@ jobs:
100100
uv run --with-requirements ci_scripts/requirements.txt python3 \
101101
ci_scripts/check_versions.py --declare --base-file "$RUNNER_TEMP/base-$slug.yaml" "$package" || return 1
102102
103-
if git diff --quiet -- docs/packages patches ci_scripts/packages.txt ci_scripts/deprecated.txt; then
103+
if git diff --quiet -- docs/packages patches; then
104104
echo "$package: nothing to declare"
105105
return 0
106106
fi
107107
108-
git add docs/packages patches ci_scripts/packages.txt ci_scripts/deprecated.txt || return 1
108+
git add docs/packages patches || return 1
109109
git commit -s -m "$title" || return 1
110110
111111
git push origin "$branch" || return 1
@@ -148,8 +148,9 @@ jobs:
148148
- name: Verify riscv64 wheels still published
149149
run: |
150150
ret=0
151-
while IFS= read -r pkg; do
152-
[[ -z "$pkg" || "$pkg" =~ ^[[:space:]]*# ]] && continue
151+
for package_yaml in docs/packages/*.yaml; do
152+
grep -q '^deprecated:' "$package_yaml" || continue
153+
pkg=$(basename "$package_yaml" .yaml)
153154
154155
if ! output=$(curl -sf "https://pypi.org/pypi/${pkg}/json"); then
155156
echo "Failed to fetch $pkg, skipping..."
@@ -166,21 +167,9 @@ jobs:
166167
echo "$pkg does not distribute riscv64 or pure-Python wheel anymore!"
167168
ret=1
168169
fi
169-
done < ci_scripts/deprecated.txt
170+
done
170171
exit $ret
171172
172-
check_installation:
173-
runs-on: ubuntu-24.04-riscv
174-
steps:
175-
- uses: actions/checkout@v7
176-
177-
- name: Install packages inside riscv64 container
178-
working-directory: ci_scripts
179-
run: |
180-
sed -Ei 's/^scipy.*$//' packages.txt
181-
sed -Ei 's/^patchelf.*$//' packages.txt
182-
./list-packages.sh python-wheels
183-
184173
pip_audit:
185174
runs-on: ubuntu-24.04-riscv
186175
continue-on-error: true

‎ci_scripts/audit.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,10 @@ python3 -m venv packages-$1-venv
1717
. packages-$1-venv/bin/activate
1818
python -m pip install --upgrade pip
1919
python -m pip install pip-audit
20-
python -m pip_audit -r packages.txt --format json --output audit-report.json || true
20+
requirements=$(mktemp)
21+
for package_yaml in ../docs/packages/*.yaml; do
22+
grep -q '^deprecated:' "$package_yaml" || basename "$package_yaml" .yaml
23+
done > "$requirements"
24+
python -m pip_audit -r "$requirements" --format json --output audit-report.json || true
25+
rm -f "$requirements"
2126

‎ci_scripts/audit_report.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import json
1919
import os
20+
import re
2021
import subprocess
2122
import sys
2223
from pathlib import Path
@@ -25,6 +26,10 @@
2526
ISSUE_LABELS = ["pip-audit", "security"]
2627

2728

29+
def normalize_name(name: str) -> str:
30+
return re.sub(r"[-_.]+", "-", name).lower()
31+
32+
2833
def run_url() -> str:
2934
server = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
3035
repo = os.environ.get("GITHUB_REPOSITORY", "")
@@ -167,7 +172,7 @@ def process_report(report_path: Path) -> int:
167172

168173
# GHA warning annotation (yellow bubble on job)
169174
print(
170-
f"::warning file=ci_scripts/packages.txt::"
175+
f"::warning file=docs/packages/{normalize_name(pkg)}.yaml::"
171176
f"{pkg} {version}: {vuln_id} (fix: {fixes})"
172177
)
173178

‎ci_scripts/check_versions.py‎

Lines changed: 19 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@
3333
ROOT_DIR = Path(__file__).resolve().parent.parent
3434
DOCS_DIR = ROOT_DIR / "docs" / "packages"
3535
PATCHES_DIR = ROOT_DIR / "patches"
36-
PACKAGES_FILE = "ci_scripts/packages.txt"
37-
DEPRECATED_FILE = "ci_scripts/deprecated.txt"
3836
REPO = "riseproject-dev/python-wheels"
3937

4038
# Python versions (interpreter tags) a package must have upstream riscv64
@@ -44,22 +42,17 @@
4442

4543

4644
def read_packages() -> List[str]:
47-
"""Read the list of packages from packages.txt."""
45+
"""Read active package names from docs/packages/<pkg>.yaml files."""
4846
packages = []
49-
try:
50-
with open(PACKAGES_FILE, 'r') as f:
51-
for line in f:
52-
line = line.strip()
53-
if line and not line.startswith('#'):
54-
packages.append(line)
55-
except FileNotFoundError:
56-
print(f"Error: {PACKAGES_FILE} not found")
57-
sys.exit(1)
47+
for yaml_file in sorted(DOCS_DIR.glob("*.yaml")):
48+
package_data = yaml.safe_load(yaml_file.read_text()) or {}
49+
if "deprecated" in package_data:
50+
continue
51+
packages.append(yaml_file.stem)
5852
return packages
5953

6054

6155
def package_yaml(package: str) -> Path:
62-
# packages.txt keeps PyPI's spelling (PyYAML), the YAMLs use the normalized name.
6356
return DOCS_DIR / f"{normalize_name(package)}.yaml"
6457

6558

@@ -75,20 +68,15 @@ def declared_versions(package_data: Dict) -> Set[str]:
7568

7669

7770
def get_registry_latest_version(package_data: Dict) -> Optional[str]:
78-
"""
79-
Latest version published to the riscv64 registry, read from the package
80-
YAML. Entries without `files` are declared but not built yet.
81-
"""
82-
released = []
71+
"""Latest version tracked in the package YAML, published or pending."""
72+
tracked = []
8373
for entry in package_data.get("versions") or []:
84-
if "files" not in entry:
85-
continue
8674
raw = str(entry["version"])
8775
try:
88-
released.append((version.parse(raw), raw))
76+
tracked.append((version.parse(raw), raw))
8977
except version.InvalidVersion:
9078
continue
91-
return max(released)[1] if released else None
79+
return max(tracked)[1] if tracked else None
9280

9381

9482
def get_pypi_package_info(package: str, retries: int = 3) -> Optional[Dict]:
@@ -205,28 +193,15 @@ def find_upstream_issue(package: str) -> Optional[str]:
205193

206194
def deprecate_package(package: str) -> bool:
207195
"""
208-
Deprecate our wheel in the working tree: prefix docs/packages/<pkg>.yaml
209-
with `deprecated:`, drop the package from packages.txt and add it to
210-
deprecated.txt. Returns False when it is already deprecated.
196+
Deprecate our wheel in the working tree by prefixing
197+
docs/packages/<pkg>.yaml with `deprecated:`. Returns False when it is
198+
already deprecated.
211199
"""
212200
yaml_file = package_yaml(package)
213201
content = yaml_file.read_text()
214202
if content.startswith("deprecated:"):
215203
return False
216204
yaml_file.write_text(f"deprecated:\n{content}")
217-
218-
packages_file = Path(PACKAGES_FILE)
219-
lines = [line for line in packages_file.read_text().splitlines() if line.strip() != package]
220-
packages_file.write_text("\n".join(lines) + "\n")
221-
222-
deprecated_file = Path(DEPRECATED_FILE)
223-
lines = deprecated_file.read_text().splitlines()
224-
i = next(i for i, line in enumerate(lines) if line.strip() and not line.lstrip().startswith("#"))
225-
comments, deprecated_packages = lines[:i], lines[i:]
226-
if package not in deprecated_packages:
227-
deprecated_packages.append(package)
228-
deprecated_packages.sort(key=str.lower)
229-
deprecated_file.write_text("\n".join(comments + deprecated_packages) + "\n")
230205
return True
231206

232207

@@ -440,7 +415,7 @@ def check_package(
440415
With `since`, every stable release uploaded since that date and missing
441416
from the YAML counts as an upgrade, not just the ones newer than the
442417
registry. With `declare`, the upgrade, deprecation or yanked marking is
443-
written to docs/packages/<pkg>.yaml (and packages.txt/deprecated.txt) for
418+
written to docs/packages/<pkg>.yaml for
444419
the caller to commit. Versions yanked upstream are reported in the
445420
result's `yanked` alongside the upgrade/deprecation status.
446421
"""
@@ -452,8 +427,8 @@ def check_package(
452427

453428
registry_version = get_registry_latest_version(package_data)
454429
if registry_version is None:
455-
print(f"[X] No released version in {package_yaml(package).relative_to(ROOT_DIR)}")
456-
return {"status": "error", "package": package, "error": "No released version in YAML"}
430+
print(f"[X] No tracked version in {package_yaml(package).relative_to(ROOT_DIR)}")
431+
return {"status": "error", "package": package, "error": "No tracked version in YAML"}
457432

458433
pypi_info = get_pypi_package_info(package)
459434
if pypi_info is None:
@@ -570,7 +545,7 @@ def check_upgrade(
570545
def deprecation_result(package: str, registry_version: str, pypi_version: str, reason: str, declare: bool) -> Dict[str, any]:
571546
upstream_issue = find_upstream_issue(package)
572547
if declare and deprecate_package(package):
573-
print(f" [+] deprecated {package} in {package_yaml(package).relative_to(ROOT_DIR)}, packages.txt and deprecated.txt")
548+
print(f" [+] deprecated {package} in {package_yaml(package).relative_to(ROOT_DIR)}")
574549
return {
575550
"status": "can_deprecate",
576551
"package": package,
@@ -809,13 +784,13 @@ def main():
809784
import argparse
810785

811786
parser = argparse.ArgumentParser(description="Check package versions between riscv64 registry and PyPI")
812-
parser.add_argument("packages", nargs="*", help="Specific packages to check (default: all packages from packages.txt)")
787+
parser.add_argument("packages", nargs="*", help="Specific packages to check (default: active docs/packages/*.yaml files)")
813788
parser.add_argument("--since", type=parse_since, metavar="DATE|Nd|Nm",
814789
help="Treat every stable release uploaded since this date (YYYY-MM-DD) or duration (e.g. 12m, 90d) "
815790
"that docs/packages/<pkg>.yaml does not list as a version to add, uncapped")
816791
parser.add_argument("--declare", action="store_true",
817792
help="Apply the outcome to the working tree: declare new versions and mark yanked ones in "
818-
"docs/packages/<pkg>.yaml, deprecate packages in the YAML, packages.txt and deprecated.txt")
793+
"docs/packages/<pkg>.yaml, and deprecate packages in the YAML")
819794
parser.add_argument("--base-file", type=Path, metavar="FILE",
820795
help="Base branch copy of the package YAML (as for pending_versions.py): with --declare, "
821796
"only versions it does not declare get the previous version's patches")

‎ci_scripts/deprecated.txt‎

Lines changed: 0 additions & 71 deletions
This file was deleted.

‎ci_scripts/list-packages.sh‎

Lines changed: 0 additions & 28 deletions
This file was deleted.

0 commit comments

Comments
 (0)