Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 17 additions & 7 deletions tools/robostack.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import subprocess
import sys
import time
from collections.abc import Callable
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
Expand Down Expand Up @@ -429,22 +430,31 @@ def upload(distro: str, files: list[str], cwd: Path) -> int:
print(f"no ANACONDA_API_TOKEN (repository secret {anaconda_token_secret(distro)}): not uploading to "
f"anaconda.org/{channel}", file=sys.stderr)
return 1
missing = [f for f in files if not on_anaconda(channel, Path(cwd, f))]
if missing:
rc |= retry(["rattler-build", "upload", "anaconda", "-o", channel, "--api-key", token, *missing], cwd,
"anaconda.org")
return rc

def anaconda_upload() -> list[str] | None:
# asked again before every attempt: an upload that timed out may still have
# arrived, and uploading it again fails with "already exists"
missing = [f for f in files if not on_anaconda(channel, Path(cwd, f))]
return ["rattler-build", "upload", "anaconda", "-o", channel, "--api-key", token, *missing] if missing else None

return rc | retry(anaconda_upload, cwd, "anaconda.org")


UPLOAD_ATTEMPTS = 5
UPLOAD_WAIT = 30 # doubled on each retry: 30 s, 1, 2, 4 minutes (anaconda.org's 503s can last minutes)


def retry(cmd: list[str], cwd: Path, what: str) -> int:
def retry(cmd: list[str] | Callable[[], list[str] | None], cwd: Path, what: str) -> int:
"""Runs an upload, retrying when it fails. `cmd` may be a function that returns the
command for what is still to upload, or None when nothing is."""
# the channels (and Sigstore, for attestations) fail now and then; uploads are
# idempotent, so retry
rc = 0
for attempt in range(1, UPLOAD_ATTEMPTS + 1):
rc = run(cmd, cwd)
command = cmd() if callable(cmd) else cmd
if command is None:
return 0
rc = run(command, cwd)
if rc == 0 or attempt == UPLOAD_ATTEMPTS:
return rc
wait = UPLOAD_WAIT * 2 ** (attempt - 1)
Expand Down
20 changes: 20 additions & 0 deletions tools/tests/test_bot_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,26 @@ def test_uploads_retry_five_times_waiting_longer(monkeypatch):
assert waits == [30, 60, 120, 240]


def test_anaconda_retry_skips_an_upload_that_arrived_despite_failing(monkeypatch):
# the first upload times out (HTTP 524) although anaconda.org stored the file:
# the retry must not upload it again ("already exists") but find nothing left
calls, uploaded = [], set()

def run(cmd, cwd):
calls.append(cmd)
uploaded.update(f for f in cmd if f.endswith(".conda"))
return 1

monkeypatch.setattr(rs, "run", run)
monkeypatch.setattr(rs, "on_anaconda", lambda channel, path: str(path.relative_to(rs.ROOT)) in uploaded)
monkeypatch.setattr(rs, "upload_hosts", lambda distro: ["anaconda"])
monkeypatch.setattr(rs.time, "sleep", lambda seconds: None)
monkeypatch.setenv("ANACONDA_API_TOKEN", "test")
monkeypatch.delenv("ROBOSTACK_UPLOAD_CHANNEL", raising=False)
assert rs.upload("lyrical", ["osx-64/ros2-a-1.0.0-h_30.conda"], rs.ROOT) == 0
assert len(calls) == 1


def test_update_pixi_parsing():
assert jobs("@robostack-bot update-pixi") == [{"command": "update-pixi", "distro": "", "args": "", "preview": False}]
assert maintenance.parse_command("@robostack-bot update-pixi", "NONE") == []
Expand Down
Loading