Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 26 additions & 10 deletions .github/scripts/fork-checks-ran.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,14 @@
pull_request_target run of the configured workflow file that belongs to THIS pull request must have a job (= check run)
of the configured name (`check`) whose latest attempt concluded success. Runs of other events do not count: they run
the pull request's own copy of the file. A run belongs to this pull request when its head repository is this
repository and its pull_requests list holds PR_NUMBER (#433): anyone can open a pull request from their own fork at
the same public commit (its run skips the job), and a second pull request at the same commit may target a base branch
whose fork-checks.yml is a no-op. GitHub leaves pull_requests empty on runs of a pull request from another
repository, so such a run is never counted (fail closed). Two pull requests from the same head branch may both be
listed on each other's runs; both come from someone who can push to this repository. It waits while that job is queued or running; failed,
repository and its pull_requests list holds PR_NUMBER (#433) with that entry's head branch and head repository the
run's own (#443): anyone can open a pull request from their own fork at the same public commit (its run skips the
job), and a second pull request at the same commit may target a base branch whose fork-checks.yml is a no-op. GitHub
lists on a run the open pull requests whose head SHA or head branch matches it, so a second pull request from
another branch at the same commit may be listed on this one's runs; its entry names the other branch, so it is not
counted. GitHub leaves pull_requests empty on runs of a pull request from another repository, so such a run is never
counted (fail closed). Two pull requests from the same head branch may both be listed on each other's runs; both
come from someone who can push to this repository. It waits while that job is queued or running; failed,
cancelled or skipped fails at once, and so does a finished run without the job (a blocked event leaves one). The
Actions API needs only actions: read, which every caller of fork-tests.yml grants.

Expand Down Expand Up @@ -140,11 +143,24 @@ def config_problems(cfg):


def of_this_pull_request(run, repo, number):
"""A run from this repository whose pull_requests list holds this pull request. An empty or missing list (what
GitHub gives a pull request from another repository) is not this pull request's run: never counted."""
head, prs = run.get('head_repository'), run.get('pull_requests')
return (isinstance(head, dict) and head.get('full_name') == repo and isinstance(prs, list)
and any(isinstance(p, dict) and p.get('number') == number for p in prs))
"""A run from this repository whose pull_requests list holds this pull request from the run's own head branch
and head repository. GitHub lists the open pull requests whose head SHA or head branch matches the run, so a
second pull request at the same commit can be listed on this one's runs (#443): its entry is counted only if its
head.ref is the run's head_branch and its head.repo.id is the run's head repository's id (each entry's
head.repo has an id, a short name and an API url, no full_name). An empty or missing list (what GitHub gives a
pull request from another repository), or a missing field on either side, is never counted."""
head, prs, branch = run.get('head_repository'), run.get('pull_requests'), run.get('head_branch')
if not (isinstance(head, dict) and head.get('full_name') == repo and isinstance(prs, list)):
return False
repo_id = head.get('id')
if not (isinstance(branch, str) and branch and type(repo_id) is int): # not bool, not None
return False

def from_this_run(p):
h = p.get('head') if isinstance(p, dict) else None
r = h.get('repo') if isinstance(h, dict) else None
return isinstance(r, dict) and h.get('ref') == branch and r.get('id') == repo_id
return any(isinstance(p, dict) and p.get('number') == number and from_this_run(p) for p in prs)


def look(api, sha, name, workflow, repo, number):
Expand Down
72 changes: 67 additions & 5 deletions .github/scripts/fork-checks-ran.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,24 @@ def job(name, status='completed', conclusion='success'):
'html_url': f'https://github.invalid/job/{abs(hash(name)) % 1000}'}


def wr(rid, jobs, path=WF, event='pull_request_target', status='completed', prs=(7,), head_repo='o/r'):
"""A run of fork-checks.yml on SHA; by default one of pull request #7 (PR_ENV) from this repository."""
return {'id': rid, 'path': path, 'event': event, 'status': status, 'jobs': jobs,
'pull_requests': [{'number': n, 'head': {'sha': SHA}} for n in prs],
'head_repository': {'full_name': head_repo}}
REPO_ID, OTHER_ID = 101, 202 # GitHub's ids of this repository (o/r) and of another one


def pr(number, ref='feature', repo_id=REPO_ID):
"""A pull_requests entry as GitHub gives it: its head repo has an id, a short name and an API url, no full_name."""
return {'id': 5000 + number, 'number': number, 'url': f'https://api.github.invalid/repos/o/r/pulls/{number}',
'head': {'ref': ref, 'sha': SHA, 'repo': {'id': repo_id, 'name': 'r',
'url': 'https://api.github.invalid/repos/o/r'}},
'base': {'ref': 'main', 'sha': BASE, 'repo': {'id': REPO_ID, 'name': 'r',
'url': 'https://api.github.invalid/repos/o/r'}}}


def wr(rid, jobs, path=WF, event='pull_request_target', status='completed', prs=(7,), head_repo='o/r', branch='feature'):
"""A run of fork-checks.yml on SHA; by default one of pull request #7 (PR_ENV) from this repository's branch
`feature`. prs: numbers (entries from that branch) or whole entries."""
return {'id': rid, 'path': path, 'event': event, 'status': status, 'jobs': jobs, 'head_branch': branch,
'pull_requests': [p if isinstance(p, dict) else pr(p, branch) for p in prs],
'head_repository': {'full_name': head_repo, 'id': REPO_ID if head_repo == 'o/r' else OTHER_ID}}


class Clock:
Expand Down Expand Up @@ -248,6 +261,55 @@ def test_a_run_with_an_empty_or_missing_pull_request_list_never_passes(self):
self.assertIn('fork-checks did not run', out)
self.assertNotIn('actions/runs/5/jobs', ' '.join(fake.paths))

# --- GitHub lists a pull request on runs whose head SHA OR head branch matches it (#443) ------------------------
def test_a_second_pull_request_from_another_branch_at_the_same_commit_does_not_count(self):
# PR #11 from branch `other` at the same commit, against a base whose fork-checks.yml is a no-op: GitHub may
# list this PR (#7, branch `feature`) on #11's run too. Only the entry from the run's own branch counts.
other = wr(9, [job(CFG['check'])], branch='other', prs=(pr(7, 'feature'), pr(11, 'other')))
for runs in ([wr(3, [job(CFG['check'], 'completed', 'failure')]), other], [other]):
fake = Fake([runs])
code, out, _ = check(fake)
self.assertEqual(code, 1, out)
self.assertNotIn('actions/runs/9/jobs', ' '.join(fake.paths))
# ... while this PR's own run, which lists #11 too, still counts for #7.
own = wr(3, [job(CFG['check'])], prs=(pr(11, 'other'), pr(7, 'feature')))
self.assertEqual(check(Fake([[other, own]]))[0], 0)

def test_an_entry_whose_head_is_another_repository_does_not_count(self):
code, out, _ = check(Fake([[wr(5, [job(CFG['check'])], prs=(pr(7, 'feature', OTHER_ID),))]]))
self.assertEqual(code, 1, out)
self.assertIn('fork-checks did not run', out)

def test_missing_head_fields_fail_closed(self):
# Without the head branch and repository on both sides the entry cannot be tied to this run: never counted.
entry, run = pr(7), wr(5, [job(CFG['check'])])
bad_entries = [{k: v for k, v in entry.items() if k != 'head'}, {**entry, 'head': None}, {**entry, 'head': 'feature'},
{**entry, 'head': {**entry['head'], 'ref': None}}, {**entry, 'head': {**entry['head'], 'ref': ''}},
{**entry, 'head': {k: v for k, v in entry['head'].items() if k != 'ref'}},
{**entry, 'head': {k: v for k, v in entry['head'].items() if k != 'repo'}},
{**entry, 'head': {**entry['head'], 'repo': None}},
{**entry, 'head': {**entry['head'], 'repo': {'name': 'r'}}},
{**entry, 'head': {**entry['head'], 'repo': {'id': str(REPO_ID)}}},
{**entry, 'head': {**entry['head'], 'repo': {'id': True}}}]
bad_runs = [{**run, 'pull_requests': [e]} for e in bad_entries]
bad_runs += [{k: v for k, v in run.items() if k != 'head_branch'}, {**run, 'head_branch': None},
{**run, 'head_branch': ''}, {**run, 'head_repository': {'full_name': 'o/r'}},
{**run, 'head_repository': {'full_name': 'o/r', 'id': None}}]
# Both sides missing must not match each other either (None == None).
bad_runs += [{**run, 'head_branch': None, 'pull_requests': [{**entry, 'head': {**entry['head'], 'ref': None}}]},
{**run, 'head_branch': '', 'pull_requests': [{**entry, 'head': {**entry['head'], 'ref': ''}}]},
{**run, 'head_repository': {'full_name': 'o/r', 'id': True},
'pull_requests': [{**entry, 'head': {**entry['head'], 'repo': {'id': True}}}]},
{**run, 'head_repository': {'full_name': 'o/r'},
'pull_requests': [{**entry, 'head': {**entry['head'], 'repo': {'name': 'r'}}}]}]
for bad in bad_runs:
fake = Fake([[bad]])
code, out, _ = check(fake)
self.assertEqual(code, 1, (bad, out))
self.assertIn('fork-checks did not run', out)
self.assertNotIn('actions/runs/5/jobs', ' '.join(fake.paths), bad)
self.assertEqual(check(Fake([[run]]))[0], 0) # the same run, whole, passes

def test_no_run_of_this_pull_request_fails_with_a_clear_message(self):
code, out, _ = check(Fake([[wr(9, [job(CFG['check'])], prs=(11,))]]))
self.assertEqual(code, 1, out)
Expand Down
54 changes: 42 additions & 12 deletions .github/scripts/fork-contracts.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,9 @@
fork-checks' check names are its jobs' names, and fork-tests.yml's fork_checks_ran is
exactly the pinned job (#430): its if:, permissions, the shared checkout and the step
running fork-checks-ran.py with the pinned env (PR_NUMBER too, #433), no
continue-on-error, and a timeout that holds the worst case: the checkout, the wait cap (or
the script's git calls, if longer) and one last poll, from the script's own limits.
continue-on-error, a checkout with its own timeout, and a job timeout that holds the worst
case: the checkout's timeout, the wait cap (or the script's git calls, if longer) and one
last poll, from the script's own limits.

python3 .github/scripts/fork-contracts.test.py

Expand Down Expand Up @@ -349,9 +350,12 @@ def checkout_problems(wfs):
for name, job_id, step in shared:
env = step.get("env") or {}
same = {k: step.get(k) for k in ("name", "shell", "run")} == {k: first.get(k) for k in ("name", "shell", "run")}
if not same or set(step) != {"name", "shell", "env", "run"} or set(env) != {"REF", "HISTORY", "SPARSE"} \
or env.get("HISTORY") not in ("full", "tip"):
if not same or set(step) - {"timeout-minutes"} != {"name", "shell", "env", "run"} \
or set(env) != {"REF", "HISTORY", "SPARSE"} or env.get("HISTORY") not in ("full", "tip"):
problems.append(f"{name}/{job_id}: not the shared checkout step")
# A job may bound its checkout (fork_checks_ran does, #443): a whole number of minutes, at least 1.
if "timeout-minutes" in step and not (type(step["timeout-minutes"]) is int and step["timeout-minutes"] > 0):
problems.append(f"{name}/{job_id}: the shared checkout's timeout-minutes is not a positive whole number")
return problems


Expand Down Expand Up @@ -386,6 +390,14 @@ def first_step(wfs, name, job):
check("checkout: a copy that drifts from the shared step fails",
checkout_mutant(lambda w: first_step(w, "fork-follow.yml", "advance").__setitem__(
"run", first_step(w, "fork-follow.yml", "advance")["run"].replace(" --no-recurse-submodules", ""))))
for bad in ("3", True, 0, -1):
check(f"checkout: a copy with timeout-minutes {bad!r} fails",
checkout_mutant(lambda w, bad=bad: first_step(w, "fork-follow.yml", "advance").__setitem__(
"timeout-minutes", bad)))
check("checkout: a copy with another key (continue-on-error) fails",
checkout_mutant(lambda w: first_step(w, "fork-follow.yml", "advance").__setitem__("continue-on-error", True)))
check("checkout: a copy with timeout-minutes 3 passes",
not checkout_mutant(lambda w: first_step(w, "fork-follow.yml", "advance").__setitem__("timeout-minutes", 3)))
check("checkout: a copy with a token in its env fails",
checkout_mutant(lambda w: first_step(w, "fork-release.yml", "assemble")["env"].__setitem__(
"GH_TOKEN", "${{ github.token }}")))
Expand Down Expand Up @@ -1646,7 +1658,6 @@ def result_mutant(mutate):
}
POLICY_CHECKOUT_ENV = {"REF": "${{ github.sha }}", "HISTORY": "tip", "SPARSE": "/.github/scripts/"}
POLICY_JOB_KEYS = {"name", "if", "runs-on", "timeout-minutes", "permissions", "steps"}
POLICY_CHECKOUT_BUDGET = 180 # seconds for the token-free sparse checkout of one commit (a few seconds in CI)
_fcr_spec = importlib.util.spec_from_file_location("fork_checks_ran", root / ".github/scripts/fork-checks-ran.py")
_fcr = importlib.util.module_from_spec(_fcr_spec)
sys.dont_write_bytecode = True # no __pycache__ in the tree
Expand All @@ -1655,10 +1666,10 @@ def result_mutant(mutate):
POLICY_LIMITS = {k: getattr(_fcr, k) for k in ("GIT_TIMEOUT", "GIT_CALLS", "API_TIMEOUT", "API_CALLS_PER_POLL")}


def policy_worst_case(cap, limits):
"""Seconds fork_checks_ran can take: the checkout, then the cap (the git calls run inside it, or past it if they
take longer), then one last poll after the deadline."""
return (POLICY_CHECKOUT_BUDGET + max(cap, limits["GIT_TIMEOUT"] * limits["GIT_CALLS"])
def policy_worst_case(checkout, cap, limits):
"""Seconds fork_checks_ran can take: the checkout (its step's own timeout, #443), then the cap (the git calls run
inside it, or past it if they take longer), then one last poll after the deadline."""
return (checkout + max(cap, limits["GIT_TIMEOUT"] * limits["GIT_CALLS"])
+ limits["API_TIMEOUT"] * limits["API_CALLS_PER_POLL"])


Expand All @@ -1674,18 +1685,25 @@ def policy_job_problems(job, fc, limits=POLICY_LIMITS):
if job.get("permissions") != {"actions": "read", "contents": "read"}:
problems.append(f"{where}: permissions are not exactly actions: read, contents: read")
steps_ = job.get("steps") or []
checkout_minutes = None
if len(steps_) != 2:
problems.append(f"{where}: not exactly two steps (checkout, script)")
else:
checkout, script = steps_
# Its own timeout, so a slow checkout fails as that step, not as the job cancelled (#443).
checkout_minutes = checkout.get("timeout-minutes")
if type(checkout_minutes) is not int or checkout_minutes <= 0:
problems.append(f"{where}: the checkout step has no timeout-minutes of its own (a positive whole number)")
checkout_minutes = None
if "git init" not in str(checkout.get("run", "")) or checkout.get("env") != POLICY_CHECKOUT_ENV:
problems.append(f"{where}: the first step is not the shared checkout of github.sha with only the script")
if set(script) != {"name", "env", "run"} or str(script.get("run", "")).strip() != POLICY_SCRIPT \
or script.get("env") != POLICY_ENV:
problems.append(f"{where}: the second step is not `{POLICY_SCRIPT}` with the pinned env and nothing else")
timeout, cap = job.get("timeout-minutes"), fc.get("wait_seconds")
if not isinstance(timeout, int) or not isinstance(cap, int) or policy_worst_case(cap, limits) > timeout * 60:
problems.append(f"{where}: its worst case (checkout {POLICY_CHECKOUT_BUDGET} s, fork_checks.wait_seconds "
if not isinstance(timeout, int) or not isinstance(cap, int) or checkout_minutes is None \
or policy_worst_case(checkout_minutes * 60, cap, limits) > timeout * 60:
problems.append(f"{where}: its worst case (checkout {checkout_minutes!r} min, fork_checks.wait_seconds "
f"{cap!r} or the git calls {limits}, one last poll) does not fit the timeout ({timeout!r} min), "
"so a timeout would cancel the job before its message")
return problems
Expand Down Expand Up @@ -1780,7 +1798,19 @@ def policy_mutant(mutate, limits=POLICY_LIMITS):
"wait_seconds", t["jobs"]["fork_checks_ran"]["timeout-minutes"] * 60),
"a job timeout cut below the cap": lambda p, w, t: t["jobs"]["fork_checks_ran"].__setitem__("timeout-minutes", 10),
"a cap that leaves no room for the last poll": lambda p, w, t: p["fork_checks"].__setitem__(
"wait_seconds", t["jobs"]["fork_checks_ran"]["timeout-minutes"] * 60 - POLICY_CHECKOUT_BUDGET - 1),
"wait_seconds", (t["jobs"]["fork_checks_ran"]["timeout-minutes"]
- (t["jobs"]["fork_checks_ran"]["steps"][0].get("timeout-minutes") or 0)) * 60 - 1),
# The checkout's own timeout (#443): there, a whole number, and counted in the worst case.
"fork_checks_ran's checkout without its own timeout": lambda p, w, t: t["jobs"]["fork_checks_ran"]["steps"][0]
.pop("timeout-minutes", None),
"fork_checks_ran's checkout timeout as text": lambda p, w, t: t["jobs"]["fork_checks_ran"]["steps"][0]
.__setitem__("timeout-minutes", "3"),
"fork_checks_ran's checkout timeout true": lambda p, w, t: t["jobs"]["fork_checks_ran"]["steps"][0]
.__setitem__("timeout-minutes", True),
"fork_checks_ran's checkout timeout zero": lambda p, w, t: t["jobs"]["fork_checks_ran"]["steps"][0]
.__setitem__("timeout-minutes", 0),
"a checkout timeout that breaks the budget": lambda p, w, t: t["jobs"]["fork_checks_ran"]["steps"][0]
.__setitem__("timeout-minutes", (t["jobs"]["fork_checks_ran"]["steps"][0].get("timeout-minutes") or 0) + 2),
"fork_checks_ran without PR_NUMBER": lambda p, w, t: t["jobs"]["fork_checks_ran"]["steps"][-1]["env"]
.pop("PR_NUMBER"),
"fork_checks_ran with PR_NUMBER from elsewhere": lambda p, w, t: t["jobs"]["fork_checks_ran"]["steps"][-1]["env"]
Expand Down
6 changes: 5 additions & 1 deletion .github/workflows/fork-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,10 @@ jobs:
# config on the pull request's base commit (#430), and fork_ci_result needs it. Other events: a called run (release) logs a skip; a direct run passes
# only on the default branch. Checks out without a token (the shared plain-git step: actions/checkout
# with persist-credentials: false fails on upstream's gitlinks); the script fetches the base commit.
# fork-contracts.test.py pins this job (if, env, steps, no continue-on-error, cap below the timeout).
# fork-contracts.test.py pins this job (if, env, steps, no continue-on-error) and checks its worst case
# fits the job timeout: the checkout step's own timeout + the wait cap (or the script's git calls, if
# longer) + one last poll (today 3 + 10 + 1 = 14 min of 15). The checkout's timeout makes a slow fetch fail
# as a timed-out Checkout step, not as a job cancelled before the script's message.
# The checkout and the base fetch use no token, so this job works only in a public repository.
fork_checks_ran:
name: Fork checks ran
Expand All @@ -265,6 +268,7 @@ jobs:
steps:
- name: Checkout
shell: bash
timeout-minutes: 3
env:
REF: ${{ github.sha }}
HISTORY: tip
Expand Down
Loading