Keep full backend matrices for relevant changes and sample unrelated main pushes

.github/backend-ci-policy.md

@@ -0,0 +1,19 @@
+# Backend CI coverage
+
+QNN, Arm (including VGF), XNNPACK and Vulkan use full matrices for relevant
+backend and shared export/runtime/test/build/dependency changes. Unrelated PRs
+and unsampled main pushes use the reduced matrices. The existing CI run
+decision supplies the main sample; its rules are not modified.
+
+| Event | Matrix |
+| --- | --- |
+| Relevant PR or main-push change | Full |
+| Unrelated PR | Reduced |
+| Unrelated sampled main push | Full |
+| Unrelated unsampled main push | Reduced |
+| Schedule, existing release/tag/manual run | Full, as before |
+| Unknown/unavailable change information | Full |
+
+Git diffs include deleted and renamed paths and do not have the API's 300-file
+limit. Callers without a reduced matrix continue to use their full flow list.
+This change does not alter release publication or viable/strict rules.

.github/scripts/select_backend_flows.py

@@ -0,0 +1,120 @@
+import fnmatch
+import json
+import os
+import re
+import subprocess
+from pathlib import Path
+
+
+SHARED_PATHS = (
+    ".ci/**",
+    ".github/workflows/_test_backend.yml",
+    ".github/scripts/select_backend_flows.py",
+    ".github/scripts/test_select_backend_flows.py",
+    "backends/test/**",
+    "backends/transforms/**",
+    "codegen/**",
+    "exir/**",
+    "runtime/**",
+    "kernels/**",
+    "extension/**",
+    "schema/**",
+    "third-party/**",
+    "examples/models/**",
+    "tools/cmake/**",
+    "test/**",
+    "CMakeLists.txt",
+    "**/CMakeLists.txt",
+    "CMakePresets.json",
+    "setup.py",
+    "pyproject.toml",
+    "install_*",
+    "requirements*",
+    "torch_pin.py",
+    "version.txt",
+    ".gitmodules",
+)
+
+
+def select_flows(full, reduced, backend_paths, event, ref, changed, sampled):
+    if (
+        not isinstance(full, list)
+        or not full
+        or not all(isinstance(f, str) and f for f in full)
+    ):
+        raise ValueError("Full flows must be a nonempty list of flow names")
+    if not isinstance(reduced, list) or not reduced or not set(reduced).issubset(full):
+        raise ValueError("Reduced flows must be a nonempty subset of full flows")
+    if event in ("schedule", "workflow_dispatch") or ref.startswith(
+        ("refs/heads/release/", "refs/tags/")
+    ):
+        return full, "scheduled, release or forced run"
+    if sampled:
+        return full, "existing full-run sample"
+    if event not in ("pull_request", "push") or (
+        event == "push" and ref != "refs/heads/main"
+    ):
+        return full, "unrecognized event or branch"
+    if changed is None:
+        return full, "changed files unavailable"
+    if (
+        not isinstance(backend_paths, list)
+        or not backend_paths
+        or not all(isinstance(p, str) and p for p in backend_paths)
+    ):
+        return full, "backend paths unavailable"
+    for path in changed:
+        if any(
+            fnmatch.fnmatchcase(path, pattern)
+            for pattern in (*SHARED_PATHS, *backend_paths)
+        ):
+            return full, f"relevant change: {path}"
+    return reduced, "unrelated changes"
+
+
+def changed_files(event, payload, source_sha):
+    if event == "pull_request":
+        pr = payload.get("pull_request", {})
+        base = pr.get("base", {}).get("sha", "")
+        head = pr.get("head", {}).get("sha", "")
+        revision = f"{base}...{head}"
+    elif event == "push":
+        base = payload.get("before", "")
+        head = payload.get("after", "")
+        revision = f"{base}..{head}"
+    else:
+        return None
+    if head != source_sha or not all(
+        re.fullmatch(r"[0-9a-f]{40}", sha) and sha != "0" * 40 for sha in (base, head)
+    ):
+        return None
+    try:
+        result = subprocess.run(
+            ["git", "diff", "--name-only", "--no-renames", "-z", revision, "--"],
+            check=True,
+            capture_output=True,
+        )
+    except subprocess.CalledProcessError:
+        return None
+    return [path.decode() for path in result.stdout.split(b"\0") if path]
+
+
+def main():
+    event = os.environ["GITHUB_EVENT_NAME"]
+    payload = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text())
+    flows, reason = select_flows(
+        json.loads(os.environ["FULL_FLOWS"]),
+        json.loads(os.environ["REDUCED_FLOWS"]),
+        json.loads(os.environ["BACKEND_PATHS"]),
+        event,
+        os.environ["GITHUB_REF"],
+        changed_files(event, payload, os.environ["SOURCE_SHA"]),
+        os.environ["IS_FULL_RUN"] == "true",
+    )
+    print(reason)
+    with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+        output.write("flows=" + json.dumps(flows, separators=(",", ":")) + "\n")
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/test_select_backend_flows.py

@@ -0,0 +1,144 @@
+import json
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+from select_backend_flows import changed_files, select_flows
+
+
+class BackendFlowPolicyTest(unittest.TestCase):
+    def select(
+        self,
+        paths,
+        event="pull_request",
+        ref="refs/pull/1/merge",
+        sampled=False,
+        backend_paths=None,
+    ):
+        return select_flows(
+            ["fp", "int"],
+            ["fp"],
+            backend_paths if backend_paths is not None else ["backends/demo/**"],
+            event,
+            ref,
+            paths,
+            sampled,
+        )[0]
+
+    def test_related_paths_require_quantized_flows(self):
+        for path in [
+            "backends/demo/operator.py",
+            "exir/pass.py",
+            "runtime/core.cc",
+            "extension/pybindings/module.cpp",
+            "backends/test/suite/flows/demo.py",
+            "requirements-examples.txt",
+            ".ci/docker/ci_commit_pins/pytorch.txt",
+            "tools/cmake/Utils.cmake",
+            ".github/scripts/select_backend_flows.py",
+        ]:
+            with self.subTest(path=path):
+                self.assertEqual(self.select([path]), ["fp", "int"])
+
+    def test_unrelated_changes_use_reduced_flows(self):
+        self.assertEqual(self.select(["docs/source/tutorial.md"]), ["fp"])
+        self.assertEqual(
+            self.select(["docs/source/tutorial.md"], "push", "refs/heads/main"), ["fp"]
+        )
+
+    def test_sampling_and_existing_full_events_require_all_flows(self):
+        self.assertEqual(
+            self.select(["docs/source/tutorial.md"], "push", "refs/heads/main", True),
+            ["fp", "int"],
+        )
+        for event, ref in [
+            ("schedule", "refs/heads/main"),
+            ("workflow_dispatch", "refs/heads/main"),
+            ("push", "refs/heads/release/1.6"),
+            ("push", "refs/tags/ciflow/nightly/123"),
+            ("push", "refs/tags/v1.6.0-rc1"),
+            ("unexpected", "refs/heads/main"),
+        ]:
+            with self.subTest(event=event, ref=ref):
+                self.assertEqual(self.select([], event, ref), ["fp", "int"])
+
+    def test_unknown_changes_and_missing_path_configuration_require_all_flows(self):
+        self.assertEqual(self.select(None), ["fp", "int"])
+        self.assertEqual(self.select(["docs/a.md"], backend_paths=[]), ["fp", "int"])
+        self.assertEqual(self.select([], "push", "refs/heads/unknown"), ["fp", "int"])
+
+    def test_invalid_matrices_cannot_report_green_without_tests(self):
+        for full, reduced in [
+            ([], ["fp"]),
+            (["fp"], []),
+            (["fp"], ["int"]),
+            ([1], ["fp"]),
+        ]:
+            with self.subTest(full=full, reduced=reduced), self.assertRaises(
+                ValueError
+            ):
+                select_flows(
+                    full,
+                    reduced,
+                    ["backends/demo/**"],
+                    "push",
+                    "refs/heads/main",
+                    [],
+                    False,
+                )
+
+    def test_deleted_and_renamed_paths_and_large_changes_are_not_lost(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+
+            def git(*args):
+                return subprocess.check_output(
+                    ["git", *args], cwd=root, text=True
+                ).strip()
+
+            git("init", "-q")
+            git("config", "user.name", "CI test")
+            git("config", "user.email", "ci-test@example.invalid")
+            (root / "backends/demo").mkdir(parents=True)
+            (root / "backends/demo/removed.py").write_text("old")
+            (root / "setup.py").write_text("old setup")
+            git("add", ".")
+            git("commit", "-qm", "base fixture")
+            base = git("rev-parse", "HEAD")
+            (root / "backends/demo/removed.py").unlink()
+            (root / "docs").mkdir()
+            (root / "setup.py").rename(root / "docs/moved.py")
+            for number in range(310):
+                (root / f"docs/{number}.md").write_text("new")
+            git("add", "-A")
+            git("commit", "-qm", "changed fixture")
+            head = git("rev-parse", "HEAD")
+            previous = Path.cwd()
+            os.chdir(root)
+            try:
+                paths = changed_files(
+                    "pull_request",
+                    {"pull_request": {"base": {"sha": base}, "head": {"sha": head}}},
+                    head,
+                )
+            finally:
+                os.chdir(previous)
+            self.assertIn("backends/demo/removed.py", paths)
+            self.assertIn("setup.py", paths)
+            self.assertIn("docs/moved.py", paths)
+            self.assertGreater(len(paths), 300)
+            self.assertEqual(self.select(paths), ["fp", "int"])
+
+    def test_unknown_push_base_and_source_mismatch_use_full_coverage(self):
+        self.assertIsNone(
+            changed_files("push", {"before": "0" * 40, "after": "a" * 40}, "a" * 40)
+        )
+        self.assertIsNone(
+            changed_files("push", {"before": "b" * 40, "after": "a" * 40}, "c" * 40)
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/workflows/_test_backend.yml

@@ -11,6 +11,16 @@ on:
         description: 'JSON array of flows to test'
         required: true
         type: string
+      reduced-flows:
+        description: 'Nonempty subset of flows for unrelated PRs and unsampled main pushes'
+        required: false
+        type: string
+        default: ''
+      backend-paths:
+        description: 'JSON array of backend paths that require all flows'
+        required: false
+        type: string
+        default: '[]'
       exclude:
         description: 'JSON array of {flow, suite} pairs to drop from the matrix'
         required: false
@@ -53,18 +63,64 @@ on:
         default: false
 
 jobs:
+  run-decision:
+    if: ${{ inputs.reduced-flows != '' }}
+    uses: ./.github/workflows/_ci-run-decision.yml
+
+  select-flows:
+    needs: run-decision
+    if: ${{ !cancelled() && (needs.run-decision.result == 'success' || needs.run-decision.result == 'skipped') }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      flows: ${{ steps.full.outputs.flows || steps.select.outputs.flows }}
+    steps:
+      - name: Select required full matrix
+        id: full
+        if: ${{ inputs.reduced-flows == '' }}
+        env:
+          FULL_FLOWS: ${{ inputs.flows }}
+        run: |
+          python3 - <<'PY'
+          import json
+          import os
+          flows = json.loads(os.environ['FULL_FLOWS'])
+          if not isinstance(flows, list) or not flows or not all(isinstance(f, str) and f for f in flows):
+              raise ValueError('Full flows must be a nonempty list of flow names')
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as output:
+              output.write('flows=' + json.dumps(flows, separators=(',', ':')) + '\n')
+          PY
+      - name: Checkout flow policy and change history
+        if: ${{ inputs.reduced-flows != '' }}
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          fetch-depth: 0
+          sparse-checkout: .github/scripts
+      - name: Select flows for this change
+        id: select
+        if: ${{ inputs.reduced-flows != '' }}
+        env:
+          FULL_FLOWS: ${{ inputs.flows }}
+          REDUCED_FLOWS: ${{ inputs.reduced-flows }}
+          BACKEND_PATHS: ${{ inputs.backend-paths }}
+          IS_FULL_RUN: ${{ needs.run-decision.outputs.is-full-run }}
+          SOURCE_SHA: ${{ inputs.ref }}
+        run: python3 .github/scripts/select_backend_flows.py
+
   docker-image:
     name: Resolve CI docker image
     if: ${{ inputs.run-linux }}
     uses: ./.github/workflows/_docker-image.yml
 
   test-backend-linux:
-    needs: docker-image
+    needs: [docker-image, select-flows]
     if: ${{ inputs.run-linux }}
     strategy:
       fail-fast: false
       matrix:
-        flow: ${{ fromJSON(inputs.flows) }}
+        flow: ${{ fromJSON(needs.select-flows.outputs.flows) }}
         suite: [models, operators]
         exclude: ${{ fromJSON(inputs.exclude) }}
 
@@ -149,11 +205,12 @@ jobs:
           path: golden_artifacts_*.zip
 
   test-backend-macos:
+    needs: select-flows
     if: ${{ inputs.run-macos }}
     strategy:
       fail-fast: false
       matrix:
-        flow: ${{ fromJSON(inputs.flows) }}
+        flow: ${{ fromJSON(needs.select-flows.outputs.flows) }}
         suite: [models, operators]
         exclude: ${{ fromJSON(inputs.exclude) }}