Move the rollback workflow's steps into tested scripts and enforce the one-line run: rule

.github/scripts/rollback-read-release-identity.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Read the release identity latest now routes to from the routing registry.
+# Run from projects/policyengine-simulation-executor.
+# Required env vars: MODAL_ENVIRONMENT, and the runner's GITHUB_OUTPUT. The
+# Modal client reads MODAL_TOKEN_ID and MODAL_TOKEN_SECRET.
+# Outputs: policyengine_version, app_name, uk_model_version and
+# uk_dataset_sha256 (when the release recorded one) in GITHUB_OUTPUT.
+
+set -euo pipefail
+
+uv run --frozen python -m src.modal.utils.update_version_registry \
+  --environment "${MODAL_ENVIRONMENT}" --print-identity \
+  | grep -E '^(policyengine_version|app_name|uk_model_version|uk_dataset_sha256)=' \
+  | tee -a "$GITHUB_OUTPUT"

.github/scripts/rollback-resolve-pending-publication.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Print the target's unfinished routing publication from the durable routing
+# journal, then restore what it replaced, as a publication whose release
+# checks failed. Needs no artifact from the deployment run.
+# Run from projects/policyengine-simulation-executor.
+# Required env vars: TARGET, MODAL_ENVIRONMENT. The Modal client reads
+# MODAL_TOKEN_ID and MODAL_TOKEN_SECRET.
+
+set -euo pipefail
+
+uv run --frozen python -m src.modal.utils.update_version_registry \
+  --environment "${MODAL_ENVIRONMENT}" --print-pending-plan
+uv run --frozen python -m src.modal.utils.update_version_registry \
+  --environment "${MODAL_ENVIRONMENT}" --finalize-publication \
+  --release-environment "${TARGET}" --gates-passed false

.github/scripts/rollback-restore-entrypoint-traffic.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Route all stable Cloud Run service traffic to the requested retained
+# revision, unless it already serves it.
+# Required env vars: SERVICE, SIMULATION_ENTRYPOINT_TARGET_REVISION,
+# SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION, and the other
+# SIMULATION_ENTRYPOINT_* variables set-cloud-run-simulation-entry-revision.sh
+# reads. gcloud uses the job's Google credentials.
+
+set -euo pipefail
+
+script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+if [[ "${SIMULATION_ENTRYPOINT_TARGET_REVISION}" == "${SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION}" ]]; then
+  echo "${SERVICE} already serves ${SIMULATION_ENTRYPOINT_TARGET_REVISION}."
+else
+  "${script_directory}/set-cloud-run-simulation-entry-revision.sh"
+fi

.github/scripts/rollback-restore-routing.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Restore the target's Modal routing: undo a deployment run's routing
+# changes from the plans it kept, or point latest at a published release.
+# With DEPLOY_RUN_ID, every attempt of that run whose routing job kept a plan
+# must have its plan under $RUNNER_TEMP/routing-plans, downloaded by the
+# workflow; the plans are passed oldest first. Without it, latest is
+# restored to POLICYENGINE_VERSION, refusing unless latest is currently
+# EXPECTED_LATEST when that is given.
+# Run from projects/policyengine-simulation-executor.
+# Required env vars: TARGET, DEPLOY_RUN_ID, POLICYENGINE_VERSION,
+# EXPECTED_LATEST, MODAL_ENVIRONMENT, GH_TOKEN (read by gh), and the runner's
+# GITHUB_REPOSITORY and RUNNER_TEMP. The Modal client reads MODAL_TOKEN_ID and
+# MODAL_TOKEN_SECRET.
+
+set -euo pipefail
+
+if [[ -n "${DEPLOY_RUN_ID}" ]]; then
+  # Every attempt whose routing job kept a plan must have one here:
+  # restoring without an attempt's plan would stop short of what
+  # the run replaced and still look complete.
+  attempts="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${DEPLOY_RUN_ID}" --jq .run_attempt)"
+  for attempt in $(seq 1 "${attempts}"); do
+    kept="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${DEPLOY_RUN_ID}/attempts/${attempt}/jobs" \
+      --jq "[.jobs[] | select(.name == \"Complete ${TARGET} deployment / Publish routing after all services pass\") | .steps[] | select(.name == \"Keep the plan for a guarded restore\" and .conclusion == \"success\")] | length" \
+      | awk '{total += $1} END {print total + 0}')"
+    if [[ "${kept}" -gt 0 && ! -f "${RUNNER_TEMP}/routing-plans/routing-plan-${TARGET}-${attempt}/plan.json" ]]; then
+      echo "Attempt ${attempt} of run ${DEPLOY_RUN_ID} kept a ${TARGET} routing plan that cannot be found; if its publication is pending, rerun this workflow with target=${TARGET} and resolve_pending_publication=true, leaving other inputs blank." >&2
+      exit 1
+    fi
+  done
+  # One plan per run attempt, passed oldest first. The restore undoes
+  # every attempt's change that is still live, newest first, and skips
+  # attempts that changed nothing.
+  plans=()
+  for attempt in $(
+    for directory in "${RUNNER_TEMP}"/routing-plans/routing-plan-"${TARGET}"-*/; do
+      name="${directory%/}"
+      name="${name##*-}"
+      if [[ "${name}" =~ ^[0-9]+$ ]]; then echo "${name}"; fi
+    done | sort -n
+  ); do
+    plans+=("${RUNNER_TEMP}/routing-plans/routing-plan-${TARGET}-${attempt}/plan.json")
+  done
+  if [[ ${#plans[@]} -eq 0 ]]; then
+    echo "Deployment run ${DEPLOY_RUN_ID} kept no ${TARGET} routing plan." >&2
+    exit 1
+  fi
+  uv run --frozen python -m src.modal.utils.update_version_registry \
+    --environment "${MODAL_ENVIRONMENT}" \
+    --restore-plan "${plans[@]}"
+else
+  args=(--environment "${MODAL_ENVIRONMENT}" --restore-latest "${POLICYENGINE_VERSION}")
+  if [[ -n "${EXPECTED_LATEST}" ]]; then
+    args+=(--expected-latest "${EXPECTED_LATEST}")
+  fi
+  uv run --frozen python -m src.modal.utils.update_version_registry "${args[@]}"
+fi

.github/scripts/rollback-stable-service-state.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Record the stable Cloud Run service's URL and the revision serving all of
+# its traffic.
+# Required env vars: SERVICE, PROJECT_ID, REGION, and the runner's
+# GITHUB_OUTPUT. gcloud uses the job's Google credentials.
+# Outputs: url and revision in GITHUB_OUTPUT.
+#
+# As in the inline step this replaced, a jq failure inside an echo argument
+# does not stop the script: without a URL it records url=null, and without
+# exactly one revision at 100 percent it records an empty revision. A gcloud
+# failure does stop it.
+
+set -euo pipefail
+
+service_json="$(gcloud run services describe "${SERVICE}" \
+  --project "${PROJECT_ID}" --region "${REGION}" --format=json)"
+echo "url=$(jq -er '.status.url' <<<"${service_json}")" >> "$GITHUB_OUTPUT"
+echo "revision=$(jq -er '
+  [.status.traffic[]? | select((.percent // 0) == 100) | .revisionName]
+  | if length == 1 then .[0] else error("no single 100 percent revision") end
+' <<<"${service_json}")" >> "$GITHUB_OUTPUT"

.github/scripts/rollback-verify-stable-service.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Verify that the stable service serves the restored release: a rollback to
+# a requested version must have made it latest, and the stable URL must
+# serve the expected revision and the whole identity latest routes to.
+# Required env vars: POLICYENGINE_VERSION (blank when restoring a deployment
+# run's plans), EXPECTED_POLICYENGINE_VERSION, STABLE_URL, EXPECTED_REVISION,
+# PREVIOUS_REVISION, and the EXPECTED_* identity variables
+# cloud-run-simulation-entry-smoke.sh reads.
+
+set -euo pipefail
+
+script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+if [[ -n "${POLICYENGINE_VERSION}" && "${EXPECTED_POLICYENGINE_VERSION}" != "${POLICYENGINE_VERSION}" ]]; then
+  echo "latest routes to ${EXPECTED_POLICYENGINE_VERSION}, not ${POLICYENGINE_VERSION}" >&2
+  exit 1
+fi
+"${script_directory}/cloud-run-simulation-entry-smoke.sh" \
+  "${STABLE_URL}" "${EXPECTED_REVISION}" "${PREVIOUS_REVISION}"

.github/workflows/simulation-rollback.yml

@@ -113,61 +113,15 @@ jobs:
           MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
           MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
           GH_TOKEN: ${{ github.token }}
-        run: |
-          if [[ -n "${DEPLOY_RUN_ID}" ]]; then
-            # Every attempt whose routing job kept a plan must have one here:
-            # restoring without an attempt's plan would stop short of what
-            # the run replaced and still look complete.
-            attempts="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${DEPLOY_RUN_ID}" --jq .run_attempt)"
-            for attempt in $(seq 1 "${attempts}"); do
-              kept="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${DEPLOY_RUN_ID}/attempts/${attempt}/jobs" \
-                --jq "[.jobs[] | select(.name == \"Complete ${TARGET} deployment / Publish routing after all services pass\") | .steps[] | select(.name == \"Keep the plan for a guarded restore\" and .conclusion == \"success\")] | length" \
-                | awk '{total += $1} END {print total + 0}')"
-              if [[ "${kept}" -gt 0 && ! -f "${RUNNER_TEMP}/routing-plans/routing-plan-${TARGET}-${attempt}/plan.json" ]]; then
-                echo "Attempt ${attempt} of run ${DEPLOY_RUN_ID} kept a ${TARGET} routing plan that cannot be found; if its publication is pending, rerun this workflow with target=${TARGET} and resolve_pending_publication=true, leaving other inputs blank." >&2
-                exit 1
-              fi
-            done
-            # One plan per run attempt, passed oldest first. The restore undoes
-            # every attempt's change that is still live, newest first, and skips
-            # attempts that changed nothing.
-            plans=()
-            for attempt in $(
-              for directory in "${RUNNER_TEMP}"/routing-plans/routing-plan-"${TARGET}"-*/; do
-                name="${directory%/}"
-                name="${name##*-}"
-                if [[ "${name}" =~ ^[0-9]+$ ]]; then echo "${name}"; fi
-              done | sort -n
-            ); do
-              plans+=("${RUNNER_TEMP}/routing-plans/routing-plan-${TARGET}-${attempt}/plan.json")
-            done
-            if [[ ${#plans[@]} -eq 0 ]]; then
-              echo "Deployment run ${DEPLOY_RUN_ID} kept no ${TARGET} routing plan." >&2
-              exit 1
-            fi
-            uv run --frozen python -m src.modal.utils.update_version_registry \
-              --environment "${MODAL_ENVIRONMENT}" \
-              --restore-plan "${plans[@]}"
-          else
-            args=(--environment "${MODAL_ENVIRONMENT}" --restore-latest "${POLICYENGINE_VERSION}")
-            if [[ -n "${EXPECTED_LATEST}" ]]; then
-              args+=(--expected-latest "${EXPECTED_LATEST}")
-            fi
-            uv run --frozen python -m src.modal.utils.update_version_registry "${args[@]}"
-          fi
+        run: ../../.github/scripts/rollback-restore-routing.sh
 
       - name: Resolve the unfinished routing publication
         if: ${{ inputs.resolve_pending_publication }}
         working-directory: projects/policyengine-simulation-executor
         env:
           MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
           MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
-        run: |
-          uv run --frozen python -m src.modal.utils.update_version_registry \
-            --environment "${MODAL_ENVIRONMENT}" --print-pending-plan
-          uv run --frozen python -m src.modal.utils.update_version_registry \
-            --environment "${MODAL_ENVIRONMENT}" --finalize-publication \
-            --release-environment "${TARGET}" --gates-passed false
+        run: ../../.github/scripts/rollback-resolve-pending-publication.sh
 
       - name: Read the restored release identity
         if: ${{ !inputs.resolve_pending_publication }}
@@ -176,11 +130,7 @@ jobs:
         env:
           MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
           MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
-        run: |
-          uv run --frozen python -m src.modal.utils.update_version_registry \
-            --environment "${MODAL_ENVIRONMENT}" --print-identity \
-            | grep -E '^(policyengine_version|app_name|uk_model_version|uk_dataset_sha256)=' \
-            | tee -a "$GITHUB_OUTPUT"
+        run: ../../.github/scripts/rollback-read-release-identity.sh
 
       - uses: google-github-actions/auth@v2
         if: ${{ !inputs.resolve_pending_publication }}
@@ -194,14 +144,7 @@ jobs:
       - name: Resolve the stable service
         if: ${{ !inputs.resolve_pending_publication }}
         id: service
-        run: |
-          service_json="$(gcloud run services describe "${SERVICE}" \
-            --project "${PROJECT_ID}" --region "${REGION}" --format=json)"
-          echo "url=$(jq -er '.status.url' <<<"${service_json}")" >> "$GITHUB_OUTPUT"
-          echo "revision=$(jq -er '
-            [.status.traffic[]? | select((.percent // 0) == 100) | .revisionName]
-            | if length == 1 then .[0] else error("no single 100 percent revision") end
-          ' <<<"${service_json}")" >> "$GITHUB_OUTPUT"
+        run: .github/scripts/rollback-stable-service-state.sh
 
       - name: Restore entrypoint traffic
         if: ${{ inputs.entrypoint_revision != '' }}
@@ -211,12 +154,7 @@ jobs:
           SIMULATION_ENTRYPOINT_SERVICE: ${{ env.SERVICE }}
           SIMULATION_ENTRYPOINT_TARGET_REVISION: ${{ inputs.entrypoint_revision }}
           SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION: ${{ steps.service.outputs.revision }}
-        run: |
-          if [[ "${SIMULATION_ENTRYPOINT_TARGET_REVISION}" == "${SIMULATION_ENTRYPOINT_EXPECTED_CURRENT_REVISION}" ]]; then
-            echo "${SERVICE} already serves ${SIMULATION_ENTRYPOINT_TARGET_REVISION}."
-          else
-            .github/scripts/set-cloud-run-simulation-entry-revision.sh
-          fi
+        run: .github/scripts/rollback-restore-entrypoint-traffic.sh
 
       - name: Verify the stable service serves the restored release
         if: ${{ !inputs.resolve_pending_publication }}
@@ -232,10 +170,4 @@ jobs:
           STABLE_URL: ${{ steps.service.outputs.url }}
           EXPECTED_REVISION: ${{ inputs.entrypoint_revision }}
           PREVIOUS_REVISION: ${{ steps.service.outputs.revision }}
-        run: |
-          if [[ -n "${POLICYENGINE_VERSION}" && "${EXPECTED_POLICYENGINE_VERSION}" != "${POLICYENGINE_VERSION}" ]]; then
-            echo "latest routes to ${EXPECTED_POLICYENGINE_VERSION}, not ${POLICYENGINE_VERSION}" >&2
-            exit 1
-          fi
-          .github/scripts/cloud-run-simulation-entry-smoke.sh \
-            "${STABLE_URL}" "${EXPECTED_REVISION}" "${PREVIOUS_REVISION}"
+        run: .github/scripts/rollback-verify-stable-service.sh

changelog.d/1005.changed.md

@@ -0,0 +1 @@
+The manual rollback workflow's steps now run tested scripts under `.github/scripts/` instead of inline shell, with unchanged inputs, outputs and exit statuses. Rule `sim.ci.multiline-run-is-script` is now enforced with no baseline: any multi-line `run:` block in a workflow fails the quality-guards job.

docs/engineering/skills/ci-workflows.md

@@ -41,15 +41,19 @@ at least one failure path.
 Examples: `projects/policyengine-simulation-executor/tests/test_modal_scripts.py`
 and `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`.
 
-## Enforcement and baselines
+## Enforcement
 
 The `ci-scripts` quality guard (`python scripts/run_quality_guards.py`, or
-`python -m scripts.rules ci-scripts`) checks all three rules above. Existing
-violations are listed in `rules/baselines/ci-*.txt`. Anything new fails, and so
-does a baseline entry that no longer applies, so each list only shrinks. When
-you convert an existing multi-line block into a script, or add a missing test,
-remove its baseline entry and lower the rule's `baseline.count` in
-`rules/registry.toml`.
+`python -m scripts.rules ci-scripts`) checks all three rules above, in the
+`quality-guards` CI job, and all three are enforced:
+
+- Any multi-line `run:` block fails. The rule has no baseline, so
+  `[ci_scripts]` in `rules/registry.toml` lists none for it.
+- The baselines it lists for the other two checks,
+  `rules/baselines/ci-untested-scripts.txt` and
+  `rules/baselines/ci-shellcheck.txt`, are empty, and stay so:
+  `python -m scripts.rules validate`, run by the `rules-registry` guard,
+  fails if a check whose rule is enforced lists a baseline with an entry.
 
 ## Pushing to main from a workflow
 

docs/generated/rules.md

@@ -8,7 +8,7 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf
 
 | Tier | enforced | ratcheted | advisory | doc-only | Total |
 | --- | ---: | ---: | ---: | ---: | ---: |
-| mechanical | 57 | 2 | 0 | 14 | 73 |
+| mechanical | 58 | 1 | 0 | 14 | 73 |
 | metric | 1 | 0 | 0 | 0 | 1 |
 | judgment | 0 | 0 | 0 | 0 | 0 |
 
@@ -64,7 +64,6 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf
 
 ### Ratcheted (existing violations still tolerated)
 
-- `sim.ci.multiline-run-is-script`: 6 remaining in `rules/baselines/ci-multiline-run.txt`
 - `sim.microdata.weight-stripping-confined`: 0 remaining in `rules/baselines/sim-microdata-weight-stripping.txt`
 
 ### Exceptions and review dates
@@ -441,13 +440,13 @@ flowchart LR
 
 | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof |
 | --- | --- | --- | --- | --- | --- | --- | --- |
-| `sim.ci.multiline-run-is-script` | A GitHub Actions run: block holds one line of code and no chained commands (&&, \|\|, ;); anything more lives in a tested script under .github/scripts/, called from a single-line step. | mechanical | ratcheted | yes | quality guard `ci-scripts` | `pr.yml:quality-guards` | yes |
+| `sim.ci.multiline-run-is-script` | A GitHub Actions run: block holds one line of code and no chained commands (&&, \|\|, ;); anything more lives in a tested script under .github/scripts/, called from a single-line step. | mechanical | enforced | yes | quality guard `ci-scripts` | `pr.yml:quality-guards` | yes |
 | `sim.ci.shell-scripts-pass-shellcheck` | Every shell script under .github/scripts/ passes shellcheck. | mechanical | enforced | yes | quality guard `ci-scripts` | `pr.yml:quality-guards` | yes |
 | `sim.ci.workflow-scripts-tested` | Every shell script a workflow calls is run by at least one test. | mechanical | enforced | yes | quality guard `ci-scripts` | `pr.yml:quality-guards` | yes |
 
 ```mermaid
 flowchart LR
-  r_sim_ci_multiline_run_is_script("sim.ci.multiline-run-is-script (ratcheted)")
+  r_sim_ci_multiline_run_is_script("sim.ci.multiline-run-is-script (enforced)")
   e_quality_guard_ci_scripts["quality guard ci-scripts"]
   j_CI_pr_yml_quality_guards["CI pr.yml:quality-guards"]
   r_sim_ci_shell_scripts_pass_shellcheck("sim.ci.shell-scripts-pass-shellcheck (enforced)")

projects/policyengine-simulation-entry/tests/test_routing_workflow_invariants.py

@@ -1114,6 +1114,19 @@ def test_the_reusable_deployments_routing_writers_are_its_journal_steps():
     }
 
 
+def test_the_rollback_workflows_routing_writers_are_its_restore_steps():
+    jobs = _workflows()[".github/workflows/simulation-rollback.yml"]["jobs"]
+    writers = {
+        (job_id, step["name"])
+        for job_id, job in jobs.items()
+        for step in _writing_steps(job)
+    }
+    assert writers == {
+        ("rollback", "Restore Modal routing"),
+        ("rollback", "Resolve the unfinished routing publication"),
+    }
+
+
 # --- Characterization: routing command discovery's argv and refusals ---------