Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/scripts/run_live_staging_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@ uv run --frozen --extra dev python -m pytest \
# to staging; staging promotion waits on them, so the change that adds them
# merges only after that deployment. Their requests are short (a
# submission, then a poll every few seconds) apart from the parity tests'
# /calculate requests, but the parallel check submits 6 calculations at once:
# /calculate and /calculate-full requests, which the tests wait for longer
# than the candidate's 300 s request timeout (V1_CALCULATION_TIMEOUT_SECONDS in
# tests/fixtures/integration/live_household_calculations.py): /calculate-full
# took 224 s on staging. The parallel check submits 6 calculations at once:
# with the 3 other workers' requests, at most 3 wait for a free slot, as above.
uv run --frozen --extra dev python -m pytest \
-n 4 \
Expand Down
1 change: 1 addition & 0 deletions changelog.d/live-household-parity-checks.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix the live staging household parity checks: the US household's `income_tax` is on its tax unit, where `/calculate` expects it, and the checks wait longer than the candidate's 300 s request timeout for `/calculate-full`.
12 changes: 10 additions & 2 deletions docs/migration/stage-11-v2-households.md
Original file line number Diff line number Diff line change
Expand Up @@ -380,5 +380,13 @@ cycle, six calculations submitted at once (each accepted within
`STAGING_HOUSEHOLD_SUBMIT_LIMIT_SECONDS`, 10 by default, and finished within
`STAGING_HOUSEHOLD_FINISH_LIMIT_SECONDS`, 300), a household across 401 axis
points, and a parity set of three US and three UK households compared value
by value with `/calculate` and `/calculate-full`. The checks need the
simulation service's household calculation route deployed to staging first.
by value with `/calculate` and `/calculate-full`. The parity set waits 320 s
for each `/calculate` or `/calculate-full` answer, longer than the candidate's
300 s Cloud Run request timeout: `/calculate-full` calculates every variable
the household leaves out, which took 224 s on staging. A unit test sends each
parity household through those routes with the installed country models and
replaces only the calculation, so a household the routes refuse before
calculating, in their input checks or situation parsing, fails before a
release. The checks
need the simulation service's household calculation route deployed to staging
first.
16 changes: 14 additions & 2 deletions tests/fixtures/integration/live_household_calculations.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,13 @@
TRANSIENT_LIMIT_SECONDS = float(
os.environ.get("STAGING_HOUSEHOLD_TRANSIENT_LIMIT_SECONDS", "90")
)
# How long the check waits for the Public API's own /calculate or
# /calculate-full answer: longer than the candidate's Cloud Run request timeout
# (CLOUD_RUN_TIMEOUT in .github/scripts/cloud_run_env.sh, 300 s), so the check
# gets the candidate's own answer, a 504 included. /calculate-full calculates
# every variable the household leaves out and can take most of that: 224 s on
# staging, and up to 300 s in production.
V1_CALCULATION_TIMEOUT_SECONDS = 320.0
FINISHED_STATUSES = {"succeeded", "failed"}
YEAR = "2026"
REFORM_START = f"{YEAR}-01-01"
Expand Down Expand Up @@ -279,7 +286,8 @@ def calculate_with_v1(client: httpx.Client, household: LiveHousehold) -> dict:
"""Return the Public API's own answer for the same household and reform.

``/calculate-full`` answers when the household adds missing variables,
``/calculate`` otherwise.
``/calculate`` otherwise. The answer is awaited for
``V1_CALCULATION_TIMEOUT_SECONDS``.
"""
route = "calculate-full" if household.add_missing_variables else "calculate"
request: dict[str, Any] = {"household": household.situation()}
Expand All @@ -288,7 +296,11 @@ def calculate_with_v1(client: httpx.Client, household: LiveHousehold) -> dict:
parameter_name: {f"{REFORM_START}.{OPEN_END}": value}
for parameter_name, value in household.reform.items()
}
response = client.post(f"/{household.country_id}/{route}", json=request)
response = client.post(
f"/{household.country_id}/{route}",
json=request,
timeout=V1_CALCULATION_TIMEOUT_SECONDS,
)
assert response.status_code == 200, response.text
payload = response.json()
assert payload["status"] == "ok", payload
Expand Down
11 changes: 8 additions & 3 deletions tests/integration/test_live_v2_household_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,18 @@ def parity_households(run_amount: int) -> list[LiveHousehold]:
"employment_income": {YEAR: 42_000 + run_amount},
# Deprecated: dropped with a warning by both calculations.
"medical_out_of_pocket_expenses": {YEAR: 1_000},
"income_tax": CALCULATED,
}
},
groups={
"household": {
"state_name": {YEAR: "CA"},
"household_net_income": CALCULATED,
},
"tax_unit": {"eitc": CALCULATED, "ctc": CALCULATED},
"tax_unit": {
"income_tax": CALCULATED,
"eitc": CALCULATED,
"ctc": CALCULATED,
},
"spm_unit": {"snap": CALCULATED},
},
expects_warnings=True,
Expand Down Expand Up @@ -174,9 +177,11 @@ def test_a_v2_household_calculation_matches_api_v1(
policy_id = store_reform(api_client, household)

finished = calculate(api_client, household, household_id, policy_id)
# Checked first, so a failed job is reported without waiting for the
# Public API's own calculation, which can take minutes.
assert finished.item["status"] == "succeeded", finished.item["error"]
v1_answer = calculate_with_v1(api_client, household)

assert finished.item["status"] == "succeeded", finished.item["error"]
result = finished.item["result"]
differences = household_differences(v1_answer["result"], result["household"])
assert differences == [], "\n".join(differences[:50])
Expand Down
133 changes: 133 additions & 0 deletions tests/unit/v2/test_live_household_parity_requests.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
"""The live parity checks' requests to the Public API's own calculation routes.

Each parity household goes to ``/calculate`` (``/calculate-full`` when it adds
missing variables) as the check sends it. The real routes check its inputs and
parse its situation with the installed country models, and only the
calculation itself is left out, so a household the routes refuse before
calculating fails here rather than in a release. A refusal during the
calculation, such as an SPM error, is not covered. The check waits longer than
the candidate's Cloud
Run request timeout (``CLOUD_RUN_TIMEOUT`` in
``.github/scripts/cloud_run_env.sh``) for the answer: ``/calculate-full`` can
take most of it, and the check should get the candidate's own answer.
"""

from __future__ import annotations

from typing import TYPE_CHECKING

import httpx
import pytest
from flask import Flask

from policyengine_api.country import COUNTRIES
from policyengine_api.extensions import cache
from policyengine_api.routes import household_routes
from policyengine_api.services.household_calculation_service import (
HouseholdCalculationResult,
HouseholdCalculationService,
PreparedHouseholdCalculation,
)
from tests.fixtures.integration.live_household_calculations import (
LiveHousehold,
calculate_with_v1,
)
from tests.integration.test_live_v2_household_parity import parity_households
from tests.unit.cloud_run_deploy_support import run_bash, script_env

if TYPE_CHECKING:
from collections.abc import Iterator

PARITY_HOUSEHOLDS = parity_households(0)


class UncalculatedHouseholds(HouseholdCalculationService):
"""The routes' household service, answering each accepted household uncalculated.

Input checks and situation parsing run as in the Public API; the
calculation, which takes minutes for ``/calculate-full``, does not. The
answer is the household as the routes prepared it: deprecated inputs
dropped and, for ``/calculate-full``, the missing variables added.
"""

def calculate_prepared_household(
self, prepared: PreparedHouseholdCalculation
) -> HouseholdCalculationResult:
return HouseholdCalculationResult(
household=prepared.household_json, warnings=prepared.warnings
)


@pytest.fixture
def public_api(monkeypatch: pytest.MonkeyPatch) -> Iterator[httpx.Client]:
"""Call the Public API's calculation routes with the installed country models."""
monkeypatch.setattr(
household_routes,
"household_calculation_service",
UncalculatedHouseholds(country_provider=lambda: COUNTRIES),
)
app = Flask(__name__)
app.config.update(TESTING=True, CACHE_TYPE="NullCache")
cache.init_app(app)
app.register_blueprint(household_routes.household_bp)
with httpx.Client(
base_url="http://public-api.test", transport=httpx.WSGITransport(app=app)
) as client:
yield client


def candidate_request_timeout_seconds() -> float:
"""Return how long Cloud Run lets a request to the candidate run."""
result = run_bash(
"-c",
"source .github/scripts/cloud_run_env.sh && cloud_run_set_defaults"
' && printf "%s" "$CLOUD_RUN_TIMEOUT"',
env=script_env(),
)
assert result.returncode == 0, result.stderr
return float(result.stdout)


@pytest.mark.parametrize(
"household",
PARITY_HOUSEHOLDS,
ids=[parity_household.name for parity_household in PARITY_HOUSEHOLDS],
)
def test_the_public_api_accepts_each_parity_household(
public_api: httpx.Client, household: LiveHousehold
) -> None:
answer = calculate_with_v1(public_api, household)

assert answer["status"] == "ok"


@pytest.mark.parametrize(
"household",
[
LiveHousehold(name="calculate", country_id="us", people={"you": {}}),
LiveHousehold(
name="calculate-full",
country_id="us",
people={"you": {}},
add_missing_variables=True,
),
],
ids=["calculate", "calculate-full"],
)
def test_the_check_waits_longer_than_the_candidate_lets_a_request_run(
household: LiveHousehold,
) -> None:
read_timeouts: list[float | None] = []

def answer(request: httpx.Request) -> httpx.Response:
read_timeouts.append(request.extensions["timeout"]["read"])
return httpx.Response(200, json={"status": "ok", "result": {}})

with httpx.Client(
base_url="https://public-api.example", transport=httpx.MockTransport(answer)
) as client:
calculate_with_v1(client, household)

(read_timeout,) = read_timeouts
assert read_timeout is not None
assert read_timeout > candidate_request_timeout_seconds()
Loading