Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/805-spm-forecast-2036.changed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
US simulations for 2036 now run: the worker uses policyengine.py 6.20.0, with policyengine-us 2.2.2 and spm-calculator 1.1.0, whose SPM forecast covers 2022–2036. 2037 is now the first year the worker refuses with `SPM_YEAR_UNAVAILABLE`. Stage 12 results for US national runs keep reporting the release's data version, as the main path does, now that the bundle addresses the national dataset by a Hugging Face commit.
1 change: 1 addition & 0 deletions changelog.d/update-policyengine-6.20.0.changed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Updated policyengine.py from 6.17.1 to 6.20.0 in the simulation runtime.
4 changes: 2 additions & 2 deletions projects/policyengine-simulation-executor/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ dependencies = [
"policyengine-stage12-persistence",
# PolicyEngine.py's models extra is the only selector for core, country
# models, and the SPM calculator. Do not duplicate those versions here.
"policyengine[models]==6.17.1",
"policyengine[models]==6.20.0",
"tables>=3.10.2",
"modal>=0.73.0",
"policyengine-observability[fastapi,google,otlp-grpc]>=3.0.7,<4",
Expand Down Expand Up @@ -49,7 +49,7 @@ packages = ["src/policyengine_simulation_executor"]
# certified datasets and writes their receipt.
[dependency-groups]
modal-simulation-image = [
"policyengine[models]==6.17.1",
"policyengine[models]==6.20.0",
"fastapi>=0.115.0",
"tables>=3.10.2",
"policyengine-observability[fastapi,google,otlp-grpc]>=3.0.7,<4",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from hashlib import sha256
from typing import Any, Literal
from typing import TYPE_CHECKING, Any, Literal

import pandas as pd
from policyengine_observability import (
Expand Down Expand Up @@ -55,13 +55,17 @@
stage12_region_groups_for_model,
)
from policyengine_simulation_executor.stage12_runtime.aggregation import (
reported_data_version,
validate_uk_local_authority_metadata,
)
from policyengine_simulation_executor.stage12_runtime.output_planning import (
plan_simulation_input,
resolve_report_output_plan,
)

if TYPE_CHECKING:
from policyengine_simulation_contract.stage12_bundle import CountryId


class Stage12ParityReceipt(BaseModel):
"""Value-free evidence for one completed parity qualification."""
Expand Down Expand Up @@ -335,15 +339,22 @@ def run_existing(request: dict[str, Any]) -> Mapping[str, Any]:
def _validate_existing_provenance(
existing_result: Mapping[str, Any],
bundle: BundleProvenance,
country: CountryId,
) -> None:
"""Require the existing result's model and data versions to be the bundle's."""
"""Require the existing result's model and data versions to be the bundle's.

The data version must be ``reported_data_version`` of the bundle's
dataset.
"""
if existing_result.get("model_version") != bundle.country_package_version:
error_code = "country_package_provenance_mismatch"
raise ParityMismatchError(
error_code,
"report.model_version",
)
if existing_result.get("data_version") != bundle.dataset.artifact_revision:
if existing_result.get("data_version") != reported_data_version(
country, bundle.dataset
):
error_code = "dataset_provenance_mismatch"
raise ParityMismatchError(
error_code,
Expand Down Expand Up @@ -430,7 +441,9 @@ def qualify_report_parity(
existing_baseline = _existing_frames(microdata, side="baseline")
existing_reform = _existing_frames(microdata, side="reform")
bundle = report.baseline.bundle
_validate_existing_provenance(existing_result, bundle)
_validate_existing_provenance(
existing_result, bundle, report.baseline.geography.country
)

baseline_calculation, reform_calculation = _run_v2_simulations(
runners.single_simulation,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,34 @@
from policyengine.tax_benefit_models.uk.datasets import PolicyEngineUKDataset
from policyengine.tax_benefit_models.us.datasets import PolicyEngineUSDataset
from policyengine_simulation_contract.stage12_bundle import CountryId
from policyengine_simulation_contract.stage12_execution import DatasetProvenance
from policyengine_simulation_contract.uk_geography import UKLocalAuthorityMetadata
from pydantic import JsonValue


def reported_data_version(country: CountryId, dataset: DatasetProvenance) -> str:
"""Return the ``data_version`` the main path reports for a run on ``dataset``.

For a run on the country's default dataset the main path reports the
installed release's data version, and so does this. For any other dataset
this returns the dataset's revision, which is what the main path reports
for a US state or district run on the regional dataset and for a UK
local-area breakdown. A UK constituency or local-authority report on the
local-area dataset is the exception: the main path reports the release's
data version for it. Until policyengine.py 6.20.0 the default dataset's
revision was the release's data version; since then the US one is a
Hugging Face commit, while the release keeps its version name.
"""
from policyengine_simulation_executor.release_bundle import ( # noqa: PLC0415 # resolved per call so tests can patch release_bundle.get_country_release_bundle
get_country_release_bundle,
)

release = get_country_release_bundle(country)
if dataset.identity == release.default_dataset:
return release.data_version
return dataset.artifact_revision


def validate_uk_local_authority_metadata(
country: CountryId,
baseline: UKLocalAuthorityMetadata | None,
Expand Down Expand Up @@ -242,7 +266,9 @@ def stand_in(
dataset=datasets["baseline"],
baseline=stand_in(datasets["baseline"], report.baseline.policy),
reform=stand_in(datasets["reform"], report.reform.policy),
resolved_data_version=report.baseline.bundle.dataset.artifact_revision,
resolved_data_version=reported_data_version(
country, report.baseline.bundle.dataset
),
resolved_region_code=report.baseline.geography.region,
uk_local_authority_metadata=uk_local_authority_metadata,
omit_uk_area_breakdowns=not local_area_report
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from policyengine_simulation_contract.spm import SPMInputError
from policyengine_simulation_executor import household_spm

FORECAST_SHA = "3d86d5c4c0423480e6b69b75d222ffa4a7a2639e4094df5ba2504af01be17173"
FORECAST_SHA = "335e6755dc8e0bd883e09b76c2bd53a5315683773e44d50085c7d6b8832b6291"
DEFAULTS = {
"as_of": None,
"county_vintage": "2020",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -188,14 +188,14 @@ def test_resolve_bundle_dataset_uri_preserves_explicit_dataset_uri_and_revision(


def test_resolve_bundle_dataset_uri_maps_explicit_logical_revision_to_hf_uri():
dataset = "populace_us_2024@custom-v1"
# The repository and the path in it come from the bundle (a build folder
# since policyengine.py 6.20.0); only the revision is the requested one.
bundle = get_country_release_bundle("us")
dataset = f"{bundle.default_dataset}@custom-v1"
expected = f"{bundle.default_dataset_uri.rsplit('@', maxsplit=1)[0]}@custom-v1"

assert resolve_bundle_dataset_name("us", dataset).startswith(
"hf://policyengine/populace-us/populace_us_2024.h5@custom-v1"
)
assert resolve_bundle_dataset_uri("us", dataset).startswith(
"hf://policyengine/populace-us/populace_us_2024.h5@custom-v1"
)
assert resolve_bundle_dataset_name("us", dataset).startswith(expected)
assert resolve_bundle_dataset_uri("us", dataset).startswith(expected)


def test_resolve_bundle_dataset_uri_preserves_explicit_gcs_uri():
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import os
import re
from functools import partial
from unittest.mock import Mock

import pytest
Expand All @@ -23,7 +24,12 @@ def forecast():
return _forecast(runtime_spm_capability().defaults.forecast_content_sha256)


def _worker_entrypoint(params, monkeypatch):
def _worker_entrypoint(params, monkeypatch, observability_runtime):
"""Return the worker entrypoint for ``params`` and a mock of its next step.

The entrypoint runs under ``observability_runtime``. The mock replaces the
first step after SPM prevalidation, so a rejected request never reaches it.
"""
from policyengine_simulation_executor import region_resolution, simulation_runtime
from src.modal import budget_window_batch

Expand All @@ -35,22 +41,22 @@ def _worker_entrypoint(params, monkeypatch):
entrypoint = simulation_runtime._run_simulation_impl_core
boundary = Mock(side_effect=AssertionError("Must reject before dataset lookup"))
monkeypatch.setattr(region_resolution, "resolve_dataset_reference", boundary)
return entrypoint, boundary
return partial(entrypoint, runtime=observability_runtime), boundary


@pytest.mark.parametrize(
("period", "unavailable_year"),
[
({"time_period": "2021"}, 2021),
({"time_period": "2036"}, 2036),
({"time_period": "2037"}, 2037),
({"year": "2040"}, 2040),
({"start_year": "2021", "window_size": 1}, 2021),
({"start_year": "2036", "window_size": 1}, 2036),
({"start_year": "2035", "window_size": 2}, 2036),
({"start_year": "2037", "window_size": 1}, 2037),
({"start_year": "2036", "window_size": 2}, 2037),
],
)
def test_worker_year_boundary_matches_real_provider(
forecast, period, unavailable_year, monkeypatch
forecast, period, unavailable_year, monkeypatch, observability_runtime
):
from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail
from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider
Expand All @@ -65,7 +71,9 @@ def test_worker_year_boundary_matches_real_provider(
assert expected.code == "SPM_YEAR_UNAVAILABLE"
assert provider.provenance()["years"] == {}
params = {"country": "us", "spm": {"geography_kind": "national"}, **period}
entrypoint, boundary = _worker_entrypoint(params, monkeypatch)
entrypoint, boundary = _worker_entrypoint(
params, monkeypatch, observability_runtime
)
with pytest.raises(SPMInputError) as error:
entrypoint(params)
assert error.value.to_dict() == expected.model_dump()
Expand All @@ -80,7 +88,7 @@ def test_worker_year_boundary_matches_real_provider(
],
)
def test_worker_scenario_boundary_matches_real_calculator_contract(
forecast, period, monkeypatch
forecast, period, monkeypatch, observability_runtime
):
from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail
from spm_calculator.axiom_adapter import export_build_spec
Expand All @@ -98,26 +106,34 @@ def test_worker_scenario_boundary_matches_real_calculator_contract(
"spm": {"geography_kind": "national", "scenario": scenario},
**period,
}
entrypoint, boundary = _worker_entrypoint(params, monkeypatch)
entrypoint, boundary = _worker_entrypoint(
params, monkeypatch, observability_runtime
)
with pytest.raises(SPMInputError) as error:
entrypoint(params)
assert error.value.to_dict() == expected.model_dump()
boundary.assert_not_called()


def test_valid_worker_prevalidation_does_not_evaluate_amounts(forecast, monkeypatch):
@pytest.mark.parametrize(
"period",
[
{"start_year": "2034", "window_size": 2},
# The forecast's last year, alone and as the last year of a window.
{"time_period": "2036"},
{"start_year": "2027", "window_size": 10},
],
)
def test_valid_worker_prevalidation_does_not_evaluate_amounts(
forecast, period, monkeypatch
):
from policyengine_simulation_executor.spm import normalize_runtime_spm
from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider

calculate = Mock(side_effect=AssertionError("Prevalidation is not measurement"))
monkeypatch.setattr(PolicyEngineSPMProvider, "calculate_unit", calculate)
resolved = normalize_runtime_spm(
{
"country": "us",
"start_year": "2034",
"window_size": 2,
"spm": {"geography_kind": "national"},
}
{"country": "us", "spm": {"geography_kind": "national"}, **period}
)
assert resolved["geography_kind"] == "national"
provider = PolicyEngineSPMProvider(
Expand Down
Loading
Loading