Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/scripts/test_alembic_v2_lifecycle.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,5 @@ uv run --frozen --extra dev pytest -q \
tests/integration/test_alembic_v2_catalog_tables.py \
tests/integration/test_alembic_v2_simulation_runs.py \
tests/integration/test_alembic_v2_saved_item_keys.py \
tests/integration/test_alembic_v2_user_types.py
tests/integration/test_alembic_v2_user_types.py \
tests/integration/test_alembic_v2_withheld_area_codes.py
1 change: 1 addition & 0 deletions changelog.d/v2-withheld-area-codes.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
The v2 database gains a `withheld_area_codes` table: one row per UK constituency or local authority that disclosure control withheld from a report run's local-area results, keyed by the report run and its two simulation runs. The simulation service's Modal workers may insert its rows and the Public API may read them; no route reads it yet.
29 changes: 29 additions & 0 deletions docs/engineering/skills/alembic-migrations.md
Original file line number Diff line number Diff line change
Expand Up @@ -828,3 +828,32 @@ The revision writes no data, and the previous Public API revision, which
doesn't map the column, keeps working at it. The downgrade drops the column
with its values, which the next publication run of each PolicyEngine.py
version records again.

### API v2 withheld area codes

Revision `1e1e3f079f2c` (parent `0c136a278f29`) was autogenerated, with
`--rev-id 1e1e3f079f2c`, after the `WithheldAreaCode` model was added. It
creates `withheld_area_codes`, one row per UK area that disclosure control
withheld from a report run's local-area results: the report run (`CASCADE`),
its baseline and reform simulation runs (`RESTRICT`, as the other comparison
result tables have), the `area_type` (the new native enum `v2_uk_area_type`,
`constituency` or `local_authority`) and the area's `code`, unique per report
run and area type. The run's writer leaves a withheld area out of
`constituency_impacts` and `local_authority_impacts`; no constraint across the
tables checks that. Nothing records whether a run's list was written, or which
writer completed the run, so no rows means no area was withheld only for a
succeeded run whose writer records the list, and the database can't tell such
a run from one whose writer doesn't. Until the Modal workers write this table,
every run has no rows. The Modal worker role may read and insert its rows,
never update or delete them, and the runtime role may read them; the reconcile
step after each schema upgrade grants them.

It has one post-generation correction, for reversibility: Alembic's
generated downgrade drops the table but leaves the enum type, so the
downgrade also drops `v2_uk_area_type` after the table, as `439303be14fe`
does for its enums. Without it, upgrading again after a downgrade fails on
the existing type. The revision writes no data, and the previous Public API
revision, which doesn't map the table, keeps working at it. The downgrade
drops the table with its rows. `tests/integration/test_alembic_v2_withheld_area_codes.py`
checks that nothing else changes, that the downgrade removes the type and an
upgrade works again, and the table's keys and delete rules.
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
"""record withheld uk area codes

Revision ID: 1e1e3f079f2c
Revises: 0c136a278f29
Create Date: 2026-11-14 01:14:50.900369
Generation: uv run alembic -c alembic-v2.ini revision --autogenerate

Generated on 0c136a278f29, with --rev-id 1e1e3f079f2c. It creates the
withheld_area_codes table and its v2_uk_area_type enum type and changes no
other table, so it writes no data, and the previous Public API revision, which
doesn't map the table, keeps working. One correction after generation, for
reversibility: Alembic's generated downgrade drops the table but leaves the
enum type, so the downgrade also drops the type, after the table. The
downgrade drops the table with its rows.
"""

from typing import Sequence, Union

from alembic import op
import sqlalchemy as sa
import sqlmodel


revision: str = '1e1e3f079f2c'
down_revision: Union[str, None] = '0c136a278f29'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None


def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('withheld_area_codes',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
sa.Column('report_run_id', sa.Uuid(), nullable=False),
sa.Column('baseline_simulation_run_id', sa.Uuid(), nullable=False),
sa.Column('reform_simulation_run_id', sa.Uuid(), nullable=False),
sa.Column('area_type', sa.Enum('constituency', 'local_authority', name='v2_uk_area_type'), nullable=False),
sa.Column('code', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False),
sa.ForeignKeyConstraint(['baseline_simulation_run_id'], ['simulation_runs.id'], name=op.f('fk_withheld_area_codes_baseline_simulation_run_id_simulation_runs'), ondelete='RESTRICT'),
sa.ForeignKeyConstraint(['reform_simulation_run_id'], ['simulation_runs.id'], name=op.f('fk_withheld_area_codes_reform_simulation_run_id_simulation_runs'), ondelete='RESTRICT'),
sa.ForeignKeyConstraint(['report_run_id'], ['report_runs.id'], name=op.f('fk_withheld_area_codes_report_run_id_report_runs'), ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id', name=op.f('pk_withheld_area_codes')),
sa.UniqueConstraint('report_run_id', 'area_type', 'code', name='uq_withheld_area_codes_run_type_code')
)
# ### end Alembic commands ###


def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('withheld_area_codes')
# Post-generation reversibility correction: Alembic removes the table that
# uses this native PostgreSQL enum but leaves the schema-level type.
sa.Enum(name="v2_uk_area_type").drop(op.get_bind())
# ### end Alembic commands ###
2 changes: 1 addition & 1 deletion policyengine_api/data/v2/migration_target.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
from collections.abc import Mapping

V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST"
V2_ALEMBIC_HEAD_REVISION = "0c136a278f29"
V2_ALEMBIC_HEAD_REVISION = "1e1e3f079f2c"
DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test"
DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"})
MIGRATION_ROLE = "policyengine_v2_migrator"
Expand Down
4 changes: 4 additions & 0 deletions policyengine_api/data/v2/models/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,8 @@
OutputStatus,
Poverty,
ProgramStatistics,
UKAreaType,
WithheldAreaCode,
)
from policyengine_api.data.v2.models.simulations import ( # noqa: E402
Simulation,
Expand Down Expand Up @@ -165,6 +167,7 @@
"Stage12SimulationRole",
"TaxBenefitModel",
"TaxBenefitModelVersion",
"UKAreaType",
"User",
"UserHouseholdAssociation",
"UserIdentity",
Expand All @@ -175,4 +178,5 @@
"UserTypeChange",
"UserTypeChangeSource",
"Variable",
"WithheldAreaCode",
]
5 changes: 5 additions & 0 deletions policyengine_api/data/v2/models/reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
LaborSupplyResponse,
LocalAuthorityImpact,
ProgramStatistics,
WithheldAreaCode,
)

# A budget window compares at most this many years, none after LAST_YEAR.
Expand Down Expand Up @@ -376,3 +377,7 @@ class ReportRun(TimestampedModel, table=True):
back_populates="report_run",
cascade_delete=True,
)
withheld_area_codes: list["WithheldAreaCode"] = Relationship(
back_populates="report_run",
cascade_delete=True,
)
48 changes: 48 additions & 0 deletions policyengine_api/data/v2/models/run_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,13 @@ class DecileType(StrEnum):
WEALTH = "wealth"


class UKAreaType(StrEnum):
"""The level of UK area a local-area breakdown reports."""

CONSTITUENCY = "constituency"
LOCAL_AUTHORITY = "local_authority"


class LaborSupplyResponse(IdentifiedModel, table=True):
"""The labour supply response to the reform in one report run.

Expand Down Expand Up @@ -552,3 +559,44 @@ class LocalAuthorityImpact(UKAreaImpactBase, table=True):
local_authority_name: str = Field(max_length=255)

report_run: ReportRun = Relationship(back_populates="local_authority_impacts")


class WithheldAreaCode(IdentifiedModel, table=True):
"""A UK area whose results disclosure control withheld from a report run.

One row per report run, area type and area code. The run's writer leaves a
withheld area out of ``constituency_impacts`` and
``local_authority_impacts``; no constraint across the tables checks that.
Nothing records whether a run's list was written, or which writer completed
the run, so no rows means no area was withheld only for a succeeded run
whose writer records the list, and the database can't tell such a run from
one whose writer doesn't. Until the Modal workers write this table, every
run has no rows.
"""

__tablename__ = "withheld_area_codes"
__table_args__ = (
sa.UniqueConstraint(
"report_run_id",
"area_type",
"code",
name="uq_withheld_area_codes_run_type_code",
),
)

report_run_id: UUID = Field(
foreign_key="report_runs.id",
ondelete="CASCADE",
)
baseline_simulation_run_id: UUID = Field(
foreign_key="simulation_runs.id",
ondelete="RESTRICT",
)
reform_simulation_run_id: UUID = Field(
foreign_key="simulation_runs.id",
ondelete="RESTRICT",
)
area_type: UKAreaType = Field(sa_type=enum_type(UKAreaType, "v2_uk_area_type"))
code: str = Field(max_length=64)

report_run: ReportRun = Relationship(back_populates="withheld_area_codes")
2 changes: 2 additions & 0 deletions policyengine_api/data/v2/privileges.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,8 @@ def _run_table(
"local_authority_impacts": _RESULT,
"poverty": _RESULT,
"program_statistics": _RESULT,
# The UK areas disclosure control withheld from a local-area run's results.
"withheld_area_codes": _RESULT,
# Metadata catalog: the catalog writer publishes it, the Public API reads it.
"dataset_versions": _access(catalog_writer=READ),
"datasets": _access(
Expand Down
6 changes: 4 additions & 2 deletions tests/integration/test_alembic_v2_simulation_runs.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,11 @@
}
)
# The tables the simulation service and its Modal workers write; the Public API
# writes every other table.
# writes every other table. withheld_area_codes came in a later revision.
SIMULATION_SIDE_TABLES = (
CHANGED_TABLES | NEW_TABLES | {"aggregates", "change_aggregates"}
CHANGED_TABLES
| NEW_TABLES
| {"aggregates", "change_aggregates", "withheld_area_codes"}
)
# The foreign keys from the simulation service's tables to the Public API's
# rows, with their delete rules.
Expand Down
Loading
Loading