Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/economy-retry-after-seconds.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
A failed economy calculation's 502 body now has `retry_after_seconds`: the longest time the Public API keeps returning the stored failure before a request starts a new run, read from the cache's own expiry of the stored failure and its lookup index, or null when no failure is stored or its lifetime cannot be read.
8 changes: 7 additions & 1 deletion docs/canonical-spm.md
Original file line number Diff line number Diff line change
Expand Up @@ -372,7 +372,13 @@ state can be requested again, so a client can retry one state of a
nationwide report's congressional districts. Any other failed job, such as
a national, congressional district or place job, is still stored and
returned again for the cache lifetime of 30 days, and so is a typed SPM
failure.
failure. A 502 for a stored failure also carries `retry_after_seconds`: the
whole seconds, rounded up, until lookups stop finding that failure (the
earlier of its record's expiry and its scope index's), after which the next
request submits a new job. It is null when no failure is stored, when the
next request would answer from a newer job (a late poll of a job that a newer
one replaced), or when the cache cannot be read. It is the longest wait: a deploy that changes the
versions or worker app a request resolves to starts a new run sooner.

Before cache access or submission,
the API validates the selected worker application's `canonical-spm-v1` capability
Expand Down
10 changes: 8 additions & 2 deletions policyengine_api/routes/economy_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,12 @@ def get_economic_impact(
simulation finishes, and while computing the body also carries the stage
fields the Simulation Entrypoint reports. Invalid query parameters give 400,
a policy ID the country does not have gives 404 and a failed calculation
gives 502.
gives 502. A 502 body's ``retry_after_seconds`` is how long the Public API
keeps returning the stored failure before a request starts a new run, or
null when no failure is stored, the next request would answer from a
newer job, or the failure's lifetime cannot be read. It is the longest
wait: a deploy that changes the versions or worker app a request resolves
to lets a request start a new run sooner.
"""
policy_id = int(policy_id or get_current_law_policy_id(country_id))
baseline_policy_id = int(
Expand Down Expand Up @@ -124,13 +129,14 @@ def get_economic_impact(
except PolicyNotFoundError as error:
return _policy_not_found_response(error)

result_dict: dict[str, str | dict | None] = economic_impact_result.to_dict()
result_dict: dict[str, str | dict | int | None] = economic_impact_result.to_dict()

if result_dict["status"] == "error":
return _make_error_response(
result_dict["message"],
HTTPStatus.BAD_GATEWAY,
result=None,
retry_after_seconds=result_dict.get("retry_after_seconds"),
)

body = {
Expand Down
9 changes: 9 additions & 0 deletions policyengine_api/runtime_cache/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ def execute(self) -> list[object]:
...


# Redis's PTTL replies for a key that does not exist and for one that never expires.
PTTL_MISSING_KEY = -2
PTTL_NO_EXPIRY = -1


class CacheBackend(Protocol):
"""The Redis client subset the runtime cache uses; fakes implement it too."""

Expand All @@ -73,6 +78,10 @@ def delete(self, *keys: str) -> object:
"""Delete every given key and return how many existed."""
...

def pttl(self, key: str) -> object:
"""Return the key's milliseconds to live; -2 if missing, -1 if no expiry."""
...

def pipeline(self, *, transaction: bool = True) -> CachePipeline:
"""Return a pipeline whose commands run together on ``execute``."""
...
Expand Down
19 changes: 19 additions & 0 deletions policyengine_api/runtime_cache/fake.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@

from __future__ import annotations

import math
from threading import RLock
from typing import TYPE_CHECKING, Self

from typing_extensions import override

from policyengine_api.runtime_cache.core import PTTL_MISSING_KEY, PTTL_NO_EXPIRY

if TYPE_CHECKING:
from collections.abc import Callable

Expand Down Expand Up @@ -104,6 +107,17 @@ def expire(self, key: str, seconds: int) -> bool:
self._expires[key] = self._now() + seconds
return True

def pttl(self, key: str) -> int:
"""Emulate PTTL: milliseconds left, -2 for a missing key, -1 for no expiry."""
with self._lock:
self._purge(key)
if key not in self._values and key not in self._sorted_sets:
return PTTL_MISSING_KEY
expires_at = self._expires.get(key)
if expires_at is None:
return PTTL_NO_EXPIRY
return math.ceil((expires_at - self._now()) * 1000)

def zadd(self, key: str, mapping: dict[str, float]) -> int:
"""Emulate ZADD without flags: upsert scores and count new members only."""
with self._lock:
Expand Down Expand Up @@ -262,6 +276,11 @@ def expire(self, key: str, seconds: int) -> bool:
"""Return False, as Redis does for a key that does not exist."""
return False

@override
def pttl(self, key: str) -> int:
"""Return -2, as Redis does for a key that does not exist."""
return PTTL_MISSING_KEY

@override
def eval(
self,
Expand Down
33 changes: 33 additions & 0 deletions policyengine_api/runtime_cache/reform_impacts.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,19 @@
"""Recoverable reform-impact caching and submission coordination."""

import hashlib
import math
import re
import time
from dataclasses import asdict, dataclass, replace
from datetime import UTC, datetime
from typing import Any

from redis.exceptions import RedisError

from policyengine_api.observability.identifiers import normalize_observability_id
from policyengine_api.runtime_cache.claims import ExpiringClaimStore
from policyengine_api.runtime_cache.core import (
PTTL_NO_EXPIRY,
CacheBackend,
CacheCoordinationError,
CacheNamespace,
Expand Down Expand Up @@ -427,6 +431,35 @@ def get_by_execution_id(self, execution_id: str) -> CachedReformImpact | None:
)
)

def seconds_left(self, impact: CachedReformImpact) -> int | None:
"""Return the whole seconds, rounded up, until lookups stop finding the impact.

A lookup finds a record only through its scope index, so this is the
earlier of the record's expiry and the index's, read from the cache's
own expiry whatever lifetime each was written with; 0 once the index
has expired. Return None when the impact has no worker handle, its
record is missing or never expires, or the cache cannot be read.
"""
if not impact.execution_id:
return None
try:
record_milliseconds = self.client.pttl(
self._record_key(impact.execution_id)
)
index_milliseconds = self.client.pttl(self._scope_index(impact))
except (RedisError, OSError):
return None
if not isinstance(record_milliseconds, int) or record_milliseconds < 0:
return None
if (
not isinstance(index_milliseconds, int)
or index_milliseconds == PTTL_NO_EXPIRY
):
return math.ceil(record_milliseconds / 1000)
if index_milliseconds < 0:
return 0
return math.ceil(min(record_milliseconds, index_milliseconds) / 1000)

def _from_index(self, index: str, limit: int) -> list[CachedReformImpact]:
try:
keys = self.client.zrevrange(index, 0, max(limit - 1, 0)) # type: ignore[attr-defined]
Expand Down
34 changes: 22 additions & 12 deletions policyengine_api/services/economy_impact_execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def _handle_execution_state(
retry_note=retry_note,
)

self._set_reform_impact_error(
stored_failure = self._set_reform_impact_error(
setup_options=setup_options,
message=error_message,
execution_id=reform_impact.execution_id,
Expand All @@ -108,7 +108,12 @@ def _handle_execution_state(
{"message": error_message},
severity="ERROR",
)
return EconomicImpactResult.error(message=error_message)
return self._stored_failure_result(
error_message,
self._failure_the_next_request_answers_from(
stored_failure, setup_options
),
)

if execution_state in EXECUTION_STATUSES_PENDING:
logger.log_struct(
Expand Down Expand Up @@ -179,19 +184,24 @@ def _record_retryable_failure(
``RETRYABLE_FAILURE_SECONDS``. Until then every request gets the same
error, which ends with ``retry_note`` to say when to try again; after
it, the next request finds no record and submits a new job. No other
job's record is read or changed, and no submission claim is held at
this point: it was released once the job was handed off. Other failed
jobs, and typed SPM failures, are stored as before.
job's record is changed (the scope's records are read only to check
that the next request answers from this one), and no submission claim
is held at this point: it was released once the job was handed off.
Other failed jobs, and typed SPM failures, are stored as before.
"""
message = f"{reason}. {retry_note}"
stored_failure = None
if reform_impact.execution_id:
self._set_reform_impact_error(
stored_failure = self._set_reform_impact_error(
setup_options=setup_options,
message=message,
execution_id=reform_impact.execution_id,
keep_seconds=RETRYABLE_FAILURE_SECONDS,
)
return EconomicImpactResult.error(message=message)
return self._stored_failure_result(
message,
self._failure_the_next_request_answers_from(stored_failure, setup_options),
)

@observability_runtime.span(
ECONOMY_ANNUAL_STAGES.name(Stage.ECONOMY_READ_COMPLETED)
Expand Down Expand Up @@ -269,11 +279,11 @@ def _handle_failed_impact(
most_recent_impact.error_code, most_recent_impact.message
)
# Failed executions have no successful output or SPM receipts to
# validate. Replay their original failure on every later poll.
return EconomicImpactResult.error(
message=(
most_recent_impact.message or "Simulation entrypoint execution failed"
)
# validate. Replay their original failure on every later poll, with
# the seconds until its record expires and a new job can start.
return self._stored_failure_result(
most_recent_impact.message or "Simulation entrypoint execution failed",
most_recent_impact,
)

@observability_runtime.span(ECONOMY_ANNUAL_STAGES.name(Stage.ECONOMY_POLL_ACTIVE))
Expand Down
22 changes: 18 additions & 4 deletions policyengine_api/services/economy_impact_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,24 +158,31 @@ class EconomicImpactResult(BaseModel):
"""The status of an economic impact calculation and, once complete, its result.

``data`` holds the reform impact JSON when the status is ok, and ``message``
explains an error.
explains an error. For an error, ``retry_after_seconds`` is how long the
stored failure is still returned: once it has passed, a request finds no
record and submits a new job. It is the longest wait, since a deploy that
changes the versions or worker app a request resolves to stops the record
being found sooner. It is None when no failure is stored, the next request
would answer from a newer job, or the failure's lifetime cannot be read.
"""

status: ImpactStatus
data: dict | None = None
message: str | None = None
retry_after_seconds: int | None = None
# Stage fields relayed from the Simulation Entrypoint while computing.
# Kept out of to_dict(); the route adds them only when present.
run_progress: SimulationRunProgress | None = None

model_config = {"frozen": True} # Make model immutable

def to_dict(self) -> dict[str, str | dict | None]:
def to_dict(self) -> dict[str, str | dict | int | None]:
"""Convert the EconomicImpactResult to a dictionary."""
return {
"status": self.status.value,
"data": self.data,
"message": self.message,
"retry_after_seconds": self.retry_after_seconds,
}

@classmethod
Expand All @@ -191,10 +198,17 @@ def computing(
return cls(status=ImpactStatus.COMPUTING, data=None, run_progress=run_progress)

@classmethod
def error(cls, message: str) -> "EconomicImpactResult":
def error(
cls, message: str, *, retry_after_seconds: int | None = None
) -> "EconomicImpactResult":
"""Create an EconomicImpactResult for an error in the impact calculation."""
logger.log_struct({"message": message}, severity="ERROR")
return cls(status=ImpactStatus.ERROR, data=None, message=message)
return cls(
status=ImpactStatus.ERROR,
data=None,
message=message,
retry_after_seconds=retry_after_seconds,
)


class BudgetWindowEconomicImpactResult(BaseModel):
Expand Down
50 changes: 47 additions & 3 deletions policyengine_api/services/economy_stored_impacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
)
from policyengine_api.services.economy_impact_types import (
LOCAL_AREA_BREAKDOWN_RESULT_SCHEMA_VERSION,
EconomicImpactResult,
EconomicImpactSetupOptions,
ImpactAction,
ImpactStatus,
Expand Down Expand Up @@ -350,6 +351,48 @@ def _set_reform_impact_complete(
)
raise

def _failure_the_next_request_answers_from(
self,
stored_failure: CachedReformImpact | None,
setup_options: EconomicImpactSetupOptions,
) -> CachedReformImpact | None:
"""Return the stored failure when the next request would answer from it.

A late poll of a job that a newer one has replaced fails onto a record
the next request does not select: that request answers from the newer
job. Then None, since this failure's lifetime says nothing about when
a new run can start. The check is the cache lookup alone, with no
request to the simulation API; a deploy that changes the worker app a
request resolves to can still start a new run sooner.
"""
if stored_failure is None:
return None
next_answer = self._get_most_recent_impact(setup_options)
if (
next_answer is None
or next_answer.execution_id != stored_failure.execution_id
):
return None
return stored_failure

def _stored_failure_result(
self, message: str, stored_failure: CachedReformImpact | None
) -> EconomicImpactResult:
"""Return a failure with the seconds the Public API keeps returning it.

Read from the stored failure after it is written, or as it is
replayed, so a fresh failure and a replayed one both say how long a
lookup still finds it. None when no failure was stored.
"""
retry_after_seconds = (
self._reform_impacts.get_reform_impact_seconds_left(stored_failure)
if stored_failure is not None
else None
)
return EconomicImpactResult.error(
message=message, retry_after_seconds=retry_after_seconds
)

@observability_runtime.span(
ECONOMY_ANNUAL_STAGES.name(Stage.ECONOMY_PERSIST_FAILED)
)
Expand All @@ -360,11 +403,12 @@ def _set_reform_impact_error(
execution_id: str,
error_code: str | None = None,
keep_seconds: int | None = None,
) -> None:
) -> CachedReformImpact | None:
"""Record the impact as "error" with its message; log and re-raise errors.

``keep_seconds`` keeps the error for that long instead of the usual
30 days.
30 days. Return the stored failure, or None when the cache did not
store it.
"""
# Optional fields are passed only when set, as before keep_seconds existed.
optional_fields: dict[str, Any] = {}
Expand All @@ -373,7 +417,7 @@ def _set_reform_impact_error(
if keep_seconds is not None:
optional_fields["keep_seconds"] = keep_seconds
try:
self._reform_impacts.set_error_reform_impact(
return self._reform_impacts.set_error_reform_impact(
execution_id=execution_id,
message=message,
**optional_fields,
Expand Down
9 changes: 9 additions & 0 deletions policyengine_api/services/reform_impacts_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,15 @@ def set_error_reform_impact(
end_time=self._now(),
)

def get_reform_impact_seconds_left(self, impact: CachedReformImpact) -> int | None:
"""Return the whole seconds until a lookup stops finding this cached impact.

Rounded up, so a caller that waits this long finds no record and its
next request submits a new job. None when the record is missing or the
cache cannot be read.
"""
return self._cache.seconds_left(impact)

def set_complete_reform_impact(
self,
execution_id: str,
Expand Down
1 change: 1 addition & 0 deletions tests/fixtures/services/economy_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ def mock_reform_impacts_service():
mock_service.set_reform_impact.return_value = None
mock_service.set_complete_reform_impact.return_value = None
mock_service.set_error_reform_impact.return_value = None
mock_service.get_reform_impact_seconds_left.return_value = None

with patch(
"policyengine_api.services.economy_service.ReformImpactsService",
Expand Down
Loading
Loading