Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@
validate_adjustment_currency,
validate_managed_revision_rows,
validate_revision_currency,
validate_revision_rows_binding,
)
from adcp_reporting_storage.ledger.row_storage import (
REPORTING_ROW_STORAGE_DDL_PATH,
Expand Down Expand Up @@ -1837,10 +1838,7 @@ async def commit_provisional_observation(
).fetchone()
if existing is not None:
retained_observation = ProvisionalObservation.from_wire(existing[1])
if (
retained_observation.acquisition != acquisition
or existing[0] != observation.revision_id
):
if retained_observation.acquisition != acquisition:
raise LedgerConflictError("OBSERVATION_CONFLICT", "observation replay differs")
retained_revision = await self.get_revision(
account_id=acquisition.account_id, reporting_revision_id=existing[0]
Expand All @@ -1849,6 +1847,18 @@ async def commit_provisional_observation(
raise LedgerConflictError(
"HISTORY_UNAVAILABLE", "observation revision is missing"
)
if existing[0] != observation.revision_id:
if (
revision.finality != "snapshot"
or revision.supersedes_reporting_revision_id != existing[0]
or revision.source_manifest_sha256
!= retained_revision.source_manifest_sha256
):
raise LedgerConflictError(
"OBSERVATION_CONFLICT", "observation replay differs"
)
validate_revision_rows_binding(retained_revision, rows)
revision = retained_revision
# Reuse the immutable publication replay checks while holding
# the observation's account lock and transaction.
return await self.commit_revision(revision, rows)
Expand All @@ -1867,6 +1877,24 @@ async def commit_provisional_observation(
)
if checkpoint is not None and checkpoint.next_observation != acquisition.ordinal:
raise LedgerConflictError("OBSERVATION_CONFLICT", "observation ordinal changed")
predecessor_id = revision.supersedes_reporting_revision_id
if revision.finality == "snapshot" and predecessor_id is not None:
await self._require_current_leaf(connection, revision)
predecessor = await self.get_revision(
account_id=acquisition.account_id,
reporting_revision_id=predecessor_id,
)
if predecessor is None:
raise LedgerConflictError(
"HISTORY_UNAVAILABLE", "observation predecessor is missing"
)
if (
predecessor.readable
and predecessor.source_manifest_sha256 == revision.source_manifest_sha256
):
validate_revision_rows_binding(predecessor, rows)
revision = predecessor
observation = replace(observation, revision_id=predecessor_id)
committed = await self.commit_revision(revision, rows)
await self.record_restatement_checkpoint(
RestatementCheckpoint(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS reporting_provisional_observations (
account_id TEXT COLLATE "C" NOT NULL,
reporting_obligation_id TEXT COLLATE "C" NOT NULL,
ordinal BIGINT NOT NULL,
reporting_revision_id TEXT COLLATE "C" NOT NULL UNIQUE,
reporting_revision_id TEXT COLLATE "C" NOT NULL,
payload JSONB NOT NULL,
PRIMARY KEY (account_id, reporting_obligation_id, ordinal),
CONSTRAINT provisional_observation_revision_fk
Expand All @@ -32,6 +32,12 @@ CREATE TABLE IF NOT EXISTS reporting_provisional_observations (
(account_id, reporting_obligation_id, ordinal)
);

-- Source observations and immutable revisions have many-to-one cardinality:
-- successful observations with identical content retain one revision.
ALTER TABLE reporting_provisional_observations
DROP CONSTRAINT IF EXISTS
reporting_provisional_observations_reporting_revision_id_key;

CREATE OR REPLACE FUNCTION reporting_provisional_immutable()
RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,6 @@
"enabled": true,
"fingerprint": "6e0009dfd0c8c76adc077b6f9e8bc866b476342946a107c8cba342ee3c6ec602"
},
"constraint:reporting_provisional_observations.reporting_provisional_observations_reporting_revision_id_key": {
"enabled": true,
"fingerprint": "dface19184e3b1196ac3f05865ce9fd2c1fde32593a604fd8e59e0e7b37525e7"
},
"function:reporting_provisional_immutable()": {
"enabled": true,
"fingerprint": "5f46da14407d6b8acab20bf770940bcae96698b27a941eab2d1f4aa5c271484b"
Expand All @@ -87,10 +83,6 @@
"enabled": true,
"fingerprint": "cc1a138288cf600afcffe140e7001d4e44d1a762ba2b6feeea6c5615c2e1c204"
},
"index:reporting_provisional_observations.reporting_provisional_observations_reporting_revision_id_key": {
"enabled": true,
"fingerprint": "423b8e84d59c9f7c8a580f1b8f97058e964b191a9c4a2dfd0e1e95e00b1872c2"
},
"table:reporting_provisional_acquisitions": {
"enabled": true,
"fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda"
Expand Down
3 changes: 2 additions & 1 deletion src/adcp/reporting/ledger/producer.py
Original file line number Diff line number Diff line change
Expand Up @@ -1874,7 +1874,8 @@ async def commit_revision_from_manifest(
finally:
if staged and staging is not None:
staging.discard_staged_revision_rows(revision)
turn.revisions_committed.append(committed.reporting_revision_id)
if committed.reporting_revision_id == revision.reporting_revision_id:
turn.revisions_committed.append(committed.reporting_revision_id)
return committed

async def _commit_publication(
Expand Down
6 changes: 6 additions & 0 deletions src/adcp/reporting/ledger/provisional.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,4 +205,10 @@ async def commit_provisional_observation(
revision: ReportingRevisionRecord,
rows: Sequence[dict[str, Any]],
) -> ReportingRevisionRecord:
"""Atomically publish an observation and advance its checkpoint.

An unchanged snapshot must reuse its readable current predecessor instead
of emitting a revision. Implementations make that decision under the same
account lock that commits changed content and the observation.
"""
pass
48 changes: 43 additions & 5 deletions src/adcp/reporting/ledger/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -700,6 +700,18 @@ def validate_managed_revision_rows(
)


def validate_revision_rows_binding(
revision: ReportingRevisionRecord, rows: Sequence[dict[str, Any]]
) -> None:
"""Verify submitted rows against an already retained revision's Core binding."""
from adcp.reporting.ledger.producer import recompute_revision_content_sha256

if recompute_revision_content_sha256(revision, rows) != revision.revision_content_sha256:
raise LedgerConflictError(
"REVISION_CONTENT_MISMATCH", "revision row content does not match its immutable binding"
)


def _consumer_status_identity(status: ConsumerStatusRecord) -> str:
payload: dict[str, Any] = {
"chain": list(status.chain_key),
Expand Down Expand Up @@ -1308,15 +1320,25 @@ async def commit_provisional_observation(
"observation replay differs: acquisition does not bind this obligation",
)
if existing is not None:
if (
existing.acquisition != acquisition
or existing.revision_id != observation.revision_id
):
if existing.acquisition != acquisition:
raise LedgerConflictError("OBSERVATION_CONFLICT", "observation replay differs")
if existing.revision_id not in self._revisions:
retained_revision = self._revisions.get(existing.revision_id)
if retained_revision is None:
raise LedgerConflictError(
"HISTORY_UNAVAILABLE", "observation revision is missing"
)
if existing.revision_id != observation.revision_id:
if (
revision.finality != "snapshot"
or revision.supersedes_reporting_revision_id != existing.revision_id
or revision.source_manifest_sha256
!= retained_revision.source_manifest_sha256
):
raise LedgerConflictError(
"OBSERVATION_CONFLICT", "observation replay differs"
)
validate_revision_rows_binding(retained_revision, rows)
revision = retained_revision
# A matching observation identity does not make conflicting
# publication content an idempotent replay.
return await self.commit_revision(revision, rows)
Expand All @@ -1326,6 +1348,22 @@ async def commit_provisional_observation(
checkpoint = self._restatement_checkpoints.get(acquisition.obligation_id)
if checkpoint is not None and checkpoint.next_observation != acquisition.ordinal:
raise LedgerConflictError("OBSERVATION_CONFLICT", "observation ordinal changed")
predecessor_id = revision.supersedes_reporting_revision_id
if revision.finality == "snapshot" and predecessor_id is not None:
siblings = [
item
for item in self._revisions.values()
if item.reporting_obligation_id == revision.reporting_obligation_id
]
self._require_current_leaf(revision, siblings)
predecessor = self._revisions[predecessor_id]
if (
predecessor.readable
and predecessor.source_manifest_sha256 == revision.source_manifest_sha256
):
validate_revision_rows_binding(predecessor, rows)
revision = predecessor
observation = replace(observation, revision_id=predecessor_id)
committed = await self.commit_revision(revision, rows)
await self.record_restatement_checkpoint(
RestatementCheckpoint(
Expand Down
2 changes: 1 addition & 1 deletion tests/conformance/reporting/_provisional_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@ def _storage_split():
PROVISIONAL_REQUIRED_OBJECTS as PROVISIONAL_OBJECTS,
)

assert len(PROVISIONAL_OBJECTS) == 27
assert len(PROVISIONAL_OBJECTS) == 25
45 changes: 31 additions & 14 deletions tests/conformance/reporting/test_reporting_core_lifecycle.py
Original file line number Diff line number Diff line change
Expand Up @@ -439,9 +439,19 @@ async def test_reporting_core_lifecycle(ledger: PgReportingLedgerStore) -> None:
first_revisions = await ledger.list_revisions(
account_id=ACCOUNT, reporting_obligation_id=obligation.reporting_obligation_id
)
assert len(first_revisions) == 2
automatic = next(item for item in first_revisions if item.reporting_revision_id != revision_id)
assert automatic.supersedes_reporting_revision_id == revision_id
assert first_revisions == (revision,)
second_observation = await ledger.get_provisional_observation(
account_id=ACCOUNT,
reporting_obligation_id=obligation.reporting_obligation_id,
)
assert second_observation is not None
assert second_observation.acquisition.ordinal == 1
assert second_observation.revision_id == revision.reporting_revision_id
second_checkpoint = await ledger.get_restatement_checkpoint(
account_id=ACCOUNT,
reporting_obligation_id=obligation.reporting_obligation_id,
)
assert second_checkpoint is not None and second_checkpoint.next_observation == 2

zero_obligation = await ledger.find_obligation(
account_id=ACCOUNT,
Expand All @@ -456,10 +466,7 @@ async def test_reporting_core_lifecycle(ledger: PgReportingLedgerStore) -> None:
account_id=ACCOUNT, reporting_obligation_id=zero_obligation.reporting_obligation_id
)
assert [item.row_count for item in zero_revisions] == [0]
assert set(turn.revisions_committed) == {
automatic.reporting_revision_id,
zero_revisions[0].reporting_revision_id,
}
assert turn.revisions_committed == [zero_revisions[0].reporting_revision_id]

settled = await _reconcile(ledger, expected=_expected_periods(2))
assert settled.definitive is True, settled.obligations
Expand Down Expand Up @@ -503,14 +510,24 @@ async def test_reporting_core_lifecycle(ledger: PgReportingLedgerStore) -> None:
)
== ()
)
# At this later turn the first period's next hourly observation is due
# too. Preserve that automatic revision as the next restatement's parent.
# At this later turn the first period's next hourly observation is due too.
# Its unchanged content keeps the existing revision as the restatement parent.
refreshed_first = await ledger.list_revisions(
account_id=ACCOUNT, reporting_obligation_id=obligation.reporting_obligation_id
)
assert len(refreshed_first) == 3
automatic_at_ready = max(refreshed_first, key=lambda item: item.created_at)
assert automatic_at_ready.supersedes_reporting_revision_id == automatic.reporting_revision_id
assert refreshed_first == (revision,)
ready_observation = await ledger.get_provisional_observation(
account_id=ACCOUNT,
reporting_obligation_id=obligation.reporting_obligation_id,
)
assert ready_observation is not None
assert ready_observation.acquisition.ordinal == 2
assert ready_observation.revision_id == revision.reporting_revision_id
ready_checkpoint = await ledger.get_restatement_checkpoint(
account_id=ACCOUNT,
reporting_obligation_id=obligation.reporting_obligation_id,
)
assert ready_checkpoint is not None and ready_checkpoint.next_observation == 3

# The zero-row period and the silent period are distinguishable, which is
# the entire operational point of the tier.
Expand Down Expand Up @@ -560,8 +577,8 @@ async def test_reporting_core_lifecycle(ledger: PgReportingLedgerStore) -> None:
chain = await ledger.list_revisions(
account_id=ACCOUNT, reporting_obligation_id=obligation.reporting_obligation_id
)
assert len(chain) == 4
assert restated.supersedes_reporting_revision_id == automatic_at_ready.reporting_revision_id
assert len(chain) == 2
assert restated.supersedes_reporting_revision_id == revision.reporting_revision_id
# The superseded revision is still retained and still readable: a consumer
# that already cited it must be able to fetch exactly what it cited.
superseded = await ledger.read_revision_rows(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -601,7 +601,7 @@ async def test_installed_pg_extra_migrates_commits_and_restarts(installed_distri

values = json.load(sys.stdin)
assert len(values["waiver_objects"]) == 10
assert len(values["provisional_objects"]) == 27
assert len(values["provisional_objects"]) == 25
assert json.loads(files("adcp_reporting_storage.ledger").joinpath(
"required_provisional_schema.json").read_text()) == values["provisional_objects"]
expected_objects = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ async def test_progress_retains_policy_until_terminal_publication(
assert len(source.requests) == 1
h.source_clock.now = END + timedelta(hours=1)
unchanged = await source_turn(h.production)
assert len(unchanged.revisions_committed) == 1
assert not unchanged.revisions_committed
assert len(source.requests) == 2
checkpoint = await h.store.get_restatement_checkpoint(
account_id=h.item.config.account_id,
Expand All @@ -111,9 +111,8 @@ async def test_progress_retains_policy_until_terminal_publication(
changed = await source_turn(h.production)
assert len(changed.revisions_committed) == 1
history = await revisions(h)
assert len(history) == 3
assert len(history) == 2
assert history[1].supersedes_reporting_revision_id == history[0].reporting_revision_id
assert history[2].supersedes_reporting_revision_id == history[1].reporting_revision_id

h.source_clock.now = END + timedelta(hours=3)
await source_turn(h.production)
Expand All @@ -135,7 +134,6 @@ async def test_progress_retains_policy_until_terminal_publication(
completed = await source_turn(h.production)
assert len(completed.revisions_committed) == 1
assert [r.finality for r in await revisions(h)] == [
"snapshot",
"snapshot",
"snapshot",
"official",
Expand Down Expand Up @@ -163,7 +161,7 @@ async def test_settling_checkpoint_and_pending_work_survive_fresh_service(backen
assert len(initial.revisions_committed) == 1
h.source_clock.now = END + timedelta(hours=1)
noop = await source_turn(h.production)
assert len(noop.revisions_committed) == 1
assert not noop.revisions_committed
original_source = h.production.offerings[0].producer._source
assert len(original_source.requests) == 2
prior_keys = {r.identity.source_execution_key for r in original_source.requests}
Expand Down Expand Up @@ -199,6 +197,6 @@ async def test_settling_checkpoint_and_pending_work_survive_fresh_service(backen
reporting_obligation_id=fresh.item.obligation.reporting_obligation_id,
)
assert checkpoint is not None and checkpoint.next_observation == 3
assert len(await revisions(fresh)) == 3
assert len(await revisions(fresh)) == 2
unfinished = await pending(fresh)
assert unfinished == (fresh.item.obligation.reporting_obligation_id,)
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,30 @@ async def test_both_observation_tables_reject_update_and_delete():
assert await latest(store) == before


async def test_observation_schema_upgrade_removes_revision_uniqueness():
async with isolated_reporting_pool() as pool:
store = PgReportingLedgerStore(pool=pool)
await store.create_schema()
async with pool.connection() as connection:
await connection.execute(
"ALTER TABLE reporting_provisional_observations "
"ADD CONSTRAINT "
"reporting_provisional_observations_reporting_revision_id_key "
"UNIQUE (reporting_revision_id)"
)
await store.create_schema()
async with pool.connection() as connection:
retained = await (
await connection.execute(
"SELECT 1 FROM pg_constraint "
"WHERE conrelid='reporting_provisional_observations'::regclass "
"AND conname="
"'reporting_provisional_observations_reporting_revision_id_key'"
)
).fetchone()
assert retained is None


async def test_observation_foreign_keys_bind_both_obligation_and_revision_accounts():
psycopg = pytest.importorskip("psycopg")
async with isolated_reporting_pool() as pool:
Expand Down
Loading
Loading