Skip to content
139 changes: 139 additions & 0 deletions src/relay-local-recovery.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
/** Explicit operator-selected loss of unpublished local moves, not fabricated relay refusals [H16,H21]. */
import { createHash } from 'node:crypto';
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { appliedSequence } from './engine.js';
import { exportBoard, restoreRelaySnapshot, semanticBoardDigest } from './exchange.js';
import { outboxFile, readOutbox } from './relay-outbox.js';
import { withOutboxGuard } from './relay-outbox-guard.js';
import { readBoardKey } from './relay-key.js';
import { Refused } from './refused.js';

const digest = bytes => createHash('sha256').update(bytes).digest('hex');
const refusal = text => new Refused('RELAY_LOCAL_RECOVERY', text);
const nativeDigest = board => semanticBoardDigest(exportBoard(board));

/** Bind all retained link inputs across a held retry, not only its recovery marker. */
function linkInputs(state) {
const { recovery, ...inputs } = state;
return digest(JSON.stringify(inputs));
}

/** Preserve every existing ordered outcome exactly; loss permission applies only to local unpublished work. */
function preservesReceipts(before, after) {
const meta = new Map(after.tables.board_meta.map(row => [row.meta_key, row.meta_value]));
if (before.tables.board_meta.some(row => /^relay_(receipt|refusal)_/u.test(row.meta_key) && meta.get(row.meta_key) !== row.meta_value)) {
throw refusal('the authenticated prefix disagrees with an existing receipt; no local recovery was applied');
}
}

/**
* Caller holds the ordinary relay lock and has quiesced native writers, checkpoint publishers and old callers.
* The exact queue digest is the loss selection, not an authentication credential. Only the existing machine's
* authenticated relay prefix is adopted. No POST/PUT, queued operation, person event or invented refusal is made.
* A durable existing recovery hold fences ordinary traffic across each local crash cut; retry the same selection.
* io.recoveryCut is an injectable fixture cut, never an alternate persistence or authority path.
*/
export async function discardUnpublished(link, { queueSha256, decision }) {
const { root, file, state, io, localRecords, saveLink, loadLink } = link;
if (!/^[0-9a-f]{64}$/u.test(queueSha256 ?? '') || typeof decision !== 'string' || !decision.trim() || decision.length > 2000) {
throw refusal('supply the exact inspected queue SHA-256 and attributable loss decision');
}
if (state.mode !== 'ordered' || state.unlinking || state.environment || state.baselineAccepted === false) {
throw refusal('recover only an existing accepted machine-owned ordered board');
}
const directory = join(dirname(file), 'local-recovery', queueSha256);
const queueFile = outboxFile(file), archive = join(directory, 'outbox.sealed');
const planFile = join(directory, 'plan.json');
const key = readBoardKey(state.board);
if (!state.recovery && state.localRecovery?.kind === 'discard-unpublished' && state.localRecovery.queueSha256 === queueSha256 && state.localRecovery.decision === decision) {
if (!existsSync(archive) || digest(readFileSync(archive)) !== queueSha256 || digest(readFileSync(planFile)) !== state.localRecovery.planSha256) {
throw refusal('the completed recovery custody changed; do not repeat the reset');
}
return { board: state.board, sequence: state.localRecovery.sequence, nativeEventCursor: state.localRecovery.nativeEventCursor, targetDigest: state.localRecovery.targetDigest, queueSha256, alreadyCompleted: true, archive, plan: planFile,
disposition: 'read back the original recovery completion; later native work was not reset' };
}
let marker = state.recovery;
if (marker && (marker.kind !== 'discard-unpublished' || marker.queueSha256 !== queueSha256 || marker.decision !== decision)) {
throw refusal('another recovery selection is pending; preserve it and finish that exact selection');
}
if (!marker) {
if (!existsSync(queueFile) || digest(readFileSync(queueFile)) !== queueSha256) throw refusal('the local queue changed or is absent; no recovery applied');
const original = localRecords(root, exportBoard);
const sourceDigest = semanticBoardDigest(original);
const initialLink = digest(readFileSync(file));
const queue = await readOutbox(queueFile, key, state.board);
if (!queue?.moves?.length) throw refusal('this selection has no queued moves');
let checkpoint = await link.fetchCheckpoint(key);
const records = [];
let after = checkpoint.sequence;
for (;;) {
const page = await link.request(state, '/api/v1/boards/' + state.board + '/events?after=' + after, { method: 'GET' }, io);
if (!Array.isArray(page.events) || page.events.some((record, i) => record?.event_id !== after + i + 1 || !['move', 'request'].includes(record.kind) || typeof record.sealed !== 'string')) {
throw refusal('the relay did not supply a contiguous authenticated-prefix candidate');
}
if (!page.events.length) break;
records.push(...page.events); after = records.at(-1).event_id;
}
const minimum = Math.max(queue.baseSequence, localRecords(root, appliedSequence), checkpoint.sequence);
checkpoint = await link.advanceCheckpoint(key, checkpoint, records, minimum);
preservesReceipts(original, checkpoint.native);
const targetDigest = semanticBoardDigest(checkpoint.native);
const receipts = new Map(checkpoint.native.tables.board_meta.map(row => [row.meta_key, row.meta_value]));
// The individual identities remain in encrypted custody; the report never calls abandonment absence or success.
const plan = { version: 1, board: state.board, queueSha256, decision, sourceDigest, targetDigest,
sequence: checkpoint.sequence, queued: queue.moves.length, native: checkpoint.native,
previousLinkState: { pending: state.pending ?? null, recovered: state.recovered ?? null,
snapshot: state.snapshot ?? null, checkpoint: state.checkpoint ?? null },
originalReceiptCount: [...receipts.keys()].filter(key => key.startsWith('relay_receipt_')).length };
await withOutboxGuard(file, () => {
if (digest(readFileSync(file)) !== initialLink || digest(readFileSync(queueFile)) !== queueSha256 || localRecords(root, nativeDigest) !== sourceDigest) {
throw refusal('local state changed during preparation; no recovery applied');
}
// Reusing an uncommitted staged plan is not a state effect. It is safe to replace only under unchanged inputs.
mkdirSync(directory, { recursive: true, mode: 0o700 });
const bytes = JSON.stringify(plan);
const temporary = planFile + '.tmp';
writeFileSync(temporary, bytes, { mode: 0o600 }); renameSync(temporary, planFile);
marker = { kind: 'discard-unpublished', queueSha256, decision, planSha256: digest(bytes), sourceDigest, targetDigest, sequence: checkpoint.sequence, linkInputsSha256: linkInputs(state) };
state.recovery = marker; saveLink(file, state);
});
await io.recoveryCut?.('held');
}
return withOutboxGuard(file, async () => {
const current = loadLink(file);
if (JSON.stringify(current.recovery) !== JSON.stringify(marker)) throw refusal('the selected recovery hold changed');
if (linkInputs(current) !== marker.linkInputsSha256) throw refusal('retained link inputs changed after the recovery hold; reconcile that writer before resuming');
const bytes = readFileSync(planFile);
if (digest(bytes) !== marker.planSha256) throw refusal('the retained recovery plan changed; keep its hold');
const plan = JSON.parse(bytes);
if (plan.board !== state.board || plan.queueSha256 !== queueSha256 || plan.decision !== decision || plan.sourceDigest !== marker.sourceDigest || plan.targetDigest !== marker.targetDigest || plan.sequence !== marker.sequence || semanticBoardDigest(plan.native) !== marker.targetDigest) {
throw refusal('the retained plan does not match this board and recovery selection');
}
let actual = localRecords(root, nativeDigest);
if (![marker.sourceDigest, marker.targetDigest].includes(actual)) throw refusal('native state changed after the recovery hold; keep it and reconcile that writer');
if (existsSync(queueFile)) {
if (digest(readFileSync(queueFile)) !== queueSha256 || existsSync(archive)) throw refusal('the active or archived queue changed; no replacement is allowed');
renameSync(queueFile, archive);
}
if (!existsSync(archive) || digest(readFileSync(archive)) !== queueSha256) throw refusal('original encrypted queue custody is missing; keep the recovery hold');
await io.recoveryCut?.('archived');
if (actual !== marker.targetDigest) {
localRecords(root, board => restoreRelaySnapshot(board, plan.native, plan.sequence, marker.sourceDigest));
actual = localRecords(root, nativeDigest);
if (actual !== marker.targetDigest) throw refusal('native recovery readback differs; keep the recovery hold');
}
await io.recoveryCut?.('restored');
rmSync(join(dirname(file), 'relay-outbox.summary.json'), { force: true });
for (const name of ['pending', 'recovered', 'snapshot', 'checkpoint', 'outbox', 'recovery']) delete current[name];
current.sequence = plan.sequence;
// Mirror cursor names native event IDs, not relay positions; retain that distinction after a lossy rebase.
current.cursor = localRecords(root, board => board.db.prepare('SELECT COALESCE(MAX(event_id), 0) AS id FROM event').get().id);
current.localRecovery = { kind: 'discard-unpublished', queueSha256, decision, sequence: plan.sequence, nativeEventCursor: current.cursor, targetDigest: plan.targetDigest, planSha256: marker.planSha256 };
saveLink(file, current);
Object.keys(state).forEach(key => delete state[key]); Object.assign(state, current);
return { board: state.board, sequence: plan.sequence, nativeEventCursor: current.cursor, targetDigest: plan.targetDigest, queueSha256, retiredQueuedEntries: plan.queued,
archive, plan: planFile, originalReceiptCount: plan.originalReceiptCount,
disposition: 'authenticated ordered baseline restored; unpublished local messages abandoned, not declared absent or successful' };
});
}
36 changes: 33 additions & 3 deletions src/relay-ordering.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ import {
appliedSequence,
engineReceipt,
} from './engine.js';
import { restoreRelaySnapshot } from './exchange.js';
import { exportBoard, importBoard, restoreRelaySnapshot } from './exchange.js';
import { ENGINE_VERSION } from './machine.js';
import {
codeIsFromMain,
repoInfo,
Expand Down Expand Up @@ -39,7 +40,7 @@ import {
*
* @param {{ request: Function, localRecords: Function, decoded: Function, replayRelayRecord: Function,
* reportRecoveredReply: Function, saveLink: Function, sealedRecord: Function, flush: Function }} relay
* @returns {{ applyRecords: Function, fetchCheckpoint: Function, catchUp: Function, postPending: Function, sendPending: Function }}
* @returns {{ applyRecords: Function, advanceCheckpoint: Function, fetchCheckpoint: Function, catchUp: Function, postPending: Function, sendPending: Function }}
*/
export function relayOrdering({ request, localRecords, decoded, replayRelayRecord, reportRecoveredReply, saveLink, sealedRecord, flush }) {
/** Apply authenticated relay records in order, each in its own board transaction with a durable receipt. */
Expand All @@ -52,6 +53,35 @@ export function relayOrdering({ request, localRecords, decoded, replayRelayRecor
return outcomes;
}

/** Reconstruct a covering checkpoint in isolation; never roll the live board back or send a move. */
async function advanceCheckpoint(key, state, checkpoint, records, minimum) {
const meta = new Map((checkpoint.native?.tables?.board_meta ?? []).map(row => [row.meta_key, row.meta_value]));
if (meta.get('board_id') !== state.board || !Number.isSafeInteger(checkpoint.sequence)
|| checkpoint.sequence < 0 || String(checkpoint.sequence) !== meta.get('relay_applied_sequence')) {
throw new Refused('RELAY_SNAPSHOT', 'the checkpoint does not identify this board and prefix; keep the local board and fetch a consistent checkpoint');
}
const engine = Number(meta.get('relay_engine_version'));
if (!Number.isSafeInteger(engine) || engine < 1 || engine > ENGINE_VERSION) {
throw new Refused('ENGINE_VERSION', 'the checkpoint needs another supported engine; keep the local board and use that engine before recovery');
}
if (!Number.isSafeInteger(minimum) || minimum < checkpoint.sequence || !Array.isArray(records)
|| records.some((record, index) => record?.event_id !== checkpoint.sequence + index + 1)
|| (records.at(-1)?.event_id ?? checkpoint.sequence) < minimum) {
throw new Refused('RELAY_OUTBOX', 'the authenticated checkpoint history does not cover the current prefix; keep the local board and obtain its missing ordered records');
}
const board = store.openBoard(':memory:');
try {
importBoard(board, checkpoint.native);
for (const record of records) {
await replayRelayRecord(board, key, state, record);
if (appliedSequence(board) !== record.event_id) {
throw new Refused('RELAY_RESPONSE', 'checkpoint replay did not advance to its ordered record; keep the local board and inspect that record');
}
}
return { native: exportBoard(board), sequence: appliedSequence(board), reconstructed: true };
} finally { store.closeBoard(board); }
}

/** Read and authenticate the relay's checkpoint without touching the board; the caller restores it. */
async function fetchCheckpoint(state, io, key) {
const document = await request(state, '/api/v1/boards/' + state.board + '/state', {}, io);
Expand Down Expand Up @@ -191,5 +221,5 @@ export function relayOrdering({ request, localRecords, decoded, replayRelayRecor
if (state.pending) throw new Refused('RELAY_BUSY', 'the relay order stayed busy; retry this command after other clients finish');
}

return { applyRecords, fetchCheckpoint, catchUp, postPending, sendPending };
return { applyRecords, advanceCheckpoint, fetchCheckpoint, catchUp, postPending, sendPending };
}
38 changes: 38 additions & 0 deletions src/relay-outbox-undo.js
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,44 @@ function keyValue(encoded) {
/** Quote a SQLite identifier read from the changeset or its actual table schema. */
function identifier(name) { return `"${name.replaceAll('"', '""')}"`; }

/**
* Read the original inserted rows from a queued move's own changesets without applying them.
* Local reconstruction needs the identities already returned to callers, not newly generated fact ids.
* Schema checks use the same connection as reconstruction; malformed captures fail rather than guessing.
*/
export function capturedInserts(board, changes, wanted) {
const rows = [];
for (const change of changes ?? []) {
const bytes = Buffer.from(change, 'base64');
let columns = 0, table = null, schema = null;
for (let at = 0; at < bytes.length;) {
const marker = bytes[at];
if (marker === 0x54) {
const count = varint(bytes, at + 1);
columns = count.value;
const end = bytes.indexOf(0, count.next + columns);
if (!Number.isSafeInteger(columns) || columns < 1 || end < 0) throw new Error('incomplete captured table');
table = bytes.subarray(count.next + columns, end).toString('utf8');
schema = table === wanted ? board.db.prepare(`PRAGMA table_info(${identifier(table)})`).all() : null;
if (schema && (schema.length !== columns || schema.some((column, index) => Boolean(column.pk) !== Boolean(bytes[count.next + index])))) {
throw new Error(`captured schema differs for ${table}`);
}
at = end + 1;
continue;
}
if (!table || ![INSERT, UPDATE, DELETE].includes(marker)) throw new Error('invalid captured operation');
const before = record(bytes, at + 2, columns);
if (marker === INSERT && schema) rows.push(Object.fromEntries(schema.map((column, index) => {
const decoded = keyValue(before.values[index]);
return [column.name, typeof decoded === 'bigint' && decoded <= BigInt(Number.MAX_SAFE_INTEGER) && decoded >= BigInt(Number.MIN_SAFE_INTEGER) ? Number(decoded) : decoded];
})));
at = marker === UPDATE ? record(bytes, before.next, columns).next : before.next;
if (at > bytes.length) throw new Error('incomplete captured row');
}
}
return rows;
}

/**
* Numeric REPLACE also replaces a conflicting INSERT, which undo must refuse. Check row existence first,
* inside the same write transaction, so only changed UPDATE/DELETE values can reach that resolution.
Expand Down
Loading