Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 67 additions & 16 deletions tests/capabilities/test_research_composition_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
from __future__ import annotations

import json
import os
import shlex
import subprocess
import sys
from pathlib import Path
Expand All @@ -14,8 +16,10 @@
)
from loopx.control_plane.work_items.semantic_replan_writeback import qualify_replan_writeback
from loopx.control_plane.testing.model_tool_behavior import QuotaToolContext
from loopx.control_plane.effect_runtime import restart_effect_runtime
from loopx.capabilities.explore.research_frontier import prepare_research_replan_evidence
from tests.control_plane.canonical_authority_fixture import initialize_canonical_source_fixture
from tests.control_plane.canonical_authority_fixture import initialize_canonical_source_fixture, isolate_sqlite_runtime
from tests.control_plane.test_quota_settlement_cli import _run_generated_cli

GOAL = "research-gate-fixture"
AGENT = "fixture-agent"
Expand All @@ -33,8 +37,10 @@ def observation(node: str, target: str | None = None) -> dict:
"interaction_kind": "state_interference", "evidence_ids": [f"ev-{node}", f"ev-{target}"]}] if target else []}


@pytest.fixture
def fixture(tmp_path: Path):
@pytest.fixture(params=["file", "sqlite"])
def fixture(tmp_path: Path, request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch):
isolate_sqlite_runtime(tmp_path, monkeypatch)
monkeypatch.setenv("LOOPX_USAGE_PING", "0")
state = tmp_path / "ACTIVE_GOAL_STATE.md"
state.write_text('---\nstatus: active\nowner_mode: goal\nobjective: "Test the explicit research boundary."\n'
'updated_at: 2026-09-28T00:00:00Z\n---\n\n# Research fixture\n\n'
Expand All @@ -58,7 +64,7 @@ def fixture(tmp_path: Path):
append_research_observation(log, goal_id=GOAL, observation=observation("a", "b"))
for name in ["a", "b"]:
append_explore_result_event(log, build_explore_edge_event(goal_id=GOAL, from_node="joint", to_node=name, edge_type="depends_on"))
initialize_canonical_source_fixture(runtime, GOAL, state_path=state)
initialize_canonical_source_fixture(runtime, GOAL, state_path=state, provider=request.param)

def cli(*args: str, success: bool = True) -> dict:
result = subprocess.run([sys.executable, "-m", "loopx.cli", "--format", "json", "--registry", str(registry),
Expand All @@ -69,7 +75,10 @@ def cli(*args: str, success: bool = True) -> dict:
return QuotaToolContext.from_output(result.stdout).packet
return json.loads(result.stdout)

return cli, log, runtime, registry, state
try:
yield cli, log, runtime, registry, state
finally:
restart_effect_runtime(wait_for_exit=True)


def activate(cli) -> None:
Expand Down Expand Up @@ -100,7 +109,45 @@ def test_real_guard_and_successor_share_the_current_gap(fixture) -> None:
assert guard["effective_action"] == "autonomous_replan_required"
packet = guard["replan_action_packet"]
obligation = packet["obligation_id"]
assert packet["capability_guard"]["gap_id"] == guard["bounded_research_frontier"]["selected_gap"]["gap_id"]
receipt = guard["heartbeat_receipt"]
capability_guard = receipt["semantic_replan_capability_guard"]
gap = guard["bounded_research_frontier"]["selected_gap"]
assert capability_guard == {
"schema_version": "semantic_replan_capability_guard_v0", "capability_id": "explore",
"gap_id": gap["gap_id"], "frontier_revision": gap["frontier_revision"],
}
assert receipt["semantic_replan_obligation_id"] == obligation == gap["obligation_id"]
# The default staged entry retains the admitted duty in its original
# receipt. Follow its real readers before authoring, without a new guard.
before_reads = {path: path.read_bytes() for path in (
Comment thread
cocolord marked this conversation as resolved.
Resolved
registry, state, runtime / "goals" / GOAL / "rollout-event-log.jsonl")}

def read_stage(command):
argv = shlex.split(command)
assert argv[0] == sys.executable
module = argv.index("-m")
assert argv[module + 1] == "loopx.entrypoint"
assert all(option == "-I" for option in argv[1:module])
result = subprocess.run(argv, cwd=registry.parent, capture_output=True, text=True,
env={**os.environ, "LOOPX_USAGE_PING": "0"}, timeout=30)
assert result.returncode == 0, result.stdout + result.stderr
return json.loads(result.stdout)

review = read_stage(packet["review"])
writer = read_stage(review["next"]["writeback"])
assert review["stage"] == "review" and writer["stage"] == "writeback"
assert review["obligation_id"] == writer["obligation_id"] == obligation
assert review["context_id"] == writer["context_id"]
assert review["source_hash"] == writer["source_hash"]
successor_command = writer["writeback_contract"]["successor_command"]
successor = shlex.split(successor_command)
assert successor[successor.index("--goal-id") + 1] == GOAL
assert successor[successor.index("--claimed-by") + 1] == AGENT
assert successor[successor.index("--replan-obligation-id") + 1] == obligation
assert successor[successor.index("--target-key") + 1] == gap["successor_binding"]["target_key"]
assert writer["writeback"]["next_cli_actions"] == [successor_command]
assert writer["writeback"]["spend_allowed_now"] is False
assert {path: path.read_bytes() for path in before_reads} == before_reads
status = cli("status", "--goal-id", GOAL, "--agent-id", AGENT)
item = next(item for item in status["attention_queue"]["items"] if item["goal_id"] == GOAL)
assert item["project_asset"]["bounded_research_frontier"] == guard["bounded_research_frontier"]
Expand All @@ -109,8 +156,8 @@ def test_real_guard_and_successor_share_the_current_gap(fixture) -> None:
assert summary["research_execution_frontier"] == guard["bounded_research_frontier"]
from loopx.cli_commands.explore import render_explore_markdown
from loopx.presentation.renderers.status_markdown import render_status_markdown
assert packet["capability_guard"]["gap_id"] in render_explore_markdown(summary)
assert packet["capability_guard"]["gap_id"] in render_status_markdown(status)
assert capability_guard["gap_id"] in render_explore_markdown(summary)
assert capability_guard["gap_id"] in render_status_markdown(status)
assert "lineage_gaps" not in guard["bounded_research_frontier"]
assert "settlement_transitions" not in guard["bounded_research_frontier"]
rejected = cli("refresh-state", "--goal-id", GOAL, "--agent-id", AGENT,
Expand All @@ -126,7 +173,9 @@ def test_real_guard_and_successor_share_the_current_gap(fixture) -> None:
deferred = cli(*base, "--text", "Deferred joint experiment.", "--explore-result-node-ref", "joint",
"--status", "deferred", "--resume-when", "capacity_available:fixture", success=False)
assert "no deferral" in json.dumps(deferred)
created = cli(*base, "--text", "Run the bounded joint experiment.", "--explore-result-node-ref", "joint", "--target-key", "joint")
code, created = _run_generated_cli(
shlex.join([*successor, "--format", "json"]), registry_path=registry, cwd=registry.parent)
assert code == 0, created
assert created["replan_transition"]["recorded"]
before_closeout = state.read_bytes()
missing = cli("todo", "complete", "--goal-id", GOAL, "--todo-id", created["todo_id"],
Expand All @@ -139,7 +188,7 @@ def test_real_guard_and_successor_share_the_current_gap(fixture) -> None:
assert "current typed experiment observation" in json.dumps(retired)
assert state.read_bytes() == before_closeout
replay = cli("quota", "should-run", "--goal-id", GOAL, "--agent-id", AGENT, "--turn-instance-id", "fixture-on")
assert replay["heartbeat_receipt"]["semantic_replan_capability_guard"] == packet["capability_guard"]
assert replay["heartbeat_receipt"]["semantic_replan_capability_guard"] == capability_guard
assert replay["replan_action_packet"]["obligation_id"] == obligation
refresh = cli("refresh-state", "--goal-id", GOAL, "--agent-id", AGENT,
"--replan-obligation-id", obligation, "--turn-instance-id", "fixture-on",
Expand All @@ -157,7 +206,7 @@ def test_real_guard_and_successor_share_the_current_gap(fixture) -> None:
result["progress"]["work_item_id"] = created["todo_id"]
result["input_observations"] = guard["bounded_research_frontier"]["selected_gap"]["input_observations"]
result["execution_lineage"] = {"schema_version": "research_execution_lineage_v0", "goal_id": GOAL,
"gap_id": packet["capability_guard"]["gap_id"], "replan_obligation_id": obligation,
"gap_id": capability_guard["gap_id"], "replan_obligation_id": obligation,
"successor_todo_id": created["todo_id"], "agent_id": AGENT}
append_research_observation(log, goal_id=GOAL, observation=result, agent_id=AGENT,
registry_path=registry, runtime_root=runtime)
Expand All @@ -180,7 +229,7 @@ def test_real_guard_and_successor_share_the_current_gap(fixture) -> None:
from loopx.extensions.lark.presentation.explore_results import _node_record_values
joint = next(node for node in summary["nodes"] if node["node_id"] == "joint")
lark = _node_record_values(joint, goal_id=GOAL, source_id="synthetic-source")
assert "observed" in lark["Summary"] and packet["capability_guard"]["gap_id"] in lark["Summary"]
assert "observed" in lark["Summary"] and capability_guard["gap_id"] in lark["Summary"]


def test_selected_guard_cannot_be_erased_by_input_invalidation(fixture) -> None:
Expand Down Expand Up @@ -218,6 +267,7 @@ def test_real_replan_guard_accepts_exact_blocker_wait_and_resumes_without_closur
activate(cli)
guard = cli("quota", "should-run", "--goal-id", GOAL, "--agent-id", AGENT, "--turn-instance-id", "fixture-blocked")
packet = guard["replan_action_packet"]
capability_guard = guard["heartbeat_receipt"]["semantic_replan_capability_guard"]
task = cli("todo", "add", "--goal-id", GOAL, "--role", "agent", "--claimed-by", AGENT,
"--task-class", "advancement_task", "--action-kind", "joint_probe", "--target-key", "joint",
"--explore-result-node-ref", "joint", "--replan-obligation-id", packet["obligation_id"],
Expand All @@ -233,7 +283,7 @@ def test_real_replan_guard_accepts_exact_blocker_wait_and_resumes_without_closur
result["closure_basis"] = None
result["input_observations"] = guard["bounded_research_frontier"]["selected_gap"]["input_observations"]
result["execution_lineage"] = {"schema_version": "research_execution_lineage_v0", "goal_id": GOAL,
"gap_id": packet["capability_guard"]["gap_id"], "replan_obligation_id": packet["obligation_id"],
"gap_id": capability_guard["gap_id"], "replan_obligation_id": packet["obligation_id"],
"successor_todo_id": task["todo_id"], "agent_id": AGENT}
result["composition_resolution"] = {"schema_version": "research_composition_resolution_v0",
"disposition": "deferred", "evidence_ids": ["ev-joint"]}
Expand Down Expand Up @@ -262,8 +312,8 @@ def test_real_replan_guard_accepts_exact_blocker_wait_and_resumes_without_closur
"progress_observation": recorded["observation"]["progress"]}], state_text=state.read_text(), agent_id=AGENT,
goal_id=GOAL, registry_goal=source, guard_scoped=True,
capability_evidence=prepare_research_replan_evidence(runtime_root=runtime, goal_id=GOAL,
agent_id=AGENT, registry_goal=source, state_text=state.read_text(), capability_guard=packet["capability_guard"]),
guard_semantic_replan_obligation_id=packet["obligation_id"], guard_capability=packet["capability_guard"],
agent_id=AGENT, registry_goal=source, state_text=state.read_text(), capability_guard=capability_guard),
guard_semantic_replan_obligation_id=packet["obligation_id"], guard_capability=capability_guard,
progress_observation=recorded["observation"]["progress"])
assert repeated["accepted"] is False
cli("todo", "complete", "--goal-id", GOAL, "--todo-id", blocker["todo_id"], "--agent-id", AGENT,
Expand All @@ -284,6 +334,7 @@ def test_real_replan_guard_accepts_exact_result_source_and_candidate_dismissal(f
activate(cli)
guard = cli("quota", "should-run", "--goal-id", GOAL, "--agent-id", AGENT, "--turn-instance-id", "fixture-terminal")
packet = guard["replan_action_packet"]
capability_guard = guard["heartbeat_receipt"]["semantic_replan_capability_guard"]
task = cli("todo", "add", "--goal-id", GOAL, "--role", "agent", "--claimed-by", AGENT,
"--task-class", "advancement_task", "--action-kind", "joint_probe", "--target-key", "joint",
"--explore-result-node-ref", "joint", "--replan-obligation-id", packet["obligation_id"],
Expand All @@ -294,7 +345,7 @@ def test_real_replan_guard_accepts_exact_result_source_and_candidate_dismissal(f
result["progress"]["work_item_id"] = task["todo_id"]
result["input_observations"] = guard["bounded_research_frontier"]["selected_gap"]["input_observations"]
result["execution_lineage"] = {"schema_version": "research_execution_lineage_v0", "goal_id": GOAL,
"gap_id": packet["capability_guard"]["gap_id"], "replan_obligation_id": packet["obligation_id"],
"gap_id": capability_guard["gap_id"], "replan_obligation_id": packet["obligation_id"],
"successor_todo_id": task["todo_id"], "agent_id": AGENT}
if disposition == "dismissed":
result["progress"]["result_class"] = "no_followup"
Expand Down
31 changes: 28 additions & 3 deletions tests/control_plane/test_replan_progress_native_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import pytest

from canonical_authority_fixture import initialize_canonical_authority, isolate_sqlite_runtime
from test_quota_settlement_cli import _run_generated_cli
from tests.control_plane.codex_checkpoint_fixture import read_cli_policy_payload, selected_context_values
from loopx.control_plane.coordination.canonical_snapshot import read_canonical_snapshot
from loopx.control_plane.coordination.runtime_shadow import build_todo_runtime_shadow_projection
Expand Down Expand Up @@ -427,6 +428,12 @@ def _authority(fixture: dict) -> dict:
return result


def _quota_events(runtime: Path) -> list[dict]:
path = runtime / "goals" / GOAL / "rollout-event-log.jsonl"
return [event for line in path.read_text(encoding="utf-8").splitlines()
if (event := json.loads(line))["event_kind"] in {"quota_should_run", "quota_spend"}]


@pytest.mark.parametrize("provider", ["file", "sqlite"])
@pytest.mark.parametrize("scenario,expected_outcomes,reason_code", [
("advanced", ["new_probe_family"], None),
Expand All @@ -451,8 +458,10 @@ def test_typed_progress_cli_preserves_replan_authority(
bound = call_progress_cli(fixture, *guard_args, "--todo-id", TODO)
obligation = bound["autonomous_replan_obligation"]
assert [trigger["kind"] for trigger in obligation["triggers"]] == ["typed_progress_repeat"]
assert bound["heartbeat_receipt"]["settlement_identity"]["todo_id"] == TODO
receipt = bound["heartbeat_receipt"]
assert receipt["settlement_identity"]["todo_id"] == TODO
original = {key: fixture[key].read_bytes() for key in ("registry", "state", "index")}
original_events = _quota_events(fixture["runtime"])
authority = _authority(fixture)
result = call_progress_cli(fixture, *fixture["refresh"],
expected_code=0 if expected_outcomes else 1)
Expand Down Expand Up @@ -488,11 +497,27 @@ def test_typed_progress_cli_preserves_replan_authority(
assert result["ok"] is False and result["appended"] is False
rejection = result["replan_transition"]
assert rejection["obligation_id"] == obligation["obligation_id"]
assert rejection["host_action"] == "write_typed_semantic_delta"
assert rejection["reason_code"] == reason_code
assert rejection["next_cli_actions"]
# A refusal leaves the original Turn admitted. The next action reads
# its context before any new writeback can address the refusal.
recovery = result["settlement_recovery"]
assert rejection["host_action"] == "read_current_replan_context"
assert recovery["settlement_identity"] == receipt["settlement_identity"]
assert recovery["settlement_progress"]["next_step"] == "durable_writeback"
assert "settlement_owed" not in recovery
assert rejection["next_cli_actions"] == recovery["next_cli_actions"]
restart_effect_runtime(wait_for_exit=True)
assert call_progress_cli(fixture, *fixture["refresh"], expected_code=1) == result
[command] = rejection["next_cli_actions"]
code, recovered = _run_generated_cli(
command, registry_path=fixture["registry"], cwd=fixture["project"])
assert code == 0, recovered
replay = recovered["heartbeat_receipt"]
assert replay["status"] == "replayed"
for key in ("event_id", "recorded_at", "settlement_identity"):
assert replay[key] == receipt[key]
assert recovered["autonomous_replan_obligation"]["obligation_id"] == obligation["obligation_id"]
assert _quota_events(fixture["runtime"]) == original_events
assert {key: fixture[key].read_bytes() for key in original} == original
assert fixture["registry"].read_bytes() == original["registry"]
displayed = parse_active_state_todos(fixture["state"].read_text(), item_limit=None)["agent_todos"]["items"]
Expand Down
10 changes: 9 additions & 1 deletion tests/control_plane/test_research_checkpoint_capture.py
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,15 @@ def test_worker_keeps_work_running_when_staged_capture_declaration_fails(
"MODEL_NAME": "fixture", "REASONING_EFFORT": "high", "OPENAI_BASE_URL": "http://localhost:8123/v1",
"OPENAI_API_KEY": "fixture-key", "LOOPX_RUNTIME_ROOT": str(runtime),
"LOOPX_REGISTRY": str(registry), "LOOPX_GOAL_ID": cli.GOAL_ID, "LOOPX_AGENT_ID": cli.AGENT_ID}
monkeypatch.setattr("benchmark.runtime.worker.heartbeat_body", lambda *a, **kw: "original work contract")
from benchmark.runtime import worker
real_heartbeat_body = worker.heartbeat_body

def checked_heartbeat_body(*args, **kwargs):
# Keep the real admission owner; only the synthetic model text is fixed.
real_heartbeat_body(*args, **kwargs)
return "original work contract"

monkeypatch.setattr(worker, "heartbeat_body", checked_heartbeat_body)
off = run_once(env)
assert off["ok"] and "checkpoint" not in off
(project / "host-input.txt").unlink()
Expand Down
Loading