Skip to content

fix(analysis-ledger): drop allOf schema rules unsupported by item reg… #204

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
danielvaughn merged 1 commit into from
Oct 30, 2026
Merged

fix(analysis-ledger): drop allOf schema rules unsupported by item reg… #204

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion capabilities/analysis-ledger/capability.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
schema: 1
name: analysis-ledger
version: "0.2.1"
version: "0.2.2"
description: >
Persistent single-agent code analysis over a durable Agent Output worklist.
The agent records analysis runs, ranked targets, and evidence-bearing claims,
Expand Down
92 changes: 3 additions & 89 deletions capabilities/analysis-ledger/items.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,29 +60,7 @@
class AnalysisRun(BaseModel):
"""One bounded analysis of a repository revision."""

model_config = ConfigDict(
extra="forbid",
json_schema_extra={
"allOf": [
{
"if": {
"properties": {
"run_state": {
"enum": ["paused", "completed", "exhausted", "stale"]
}
},
"required": ["run_state"],
},
"then": {
"required": ["stop_reason"],
"properties": {
"stop_reason": {"type": "string", "minLength": 1}
},
},
}
]
},
)
model_config = ConfigDict(extra="forbid")

title: str = Field(..., min_length=1, max_length=512)
run_key: RunKey = Field(
Expand Down Expand Up @@ -157,40 +135,7 @@ def validate_run_invariants(self) -> "AnalysisRun":
class AnalysisTarget(BaseModel):
"""One bounded code unit or path waiting on the analysis frontier."""

model_config = ConfigDict(
extra="forbid",
json_schema_extra={
"allOf": [
{
"if": {
"properties": {
"target_state": {"enum": ["analyzed", "blocked", "skipped"]}
},
"required": ["target_state"],
},
"then": {
"required": ["summary"],
"properties": {"summary": {"type": "string", "minLength": 1}},
},
},
{
"if": {
"properties": {"target_state": {"const": "analyzed"}},
"required": ["target_state"],
},
"then": {
"required": ["evidence_refs"],
"properties": {
"evidence_refs": {
"type": "array",
"minItems": 1,
}
},
},
},
]
},
)
model_config = ConfigDict(extra="forbid")

title: str = Field(..., min_length=1, max_length=512)
run_ref: str = Field(
Expand Down Expand Up @@ -255,38 +200,7 @@ def require_terminal_summary(self) -> "AnalysisTarget":
class AnalysisClaim(BaseModel):
"""An evidence-bearing hypothesis or conclusion about one analysis target."""

model_config = ConfigDict(
extra="forbid",
json_schema_extra={
"allOf": [
{
"if": {
"properties": {
"disposition": {"const": "verified"},
"claim_category": {"const": "vulnerability"},
},
"required": ["disposition", "claim_category"],
},
"then": {
"required": ["weakness", "severity", "impact"],
"properties": {
"weakness": {"type": "string", "minLength": 1},
"severity": {
"enum": [
"critical",
"high",
"medium",
"low",
"info",
]
},
"impact": {"type": "string", "minLength": 1},
},
},
}
]
},
)
model_config = ConfigDict(extra="forbid")

title: str = Field(..., min_length=1, max_length=512)
run_ref: str = Field(
Expand Down
146 changes: 108 additions & 38 deletions capabilities/analysis-ledger/tests/test_analysis_ledger.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from pathlib import Path
from types import SimpleNamespace

import jsonschema
import pytest
import yaml

Expand Down Expand Up @@ -185,50 +184,121 @@ def test_item_schemas_reject_unknown_fields() -> None:
)


def test_exported_json_schemas_enforce_terminal_invariants() -> None:
UNSUPPORTED_SCHEMA_KEYWORDS = {"allOf", "if", "then", "else", "not"}


def _schema_keywords(node: t.Any) -> set[str]:
if isinstance(node, dict):
found = set(node)
for value in node.values():
found |= _schema_keywords(value)
return found
if isinstance(node, list):
found = set()
for value in node:
found |= _schema_keywords(value)
return found
return set()


@pytest.mark.parametrize(
"model_name", ["AnalysisRun", "AnalysisTarget", "AnalysisClaim"]
)
def test_exported_json_schemas_use_only_supported_keywords(model_name: str) -> None:
items = _load_module("analysis_ledger_json_schemas", "items.py")

with pytest.raises(jsonschema.ValidationError):
jsonschema.validate(
{
"title": "Authentication analysis",
"run_key": "0123456789abcdef01234567",
"objective": "Find authentication bypasses",
"repository": "/repo",
"revision": "deadbeef",
"run_state": "completed",
},
items.AnalysisRun.model_json_schema(),
schema = getattr(items, model_name).model_json_schema()

assert not _schema_keywords(schema) & UNSUPPORTED_SCHEMA_KEYWORDS


def _run_payload(**overrides: t.Any) -> dict[str, t.Any]:
run_material = "/repo\ndeadbeef\nFind authentication bypasses"
return {
"title": "Authentication analysis",
"run_key": sha256(run_material.encode()).hexdigest()[:24],
"objective": "Find authentication bypasses",
"repository": "/repo",
"revision": "deadbeef",
**overrides,
}


def _target_payload(**overrides: t.Any) -> dict[str, t.Any]:
return {
"title": "Analyze login",
"run_ref": "run-auth",
"target_key": "src/auth.py::login",
"kind": "function",
"location": "src/auth.py:40",
"rationale": "Externally reachable authentication path",
**overrides,
}


@pytest.mark.parametrize("run_state", ["paused", "completed", "exhausted", "stale"])
def test_run_validator_requires_stop_reason(run_state: str) -> None:
items = _load_module("analysis_ledger_run_invariants", "items.py")

with pytest.raises(ValueError, match="requires stop_reason"):
items.AnalysisRun.model_validate(_run_payload(run_state=run_state))

run = items.AnalysisRun.model_validate(
_run_payload(run_state=run_state, stop_reason="Frontier exhausted.")
)
assert run.stop_reason == "Frontier exhausted."


@pytest.mark.parametrize("target_state", ["analyzed", "blocked", "skipped"])
def test_target_validator_requires_summary(target_state: str) -> None:
items = _load_module("analysis_ledger_target_summary", "items.py")

with pytest.raises(ValueError, match="requires summary"):
items.AnalysisTarget.model_validate(
_target_payload(target_state=target_state, evidence_refs=["src/auth.py:42"])
)

with pytest.raises(jsonschema.ValidationError):
jsonschema.validate(
{
"title": "Verified SQL injection",
"run_ref": "run-auth",
"target_ref": "run-auth-target-login",
"claim_category": "vulnerability",
"statement": "User input reaches the query.",
"disposition": "verified",
"evidence_refs": ["src/auth.py:42"],
},
items.AnalysisClaim.model_json_schema(),

def test_target_validator_requires_evidence_refs_when_analyzed() -> None:
items = _load_module("analysis_ledger_target_evidence", "items.py")

with pytest.raises(ValueError, match="requires evidence_refs"):
items.AnalysisTarget.model_validate(
_target_payload(target_state="analyzed", summary="No bypass found.")
)

with pytest.raises(jsonschema.ValidationError):
jsonschema.validate(
{
"title": "Analyze login",
"run_ref": "run-auth",
"target_key": "src/auth.py::login",
"kind": "function",
"location": "src/auth.py:40",
"rationale": "Externally reachable authentication path",
"target_state": "analyzed",
"summary": "No bypass found.",
},
items.AnalysisTarget.model_json_schema(),
target = items.AnalysisTarget.model_validate(
_target_payload(
target_state="analyzed",
summary="No bypass found.",
evidence_refs=["src/auth.py:42"],
)
)
assert target.evidence_refs == ["src/auth.py:42"]


@pytest.mark.parametrize("missing", ["weakness", "severity", "impact"])
def test_claim_validator_requires_verified_vulnerability_fields(missing: str) -> None:
items = _load_module("analysis_ledger_claim_invariants", "items.py")
payload = {
"title": "Verified SQL injection",
"run_ref": "run-auth",
"target_ref": "run-auth-target-login",
"claim_category": "vulnerability",
"statement": "User input reaches the query.",
"disposition": "verified",
"evidence_refs": ["src/auth.py:42"],
"weakness": "CWE-89",
"severity": "high",
"impact": "Authentication bypass.",
}
items.AnalysisClaim.model_validate(payload)

del payload[missing]
with pytest.raises(
ValueError, match=f"verified vulnerability claim requires {missing}"
):
items.AnalysisClaim.model_validate(payload)


def test_manifest_builds_combined_report_item_schema() -> None:
Expand Down
Loading