fix(evaluations): send criterion errors as errorMessage and skip criteria for rows that did not generate

packages/client/README.md

@@ -196,7 +196,7 @@ A tool result is now judge-prompt input. It stays literal for the same reason th
 
 **Judges are independent AI Configs, so handlers are routed per judge.** A judge may resolve to a different provider or mode than `generation`. Each judge selects its handler from `handler` or `handlers` by its provider and mode, with the same rule as generation. There is no mode fallback: a messages-mode judge never runs on an agent handler. A judge's prompt must also match its handler's mode. Judges get no tools: the harness passes an empty tool map and removes `tools` from the judge config.
 
-Judges are resolved through flag delivery, and handlers are matched to them, **before** any evaluation records are created — a missing judge, a judge that no handler covers, or a judge whose prompt does not match its handler fails the run up front rather than after the generation spend. One error lists every judge with a problem. After that point a criterion failure never aborts the run: an unparseable judge response, an out-of-range score, a raising handler or scorer, and a row whose generation errored each become a per-criterion `ERROR` event with a cause code (`invalid_judge_output`, `invalid_score`, `handler_raised`, `scorer_raised`, `generation_incomplete`) and a top-level `errorMessage`. Event *delivery* is different: the backend needs one result per `(row, criterion)` to finish row accounting, so if tracking a criterion event fails, every remaining result is still attempted and flushed and then `run()` raises — rather than polling to its timeout with the cause hidden.
+Judges are resolved through flag delivery, and handlers are matched to them, **before** any evaluation records are created — a missing judge, a judge that no handler covers, or a judge whose prompt does not match its handler fails the run up front rather than after the generation spend. One error lists every judge with a problem. After that point a criterion failure never aborts the run: an unparseable judge response, an out-of-range score, and a raising handler or scorer each become a per-criterion `ERROR` event carrying a top-level `errorMessage` that describes the failure. A row whose generation errored is not scored at all: no judge or scorer runs for it and no criterion event is sent, because its generation `ERROR` event already records the row as failed. Event *delivery* is different: the backend needs one result per `(row, criterion)` to finish row accounting, so if tracking a criterion event fails, every remaining result is still attempted and flushed and then `run()` raises — rather than polling to its timeout with the cause hidden.
 
 The client uses **lazy initialization**: importing the package does not connect to LaunchDarkly. The singleton is created automatically on the first API call that needs it (`config().invoke()`, `graph().invoke()`, `resolve_graph()`, etc.), as long as `LD_SDK_KEY` is set in the environment.
 

packages/client/src/launchdarkly_ai_server/evaluations/events.py

@@ -1,3 +1,3 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
@@ -52,7 +52,6 @@
     evaluation_version: int | None = None
     score: float | None = None
     reason: str | None = None
-    error: dict[str, Any] | None = None
     error_message: str | None = None
 
     def to_track_payload(self) -> dict[str, Any]:
@@ -76,7 +75,6 @@
             "latencyMs": self.latency_ms,
             "score": self.score,
             "reason": self.reason,
-            "error": self.error,
             "errorMessage": self.error_message,
         }
         return {key: value for key, value in payload.items() if value is not None}

packages/client/src/launchdarkly_ai_server/evaluations/runner.py

@@ -1002,13 +1002,6 @@ async def _run_scorer_for_result(
             "kind": "scorer",
             "started_at": started.isoformat().replace("+00:00", "Z"),
         }
-        if row.get("status") != "COMPLETE":
-            return self._criterion_error_result(
-                base,
-                started_clock,
-                "generation_incomplete",
-                "generation did not complete",
-            )
         dataset_row = DatasetRow(
             row_index=row["row_index"],
             input=row.get("input"),
@@ -1072,13 +1065,6 @@ async def _run_ld_judge_for_result(
             "variation_key": resolved.variation_key,
             "version": resolved.version,
         }
-        if row.get("status") != "COMPLETE":
-            return self._criterion_error_result(
-                base,
-                started_clock,
-                "generation_incomplete",
-                "generation did not complete",
-            )
         # The config is passed unrendered: the handler owns the single
         # parse_template pass, so ``{{...}}`` sequences inside generated output
         # or dataset values are never re-expanded into the judge prompt.
@@ -1182,7 +1168,7 @@ async def _run_criteria_for_results(
         evaluation_run_id: str,
         row_spans: Mapping[int, Any] | None = None,
     ) -> list[dict[str, Any]]:
-        """Run every (row, criterion) pair, bounded by the run's concurrency.
+        """Run every (generated row, criterion) pair, bounded by the run's concurrency.
 
         Each ``evaluate`` span parents under its row's span in ``row_spans``.
         """
@@ -1202,10 +1188,6 @@ async def run_one(
         ) -> dict[str, Any]:
             await controller.acquire()
             try:
-                if row.get("status") != "COMPLETE":
-                    # A row that did not generate is never scored, so there is
-                    # no judge or scorer call for an ``evaluate`` span to wrap.
-                    return await score(row, criterion)
                 row_index = row.get("row_index")
                 index = row_index if isinstance(row_index, int) else 0
                 with bind_evaluation_row(evaluation_run_id, index):
@@ -1226,9 +1208,17 @@ async def run_one(
             finally:
                 controller.release()
 
+        # A row that did not generate gets no criterion result at all: its
+        # generation ERROR event already settled it as failed, and a criterion
+        # event for it would be unaccounted for in the backend's row counts.
+        generated = [row for row in rows if row.get("status") == "COMPLETE"]
         return list(
             await asyncio.gather(
-                *(run_one(row, criterion) for row in rows for criterion in criteria)
+                *(
+                    run_one(row, criterion)
+                    for row in generated
+                    for criterion in criteria
+                )
             )
         )
 
@@ -1280,11 +1270,15 @@ def _emit_evaluation_events(
                         input_tokens=normalized_usage["input"],
                         output_tokens=normalized_usage["output"],
                     )
+                # The cause code stays internal: ingest reads only a top-level
+                # integer ``errorCode`` and ``errorMessage``, and no integers are
+                # assigned to criterion causes yet, so only the message is sent.
                 error = result.get("error")
                 error_message: str | None = None
                 if result["status"] == "ERROR":
-                    if isinstance(error, Mapping) and error.get("message"):
-                        error_message = str(error["message"])
+                    if isinstance(error, Mapping):
+                        message = error.get("message")
+                        error_message = str(message) if message else "Unknown error"
                     else:
                         error_message = str(error) if error else "Unknown error"
                 common_payload: dict[str, Any] = {
@@ -1306,7 +1300,6 @@ def _emit_evaluation_events(
                     "latency_ms": result["latency_ms"],
                     "score": result.get("score"),
                     "reason": result.get("reason"),
-                    "error": error,
                     "error_message": error_message,
                 }
                 payload_model: CriterionEventPayload

packages/client/tests/test_evaluations_run.py

@@ -13,6 +13,12 @@
 
 from launchdarkly_ai_server import NativeTool, create_handler
 from launchdarkly_ai_server.evaluations.api import HttpResponse
+from launchdarkly_ai_server.evaluations.runner import EvaluationsRunner
+from launchdarkly_ai_server.evaluations.types import (
+    DatasetRef,
+    EvaluationRef,
+    EvaluationRunRef,
+)
 from launchdarkly_ai_server.experimental.evaluations import (
     AIConfig,
     DatasetRow,
@@ -2108,6 +2114,25 @@ def judge_run_transport(*, summary: dict[str, Any] | None = None) -> SequencedTr
     )
 
 
+def capture_criterion_results(
+    monkeypatch: pytest.MonkeyPatch,
+) -> list[dict[str, Any]]:
+    """Record the internal criterion results, which carry the cause code.
+
+    The cause never reaches the wire, so this is where a test tells causes apart.
+    """
+    captured: list[dict[str, Any]] = []
+    original = EvaluationsRunner._run_criteria_for_results
+
+    async def spy(self: EvaluationsRunner, *args: Any, **kwargs: Any) -> Any:
+        results = await original(self, *args, **kwargs)
+        captured.extend(results)
+        return results
+
+    monkeypatch.setattr(EvaluationsRunner, "_run_criteria_for_results", spy)
+    return captured
+
+
 def accuracy_judge_variation(monkeypatch: pytest.MonkeyPatch) -> None:
     async def fake_extract_variation(
         key: str, context: dict[str, Any]
@@ -2167,6 +2192,7 @@ async def test_bad_judge_output_emits_error_event_instead_of_crashing(
 ) -> None:
     transport = judge_run_transport()
     accuracy_judge_variation(monkeypatch)
+    criterion_results = capture_criterion_results(monkeypatch)
     evals = init_evaluations(
         project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport
     )
@@ -2193,14 +2219,97 @@ async def handler(
     events = [call.args[2] for call in stub_sdk_client.track.call_args_list]
     judge_event = next(event for event in events if event.get("kind") == "judge")
     assert judge_event["status"] == "ERROR"
-    assert judge_event["error"]["code"] == expected_code
     # The bad value is shown as JSON, not a Python repr, as in every SDK.
-    assert judge_event["error"]["message"] == expected_message
-    assert judge_event["errorMessage"] == judge_event["error"]["message"]
+    assert judge_event["errorMessage"] == expected_message
+    # The cause is internal taxonomy: ingest reads only a top-level integer
+    # errorCode, none is assigned yet, and a nested error object is never sent.
+    assert "error" not in judge_event
+    assert "errorCode" not in judge_event
+    assert [result["error"]["code"] for result in criterion_results] == [expected_code]
     assert "score" not in judge_event
     stub_sdk_client.flush.assert_awaited()
 
 
+@pytest.mark.parametrize(
+    ("fn", "expected_code", "expected_message"),
+    [
+        (
+            lambda row, output: 1 / 0,
+            "scorer_raised",
+            "scorer fn raised: division by zero",
+        ),
+        (
+            lambda row, output: 3,
+            "invalid_score",
+            "scorer fn score must be between 0 and 1, got 3",
+        ),
+    ],
+)
+@pytest.mark.asyncio
+async def test_scorer_error_event_sends_only_a_top_level_error_message(
+    monkeypatch: pytest.MonkeyPatch,
+    stub_sdk_client: MagicMock,
+    fn: Callable[[Any, Any], Any],
+    expected_code: str,
+    expected_message: str,
+) -> None:
+    transport = judge_run_transport()
+    criterion_results = capture_criterion_results(monkeypatch)
+    evals = init_evaluations(
+        project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport
+    )
+
+    async def handler(*args: Any) -> dict[str, Any]:
+        return {"output": "generated"}
+
+    await evals.run(
+        key="support-qa",
+        dataset="golden",
+        handler=tagged(handler),
+        generation={"provider": "OpenAI", "model": "gpt-4o", "mode": "completion"},
+        criteria=[Scorer(name="exact", fn=fn)],
+    )
+
+    events = [call.args[2] for call in stub_sdk_client.track.call_args_list]
+    scorer_event = next(event for event in events if event.get("kind") == "scorer")
+    assert scorer_event["status"] == "ERROR"
+    assert scorer_event["errorMessage"] == expected_message
+    assert "error" not in scorer_event
+    assert "errorCode" not in scorer_event
+    assert [result["error"]["code"] for result in criterion_results] == [expected_code]
+
+
+def test_criterion_error_without_a_message_falls_back_to_unknown_error() -> None:
+    client = MagicMock()
+    runner = EvaluationsRunner(MagicMock())
+    runner._emit_evaluation_events(
+        client,
+        project_key="proj",
+        evaluation=EvaluationRef(id="evaluation-id", key="support-qa", version=3),
+        evaluation_run=EvaluationRunRef(
+            id="run-id", evaluation_id="evaluation-id", state="RUNNING"
+        ),
+        dataset=DatasetRef(id="dataset-id", key="golden"),
+        results=[
+            {
+                "row_index": 0,
+                "criterion_type": "scorer:exact",
+                "kind": "scorer",
+                "status": "ERROR",
+                "error": {"code": "scorer_raised", "message": ""},
+                "started_at": "2026-01-01T00:00:00Z",
+                "evaluated_at": "2026-01-01T00:00:00Z",
+                "latency_ms": 0,
+            }
+        ],
+    )
+
+    payload = client.track.call_args.args[2]
+    assert payload["errorMessage"] == "Unknown error"
+    assert "error" not in payload
+    assert "errorCode" not in payload
+
+
 @pytest.mark.asyncio
 async def test_generated_placeholders_are_not_expanded_into_judge_prompt(
     monkeypatch: pytest.MonkeyPatch,
@@ -2403,17 +2512,46 @@ async def handler(*args: object) -> dict[str, Any]:
 
 
 @pytest.mark.asyncio
-async def test_errored_generation_row_emits_generation_incomplete_criterion_event(
+async def test_a_row_that_did_not_generate_is_never_scored(
     monkeypatch: pytest.MonkeyPatch,
     stub_sdk_client: MagicMock,
 ) -> None:
-    transport = judge_run_transport(
-        summary={"statusCounts": {"total": 1, "passed": 0, "error": 1, "pending": 0}}
+    """Its generation ERROR event is the whole story for the row.
+
+    LaunchDarkly counts a failed row as soon as its generation event lands, so
+    a criterion result for it would be unaccounted for. The other row is
+    scored as usual.
+    """
+    transport = SequencedTransport(
+        [
+            response(200, {"id": "dataset-id", "name": "golden"}),
+            response(
+                200,
+                dataset_page(
+                    [
+                        {"rowIndex": 0, "input": "fails"},
+                        {"rowIndex": 1, "input": "works"},
+                    ],
+                    total=2,
+                ),
+            ),
+            response(201, {"id": "evaluation-id", "name": "support-qa", "version": 3}),
+            response(
+                201,
+                {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"},
+            ),
+            response(
+                200,
+                {"statusCounts": {"total": 2, "passed": 1, "error": 1, "pending": 0}},
+            ),
+        ]
     )
     accuracy_judge_variation(monkeypatch)
     evals = init_evaluations(
         project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport
     )
+    judged: list[str | None] = []
+    scored: list[int] = []
 
     async def handler(
         config: dict[str, Any],
@@ -2422,22 +2560,46 @@ async def handler(
         variables: dict[str, Any],
     ) -> dict[str, Any]:
         if "Judge" in prompt_text(config):
-            raise AssertionError("judges must not run for errored generations")
-        raise RuntimeError("provider unavailable")
+            judged.append(variables["input"])
+            return {"output": '{"score": 1, "reasoning": "ok"}'}
+        if user_input == "fails":
+            raise RuntimeError("provider unavailable")
+        return {"output": "generated"}
+
+    def scorer(row: Any, output: Any) -> float:
+        scored.append(row.row_index)
+        return 1.0
 
     result = await evals.run(
         key="support-qa",
         dataset="golden",
         handler=tagged(handler),
         generation={"provider": "OpenAI", "model": "gpt-4o", "mode": "completion"},
-        criteria=[Judge(key="$ld:ai:judge:accuracy")],
+        criteria=[
+            Judge(key="$ld:ai:judge:accuracy"),
+            Scorer(name="exact", fn=scorer),
+        ],
     )
 
     assert result.passed is False
-    events = [call.args[2] for call in stub_sdk_client.track.call_args_list]
-    judge_event = next(event for event in events if event.get("kind") == "judge")
-    assert judge_event["status"] == "ERROR"
-    assert judge_event["error"]["code"] == "generation_incomplete"
+    assert judged == ["works"]
+    assert scored == [1]
+    tracked = [
+        (call.args[0], call.args[2]) for call in stub_sdk_client.track.call_args_list
+    ]
+    generation = [p for name, p in tracked if name.endswith(":generation")]
+    criterion = [p for name, p in tracked if name.endswith(":criterion")]
+    assert [(p["rowIndex"], p["status"]) for p in generation] == [
+        (0, "ERROR"),
+        (1, "COMPLETE"),
+    ]
+    assert sorted((p["rowIndex"], p["kind"]) for p in criterion) == [
+        (1, "judge"),
+        (1, "scorer"),
+    ]
+    assert all(p["status"] == "COMPLETE" for p in criterion)
+    stub_sdk_client.flush.assert_awaited()
+    assert transport.requests[-1]["url"].endswith("/summary")
 
 
 @pytest.mark.asyncio