feat: agentic wf testing in harness worker (+ fix agent-e2e)

.github/workflows/agent-e2e.yml

@@ -33,7 +33,44 @@ jobs:
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
       CONDUCTOR_SERVER_URL: http://localhost:8080/api
+      # Default model for every suite. The CI OpenAI project only allows the
+      # gpt-5.6 / gpt-6 families; gpt-6-luna is OpenAI's replacement for
+      # gpt-4o-mini, which shuts down on 2027-01-14.
+      CONDUCTOR_AGENT_LLM_MODEL: openai/gpt-6-luna
     steps:
+      # A dead key, exhausted quota, retired model or provider outage makes
+      # every agent test fail at its client timeout, three times over with
+      # reruns — the job then hits timeout-minutes before pytest prints a
+      # single traceback. Call each model the suites use directly, before any
+      # setup: the HTTP status and body name the cause in seconds.
+      - name: LLM provider check
+        timeout-minutes: 1
+        run: |
+          fail=0
+          probe() {  # label, url, curl args...
+            local label=$1 url=$2; shift 2
+            local code
+            code=$(curl -sS -m 15 -o "$RUNNER_TEMP/probe.json" -w '%{http_code}' "$url" "$@" \
+              -H 'content-type: application/json') || true
+            echo "$label: HTTP $code"
+            if [ "$code" != 200 ]; then
+              echo "::error::$label check failed (HTTP $code; 000 = no response in 15s)"
+              cat "$RUNNER_TEMP/probe.json" 2>/dev/null || true; echo
+              fail=1
+            fi
+          }
+          # Responses API, as the server uses; 16 is its minimum max_output_tokens.
+          openai_model=${CONDUCTOR_AGENT_LLM_MODEL#openai/}
+          probe "OpenAI $openai_model" https://api.openai.com/v1/responses \
+            -H "authorization: Bearer $OPENAI_API_KEY" \
+            -d "{\"model\":\"$openai_model\",\"input\":\"ping\",\"max_output_tokens\":16,\"reasoning\":{\"effort\":\"none\"}}"
+          for model in claude-sonnet-4-6 claude-sonnet-4-5; do
+            probe "Anthropic $model" https://api.anthropic.com/v1/messages \
+              -H "x-api-key: $ANTHROPIC_API_KEY" -H 'anthropic-version: 2023-06-01' \
+              -d "{\"model\":\"$model\",\"max_tokens\":5,\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}]}"
+          done
+          exit $fail
+
       - name: Checkout code
         uses: actions/checkout@v4
 
@@ -69,7 +106,12 @@ jobs:
           # mcp 2.0 dropped mcp.server.fastmcp, which mcp-testkit's server and
           # tool modules still import — hold the resolver on the 1.x line until
           # the testkit ships a 2.x-compatible release.
-          pip install pytest pytest-asyncio pytest-xdist pytest-rerunfailures mcp-testkit 'mcp<2'
+          # pytest 9.2.0 silently breaks --dist=loadgroup with pytest-xdist 3.8.0:
+          # xdist_group marks are ignored, so grouped suites (shared ports, shared
+          # server credentials) run in parallel and collide. Fixed on xdist master
+          # (pytest-dev/pytest-xdist#1388), not yet released. Once a pytest-xdist
+          # release after 3.8.0 is out, drop the pytest cap and require that xdist.
+          pip install 'pytest<9.2' pytest-asyncio pytest-xdist pytest-rerunfailures mcp-testkit 'mcp<2'
 
       - name: Start mcp-testkit
         run: |
@@ -115,6 +157,16 @@ jobs:
           pytest e2e/ -v --tb=short -n 3 --dist=loadgroup \
             --junitxml=results/junit-e2e.xml
 
+      # The uploaded server.log is lost when the run is re-run, and a run
+      # killed at timeout-minutes never prints pytest's tracebacks — keep the
+      # server's own errors in the job log, which survives both.
+      - name: Show server errors
+        if: failure() || cancelled()
+        run: |
+          [ -f server.log ] || exit 0
+          grep -nE ' (ERROR|WARN) |Exception|[Qq]uota|rate.?limit|\b(401|403|404|429)\b' server.log \
+            | grep -vE 'cannot init (Cohere|Mistral)' | tail -150
+
       # e2e/conftest.py pytest.skip()s the whole session when the server is
       # unreachable — without this guard a boot failure after the health gate
       # (or a future gate regression) would yield a green job that ran nothing.

e2e/test_suite11_langgraph.py

@@ -36,6 +36,11 @@
 TIMEOUT = 120
 SERVER_URL = os.environ.get("CONDUCTOR_SERVER_URL", "http://localhost:8080/api")
 BASE_URL = SERVER_URL.rstrip("/").replace("/api", "")
+# LangChain takes the bare OpenAI model name. No temperature: reasoning
+# models (gpt-6-luna) reject it unless reasoning effort is "none".
+OPENAI_MODEL = os.environ.get(
+    "CONDUCTOR_AGENT_LLM_MODEL", "openai/gpt-4o-mini"
+).removeprefix("openai/")
 
 
 # ═══════════════════════════════════════════════════════════════════════════
@@ -88,7 +93,7 @@ def multiply(a: int, b: int) -> str:
 # Module-level LLM instance — needed for StateGraph node functions that
 # reference `llm` via closure. The LLM detection in the serializer finds
 # module-level variables via func.__globals__ but not closure variables.
-_module_llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+_module_llm = ChatOpenAI(model=OPENAI_MODEL)
 
 # Tests
 # ═══════════════════════════════════════════════════════════════════════════
@@ -104,7 +109,7 @@ def test_framework_detection(self):
         """create_react_agent graph -> detect_framework returns 'langgraph'."""
         from langchain.agents import create_agent
 
-        llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+        llm = ChatOpenAI(model=OPENAI_MODEL)
         graph = create_agent(llm, tools=[], name="detect_test")
 
         framework = detect_framework(graph)
@@ -119,7 +124,7 @@ def test_hello_world_full_extraction(self):
         """create_agent(llm, tools=[]) serializes to full extraction path."""
         from langchain.agents import create_agent
 
-        llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+        llm = ChatOpenAI(model=OPENAI_MODEL)
         graph = create_agent(llm, tools=[], name="hello_world_test")
 
         raw_config, workers = serialize_langgraph(graph)
@@ -164,7 +169,7 @@ def test_react_tools_full_extraction(self):
         """create_agent(llm, tools=[calculate, count_words]) -> full extraction."""
         from langchain.agents import create_agent
 
-        llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+        llm = ChatOpenAI(model=OPENAI_MODEL)
         graph = create_agent(
             llm, tools=[calculate, count_words], name="react_tools_test"
         )
@@ -175,7 +180,7 @@ def test_react_tools_full_extraction(self):
         assert "model" in raw_config, (
             f"[React] 'model' missing. Keys: {list(raw_config.keys())}"
         )
-        assert "gpt-4o-mini" in str(raw_config["model"]), (
+        assert OPENAI_MODEL in str(raw_config["model"]), (
             f"[React] Wrong model: {raw_config['model']}"
         )
 
@@ -478,7 +483,7 @@ def test_tool_schema_is_json_schema(self):
         """React agent tool parameters must be valid JSON Schema."""
         from langchain.agents import create_agent
 
-        llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+        llm = ChatOpenAI(model=OPENAI_MODEL)
         graph = create_agent(
             llm, tools=[calculate, multiply], name="schema_test"
         )
@@ -533,7 +538,7 @@ def test_compile_hello_world_via_server(self):
 
         from langchain.agents import create_agent
 
-        llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+        llm = ChatOpenAI(model=OPENAI_MODEL)
         graph = create_agent(llm, tools=[], name="compile_hello_world")
 
         raw_config, _workers = serialize_langgraph(graph)
@@ -556,7 +561,7 @@ def test_compile_react_tools_via_server(self):
 
         from langchain.agents import create_agent
 
-        llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+        llm = ChatOpenAI(model=OPENAI_MODEL)
         graph = create_agent(
             llm, tools=[calculate, count_words], name="compile_react_tools"
         )
@@ -579,7 +584,7 @@ def test_compile_stategraph_via_server(self):
         if not _server_available():
             pytest.skip("Server not available")
 
-        llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+        llm = ChatOpenAI(model=OPENAI_MODEL)
 
         class State(TypedDict):
             query: str
@@ -655,5 +660,5 @@ def _make_react_agent_with_multiply():
     """Build a react agent with multiply tool for runtime execution."""
     from langchain.agents import create_agent
 
-    llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+    llm = ChatOpenAI(model=OPENAI_MODEL)
     return create_agent(llm, tools=[multiply], name="e2e_lg_runtime")

e2e/test_suite25_media_input.py

@@ -42,6 +42,7 @@
 ]
 
 TIMEOUT = 120
+OPENAI_MODEL = os.environ.get("CONDUCTOR_AGENT_LLM_MODEL", "openai/gpt-4o-mini")
 
 # ── Test image (self-contained) ───────────────────────────────────────────────
 # A 600x200 PNG rendering the exact text "MELON7391" (black on white), committed
@@ -82,7 +83,7 @@
 
 # Positive test: Anthropic is skipped (no image reaches the model — see above).
 POSITIVE_CASES = [
-    pytest.param("OPENAI_API_KEY", "openai/gpt-4o-mini", id="openai"),
+    pytest.param("OPENAI_API_KEY", OPENAI_MODEL, id="openai"),
     pytest.param(
         "ANTHROPIC_API_KEY",
         "anthropic/claude-sonnet-4-5",
@@ -94,7 +95,7 @@
 # Counterfactual: both providers should COMPLETE and simply not emit the token
 # (no media is sent at all), so neither is expected to fail.
 COUNTERFACTUAL_CASES = [
-    pytest.param("OPENAI_API_KEY", "openai/gpt-4o-mini", id="openai"),
+    pytest.param("OPENAI_API_KEY", OPENAI_MODEL, id="openai"),
     pytest.param("ANTHROPIC_API_KEY", "anthropic/claude-sonnet-4-5", id="anthropic"),
 ]
 

e2e/test_suite7_media_tools.py

@@ -116,7 +116,10 @@ class TestSuite7MediaTools:
 
     # ── Image: OpenAI ─────────────────────────────────────────────────
 
-    @pytest.mark.xfail(reason="OpenAI removed dall-e-2 default; model passthrough issue in Conductor runtime")
+    @pytest.mark.skip(
+        reason="dall-e-3 has been removed from OpenAI's API ('model does not exist'), and "
+        "its gpt-image-* replacements are not enabled in the CI OpenAI project"
+    )
     def test_image_openai(self, runtime, model):
         """Generate image via OpenAI DALL-E 3."""
         if not os.environ.get("OPENAI_API_KEY"):
@@ -175,6 +178,7 @@ def test_image_gemini(self, runtime, model):
 
     # ── Audio: OpenAI ─────────────────────────────────────────────────
 
+    @pytest.mark.skip(reason="The CI OpenAI project does not have access to tts-1 (HTTP 403)")
     def test_audio_openai(self, runtime, model):
         """Generate audio via OpenAI TTS-1."""
         if not os.environ.get("OPENAI_API_KEY"):