Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions .github/workflows/agent-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,42 @@ jobs:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
CONDUCTOR_SERVER_URL: http://localhost:8080/api
# Default model for every suite. The Orkes-Developers OpenAI project only
# allows the gpt-5.6 / gpt-6 families; gpt-6-luna is OpenAI's replacement
# for gpt-4o-mini, which shuts down on 2027-01-14.
CONDUCTOR_AGENT_LLM_MODEL: openai/gpt-6-luna
SKIP_IMAGE_GEN: 'true' # no image models in the Orkes-Developers OpenAI project; dall-e-3 is retired, so re-enabling also needs gpt-image-1
SKIP_TTS: 'true' # tts-1 is not enabled in the Orkes-Developers OpenAI project (HTTP 403)
steps:
# A dead key, exhausted quota, retired model or provider outage makes
# every agent test wait out its timeout — the job then hits timeout-minutes
# before the runner reports a single failure. Call the model the suites use
# directly, before any setup: the HTTP status and body name the cause in
# seconds. No suite calls Anthropic through the server, so only OpenAI is
# checked.
- name: LLM provider check
timeout-minutes: 1
run: |
fail=0
probe() { # label, url, curl args...
local label=$1 url=$2; shift 2
local code
code=$(curl -sS -m 15 -o "$RUNNER_TEMP/probe.json" -w '%{http_code}' "$url" "$@" \
-H 'content-type: application/json') || true
echo "$label: HTTP $code"
if [ "$code" != 200 ]; then
echo "::error::$label check failed (HTTP $code; 000 = no response in 15s)"
cat "$RUNNER_TEMP/probe.json" 2>/dev/null || true; echo
fail=1
fi
}
# Responses API, as the server uses; 16 is its minimum max_output_tokens.
openai_model=${CONDUCTOR_AGENT_LLM_MODEL#openai/}
probe "OpenAI $openai_model" https://api.openai.com/v1/responses \
-H "authorization: Bearer $OPENAI_API_KEY" \
-d "{\"model\":\"$openai_model\",\"input\":\"ping\",\"max_output_tokens\":16,\"reasoning\":{\"effort\":\"none\"}}"
exit $fail

- name: Checkout code
uses: actions/checkout@v4

Expand Down Expand Up @@ -119,6 +154,16 @@ jobs:
# 45-minute time budget. Matrix-suite concurrency is intentional.
npm run test:agent-e2e -- --maxWorkers=2

# The uploaded server.log is lost when the run is re-run, and a run
# killed at timeout-minutes never reports its test failures — keep the
# server's own errors in the job log, which survives both.
- name: Show server errors
if: failure() || cancelled()
run: |
[ -f server.log ] || exit 0
grep -nE ' (ERROR|WARN) |Exception|[Qq]uota|rate.?limit|\b(401|403|404|429)\b' server.log \
| grep -vE 'cannot init (Cohere|Mistral)' | tail -150 || true

# The TS suites fail hard when the server is down (no session-skip),
# so this guard is defense-in-depth against a future gate regression
# or a secrets-less fork run skipping everything.
Expand Down
19 changes: 11 additions & 8 deletions e2e/test_suite11_langgraph.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

import { describe, expect, beforeAll, afterAll, jest } from '@jest/globals';
import { AgentRuntime } from '@io-orkes/conductor-javascript/agents';
import { checkServerHealth, itSkipIf, expectMsg } from './helpers';
import { checkServerHealth, itSkipIf, expectMsg, MODEL } from './helpers';


jest.setTimeout(300_000); // ported from vitest describe({ timeout }) options
Expand All @@ -42,6 +42,9 @@ let detectFramework: any;

const SERVER_URL = process.env.CONDUCTOR_SERVER_URL ?? 'http://localhost:8080/api';
const BASE_URL = SERVER_URL.replace(/\/api$/, '');
// LangChain takes the bare OpenAI model name. No temperature: reasoning
// models (gpt-6-luna) reject it unless reasoning effort is "none".
const OPENAI_MODEL = MODEL.replace(/^openai\//, '');

try {
// Synchronous require instead of upstream top-level await import — this file
Expand Down Expand Up @@ -97,7 +100,7 @@ describe('Suite 11: LangGraph Integration', () => {
// ── 1. Framework detection ──────────────────────────────────────────

itSkipIf(!langGraphAvailable)('framework detection identifies langgraph', () => {
const llm = new ChatOpenAI({ model: 'gpt-4o-mini', temperature: 0 });
const llm = new ChatOpenAI({ model: OPENAI_MODEL });
const graph = createReactAgent({ llm, tools: [], name: 'detect_test' });

const framework = detectFramework(graph);
Expand Down Expand Up @@ -128,7 +131,7 @@ describe('Suite 11: LangGraph Integration', () => {
func: async ({ text }: { text: string }) => text.split('').reverse().join(''),
});

const llm = new ChatOpenAI({ model: 'gpt-4o-mini', temperature: 0 });
const llm = new ChatOpenAI({ model: OPENAI_MODEL });
const graph = createReactAgent({
llm,
tools: [calculateTool, countWordsTool, reverseTool],
Expand All @@ -148,7 +151,7 @@ describe('Suite 11: LangGraph Integration', () => {
expectMsg(
String(rawConfig.model),
'[FullExtract] wrong model',
).toContain('gpt-4o-mini');
).toContain(OPENAI_MODEL);

// Tools must be present with correct names
const toolsList = rawConfig.tools as Record<string, unknown>[];
Expand Down Expand Up @@ -182,7 +185,7 @@ describe('Suite 11: LangGraph Integration', () => {
func: async ({ a, b }: { a: number; b: number }) => String(a * b),
});

const llm = new ChatOpenAI({ model: 'gpt-4o-mini', temperature: 0 });
const llm = new ChatOpenAI({ model: OPENAI_MODEL });
const graph = createReactAgent({ llm, tools: [multiplyTool], name: 'e2e_lg_schema' });

(graph as any)._agentspan = {
Expand Down Expand Up @@ -454,7 +457,7 @@ describe('Suite 11: LangGraph Integration', () => {
return;
}

const llm = new ChatOpenAI({ model: 'gpt-4o-mini', temperature: 0 });
const llm = new ChatOpenAI({ model: OPENAI_MODEL });
const graph = createReactAgent({ llm, tools: [], name: 'compile_hello_world' });

(graph as any)._agentspan = {
Expand Down Expand Up @@ -499,7 +502,7 @@ describe('Suite 11: LangGraph Integration', () => {
func: async ({ text }: { text: string }) => `${text.split(/\s+/).length} words`,
});

const llm = new ChatOpenAI({ model: 'gpt-4o-mini', temperature: 0 });
const llm = new ChatOpenAI({ model: OPENAI_MODEL });
const graph = createReactAgent({
llm,
tools: [calculateTool, countWordsTool],
Expand Down Expand Up @@ -593,7 +596,7 @@ describe('Suite 11: LangGraph Integration', () => {
func: async ({ a, b }: { a: number; b: number }) => String(a * b),
});

const llm = new ChatOpenAI({ model: 'gpt-4o-mini', temperature: 0 });
const llm = new ChatOpenAI({ model: OPENAI_MODEL });
const graph = createReactAgent({
llm,
tools: [multiplyTool],
Expand Down
13 changes: 11 additions & 2 deletions e2e/test_suite16_streaming.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -251,9 +251,18 @@ describe('Suite 16: Streaming — HITL', () => {
const agent = new Agent({
name: uniqueName('s16_hitl_approve'),
model: MODEL,
// After a tool result the server re-sends the original user request, so
// gpt-6-luna often reads it as a new request and publishes a second
// article, which waits for an approval this test never gives. Asking it
// to count earlier calls stopped that in 4 of 4 CI runs; softer wording
// ("publish once", "if already published, stop") still failed.
instructions:
'You are a blog writer. Write a very short article (one paragraph) ' +
'about Python and publish it using the publish_article tool.',
'You are a blog writer. Write the requested post as one short paragraph ' +
'and publish it with the publish_article tool. The publish_article tool ' +
'can only ever be called once in this chat session. Keep track of how ' +
'many times it has been called, counting any earlier publish_article ' +
'calls and [TOOL RESULTS]. If that count is more than zero, do not call ' +
Comment thread
chrishagglund-ship-it marked this conversation as resolved.
Resolved
'it again: end the session by replying with a one-sentence confirmation.',
tools: [publishArticle],
});
const stream = await runtime.stream(agent, 'Write a short blog post about Python programming');
Expand Down
11 changes: 9 additions & 2 deletions e2e/test_suite7_media_tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,15 @@ function assertToolCompiled(
expectMsg(model, `[${stepName}] wrong model`).toBe(expectedModel);
}

const envFlag = (name: string) => process.env[name]?.toLowerCase() === 'true';

// ── Tests ───────────────────────────────────────────────────────────────

describe('Suite 7: Media Tools', () => {
(!process.env.OPENAI_API_KEY ? it.skip : it.failing)('image — OpenAI DALL-E 3', async () => {
// SKIP_IMAGE_GEN=true skips this: dall-e-3 has been removed from OpenAI's API
// ('model does not exist'), and its gpt-image-* replacements are not enabled
// in the Orkes-Developers OpenAI project.
(!process.env.OPENAI_API_KEY || envFlag('SKIP_IMAGE_GEN') ? it.skip : it.failing)('image — OpenAI DALL-E 3', async () => {
const img = imageTool({
name: 'gen_image',
description: 'Generate an image from text.',
Expand Down Expand Up @@ -143,7 +148,9 @@ describe('Suite 7: Media Tools', () => {
await assertMediaGenerated(result, 'Image/Gemini', 'GENERATE_IMAGE');
});

itSkipIf(!process.env.OPENAI_API_KEY)('audio — OpenAI TTS-1', async () => {
// SKIP_TTS=true skips this: the Orkes-Developers OpenAI project does not have
// access to tts-1 (HTTP 403).
itSkipIf(!process.env.OPENAI_API_KEY || envFlag('SKIP_TTS'))('audio — OpenAI TTS-1', async () => {
const aud = audioTool({
name: 'gen_audio',
description: 'Convert text to speech.',
Expand Down
Loading