Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion contracts/agents-api/core-metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ The response has `object: "core.metrics"`, `range`, `service`, `execution`, `dat
| `service.revision` | The full source commit injected at build time; null for builds without one |
| `service.started_at` | When the process initialized |
| `service.execution_owner` | Whether this process holds the execution worker's database lease |
| `execution.slots_in_use`, `execution.slots_total` | Active Session reservations of the execution worker, and its capacity: [`OAC_EXECUTION_CONCURRENCY`](../../docs/configuration.md#settings). Environment input, Turns and file work share the slots; native Harness subprocesses are not counted |
| `execution.slots_in_use`, `execution.slots_total` | Execution slots in use and the execution worker's capacity: [`OAC_EXECUTION_CONCURRENCY`](../../docs/configuration.md#settings). Environment input, Turns and file work share the slots. A Turn waiting for function results holds none until it resumes, so resumed Turns can briefly take the count above the capacity. Native Harness subprocesses are not counted |
| `execution.queued_turns`, `execution.in_progress_turns` | Root Turns in those states, including Turns of deleted Sessions. Subagent Turns and input reserved for a preparing Environment are not counted |
| `execution.waiting_for_daemon` | Queued Turns whose Session's device is not connected |
| `execution.oldest_queued_seconds` | Age of the oldest queued Turn, from its `created_at` |
Expand Down
4 changes: 2 additions & 2 deletions contracts/agents-api/zh/core-metrics.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
---
title: "Core 指标"
source: contracts/agents-api/core-metrics.md
source_hash: 8be639e4dfa07567ea3c79873f6b7f3aee5ffb9889dc506ac334a2ceb5f686b9
source_hash: ffcae76e68ab3e8e5a8a90e3618a61c597661d38dcca647f6f96d23cb7a127a7
---

`GET /core/v1/metrics?range=1h|6h|24h|7d` 报告 Core 自身的健康状况:进程、执行队列与槽位、PostgreSQL 和后台任务。它要求 Core 密钥([Core 管理 API](admin-api.md))。
Expand Down Expand Up @@ -35,7 +35,7 @@ source_hash: 8be639e4dfa07567ea3c79873f6b7f3aee5ffb9889dc506ac334a2ceb5f686b9
| `service.revision` | 构建时注入的完整源代码提交;未注入时为 null |
| `service.started_at` | 进程初始化时间 |
| `service.execution_owner` | 此进程是否持有执行 worker 的数据库租约 |
| `execution.slots_in_use`, `execution.slots_total` | 执行 worker 的活动 Session 预留数量及容量:[`OAC_EXECUTION_CONCURRENCY`](../../../docs/zh/configuration.md#settings)。Environment 输入、Turn 和文件工作共享槽位;不统计原生 Harness 子进程 |
| `execution.slots_in_use`, `execution.slots_total` | 执行 worker 已占用的执行槽位及其容量:[`OAC_EXECUTION_CONCURRENCY`](../../../docs/zh/configuration.md#settings)。Environment 输入、Turn 和文件工作共享槽位。等待函数结果的 Turn 在恢复前不占槽位,因此恢复的 Turn 可能让占用数短暂超过容量。不统计原生 Harness 子进程 |
| `execution.queued_turns`, `execution.in_progress_turns` | 处于相应状态的根 Turn,包括已删除 Session 的 Turn。不统计 Subagent Turn 和为准备中 Environment 预留的输入 |
| `execution.waiting_for_daemon` | Session 设备未连接的排队 Turn |
| `execution.oldest_queued_seconds` | 最早排队 Turn 自 `created_at` 起的年龄 |
Expand Down
17 changes: 17 additions & 0 deletions services/core/internal/execution/delivery.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,8 @@ func (d *Dispatcher) deliver(ctx context.Context, tenantID, sessionID string, pe
var cancelSent time.Time
var cancelReply <-chan cancellationResult
functions := &functionExchange{kind: request.AgentKind, turns: d.SessionsReader, sessions: d.sessionExecution, tenant: tenantID, session: sessionID, turn: runID, tools: request.FunctionTools, abandoned: map[string]bool{}}
waiting := false
defer d.markWaiting(&waiting, false)
done := false
cancelCtx, stopCancellation := context.WithCancel(ctx)
defer stopCancellation()
Expand Down Expand Up @@ -285,6 +287,7 @@ func (d *Dispatcher) deliver(ctx context.Context, tenantID, sessionID string, pe
result.ErrorCode = "execution_state_changed"
return
}
d.markWaiting(&waiting, turn.Status == sessions.TurnWaiting)
if !turn.CancelRequestedAt.IsZero() {
cancelReply = requestCancellation(cancelCtx, peer, runID)
cancelSent = time.Now()
Expand Down Expand Up @@ -343,6 +346,20 @@ func (d *Dispatcher) deliver(ctx context.Context, tenantID, sessionID string, pe
}
}

// markWaiting records whether the delivered Turn waits for function results,
// which frees the Worker's execution slot until it resumes.
func (d *Dispatcher) markWaiting(current *bool, waiting bool) {
if d.waitingTurns == nil || *current == waiting {
return
}
*current = waiting
if waiting {
d.waitingTurns.Add(1)
} else {
d.waitingTurns.Add(-1)
}
}

func (r *Result) mergeObservation(env proto.Envelope) error {
switch env.Type {
case proto.TypeError:
Expand Down
4 changes: 4 additions & 0 deletions services/core/internal/execution/dispatcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"errors"
"sync/atomic"
"time"

v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1"
Expand All @@ -28,6 +29,9 @@ type Snapshot struct {

type Dispatcher struct {
notifications *executionNotifications
// waitingTurns counts the Worker's delivered Turns that wait for function
// results; their Sessions stay reserved but hold no execution slot.
waitingTurns *atomic.Int64
// sessionExecution runs the Session execution operations on the lease;
// Bind sets it from Owner.Sessions.
sessionExecution *sessions.ExecutionOperations
Expand Down
35 changes: 22 additions & 13 deletions services/core/internal/execution/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"errors"
"slices"
"sync"
"sync/atomic"
"time"

v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1"
Expand Down Expand Up @@ -82,6 +83,7 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher, owner Owner) (_ *W
return nil, errors.New("execution worker requires the deployment execution operations")
}
owned.notifications = &executionNotifications{}
owned.waitingTurns = new(atomic.Int64)
worker := &Worker{concurrency: dispatcher.MaxConcurrentExecutions, dispatcher: owned, lease: owner.Lease, directoryReads: make(chan directoryReadRequest), fileWrites: make(chan fileWriteRequest), stopped: make(chan struct{}), scheduleWake: make(chan struct{}, 1), enrolledConnections: make(map[string]*runtimeConnection)}
worker.runtimes, err = newRuntimeManager(owner, owned.Deployment, owned.DeploymentReader, owned.SessionsReader, owned.Registry, owned.ManagedRuntimes)
if err != nil {
Expand Down Expand Up @@ -271,7 +273,9 @@ func (w *Worker) Run(ctx context.Context) (runErr error) {
w.observeWorkerClosed(closeLease(ctx, w.lease))
}()
active := make(map[string]bool)
w.observeSlots(len(active))
// A Session waiting for function results stays reserved but holds no slot.
slots := func() int { return len(active) - int(w.dispatcher.waitingTurns.Load()) }
w.observeSlots(slots())
type completion struct {
id string
err error
Expand Down Expand Up @@ -315,12 +319,12 @@ func (w *Worker) Run(ctx context.Context) (runErr error) {
case err := <-lifecycleDone:
return err
case request := <-w.fileWrites:
if request.ctx.Err() != nil || active[request.environment.AttachedSessionID] || len(active) == w.executionConcurrency() {
if request.ctx.Err() != nil || active[request.environment.AttachedSessionID] || slots() >= w.executionConcurrency() {
request.result <- fileWriteResult{err: ErrExecutionUnavailable}
continue
}
active[request.environment.AttachedSessionID] = true
w.observeSlots(len(active))
w.observeSlots(slots())
running.Add(1)
go func() {
defer running.Done()
Expand All @@ -329,21 +333,21 @@ func (w *Worker) Run(ctx context.Context) (runErr error) {
continue
case write := <-writesCompleted:
delete(active, write.request.environment.AttachedSessionID)
w.observeSlots(len(active))
w.observeSlots(slots())
write.request.result <- write.result
if !rescanOnCompletion {
continue
}
rescanOnCompletion = false
case request := <-w.directoryReads:
if request.ctx.Err() != nil || reads == w.executionConcurrency() || (!active[request.environment.AttachedSessionID] && len(active) == w.executionConcurrency()) {
if request.ctx.Err() != nil || reads == w.executionConcurrency() || (!active[request.environment.AttachedSessionID] && slots() >= w.executionConcurrency()) {
request.reply(directoryReadResult{err: ErrExecutionUnavailable})
continue
}
reserved := !active[request.environment.AttachedSessionID]
if reserved {
active[request.environment.AttachedSessionID] = true
w.observeSlots(len(active))
w.observeSlots(slots())
}
reads++
running.Add(1)
Expand All @@ -361,7 +365,7 @@ func (w *Worker) Run(ctx context.Context) (runErr error) {
reads--
if read.id != "" {
delete(active, read.id)
w.observeSlots(len(active))
w.observeSlots(slots())
}
read.request.reply(read.result)
if read.id == "" || !rescanOnCompletion {
Expand All @@ -370,7 +374,7 @@ func (w *Worker) Run(ctx context.Context) (runErr error) {
rescanOnCompletion = false
case result := <-completed:
delete(active, result.id)
w.observeSlots(len(active))
w.observeSlots(slots())
if result.err != nil {
return result.err
}
Expand Down Expand Up @@ -400,7 +404,8 @@ func (w *Worker) Run(ctx context.Context) (runErr error) {
return err
}
}
if len(active) == w.executionConcurrency() {
w.observeSlots(slots())
if slots() >= w.executionConcurrency() {
w.observeSchedulerPoll(0, nil)
continue
}
Expand All @@ -412,14 +417,14 @@ func (w *Worker) Run(ctx context.Context) (runErr error) {
if !maintenance {
schedule.nextEnvironmentScan = time.Time{}
}
work, err := schedule.selectWork(ctx, w, devices, active)
w.observeSlots(len(active))
work, err := schedule.selectWork(ctx, w, devices, active, w.executionConcurrency()-slots())
w.observeSlots(slots())
if err != nil {
w.observeSchedulerPoll(0, err)
return err
}
w.observeSchedulerPoll(len(work), nil)
rescanOnCompletion = rescanOnCompletion && (len(active) > len(work) || len(active) == w.executionConcurrency())
rescanOnCompletion = rescanOnCompletion && (len(active) > len(work) || slots() >= w.executionConcurrency())
for _, item := range work {
running.Add(1)
go func() {
Expand All @@ -431,7 +436,11 @@ func (w *Worker) Run(ctx context.Context) (runErr error) {
err = w.runClaim(ctx, item.ExecutionWork)
}
w.dispatcher.notifications.notify(item.TenantID, item.SessionID)
completed <- completion{id: item.SessionID, err: err}
// Waiting Turns hold no slot, so more Turns can run than the buffer holds.
select {
case completed <- completion{id: item.SessionID, err: err}:
case <-ctx.Done():
}
}()
}
}
Expand Down
4 changes: 2 additions & 2 deletions services/core/internal/execution/worker_schedule.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ type scheduledWork struct {
reservationID string
}

func (s *workerSchedule) selectWork(ctx context.Context, w *Worker, devices []string, active map[string]bool) ([]scheduledWork, error) {
func (s *workerSchedule) selectWork(ctx context.Context, w *Worker, devices []string, active map[string]bool, free int) ([]scheduledWork, error) {
turns, err := w.dispatcher.SessionsReader.ListExecutionWork(ctx, s.turnCursor, []string{sessions.TurnQueued}, devices)
if err != nil {
return nil, err
Expand Down Expand Up @@ -50,7 +50,7 @@ func (s *workerSchedule) selectWork(ctx context.Context, w *Worker, devices []st
}
}
var selected []scheduledWork
for len(turns)+len(environments) > 0 && len(active) < w.executionConcurrency() {
for len(turns)+len(environments) > 0 && len(selected) < free {
var item scheduledWork
if len(environments) > 0 && (s.environmentFirst || len(turns) == 0) {
value := environments[0]
Expand Down
21 changes: 21 additions & 0 deletions services/core/tests/integration/function_worker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,3 +149,24 @@ func mcpBearerWorkerConfiguration(t *testing.T, h *dispatchHarness) (string, str
}
return string(raw), token
}

func TestWorkerWaitingTurnHoldsNoExecutionSlot(t *testing.T) {
h := newFunctionHarness(t)
h.d.MaxConcurrentExecutions = 1
asking := h.message("ask", "Ask the user")
ctx, cancel := context.WithCancel(t.Context())
worker := startWorker(t, ctx, h.s, h.d)
done := make(chan error, 1)
go func() { done <- worker.Run(ctx) }()
defer func() { cancel(); <-done }()
h.read(testExecutionRequest)
h.write(asking.TurnID, proto.TypeFunctionCall, proto.FunctionCallPayload{CallID: "ask", Name: "lookup_ticket", Arguments: json.RawMessage(`{"ticket":"42"}`)})
functionState(t, h, 1)
h.session = publicSession(t, h, "meanwhile")
other := h.message("meanwhile", "Run meanwhile")
if request := h.read(testExecutionRequest); request.ID != other.TurnID {
t.Fatal("unexpected Turn", request.ID)
}
h.write(other.TurnID, proto.TypeDone, proto.DonePayload{Content: "done"})
waitTurn(t, h, other.TurnID, sessions.TurnCompleted)
}
Loading