Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/user/pstack.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ provider. Until you run it, pstack matches its defaults to models advertised by
available providers, including models offered through another harness. Models without a reasoning
setting keep their own defaults.

Run setup in the project whose work should inform your choices. When blind model evaluations are
enabled, setup can use that project's scored subagent work to explain a recommendation. The saved
seats apply across the environment. [Model evaluations](model-evaluations.md)

List several models on a role to give pstack a pool that picks one per spawn. Arena runners,
architect runners, and interrogate reviewers are panels that run every model listed.

Expand Down
3 changes: 2 additions & 1 deletion evals/mcp-tool-use/cases/model-guidance.ndjson
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
{"id":"model-guidance-matches-delegated-task","area":"orchestration","kind":"guard","pattern":"unrelated task scores mistaken for model suitability","experience":"Model evaluation guidance is grouped by task and scope; selecting the highest unrelated coding average would ignore the requested review evidence.","privacy":"commit-safe","setup":{"role":"top","mocks":{"orchestrator_capabilities":{"parentThreadId":"33333333-3333-4333-8333-333333333333","inheritedProviderInstanceId":"claudeAgent","inheritedModel":"parent-model","inheritedOptions":[],"runtimeMode":"full-access","interactionMode":"default","providers":[{"providerInstanceId":"opencode","driverKind":"opencode","displayName":"OpenCode","models":[{"id":"lab/worker-alpha"},{"id":"lab/worker-beta"}],"canRunChildTask":true,"canRunCrossProviderChildTask":true,"constraints":[]}],"features":{"appOwnedSubagents":true,"asyncPolling":true,"cancellation":true,"batchThreadCreation":true,"threadManagement":true,"incrementalThreadRead":true,"scheduledTasks":true,"maxBatchThreads":20},"modelGuidance":[{"modelSelection":{"instanceId":"opencode","model":"lab/worker-alpha"},"evaluatorSelection":{"instanceId":"claudeAgent","model":"judge"},"rubricVersion":"1","scope":"subagent","taskCategory":"coding","sampleCount":20,"insufficientEvidenceCount":0,"meanScores":{"taskCompletion":4,"correctness":4,"instructionFollowing":4,"verification":null},"scoreCounts":{"taskCompletion":20,"correctness":20,"instructionFollowing":20,"verification":0},"confidence":"observed","promptChars":{"min":200,"median":500,"max":900},"strengths":[],"weaknesses":[],"promptAdvice":[],"latestAt":"2026-10-01T00:00:00.000Z"},{"modelSelection":{"instanceId":"opencode","model":"lab/worker-alpha"},"evaluatorSelection":{"instanceId":"claudeAgent","model":"judge"},"rubricVersion":"1","scope":"subagent","taskCategory":"review","sampleCount":1,"insufficientEvidenceCount":0,"meanScores":{"taskCompletion":1,"correctness":1,"instructionFollowing":1,"verification":null},"scoreCounts":{"taskCompletion":1,"correctness":1,"instructionFollowing":1,"verification":0},"confidence":"provisional","promptChars":{"min":200,"median":500,"max":900},"strengths":[],"weaknesses":[],"promptAdvice":[],"latestAt":"2026-10-01T00:00:00.000Z"},{"modelSelection":{"instanceId":"opencode","model":"lab/worker-beta"},"evaluatorSelection":{"instanceId":"claudeAgent","model":"judge"},"rubricVersion":"1","scope":"subagent","taskCategory":"review","sampleCount":12,"insufficientEvidenceCount":0,"meanScores":{"taskCompletion":3.5,"correctness":3.5,"instructionFollowing":3.5,"verification":null},"scoreCounts":{"taskCompletion":12,"correctness":12,"instructionFollowing":12,"verification":0},"confidence":"observed","promptChars":{"min":200,"median":500,"max":900},"strengths":[],"weaknesses":[],"promptAdvice":["Include explicit acceptance criteria and ask for evidence for each finding."],"latestAt":"2026-10-01T00:00:00.000Z"}]},"delegate_task":"default"},"world":{"repo":"acme/app","files":{"src/cache.ts":"export const cached = new Map();\n"}}},"prompt":"Delegate one read-only review of src/cache.ts. Choose a model using the relevant recent evaluation evidence in T3 orchestration and adapt the review brief to the observed prompting advice. The reviewer must report only concrete issues and must not change files.","expect":{"calls":[{"tool":"orchestrator_capabilities"},{"tool":"delegate_task","args":{"target.providerInstanceId":"opencode","target.model":"lab/worker-beta","task":"~acceptance|evidence"}}],"maxCalls":{"delegate_task":1},"notCalls":["t3_thread_launch","create_threads"]},"source":{"request":"Blind model evaluation and task-specific orchestration guidance"},"notes":"Synthetic Acme fixture. Scores are observational, not a benchmark; the guard checks task/scope relevance and a precise delegation brief."}
{"id":"model-guidance-matches-delegated-task","area":"orchestration","kind":"guard","pattern":"unrelated task scores mistaken for model suitability","experience":"Model evaluation guidance is grouped by task and scope; selecting the highest unrelated coding average would ignore the requested review evidence.","privacy":"commit-safe","setup":{"role":"top","mocks":{"orchestrator_capabilities":{"parentThreadId":"33333333-3333-4333-8333-333333333333","inheritedProviderInstanceId":"claudeAgent","inheritedModel":"parent-model","inheritedOptions":[],"runtimeMode":"full-access","interactionMode":"default","providers":[{"providerInstanceId":"opencode","driverKind":"opencode","displayName":"OpenCode","models":[{"id":"lab/worker-alpha"},{"id":"lab/worker-beta"}],"canRunChildTask":true,"canRunCrossProviderChildTask":true,"constraints":[]}],"features":{"appOwnedSubagents":true,"asyncPolling":true,"cancellation":true,"batchThreadCreation":true,"threadManagement":true,"incrementalThreadRead":true,"scheduledTasks":true,"maxBatchThreads":20},"modelGuidance":[{"modelSelection":{"instanceId":"opencode","model":"lab/worker-alpha"},"evaluatorSelection":{"instanceId":"claudeAgent","model":"judge"},"rubricVersion":"1","scope":"subagent","taskCategory":"coding","sampleCount":20,"insufficientEvidenceCount":0,"meanScores":{"taskCompletion":4,"correctness":4,"instructionFollowing":4,"verification":null},"scoreCounts":{"taskCompletion":20,"correctness":20,"instructionFollowing":20,"verification":0},"confidence":"observed","promptChars":{"min":200,"median":500,"max":900},"strengths":[],"weaknesses":[],"promptAdvice":[],"latestAt":"2026-10-01T00:00:00.000Z"},{"modelSelection":{"instanceId":"opencode","model":"lab/worker-alpha"},"evaluatorSelection":{"instanceId":"claudeAgent","model":"judge"},"rubricVersion":"1","scope":"subagent","taskCategory":"review","sampleCount":1,"insufficientEvidenceCount":0,"meanScores":{"taskCompletion":1,"correctness":1,"instructionFollowing":1,"verification":null},"scoreCounts":{"taskCompletion":1,"correctness":1,"instructionFollowing":1,"verification":0},"confidence":"provisional","promptChars":{"min":200,"median":500,"max":900},"strengths":[],"weaknesses":[],"promptAdvice":[],"latestAt":"2026-10-01T00:00:00.000Z"},{"modelSelection":{"instanceId":"opencode","model":"lab/worker-beta"},"evaluatorSelection":{"instanceId":"claudeAgent","model":"judge"},"rubricVersion":"1","scope":"subagent","taskCategory":"review","sampleCount":12,"insufficientEvidenceCount":0,"meanScores":{"taskCompletion":3.5,"correctness":3.5,"instructionFollowing":3.5,"verification":null},"scoreCounts":{"taskCompletion":12,"correctness":12,"instructionFollowing":12,"verification":0},"confidence":"observed","promptChars":{"min":200,"median":500,"max":900},"strengths":[],"weaknesses":[],"promptAdvice":["Include explicit acceptance criteria and ask for evidence for each finding."],"latestAt":"2026-10-01T00:00:00.000Z"}]},"delegate_task":"default"},"world":{"repo":"acme/app","files":{"src/cache.ts":"export const cached = new Map();\n"}}},"prompt":"Delegate one read-only review of src/cache.ts. Choose a model using the relevant recent evaluation evidence in T3 orchestration and adapt the review brief to the observed prompting advice. The reviewer must report only concrete issues and must not change files.","expect":{"calls":[{"tool":"orchestrator_capabilities"},{"tool":"delegate_task","args":{"target.providerInstanceId":"opencode","target.model":"lab/worker-beta","task":"~evidence"}}],"maxCalls":{"delegate_task":1},"notCalls":["t3_thread_launch","create_threads"]},"source":{"request":"Blind model evaluation and task-specific orchestration guidance"},"notes":"Synthetic Acme fixture. Scores are observational, not a benchmark; the guard checks task/scope relevance and a precise delegation brief."}
{"id":"model-guidance-preserves-user-choice","area":"orchestration","kind":"guard","pattern":"learned guidance overrides explicit model choice","experience":"Observed weaknesses should improve the brief without silently replacing the user's selected model.","privacy":"commit-safe","setup":{"role":"top","mocks":{"orchestrator_capabilities":{"parentThreadId":"33333333-3333-4333-8333-333333333333","inheritedProviderInstanceId":"claudeAgent","inheritedModel":"parent-model","inheritedOptions":[],"runtimeMode":"full-access","interactionMode":"default","providers":[{"providerInstanceId":"opencode","driverKind":"opencode","displayName":"OpenCode","models":[{"id":"lab/worker-alpha"},{"id":"lab/worker-beta"}],"canRunChildTask":true,"canRunCrossProviderChildTask":true,"constraints":[]}],"features":{"appOwnedSubagents":true,"asyncPolling":true,"cancellation":true,"batchThreadCreation":true,"threadManagement":true,"incrementalThreadRead":true,"scheduledTasks":true,"maxBatchThreads":20},"modelGuidance":[{"modelSelection":{"instanceId":"opencode","model":"lab/worker-alpha"},"evaluatorSelection":{"instanceId":"claudeAgent","model":"judge"},"rubricVersion":"1","scope":"subagent","taskCategory":"coding","sampleCount":20,"insufficientEvidenceCount":0,"meanScores":{"taskCompletion":4,"correctness":4,"instructionFollowing":4,"verification":null},"scoreCounts":{"taskCompletion":20,"correctness":20,"instructionFollowing":20,"verification":0},"confidence":"observed","promptChars":{"min":200,"median":500,"max":900},"strengths":[],"weaknesses":[],"promptAdvice":[],"latestAt":"2026-10-01T00:00:00.000Z"},{"modelSelection":{"instanceId":"opencode","model":"lab/worker-alpha"},"evaluatorSelection":{"instanceId":"claudeAgent","model":"judge"},"rubricVersion":"1","scope":"subagent","taskCategory":"review","sampleCount":1,"insufficientEvidenceCount":0,"meanScores":{"taskCompletion":1,"correctness":1,"instructionFollowing":1,"verification":null},"scoreCounts":{"taskCompletion":1,"correctness":1,"instructionFollowing":1,"verification":0},"confidence":"provisional","promptChars":{"min":200,"median":500,"max":900},"strengths":[],"weaknesses":[],"promptAdvice":[],"latestAt":"2026-10-01T00:00:00.000Z"},{"modelSelection":{"instanceId":"opencode","model":"lab/worker-beta"},"evaluatorSelection":{"instanceId":"claudeAgent","model":"judge"},"rubricVersion":"1","scope":"subagent","taskCategory":"review","sampleCount":12,"insufficientEvidenceCount":0,"meanScores":{"taskCompletion":3.5,"correctness":3.5,"instructionFollowing":3.5,"verification":null},"scoreCounts":{"taskCompletion":12,"correctness":12,"instructionFollowing":12,"verification":0},"confidence":"observed","promptChars":{"min":200,"median":500,"max":900},"strengths":[],"weaknesses":[],"promptAdvice":["Include explicit acceptance criteria and ask for evidence for each finding."],"latestAt":"2026-10-01T00:00:00.000Z"}]},"delegate_task":"default"},"world":{"repo":"acme/app"}},"prompt":"Use delegate_task to have lab/worker-alpha on OpenCode do one read-only review of src/cache.ts. That is the model I want, even if the learned guidance prefers another one. Include an explicit requirement to cite evidence for findings.","expect":{"calls":[{"tool":"delegate_task","args":{"target.providerInstanceId":"opencode","target.model":"lab/worker-alpha","task":"~evidence"}}],"callsMustNot":[{"tool":"delegate_task","args":{"target.model":"lab/worker-beta"}}],"maxCalls":{"delegate_task":1},"notCalls":["t3_thread_launch","create_threads"]},"source":{"request":"Blind model evaluation with advisory orchestration guidance"},"notes":"Synthetic Acme fixture. A low sample review score is no authorization to change the requested model."}
{"id":"pstack-seat-uses-project-review-evidence","area":"orchestration","kind":"bad","pattern":"setup assigns seats from maker profiles alone","experience":"A model acting as a judge appeared in the catalog briefs while relevant worker evidence was outside the inventory response.","source":{"origin":"Authored from a pstack setup investigation."},"privacy":"commit-safe","setup":{"role":"top","priorTurns":[{"role":"user","text":"Use setup-pstack to recommend the first review seat. We already completed the inventory, budget and maker profiles. Both candidates qualify equally and have no effort option. Keep all other roles as they are."},{"role":"assistant","toolCalls":[{"tool":"Bash","input":{"command":"cat /home/user/pstack/skills/setup-pstack/SKILL.md"},"result":"","resultFile":"packages/pstack/upstream/skills/setup-pstack/SKILL.md"}]},{"role":"assistant","toolCalls":[{"tool":"orchestrator_capabilities","input":{"allModels":true},"result":{"parentThreadId":"33333333-3333-4333-8333-333333333333","inheritedProviderInstanceId":"claudeAgent","inheritedModel":"parent-model","inheritedOptions":[],"runtimeMode":"full-access","interactionMode":"default","providers":[{"providerInstanceId":"opencode","driverKind":"opencode","displayName":"OpenCode","models":[{"id":"lab/worker-alpha"},{"id":"lab/worker-beta"}],"canRunChildTask":true,"canRunCrossProviderChildTask":true,"constraints":[]}],"features":{"appOwnedSubagents":true,"asyncPolling":true,"cancellation":true,"batchThreadCreation":true,"threadManagement":true,"incrementalThreadRead":true,"scheduledTasks":true,"maxBatchThreads":20},"modelGuidance":[{"modelSelection":{"instanceId":"opencode","model":"lab/worker-alpha"},"evaluatorSelection":{"instanceId":"opencode","model":"lab/worker-beta"},"rubricVersion":"1","scope":"subagent","taskCategory":"coding","sampleCount":20,"insufficientEvidenceCount":0,"meanScores":{"taskCompletion":4,"correctness":4,"instructionFollowing":4,"verification":null},"scoreCounts":{"taskCompletion":20,"correctness":20,"instructionFollowing":20,"verification":0},"confidence":"observed","promptChars":{"min":200,"median":500,"max":900},"strengths":[],"weaknesses":[],"promptAdvice":[],"latestAt":"2026-10-01T00:00:00.000Z"}]}}]}],"mocks":{"orchestrator_capabilities?search=lab/worker-alpha":{"providers":[{"providerInstanceId":"opencode","models":[{"id":"lab/worker-alpha"}]}],"modelGuidance":[{"modelSelection":{"instanceId":"opencode","model":"lab/worker-alpha"},"evaluatorSelection":{"instanceId":"opencode","model":"lab/worker-beta"},"rubricVersion":"1","scope":"subagent","taskCategory":"coding","sampleCount":20,"insufficientEvidenceCount":0,"meanScores":{"taskCompletion":4,"correctness":4,"instructionFollowing":4,"verification":null},"scoreCounts":{"taskCompletion":20,"correctness":20,"instructionFollowing":20,"verification":0},"confidence":"observed","promptChars":{"min":200,"median":500,"max":900},"strengths":[],"weaknesses":[],"promptAdvice":[],"latestAt":"2026-10-01T00:00:00.000Z"},{"modelSelection":{"instanceId":"opencode","model":"lab/worker-alpha"},"evaluatorSelection":{"instanceId":"opencode","model":"lab/worker-beta"},"rubricVersion":"1","scope":"subagent","taskCategory":"review","sampleCount":12,"insufficientEvidenceCount":0,"meanScores":{"taskCompletion":3.5,"correctness":3.5,"instructionFollowing":3.5,"verification":null},"scoreCounts":{"taskCompletion":12,"correctness":12,"instructionFollowing":12,"verification":0},"confidence":"observed","promptChars":{"min":200,"median":500,"max":900},"strengths":[],"weaknesses":[],"promptAdvice":["Include explicit acceptance criteria and ask for evidence for each finding."],"latestAt":"2026-10-01T00:00:00.000Z"}]},"orchestrator_capabilities?search=lab/worker-beta":{"providers":[{"providerInstanceId":"opencode","models":[{"id":"lab/worker-beta"}]}]}}},"prompt":"Choose between lab/worker-alpha and lab/worker-beta for delegated reviews. Start with the recommended model ID on its own line. Explain what supports that choice and what we know about the other candidate. Wait before writing.","expect":{"calls":[{"tool":"orchestrator_capabilities","args":{"search":"lab/worker-alpha"}},{"tool":"orchestrator_capabilities","args":{"search":"lab/worker-beta"}}],"finalTextMust":["^[^\\n]*worker-alpha","12|twelve","review","judge|judg|evaluat"],"notCalls":["delegate_task"],"maxCalls":{"orchestrator_capabilities":2}}}
Loading
Loading