Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions docs/cli/model-routing.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,37 @@ policies.
the CLI will use an available fallback model for the current turn or the
remainder of the session.

### Numerical Routing Configuration

When numerical routing is active, Gemini CLI scores request complexity from 1
to 100. By default, the CLI uses a threshold to choose between Flash and Pro
models. You can customize this routing logic using `routing.numericalRules` in
your `settings.json` file.

This allows you to define score bands and map them to specific model IDs or
aliases:

```json
{
"routing": {
"numericalRules": [
{ "maxScore": 10, "model": "local-gemma" },
{ "maxScore": 30, "model": "gemini-2.5-flash" },
{ "maxScore": 80, "model": "gemini-3-flash-preview" },
{ "maxScore": 100, "model": "gemini-3-pro-preview" }
]
}
}
```

- **`maxScore`**: The upper complexity score boundary (inclusive, 1–100) for
which this rule applies.
- **`model`**: The model ID or alias to route to.

Rules are evaluated in ascending order of their `maxScore`. If a request's score
exceeds all configured `maxScore` bands, it falls back to the model configured
in the final score band.

### Local Model Routing (Experimental)

Gemini CLI supports using a local model for routing decisions. When configured,
Expand Down
9 changes: 9 additions & 0 deletions docs/reference/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,15 @@ their corresponding top-level category object in your `settings.json` file.

<!-- SETTINGS-AUTOGEN:START -->

#### `routing`

- **`routing.numericalRules`** (array):
- **Description:** Score bands matched in ascending maxScore order. Each
maxScore is inclusive. Scores above all configured bands use the model in
the final band.
- **Default:** `[]`
- **Requires restart:** Yes

#### `policyPaths`

- **`policyPaths`** (array):
Expand Down
1 change: 1 addition & 0 deletions packages/cli/src/config/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1104,6 +1104,7 @@ export async function loadCliConfig(
format: (argv.outputFormat ?? settings.output?.format) as OutputFormat,
},
gemmaModelRouter: settings.experimental?.gemmaModelRouter,
routing: settings.routing,
adk: settings.experimental?.adk,
fakeResponses: argv.fakeResponses,
fakeResponsesNonStrict: argv.fakeResponsesNonStrict,
Expand Down
86 changes: 86 additions & 0 deletions packages/cli/src/config/settings-validation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -538,6 +538,92 @@ describe('settings-validation', () => {
expect(formatted).toContain('...and 1 more errors.');
}
});

it('should accept valid routing.numericalRules configuration', () => {
const validSettings = {
routing: {
numericalRules: [
{ maxScore: 10, model: 'local-gemma' },
{ maxScore: 30, model: 'gemini-2.5-flash' },
{ maxScore: 80, model: 'gemini-3-flash-preview' },
{ maxScore: 100, model: 'gemini-3-pro-preview' },
],
},
};

const result = validateSettings(validSettings);
expect(result.success).toBe(true);
});

it('should accept empty routing.numericalRules array', () => {
const validSettings = {
routing: {
numericalRules: [],
},
};

const result = validateSettings(validSettings);
expect(result.success).toBe(true);
});

it('should reject routing when it is not an object', () => {
const invalidSettings = {
routing: 'invalid-string',
};

const result = validateSettings(invalidSettings);
expect(result.success).toBe(false);
expect(result.error?.issues[0]?.path).toEqual(['routing']);
});

it('should reject routing.numericalRules when it is not an array', () => {
const invalidSettings = {
routing: {
numericalRules: 'not-an-array',
},
};

const result = validateSettings(invalidSettings);
expect(result.success).toBe(false);
expect(result.error?.issues[0]?.path).toEqual([
'routing',
'numericalRules',
]);
});

it('should reject routing.numericalRules when maxScore is not a number', () => {
const invalidSettings = {
routing: {
numericalRules: [{ maxScore: 'not-a-number', model: 'flash' }],
},
};

const result = validateSettings(invalidSettings);
expect(result.success).toBe(false);
expect(result.error?.issues[0]?.path).toEqual([
'routing',
'numericalRules',
0,
'maxScore',
]);
});

it('should reject routing.numericalRules when model is not a string', () => {
const invalidSettings = {
routing: {
numericalRules: [{ maxScore: 50, model: 12345 }],
},
};

const result = validateSettings(invalidSettings);
expect(result.success).toBe(false);
expect(result.error?.issues[0]?.path).toEqual([
'routing',
'numericalRules',
0,
'model',
]);
});
});

describe('settingsZodSchema', () => {
Expand Down
44 changes: 44 additions & 0 deletions packages/cli/src/config/settingsSchema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import {
type MCPServerConfig,
type RequiredMcpServerConfig,
type BugCommandSettings,
type NumericalRoutingRule,
type TelemetrySettings,
type AuthType,
type AgentOverride,
Expand Down Expand Up @@ -173,6 +174,49 @@ const SETTINGS_SCHEMA = {
},
},

routing: {
type: 'object',
label: 'Routing',
category: 'Model',
requiresRestart: true,
default: {},
description: 'Configure numerical complexity score routing.',
showInDialog: false,
properties: {
numericalRules: {
type: 'array',
label: 'Numerical Routing Rules',
category: 'Model',
requiresRestart: true,
default: [] as NumericalRoutingRule[],
description:
'Score bands matched in ascending maxScore order. Each maxScore is inclusive. Scores above all configured bands use the model in the final band.',
showInDialog: false,
items: {
type: 'object',
properties: {
maxScore: {
type: 'number',
label: 'Maximum Score',
category: 'Model',
requiresRestart: true,
default: 100,
description: 'Maximum score for this routing rule.',
},
model: {
type: 'string',
label: 'Model ID',
category: 'Model',
requiresRestart: true,
default: '',
description: 'Model alias or ID to route to.',
},
},
},
},
},
},

policyPaths: pathArraySetting(
'Policy Paths',
'Additional policy files or directories to load.',
Expand Down
2 changes: 2 additions & 0 deletions packages/cli/src/test-utils/mockConfig.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,8 @@ export const createMockConfig = (overrides: Partial<Config> = {}): Config =>
getCompressionThreshold: vi.fn().mockResolvedValue(undefined),
getUserCaching: vi.fn().mockResolvedValue(false),
getNumericalRoutingEnabled: vi.fn().mockResolvedValue(false),
hasCustomNumericalRoutingRules: vi.fn().mockReturnValue(false),
getNumericalRoutingRules: vi.fn().mockReturnValue([]),
getClassifierThreshold: vi.fn().mockResolvedValue(undefined),
getBannerTextNoCapacityIssues: vi.fn().mockResolvedValue(''),
getBannerTextCapacityIssues: vi.fn().mockResolvedValue(''),
Expand Down
34 changes: 34 additions & 0 deletions packages/core/src/config/config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -697,6 +697,40 @@ describe('Server Config (config.ts)', () => {
});
});

describe('numericalRoutingRules', () => {
it('should return empty rules and hasCustomNumericalRoutingRules false when not configured', () => {
const config = new Config(baseParams);
expect(config.hasCustomNumericalRoutingRules()).toBe(false);
expect(config.getNumericalRoutingRules()).toEqual([]);
});

it('should return configured rules and hasCustomNumericalRoutingRules true', () => {
const rules = [
{ maxScore: 20, model: 'gemini-2.5-flash' },
{ maxScore: 80, model: 'gemini-3-pro' },
];
const config = new Config({
...baseParams,
routing: {
numericalRules: rules,
},
});
expect(config.hasCustomNumericalRoutingRules()).toBe(true);
expect(config.getNumericalRoutingRules()).toEqual(rules);
});

it('should return false for hasCustomNumericalRoutingRules when numericalRules is empty', () => {
const config = new Config({
...baseParams,
routing: {
numericalRules: [],
},
});
expect(config.hasCustomNumericalRoutingRules()).toBe(false);
expect(config.getNumericalRoutingRules()).toEqual([]);
});
});

describe('getGemini31LaunchedSync', () => {
it.each([AuthType.USE_GEMINI, AuthType.USE_VERTEX_AI, AuthType.GATEWAY])(
'should return true for %s',
Expand Down
24 changes: 24 additions & 0 deletions packages/core/src/config/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,19 @@ export interface OutputSettings {
format?: OutputFormat;
}

export interface NumericalRoutingRule {
/** Inclusive upper bound for the complexity score (1-100). */
maxScore: number;
/** Model ID or configured model alias to use for this score band. */
model: string;
}

export interface RoutingSettings {
numericalRules?: NumericalRoutingRule[];
}

export type NumericalRoutingSettings = RoutingSettings;

export interface GemmaModelRouterSettings {
enabled?: boolean;
autoStartServer?: boolean;
Expand Down Expand Up @@ -695,6 +708,7 @@ export interface ConfigParameters {
directWebFetch?: boolean;
policyUpdateConfirmationRequest?: PolicyUpdateConfirmationRequest;
output?: OutputSettings;
routing?: RoutingSettings;
gemmaModelRouter?: GemmaModelRouterSettings;
adk?: ADKSettings;
disableModelRouterForAuth?: AuthType[];
Expand Down Expand Up @@ -930,6 +944,7 @@ export class Config implements McpContext, AgentLoopContext {
| undefined;
private readonly outputSettings: OutputSettings;

private readonly routingSettings?: RoutingSettings;
private readonly gemmaModelRouter: GemmaModelRouterSettings;
private readonly agentSessionNoninteractiveEnabled: boolean;
private readonly agentSessionInteractiveEnabled: boolean;
Expand Down Expand Up @@ -1370,6 +1385,7 @@ export class Config implements McpContext, AgentLoopContext {
this.outputSettings = {
format: params.output?.format ?? OutputFormat.TEXT,
};
this.routingSettings = params.routing;
this.gemmaModelRouter = {
enabled: params.gemmaModelRouter?.enabled ?? false,
autoStartServer: params.gemmaModelRouter?.autoStartServer ?? true,
Expand Down Expand Up @@ -3920,6 +3936,14 @@ export class Config implements McpContext, AgentLoopContext {
return this.gemmaModelRouter;
}

getNumericalRoutingRules(): NumericalRoutingRule[] {
return this.routingSettings?.numericalRules ?? [];
}

hasCustomNumericalRoutingRules(): boolean {
return this.getNumericalRoutingRules().length > 0;
}

getAgentSessionNoninteractiveEnabled(): boolean {
return (
process.env['GEMINI_CLI_EXP_AGENT'] === 'true' ||
Expand Down
22 changes: 22 additions & 0 deletions packages/core/src/routing/modelRouterService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ describe('ModelRouterService', () => {
mockLocalLiteRtLmClient,
);
vi.spyOn(mockConfig, 'getNumericalRoutingEnabled').mockResolvedValue(true);
vi.spyOn(mockConfig, 'hasCustomNumericalRoutingRules').mockReturnValue(
false,
);
vi.spyOn(mockConfig, 'getResolvedClassifierThreshold').mockResolvedValue(
90,
);
Expand Down Expand Up @@ -144,6 +147,25 @@ describe('ModelRouterService', () => {
expect(compositeStrategyArgs[1]).toBe('agent-router');
});

it('should prioritize numerical routing when custom rules are configured', () => {
vi.spyOn(mockConfig, 'hasCustomNumericalRoutingRules').mockReturnValue(
true,
);
vi.mocked(CompositeStrategy).mockClear();

service = new ModelRouterService(mockConfig);

const compositeStrategyArgs = vi.mocked(CompositeStrategy).mock.calls[0];
const childStrategies = compositeStrategyArgs[0];

expect(childStrategies[0]).toBeInstanceOf(FallbackStrategy);
expect(childStrategies[1]).toBeInstanceOf(OverrideStrategy);
expect(childStrategies[2]).toBeInstanceOf(ApprovalModeStrategy);
expect(childStrategies[3]).toBeInstanceOf(NumericalClassifierStrategy);
expect(childStrategies[4]).toBeInstanceOf(ClassifierStrategy);
expect(childStrategies[5]).toBeInstanceOf(DefaultStrategy);
});

describe('route()', () => {
const strategyDecision: RoutingDecision = {
model: 'strategy-chosen-model',
Expand Down
13 changes: 11 additions & 2 deletions packages/core/src/routing/modelRouterService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,13 @@ export class ModelRouterService {
// Approval mode is next.
strategies.push(new ApprovalModeStrategy());

const hasCustomNumericalRules =
this.config.hasCustomNumericalRoutingRules?.() ?? false;
if (hasCustomNumericalRules) {
// Custom numerical rules take precedence so they can target any model.
strategies.push(new NumericalClassifierStrategy());
}

// Then, if enabled, the Gemma classifier is used.
if (this.config.getGemmaModelRouterSettings()?.enabled) {
strategies.push(new GemmaClassifierStrategy());
Expand All @@ -54,8 +61,10 @@ export class ModelRouterService {
// The generic classifier is next.
strategies.push(new ClassifierStrategy());

// The numerical classifier is next.
strategies.push(new NumericalClassifierStrategy());
// Preserve the existing default numerical routing order.
if (!hasCustomNumericalRules) {
strategies.push(new NumericalClassifierStrategy());
}

// The default strategy is the terminal strategy.
const terminalStrategy = new DefaultStrategy();
Expand Down
Loading