diff --git a/agent/provider.go b/agent/provider.go index 14cee59..4c3a538 100644 --- a/agent/provider.go +++ b/agent/provider.go @@ -44,8 +44,9 @@ type Config struct { // using the provider (0 = that window unlimited). RatePerSecond float64 RatePerMinute float64 - // ContextWindowK is the model's context window in K tokens (user-configured), - // used to size compaction thresholds. 0 = default; see CompactionWindow. + // ContextWindowK is the model's context window in K tokens (user-configured). + // CompactionWindow applies a 32K floor for compaction math; ContextWindowTokens + // does not, so routing and side-question budgets see the real window. 0 = default. ContextWindowK int // ThinkingType independently controls the thinking "on/off" field (thinking.type): // "" = don't send (default, compatible with models that don't support the field); @@ -189,6 +190,22 @@ func (c Config) CompactionWindow() int { return k * 1000 } +// ContextWindowTokens returns the model's actual configured context window in +// tokens (0/unset → 200K, capped at 1M) without the 32K compaction-math floor. +// llmpool pre-filtering and sidequestion budgets use this so an 8K/16K profile +// is not treated as 32K — that leak lets an oversize request hit the small +// model, 400, and never fail over (shouldFailover ignores 400s). +func (c Config) ContextWindowTokens() int { + k := c.ContextWindowK + if k <= 0 { + k = defaultWindowK + } + if k > maxWindowK { + k = maxWindowK + } + return k * 1000 +} + // compactionConfig builds the agent-core compaction config for a context window // in tokens. agentcore.NewSession wires the summarizer (same provider) when this // is set on Options.Compaction. diff --git a/agent/provider_window_test.go b/agent/provider_window_test.go new file mode 100644 index 0000000..c99e3a0 --- /dev/null +++ b/agent/provider_window_test.go @@ -0,0 +1,28 @@ +package agent + +import "testing" + +func TestContextWindowTokensDoesNotApplyCompactionFloor(t *testing.T) { + cases := []struct { + k, wantTokens, wantCompaction int + }{ + {0, 200_000, 200_000}, + {-1, 200_000, 200_000}, + {8, 8_000, 32_000}, + {16, 16_000, 32_000}, + {31, 31_000, 32_000}, + {32, 32_000, 32_000}, + {200, 200_000, 200_000}, + {1000, 1_000_000, 1_000_000}, + {2000, 1_000_000, 1_000_000}, + } + for _, tc := range cases { + cfg := Config{ContextWindowK: tc.k} + if got := cfg.ContextWindowTokens(); got != tc.wantTokens { + t.Errorf("ContextWindowK=%d: ContextWindowTokens()=%d, want %d", tc.k, got, tc.wantTokens) + } + if got := cfg.CompactionWindow(); got != tc.wantCompaction { + t.Errorf("ContextWindowK=%d: CompactionWindow()=%d, want %d", tc.k, got, tc.wantCompaction) + } + } +} diff --git a/server/llmpool.go b/server/llmpool.go index f892d34..a3901fc 100644 --- a/server/llmpool.go +++ b/server/llmpool.go @@ -65,7 +65,10 @@ func (s *Server) poolMember(p *db.LLMProfile, rank int) *llmpool.Member { return &llmpool.Member{ ID: p.ID, Name: p.Name, Model: p.Model, Format: p.Format, Priority: p.Priority, Active: p.IsDefault, Rank: rank, - WindowTokens: cfg.CompactionWindow(), Prov: prov, + // Actual window, not CompactionWindow(): that helper floors at 32K for + // compaction math, which would let an 8K profile accept a 16K request + // and then 400 without failover. + WindowTokens: cfg.ContextWindowTokens(), Prov: prov, } } @@ -103,7 +106,7 @@ func (s *Server) poolChain(headID int64, headProv llm.Provider, headCfg agent.Co members := []*llmpool.Member{{ ID: head.ID, Name: head.Name, Model: head.Model, Format: head.Format, Priority: head.Priority, Active: head.IsDefault, Rank: llmpool.RankActive, - WindowTokens: headCfg.CompactionWindow(), Prov: headProv, + WindowTokens: headCfg.ContextWindowTokens(), Prov: headProv, }} for _, p := range profs { if p.ID == headID { diff --git a/server/side_questions.go b/server/side_questions.go index 6b0ca08..641170b 100644 --- a/server/side_questions.go +++ b/server/side_questions.go @@ -43,7 +43,7 @@ type sideQuestionState struct { func sideModel(cfg agent.Config, id int64, name string) sidequestion.Model { b, _ := json.Marshal([]any{cfg.Format, cfg.BaseURL, cfg.Model, cfg.ThinkingType, cfg.ReasoningEffort, cfg.Stream, cfg.MaxTokensField, cfg.SessionHeaderKey, cfg.UserAgent}) h := sha256.Sum256(b) - return sidequestion.Model{ProfileID: id, Name: name, Format: string(cfg.Format), Model: cfg.Model, Identity: hex.EncodeToString(h[:]), Streaming: cfg.Stream, WindowTokens: cfg.CompactionWindow()} + return sidequestion.Model{ProfileID: id, Name: name, Format: string(cfg.Format), Model: cfg.Model, Identity: hex.EncodeToString(h[:]), Streaming: cfg.Stream, WindowTokens: cfg.ContextWindowTokens()} } func bindSideProvider(p llm.Provider, cfg agent.Config, id int64, name string) llm.Provider {