fix(server): use the real context window for llmpool and sidequestion

8K/16K profiles were treated as 32K because WindowTokens came from
CompactionWindow(). They 400 and don't fail over.
This commit is contained in:
Coff0xc
2026-11-17 12:56:13 +00:00
parent 278de8b2e0
commit 14b93cecab
4 changed files with 53 additions and 5 deletions
+19 -2
View File
@@ -44,8 +44,9 @@ type Config struct {
// using the provider (0 = that window unlimited).
RatePerSecond float64
RatePerMinute float64
// ContextWindowK is the model's context window in K tokens (user-configured),
// used to size compaction thresholds. 0 = default; see CompactionWindow.
// ContextWindowK is the model's context window in K tokens (user-configured).
// CompactionWindow applies a 32K floor for compaction math; ContextWindowTokens
// does not, so routing and side-question budgets see the real window. 0 = default.
ContextWindowK int
// ThinkingType independently controls the thinking "on/off" field (thinking.type):
// "" = don't send (default, compatible with models that don't support the field);
@@ -189,6 +190,22 @@ func (c Config) CompactionWindow() int {
return k * 1000
}
// ContextWindowTokens returns the model's actual configured context window in
// tokens (0/unset → 200K, capped at 1M) without the 32K compaction-math floor.
// llmpool pre-filtering and sidequestion budgets use this so an 8K/16K profile
// is not treated as 32K — that leak lets an oversize request hit the small
// model, 400, and never fail over (shouldFailover ignores 400s).
func (c Config) ContextWindowTokens() int {
k := c.ContextWindowK
if k <= 0 {
k = defaultWindowK
}
if k > maxWindowK {
k = maxWindowK
}
return k * 1000
}
// compactionConfig builds the agent-core compaction config for a context window
// in tokens. agentcore.NewSession wires the summarizer (same provider) when this
// is set on Options.Compaction.
+28
View File
@@ -0,0 +1,28 @@
package agent
import "testing"
func TestContextWindowTokensDoesNotApplyCompactionFloor(t *testing.T) {
cases := []struct {
k, wantTokens, wantCompaction int
}{
{0, 200_000, 200_000},
{-1, 200_000, 200_000},
{8, 8_000, 32_000},
{16, 16_000, 32_000},
{31, 31_000, 32_000},
{32, 32_000, 32_000},
{200, 200_000, 200_000},
{1000, 1_000_000, 1_000_000},
{2000, 1_000_000, 1_000_000},
}
for _, tc := range cases {
cfg := Config{ContextWindowK: tc.k}
if got := cfg.ContextWindowTokens(); got != tc.wantTokens {
t.Errorf("ContextWindowK=%d: ContextWindowTokens()=%d, want %d", tc.k, got, tc.wantTokens)
}
if got := cfg.CompactionWindow(); got != tc.wantCompaction {
t.Errorf("ContextWindowK=%d: CompactionWindow()=%d, want %d", tc.k, got, tc.wantCompaction)
}
}
}
+5 -2
View File
@@ -65,7 +65,10 @@ func (s *Server) poolMember(p *db.LLMProfile, rank int) *llmpool.Member {
return &llmpool.Member{
ID: p.ID, Name: p.Name, Model: p.Model, Format: p.Format,
Priority: p.Priority, Active: p.IsDefault, Rank: rank,
WindowTokens: cfg.CompactionWindow(), Prov: prov,
// Actual window, not CompactionWindow(): that helper floors at 32K for
// compaction math, which would let an 8K profile accept a 16K request
// and then 400 without failover.
WindowTokens: cfg.ContextWindowTokens(), Prov: prov,
}
}
@@ -103,7 +106,7 @@ func (s *Server) poolChain(headID int64, headProv llm.Provider, headCfg agent.Co
members := []*llmpool.Member{{
ID: head.ID, Name: head.Name, Model: head.Model, Format: head.Format,
Priority: head.Priority, Active: head.IsDefault, Rank: llmpool.RankActive,
WindowTokens: headCfg.CompactionWindow(), Prov: headProv,
WindowTokens: headCfg.ContextWindowTokens(), Prov: headProv,
}}
for _, p := range profs {
if p.ID == headID {
+1 -1
View File
@@ -43,7 +43,7 @@ type sideQuestionState struct {
func sideModel(cfg agent.Config, id int64, name string) sidequestion.Model {
b, _ := json.Marshal([]any{cfg.Format, cfg.BaseURL, cfg.Model, cfg.ThinkingType, cfg.ReasoningEffort, cfg.Stream, cfg.MaxTokensField, cfg.SessionHeaderKey, cfg.UserAgent})
h := sha256.Sum256(b)
return sidequestion.Model{ProfileID: id, Name: name, Format: string(cfg.Format), Model: cfg.Model, Identity: hex.EncodeToString(h[:]), Streaming: cfg.Stream, WindowTokens: cfg.CompactionWindow()}
return sidequestion.Model{ProfileID: id, Name: name, Format: string(cfg.Format), Model: cfg.Model, Identity: hex.EncodeToString(h[:]), Streaming: cfg.Stream, WindowTokens: cfg.ContextWindowTokens()}
}
func bindSideProvider(p llm.Provider, cfg agent.Config, id int64, name string) llm.Provider {