Merge pull request #313 from BaaRai/fix/task-llm-health-badge

fix(llm): record breaker health on every task path
This commit is contained in:
BaaRai
2026-11-18 01:42:49 +01:00
committed by GitHub
8 changed files with 413 additions and 27 deletions
+1
View File
@@ -36,6 +36,7 @@ All notable changes to this project are documented in this file. The format is b
### Fixed
- **LLM health badge on every task path**: `/system/llm/` read a breaker state that only the failover Pool wrote, so a profile used by an explicit task chain, by a bound agent with failover off, or as the global provider stayed "Healthy" through a real outage. Every task LLM attempt now records its outcome through the same policy the Pool uses, keyed by the profile that served it. The LLM recording page also explains an empty list when the `llm_record` toggle is off instead of looking broken.
- **Skill visibility survives restarts**: turning a skill off now writes an explicit disabled row instead of deleting it, so the startup seed, which only inserts missing rows, can no longer bring the skill back. Skills that were turned off before this version have no row to keep, so they come back once after the upgrade and must be turned off again.
- **Browser extension traffic capture**: CDP delivers `Network.responseReceived` response headers as an object, not an array of `{name, value}` records, so the extension threw on every response and never queued an exchange. It now accepts both shapes and lower-cases names before the body-capture gate, so captures work again and `content-type` correctly controls body capture.
- **bbot on arm64**: the image now builds on arm64 hosts such as Apple Silicon. bbot's `gowitness` downloaded an x86-64 Chrome and `retirejs` an x86-64 Node, which can't run there, so the build failed. On arm64 the image uses Debian's `chromium` for gowitness and an arm64 Node for retirejs, and the build check now covers `retirejs` too. amd64 builds are unchanged.
+42
View File
@@ -10,6 +10,7 @@
package llmpool
import (
"context"
"sync"
"time"
)
@@ -159,6 +160,47 @@ func (r *Registry) Trip(id int64, errMsg string, hard bool) (tripped bool) {
return tripped
}
// RecordCall applies the failover health policy to one provider call. The Pool
// calls it for every member attempt, and a caller that runs a provider directly
// (a task whose selection bypasses the Pool) calls it too, so a profile's state
// does not depend on which path the request took.
//
// stream tells a streaming call from an atomic one; committed says whether any
// event already reached the caller (streaming only). The rules are the ones the
// Pool applies:
//
// - a completed call (err == nil) clears the state with Pass;
// - a failure that does not trigger failover (context cancellation, a
// deterministic 4xx) is ignored on a stream, and clears the state on an
// atomic call, matching Complete's non-failover path;
// - a failover-eligible stream failure trips only before the first event:
// after that the caller already has output and the state is left alone;
// - a failover-eligible atomic failure trips.
//
// id 0 is a no-op (no profile to key, e.g. an environment-only configuration).
// It returns whether this call opened the breaker, and the shortened error
// stored with it.
func (r *Registry) RecordCall(ctx context.Context, id int64, stream, committed bool, err error) (tripped bool, errMsg string) {
if id == 0 {
return false, ""
}
if err == nil {
r.Pass(id)
return false, ""
}
if !shouldFailover(ctx, err) {
if !stream {
r.Pass(id)
}
return false, ""
}
if stream && committed {
return false, ""
}
msg := trimErr(err)
return r.Trip(id, msg, isHardFailure(err)), msg
}
// Reset clears one profile's state — the UI's "Recover now" action.
func (r *Registry) Reset(id int64) {
r.mu.Lock()
+119
View File
@@ -0,0 +1,119 @@
package llmpool
import (
"context"
"fmt"
"testing"
"github.com/Autumn-27/norma/llm"
)
// RecordCall is the policy a provider call applies when no Pool wraps it. A
// completed call clears the state; a failover-eligible failure trips; a
// non-failover failure neither trips nor, on a stream, clears.
func TestRecordCallPolicy(t *testing.T) {
ctx := context.Background()
soft := fmt.Errorf("openai: status 500: down")
bad := fmt.Errorf("openai: status 400: bad request")
seed := func(t *testing.T) *Registry {
t.Helper()
reg := NewRegistry(nil, nil)
reg.RecordCall(ctx, 7, false, false, soft)
if st := reg.Get(7); st.Fails != 1 || st.Trips != 0 {
t.Fatalf("seed state=%+v, want one soft failure", st)
}
return reg
}
t.Run("success clears", func(t *testing.T) {
reg := seed(t)
reg.RecordCall(ctx, 7, false, false, nil)
if st := reg.Get(7); st.Fails != 0 || st.Trips != 0 || st.Open() {
t.Fatalf("state=%+v after a success, want clean", st)
}
})
t.Run("atomic non-failover clears", func(t *testing.T) {
reg := seed(t)
reg.RecordCall(ctx, 7, false, false, bad)
if st := reg.Get(7); st.Fails != 0 || st.Trips != 0 {
t.Fatalf("state=%+v after an atomic 400, want clean (Pool.Complete policy)", st)
}
})
t.Run("stream non-failover keeps", func(t *testing.T) {
reg := seed(t)
reg.RecordCall(ctx, 7, true, false, bad)
if st := reg.Get(7); st.Fails != 1 || st.Trips != 0 {
t.Fatalf("state=%+v after a stream 400, want the failure kept", st)
}
})
t.Run("stream cancel keeps", func(t *testing.T) {
reg := seed(t)
reg.RecordCall(ctx, 7, true, false, context.Canceled)
if st := reg.Get(7); st.Fails != 1 || st.Trips != 0 {
t.Fatalf("state=%+v after a cancel, want the failure kept", st)
}
})
t.Run("committed stream failure keeps", func(t *testing.T) {
reg := seed(t)
reg.RecordCall(ctx, 7, true, true, soft)
if st := reg.Get(7); st.Fails != 1 || st.Trips != 0 {
t.Fatalf("state=%+v after a committed failure, want it ignored", st)
}
})
t.Run("pre-commit stream failure counts", func(t *testing.T) {
reg := NewRegistry(nil, nil)
reg.RecordCall(ctx, 7, true, false, soft)
if st := reg.Get(7); st.Fails != 1 || st.Trips != 0 || st.Open() {
t.Fatalf("state=%+v, want one soft failure", st)
}
})
t.Run("hard failure trips at once", func(t *testing.T) {
reg := NewRegistry(nil, nil)
reg.RecordCall(ctx, 7, false, false, fmt.Errorf("openai: status 401: bad key"))
if st := reg.Get(7); st.Trips != 1 || !st.Open() {
t.Fatalf("state=%+v, want an open breaker after one hard failure", st)
}
})
t.Run("profile 0 is a no-op", func(t *testing.T) {
reg := NewRegistry(nil, nil)
reg.RecordCall(ctx, 0, false, false, soft)
if n := len(reg.Snapshot()); n != 0 {
t.Fatalf("%d states recorded for profile 0", n)
}
})
}
// The Pool feeds the same policy: a failure before the first event counts on
// the member that failed, a failure after output reached the caller does not,
// because the Pool neither fails over nor blames the profile in that case.
func TestPoolFeedsRecordCallPolicy(t *testing.T) {
t.Run("pre-commit failure counts", func(t *testing.T) {
a := &fakeProv{name: "a", errs: []error{fmt.Errorf("status 500: down")}}
reg := NewRegistry(nil, nil)
p := New([]*Member{member(1, "a", 10, a), member(2, "b", 5, okProv("b", "full"))}, reg)
if _, err := drain(p.Stream(context.Background(), llm.CompletionRequest{})); err != nil {
t.Fatalf("stream error: %v", err)
}
if st := reg.Get(1); st.Fails != 1 || st.Trips != 0 {
t.Fatalf("state=%+v after a pre-commit failure, want one soft failure", st)
}
if st := reg.Get(2); st.Fails != 0 || st.Trips != 0 {
t.Fatalf("state=%+v on the backup member, want clean", st)
}
})
t.Run("committed failure is ignored", func(t *testing.T) {
a := &fakeProv{
name: "a",
events: [][]llm.StreamEvent{{{Type: llm.SETextDelta, Text: "partial"}}},
errs: []error{fmt.Errorf("status 500: mid-stream drop")},
}
reg := NewRegistry(nil, nil)
p := New([]*Member{member(1, "a", 10, a), member(2, "b", 5, okProv("b", "full"))}, reg)
if _, err := drain(p.Stream(context.Background(), llm.CompletionRequest{})); err == nil {
t.Fatal("mid-stream error should surface")
}
if st := reg.Get(1); st.Fails != 0 || st.Trips != 0 {
t.Fatalf("state=%+v after a committed failure, want clean", st)
}
})
}
+9 -10
View File
@@ -100,13 +100,12 @@ func (p *Pool) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[
}
}
if failed == nil {
p.health.Pass(m.ID) // completed (or failed in a non-failover way)
p.health.RecordCall(ctx, m.ID, true, false, nil) // completed: Pass
return
}
lastErr = failed
hard := isHardFailure(failed)
if p.health.Trip(m.ID, trimErr(failed), hard) {
log.Printf("[llmpool] profile %q(%s) breaker tripped: %s", m.Name, m.Model, trimErr(failed))
if tripped, msg := p.health.RecordCall(ctx, m.ID, true, false, failed); tripped {
log.Printf("[llmpool] profile %q(%s) breaker tripped: %s", m.Name, m.Model, msg)
}
if i+1 < len(order) {
n := order[i+1]
@@ -133,19 +132,19 @@ func (p *Pool) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Mes
for i, m := range order {
msg, sr, usage, err := m.Prov.Complete(ctx, req)
if err == nil {
p.health.Pass(m.ID)
p.health.RecordCall(ctx, m.ID, false, false, nil) // Pass
return msg, sr, usage, nil
}
if !shouldFailover(ctx, err) {
// Non-failover error (e.g. ctx cancel, deterministic 4xx): surface as-is
// without tripping health, matching Stream's non-failover path.
p.health.Pass(m.ID)
// without tripping health. RecordCall clears the state on an atomic
// call, matching the historical Pass here.
p.health.RecordCall(ctx, m.ID, false, false, err)
return llm.Message{}, "", llm.Usage{}, err
}
lastErr = err
hard := isHardFailure(err)
if p.health.Trip(m.ID, trimErr(err), hard) {
log.Printf("[llmpool] profile %q(%s) breaker tripped: %s", m.Name, m.Model, trimErr(err))
if tripped, msg := p.health.RecordCall(ctx, m.ID, false, false, err); tripped {
log.Printf("[llmpool] profile %q(%s) breaker tripped: %s", m.Name, m.Model, msg)
}
if i+1 < len(order) {
n := order[i+1]
+9 -2
View File
@@ -80,8 +80,12 @@ type Server struct {
llmProv llm.Provider
llmDirect llm.Provider // concrete global provider, before any failover pool
llmCfg agent.Config // current LLM config (key not exposed)
llmOn bool
llmProf string // active LLM profile name (for llmrec tagging)
// llmHealthProfileID is the llm_profiles row llmProv was built from, or 0
// when it came from environment settings. Task selections use it to key
// breaker health for the global fallback path.
llmHealthProfileID int64
llmOn bool
llmProf string // active LLM profile name (for llmrec tagging)
// chatBusy guards the per-task main-agent run: the chat handler launches the
// agent on the server's background ctx (not the request ctx) and returns
@@ -504,11 +508,13 @@ func maxTokensResolver(cfg agent.Config) func() int {
// the running engine as the GLOBAL active pair. Safe to call at runtime (UI configures LLM).
func (s *Server) applyLLM(cfg agent.Config) error {
var prov llm.Provider
var healthID int64
// A persisted active profile must use the same cached provider as task chains
// and Agent bindings, otherwise each path owns a separate rate limiter.
if active, _ := s.m.pg.ActiveProfile(); active != nil {
if activeCfg, ok := s.loadProfileConfig(active.ID); ok && activeCfg == cfg {
prov, _, _ = s.providerForProfile(active.ID)
healthID = active.ID
}
}
if prov == nil {
@@ -555,6 +561,7 @@ func (s *Server) applyLLM(cfg agent.Config) error {
s.chatAgent.SetNoaEnabled(s.m.NoaCompactionEnabled) // experimental: noa context compaction (read per run)
s.llmProv = prov
s.llmCfg = cfg
s.llmHealthProfileID = healthID
s.llmOn = true
s.cfgMu.Unlock()
s.invalidateTaskAgents()
+72 -13
View File
@@ -15,6 +15,7 @@ import (
"github.com/Autumn-27/artex/agent"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/llmpool"
"github.com/Autumn-27/artex/llmrec"
"github.com/Autumn-27/norma/llm"
"github.com/Autumn-27/norma/transcript"
@@ -137,8 +138,15 @@ func resetLLMFailStreak(taskID string, profileID int64) {
type taskLLMSelection struct {
task *Task
profileID int64
revision int64
provider llm.Provider
// healthProfileID is the llm_profiles row whose breaker state the calls on
// this selection update. It equals profileID on the task-chain path; a bound
// agent or the global provider carries its own id here while profileID stays
// 0 (no chain semantics). 0 means "do not report": either there is no
// profile (environment-only configuration) or the provider is already the
// failover Pool, which updates the registry itself.
healthProfileID int64
revision int64
provider llm.Provider
// retry holds the retry parameters resolved for the profile selected this time (profile override → global policy → built-in default).
// Same-provider safe-window retries follow it, so switching profiles switches the retry cadence.
retry agent.RetryConfig
@@ -165,6 +173,24 @@ type taskLLMStreamHooks struct {
current func(context.Context) (taskLLMSelection, error)
exhaust func(taskLLMSelection, error) (db.TaskLLMTransition, error)
transition func(taskLLMSelection, db.TaskLLMTransition, error)
// recordHealth reports one physical provider attempt to the process-wide
// breaker. It keeps /system/llm/ truthful for every selection whose provider
// is not already the failover Pool (the chain, a bound agent, the global
// profile). Optional: test literals leave it nil and the call sites skip it.
recordHealth func(ctx context.Context, profileID int64, stream, committed bool, err error)
}
// healthIDFor returns the profile whose breaker state a call on this provider
// should update. It returns 0 when there is no profile, or when the provider is
// the failover Pool, which records every member attempt itself.
func healthIDFor(prov llm.Provider, profileID int64) int64 {
if profileID == 0 {
return 0
}
if _, pooled := prov.(*llmpool.Pool); pooled {
return 0
}
return profileID
}
// current resolves the provider this role runs on, by precedence:
@@ -196,8 +222,9 @@ func (r *taskLLMRuntime) currentFor(ctx context.Context) (taskLLMSelection, erro
r.s.syncTaskLLMState(pt)
t, _ := r.s.m.Task(r.taskID)
sel := taskLLMSelection{task: t, revision: pt.LLMChainRevision}
if prov, cfg, ok := r.s.agentBindingProvider(r.agentKey); ok {
if prov, cfg, id, ok := r.s.agentBindingProvider(r.agentKey); ok {
sel.provider, sel.retry = prov, cfg.Retry
sel.healthProfileID = healthIDFor(prov, id)
return sel, nil
}
if len(pt.LLMProfileIDs) > 0 {
@@ -218,13 +245,15 @@ func (r *taskLLMRuntime) currentFor(ctx context.Context) (taskLLMSelection, erro
return sel, fmt.Errorf("LLM profile #%d is missing or invalid", sel.profileID)
}
sel.provider, sel.retry = prov, cfg.Retry
sel.healthProfileID = sel.profileID // chain selections never go through the Pool
return sel, nil
}
prov, cfg, ok := r.s.globalProvider()
prov, cfg, id, ok := r.s.globalProviderWithProfile()
if !ok {
return sel, fmt.Errorf("task %s has no available fallback LLM provider", r.taskID)
}
sel.provider, sel.retry = prov, cfg.Retry
sel.healthProfileID = healthIDFor(prov, id)
return sel, nil
}
@@ -242,7 +271,7 @@ func (r *taskLLMRuntime) activeCfg() (agent.Config, bool) {
if err != nil || pt == nil {
return agent.Config{}, false
}
if _, cfg, ok := r.s.agentBindingProvider(r.agentKey); ok {
if _, cfg, _, ok := r.s.agentBindingProvider(r.agentKey); ok {
return cfg, true
}
if len(pt.LLMProfileIDs) > 0 && pt.ActiveLLMProfileID != nil {
@@ -280,10 +309,19 @@ func parseTaskID(id string) (int64, error) {
// streamHooks builds the failover/exhaustion callbacks shared by Stream and
// Complete: how to read the current profile selection, how to mark it quota
// exhausted, and how to emit a failover transition.
// exhausted, how to emit a failover transition, and how to report breaker
// health for providers that are not the failover Pool.
func (r *taskLLMRuntime) streamHooks() taskLLMStreamHooks {
return taskLLMStreamHooks{
current: func(ctx context.Context) (taskLLMSelection, error) { return r.currentFor(ctx) },
recordHealth: func(ctx context.Context, profileID int64, stream, committed bool, err error) {
if profileID == 0 || r.s == nil || r.s.llmHealth == nil {
return
}
if tripped, msg := r.s.llmHealth.RecordCall(ctx, profileID, stream, committed, err); tripped {
log.Printf("[task-llm] profile #%d breaker tripped: %s", profileID, msg)
}
},
exhaust: func(selection taskLLMSelection, cause error) (db.TaskLLMTransition, error) {
taskNum, _ := parseTaskID(r.taskID)
transition, err := r.s.m.pg.MarkTaskLLMProfileQuotaExhaustedAtRevision(taskNum, selection.profileID, selection.revision, cause.Error())
@@ -338,6 +376,9 @@ func completeTaskLLM(ctx context.Context, taskID string, req llm.CompletionReque
for attempt := 0; ; attempt++ {
msg, sr, usage, callErr = selection.provider.Complete(ctx, req)
total.Add(usage)
if hooks.recordHealth != nil {
hooks.recordHealth(ctx, selection.healthProfileID, false, false, callErr)
}
if callErr != nil && ctx.Err() == nil &&
attempt < retries && isRetryableStreamError(callErr) {
backoff := backoffOf(attempt)
@@ -462,6 +503,9 @@ func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest
return
}
}
if hooks.recordHealth != nil {
hooks.recordHealth(ctx, selection.healthProfileID, true, committed, streamErr)
}
if streamErr != nil && !committed {
// A failed uncommitted attempt delivers none of its events (they are
// never replayed), but the usage it reported survives as one
@@ -635,7 +679,7 @@ func isRetryableStreamError(err error) bool {
// CompactionWindow mirrors current()'s precedence so the context window always
// matches the provider the role will actually stream on.
func (r *taskLLMRuntime) CompactionWindow() int {
if _, cfg, ok := r.s.agentBindingProvider(r.agentKey); ok {
if _, cfg, _, ok := r.s.agentBindingProvider(r.agentKey); ok {
return cfg.CompactionWindow()
}
taskNum, err := parseTaskID(r.taskID)
@@ -671,17 +715,19 @@ func (r *taskLLMRuntime) CompactionWindow() int {
// (agents.llm_profile_id) — the highest-precedence level for task agents. ok=false
// when the role has no binding or the bound profile no longer builds, so callers
// fall through to the task chain. A bound profile stays exclusive unless
// llm_pool_bind_fallback is on, which is what poolForBinding encodes.
func (s *Server) agentBindingProvider(agentKey string) (llm.Provider, agent.Config, bool) {
// llm_pool_bind_fallback is on, which is what poolForBinding encodes. The
// returned id is the binding's profile, used to key breaker health when the
// returned provider is not the Pool.
func (s *Server) agentBindingProvider(agentKey string) (llm.Provider, agent.Config, int64, bool) {
id := s.effectiveProfileForAgent(agentKey, nil)
if id == nil {
return nil, agent.Config{}, false
return nil, agent.Config{}, 0, false
}
prov, cfg, ok := s.providerForProfile(*id)
if !ok {
return nil, agent.Config{}, false
return nil, agent.Config{}, 0, false
}
return s.poolForBinding(*id, prov, cfg), cfg, true
return s.poolForBinding(*id, prov, cfg), cfg, *id, true
}
// globalProvider returns the process-wide provider (persisted active profile or
@@ -696,6 +742,19 @@ func (s *Server) globalProvider() (llm.Provider, agent.Config, bool) {
return s.llmProv, s.llmCfg, true
}
// globalProviderWithProfile is globalProvider plus the llm_profiles row the
// provider was built from (0 for an environment-only configuration). The id is
// read under the same lock as the provider so the pair cannot be mixed across
// an applyLLM rebuild.
func (s *Server) globalProviderWithProfile() (llm.Provider, agent.Config, int64, bool) {
s.cfgMu.Lock()
defer s.cfgMu.Unlock()
if !s.llmOn || s.llmProv == nil {
return nil, agent.Config{}, 0, false
}
return s.llmProv, s.llmCfg, s.llmHealthProfileID, true
}
// taskRuntimeAvailable reports whether every listed role can resolve a provider
// under the runtime precedence in current(): the role's own binding first, then
// the task chain, then global. An exhausted chain is a hard stop for unbound
@@ -714,7 +773,7 @@ func (s *Server) taskRuntimeAvailable(t *Task, agentKeys ...string) bool {
_, _, unboundReady = s.globalProvider()
}
for _, key := range agentKeys {
if _, _, ok := s.agentBindingProvider(key); ok {
if _, _, _, ok := s.agentBindingProvider(key); ok {
continue
}
if !unboundReady {
+150
View File
@@ -0,0 +1,150 @@
package server
import (
"context"
"fmt"
"testing"
"github.com/Autumn-27/artex/agent"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/llmpool"
"github.com/Autumn-27/norma/llm"
)
// healthHooks runs one logical call on p and reports every physical attempt to
// reg under healthID, the way streamHooks wires it in production.
func healthHooks(reg *llmpool.Registry, p llm.Provider, healthID int64) taskLLMStreamHooks {
return taskLLMStreamHooks{
current: func(context.Context) (taskLLMSelection, error) {
return taskLLMSelection{provider: p, healthProfileID: healthID, retry: agent.RetryConfig{StreamAttempts: 1}}, nil
},
exhaust: func(taskLLMSelection, error) (db.TaskLLMTransition, error) { return db.TaskLLMTransition{}, nil },
recordHealth: func(ctx context.Context, profileID int64, stream, committed bool, err error) {
reg.RecordCall(ctx, profileID, stream, committed, err)
},
}
}
// A successful call clears a degraded profile, on both call shapes.
func TestTaskLLMRecordsHealthOnSuccess(t *testing.T) {
resetLLMFailStreaksForTest()
ctx := context.Background()
for _, stream := range []bool{false, true} {
t.Run(fmt.Sprintf("stream=%v", stream), func(t *testing.T) {
reg := llmpool.NewRegistry(nil, nil)
reg.RecordCall(ctx, 7, false, false, fmt.Errorf("openai: status 500: down"))
if st := reg.Get(7); st.Fails != 1 {
t.Fatalf("seed state=%+v, want one failure", st)
}
runLogicalCall(ctx, stream, "health-success", healthHooks(reg, &scriptedLLMProvider{}, 7))
if st := reg.Get(7); st.Fails != 0 || st.Trips != 0 {
t.Fatalf("state=%+v after success, want clean", st)
}
})
}
}
// A deterministic 400 on an atomic call clears the streak, matching
// Pool.Complete; the same error on a stream leaves it alone, matching
// Pool.Stream.
func TestTaskLLMRecordsNonFailoverErrorLikeThePool(t *testing.T) {
resetLLMFailStreaksForTest()
ctx := context.Background()
bad := fmt.Errorf("openai: status 400: bad request")
reg := llmpool.NewRegistry(nil, nil)
reg.RecordCall(ctx, 7, false, false, fmt.Errorf("openai: status 500: down"))
runLogicalCall(ctx, false, "health-atomic-400", healthHooks(reg, &scriptedLLMProvider{err: bad}, 7))
if st := reg.Get(7); st.Fails != 0 || st.Trips != 0 {
t.Fatalf("atomic state=%+v after 400, want clean", st)
}
reg2 := llmpool.NewRegistry(nil, nil)
reg2.RecordCall(ctx, 8, false, false, fmt.Errorf("openai: status 500: down"))
runLogicalCall(ctx, true, "health-stream-400", healthHooks(reg2, &scriptedLLMProvider{err: bad}, 8))
if st := reg2.Get(8); st.Fails != 1 || st.Trips != 0 {
t.Fatalf("stream state=%+v after 400, want the failure kept", st)
}
}
// Every physical stream attempt is reported: a retried pre-commit 500 counts
// twice and does not trip a three-strike breaker.
func TestTaskLLMRecordsEveryStreamAttempt(t *testing.T) {
resetLLMFailStreaksForTest()
defer withZeroRetryBackoff()()
ctx := context.Background()
reg := llmpool.NewRegistry(nil, nil)
p := &scriptedLLMProvider{err: fmt.Errorf("openai: status 500: down")}
runLogicalCall(ctx, true, "health-retry", healthHooks(reg, p, 7))
if p.calls != 2 {
t.Fatalf("provider calls=%d, want one attempt plus one retry", p.calls)
}
if st := reg.Get(7); st.Fails != 2 || st.Trips != 0 {
t.Fatalf("state=%+v, want two soft failures and no trip", st)
}
}
// A failure after output reached the caller is committed: the Pool does not
// fail over there and the breaker is left alone.
func TestTaskLLMRecordsCommittedFailureAsNoHealthEvent(t *testing.T) {
resetLLMFailStreaksForTest()
reg := llmpool.NewRegistry(nil, nil)
p := &scriptedLLMProvider{
events: []llm.StreamEvent{{Type: llm.SETextDelta, Text: "partial"}},
err: fmt.Errorf("openai: status 500: mid-stream drop"),
}
runLogicalCall(context.Background(), true, "health-committed", healthHooks(reg, p, 7))
if st := reg.Get(7); st.Fails != 0 || st.Trips != 0 {
t.Fatalf("state=%+v after a committed failure, want clean", st)
}
}
// Without a health profile (environment-only configuration) nothing is
// recorded.
func TestTaskLLMRecordsNoHealthWithoutProfile(t *testing.T) {
resetLLMFailStreaksForTest()
reg := llmpool.NewRegistry(nil, nil)
p := &scriptedLLMProvider{err: fmt.Errorf("openai: status 500: down")}
runLogicalCall(context.Background(), true, "health-none", healthHooks(reg, p, 0))
if n := len(reg.Snapshot()); n != 0 {
t.Fatalf("%d states recorded with healthProfileID 0", n)
}
}
// healthIDFor keys the registry only for a provider that does not record by
// itself: a Pool already updates every member attempt.
func TestHealthIDForSkipsPooledProviders(t *testing.T) {
bare := &scriptedLLMProvider{}
pool := llmpool.New([]*llmpool.Member{{ID: 9, Prov: bare}}, nil)
if pool == nil {
t.Fatal("pool should be non-nil")
}
if got := healthIDFor(bare, 7); got != 7 {
t.Fatalf("bare provider id=%d, want 7", got)
}
if got := healthIDFor(pool, 7); got != 0 {
t.Fatalf("pooled provider id=%d, want 0", got)
}
if got := healthIDFor(bare, 0); got != 0 {
t.Fatalf("profile 0 id=%d, want 0", got)
}
}
// The production hook closure records under the given profile and no-ops on 0.
func TestStreamHooksRecordBreakerHealth(t *testing.T) {
reg := llmpool.NewRegistry(nil, nil)
r := &taskLLMRuntime{s: &Server{llmHealth: reg}}
h := r.streamHooks()
ctx := context.Background()
h.recordHealth(ctx, 0, false, false, fmt.Errorf("openai: status 500: down"))
if n := len(reg.Snapshot()); n != 0 {
t.Fatalf("%d states recorded for profile 0", n)
}
h.recordHealth(ctx, 7, false, false, fmt.Errorf("openai: status 500: down"))
if st := reg.Get(7); st.Fails != 1 {
t.Fatalf("state=%+v, want one failure", st)
}
h.recordHealth(ctx, 7, false, false, nil)
if st := reg.Get(7); st.Fails != 0 || st.Trips != 0 {
t.Fatalf("state=%+v after success, want clean", st)
}
}
@@ -133,6 +133,9 @@ export default function LLMRecordsPage() {
// Recording on/off toggle (settings.llm_record; default off). When off the
// backend records nothing.
const [recEnabled, setRecEnabled] = React.useState(false);
// recKnown flips once the settings call resolves, so the empty state does not
// claim "recording is off" while the answer is still in flight.
const [recKnown, setRecKnown] = React.useState(false);
const [recBusy, setRecBusy] = React.useState(false);
// Inline detail panel (Burp-style split, not a dialog)
@@ -167,7 +170,11 @@ export default function LLMRecordsPage() {
let alive = true;
api
.settings()
.then((s) => { if (alive) setRecEnabled(!!s.llm_record); })
.then((s) => {
if (!alive) return;
setRecEnabled(!!s.llm_record);
setRecKnown(true);
})
.catch(() => {});
return () => { alive = false; };
}, []);
@@ -400,7 +407,9 @@ export default function LLMRecordsPage() {
) : records.length === 0 ? (
<TableRow>
<TableCell colSpan={8} className="py-12 text-center text-sm text-muted-foreground">
No LLM call records yet
{!recKnown || recEnabled
? "No LLM call records yet"
: "LLM call recording is off. Turn on the Recording toggle above to start capturing calls."}
</TableCell>
</TableRow>
) : (