mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-19 14:40:53 +08:00
Merge pull request #304 from Protonmatter/protonmatter/llmpool-health-queue
Coalesce circuit-breaker health writes
This commit is contained in:
@@ -147,6 +147,9 @@ func run() int {
|
||||
if drainErr != nil {
|
||||
log.Printf("task drain: %v", drainErr)
|
||||
}
|
||||
// The health writer flushes on ctx.Done(). Wait for it before the deferred
|
||||
// store close, so the last cooling window is not written at a closed database.
|
||||
srv.FlushLLMHealth()
|
||||
if httpErr != nil || drainErr != nil {
|
||||
return 1
|
||||
}
|
||||
|
||||
+13
-16
@@ -46,7 +46,9 @@ type Registry struct {
|
||||
m map[int64]*State
|
||||
|
||||
// persist / forget mirror state to the DB so a cooling-off window survives a
|
||||
// restart. Both may be nil (no DB); both are called off the hot path.
|
||||
// restart. Both may be nil (no DB). Both run while r.mu is held, so a later
|
||||
// change to the same profile cannot enqueue ahead of this one. They must not
|
||||
// call back into the registry or wait on database I/O.
|
||||
persist func(id int64, st State)
|
||||
forget func(id int64)
|
||||
|
||||
@@ -118,16 +120,14 @@ func (r *Registry) IsOpen(id int64) bool { return r.Get(id).Open() }
|
||||
// recovers is fully trusted again (and drops the persisted row).
|
||||
func (r *Registry) Pass(id int64) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
st := r.m[id]
|
||||
if st == nil || (st.Fails == 0 && st.Trips == 0 && st.OpenUntil.IsZero()) {
|
||||
r.mu.Unlock()
|
||||
return // already clean — nothing to write
|
||||
return // already clean; nothing to write
|
||||
}
|
||||
delete(r.m, id)
|
||||
forget := r.forget
|
||||
r.mu.Unlock()
|
||||
if forget != nil {
|
||||
forget(id)
|
||||
if r.forget != nil {
|
||||
r.forget(id)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,6 +137,7 @@ func (r *Registry) Pass(id int64) {
|
||||
// true when this call is what opened the breaker, so the caller can log it once.
|
||||
func (r *Registry) Trip(id int64, errMsg string, hard bool) (tripped bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
st := r.m[id]
|
||||
if st == nil {
|
||||
st = &State{}
|
||||
@@ -152,11 +153,8 @@ func (r *Registry) Trip(id int64, errMsg string, hard bool) (tripped bool) {
|
||||
st.Fails = 0 // counted into this trip; start fresh for the half-open probe
|
||||
tripped = true
|
||||
}
|
||||
snap := *st
|
||||
persist := r.persist
|
||||
r.mu.Unlock()
|
||||
if persist != nil {
|
||||
persist(id, snap)
|
||||
if r.persist != nil {
|
||||
r.persist(id, *st)
|
||||
}
|
||||
return tripped
|
||||
}
|
||||
@@ -164,11 +162,10 @@ func (r *Registry) Trip(id int64, errMsg string, hard bool) (tripped bool) {
|
||||
// Reset clears one profile's state — the UI's "Recover now" action.
|
||||
func (r *Registry) Reset(id int64) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
delete(r.m, id)
|
||||
forget := r.forget
|
||||
r.mu.Unlock()
|
||||
if forget != nil {
|
||||
forget(id)
|
||||
if r.forget != nil {
|
||||
r.forget(id)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package llmpool
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// persist and forget run under the registry lock. A second update on the same
|
||||
// profile cannot enqueue until the first callback returns, so the queue cannot
|
||||
// observe the later state first.
|
||||
func TestHealthCallbacksStayInStateOrder(t *testing.T) {
|
||||
var entered atomic.Int32
|
||||
firstIn := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
block := func() {
|
||||
if entered.Add(1) == 1 {
|
||||
close(firstIn)
|
||||
<-release
|
||||
}
|
||||
}
|
||||
reg := NewRegistry(
|
||||
func(int64, State) { block() },
|
||||
func(int64) { block() },
|
||||
)
|
||||
go reg.Trip(1, "down", true)
|
||||
select {
|
||||
case <-firstIn:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Trip did not reach persist")
|
||||
}
|
||||
passed := make(chan struct{})
|
||||
go func() {
|
||||
reg.Pass(1)
|
||||
close(passed)
|
||||
}()
|
||||
select {
|
||||
case <-passed:
|
||||
t.Fatal("Pass enqueued while Trip's persist was still running")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
close(release)
|
||||
select {
|
||||
case <-passed:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Pass did not finish after Trip released the lock")
|
||||
}
|
||||
if entered.Load() != 2 {
|
||||
t.Fatalf("callbacks=%d, want Trip then Pass", entered.Load())
|
||||
}
|
||||
}
|
||||
+203
-16
@@ -1,7 +1,9 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
@@ -17,26 +19,19 @@ import (
|
||||
// - Circuit-breaker state is process-wide (s.llmHealth); rebuilding the pool does not clear it.
|
||||
|
||||
// newLLMHealthRegistry builds the process-wide circuit-breaker registry, mirroring
|
||||
// state into PG so a cooling-off window survives a restart. Writes are async and
|
||||
// best-effort — the in-memory copy is authoritative.
|
||||
func newLLMHealthRegistry(pg *db.DB) *llmpool.Registry {
|
||||
// state into PG so a cooling-off window survives a restart. Writes are async.
|
||||
// The in-memory copy is authoritative. The returned syncer is nil when there is
|
||||
// no database; otherwise it flushes when ctx is cancelled.
|
||||
func newLLMHealthRegistry(ctx context.Context, pg *db.DB) (*llmpool.Registry, *healthSync) {
|
||||
if pg == nil {
|
||||
return llmpool.NewRegistry(nil, nil)
|
||||
return llmpool.NewRegistry(nil, nil), nil
|
||||
}
|
||||
syncer := newHealthSync(ctx, pg.SaveLLMHealth, pg.ClearLLMHealth)
|
||||
persist := func(id int64, st llmpool.State) {
|
||||
h := db.LLMHealth{ProfileID: id, Fails: st.Fails, Trips: st.Trips, LastError: st.LastError}
|
||||
if !st.OpenUntil.IsZero() {
|
||||
t := st.OpenUntil
|
||||
h.OpenUntil = &t
|
||||
}
|
||||
go func() {
|
||||
if err := pg.SaveLLMHealth(h); err != nil {
|
||||
log.Printf("[llmpool] failed to persist circuit-breaker state: %v", err)
|
||||
}
|
||||
}()
|
||||
syncer.enqueue(id, healthOp{st: st})
|
||||
}
|
||||
forget := func(id int64) {
|
||||
go func() { _ = pg.ClearLLMHealth(id) }()
|
||||
syncer.enqueue(id, healthOp{clear: true})
|
||||
}
|
||||
reg := llmpool.NewRegistry(persist, forget)
|
||||
// Restore only windows that haven't expired (LoadLLMHealth filters), so a
|
||||
@@ -51,7 +46,199 @@ func newLLMHealthRegistry(pg *db.DB) *llmpool.Registry {
|
||||
log.Printf("[llmpool] restored circuit-breaker state: profile #%d cooling until %s", h.ProfileID, st.OpenUntil.Format(time.RFC3339))
|
||||
}
|
||||
}
|
||||
return reg
|
||||
return reg, syncer
|
||||
}
|
||||
|
||||
// healthOp is one circuit-breaker write for a profile. clear drops the row.
|
||||
// seq orders writes for that profile: a newer op wins over one still in flight.
|
||||
type healthOp struct {
|
||||
seq uint64
|
||||
clear bool
|
||||
st llmpool.State
|
||||
}
|
||||
|
||||
// healthSync writes circuit-breaker state from one goroutine. A newer update for
|
||||
// the same profile replaces one that has not been written yet, so a burst of
|
||||
// state changes becomes one database call. The database call runs without mu,
|
||||
// so enqueue does not wait on a slow write. writeMu keeps those calls in order.
|
||||
// Before each call, an op that is already older than the applied or pending
|
||||
// sequence is dropped. save and clear must not call enqueue.
|
||||
type healthSync struct {
|
||||
save func(db.LLMHealth) error
|
||||
clear func(int64) error
|
||||
retryAfter time.Duration // tests set a short delay; zero uses one second
|
||||
mu sync.Mutex
|
||||
writeMu sync.Mutex // orders database calls; enqueue does not take it
|
||||
pending map[int64]healthOp
|
||||
applied map[int64]uint64
|
||||
seq uint64
|
||||
stopped bool
|
||||
wake chan struct{}
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func newHealthSync(ctx context.Context, save func(db.LLMHealth) error, clear func(int64) error) *healthSync {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
h := &healthSync{
|
||||
save: save,
|
||||
clear: clear,
|
||||
pending: map[int64]healthOp{},
|
||||
applied: map[int64]uint64{},
|
||||
wake: make(chan struct{}, 1),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
go h.loop(ctx)
|
||||
return h
|
||||
}
|
||||
|
||||
// FlushLLMHealth waits until the circuit-breaker writer has flushed the ops
|
||||
// queued before shutdown. Call it after the server context is cancelled and
|
||||
// before the database closes.
|
||||
func (s *Server) FlushLLMHealth() {
|
||||
if s == nil || s.llmHealthSync == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-s.llmHealthSync.done:
|
||||
case <-time.After(5 * time.Second):
|
||||
log.Printf("[llmpool] timed out flushing circuit-breaker state")
|
||||
}
|
||||
}
|
||||
|
||||
func (h *healthSync) enqueue(id int64, op healthOp) {
|
||||
h.mu.Lock()
|
||||
h.seq++
|
||||
op.seq = h.seq
|
||||
if h.stopped {
|
||||
h.mu.Unlock()
|
||||
if err := h.apply(id, op); err != nil {
|
||||
log.Printf("[llmpool] write circuit-breaker profile #%d: %v", id, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if prev, ok := h.pending[id]; ok && prev.seq > op.seq {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
h.pending[id] = op
|
||||
h.mu.Unlock()
|
||||
select {
|
||||
case h.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (h *healthSync) take() map[int64]healthOp {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if len(h.pending) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := h.pending
|
||||
h.pending = map[int64]healthOp{}
|
||||
return out
|
||||
}
|
||||
|
||||
func (h *healthSync) restore(id int64, op healthOp) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.stopped || h.applied[id] >= op.seq {
|
||||
return
|
||||
}
|
||||
if cur, ok := h.pending[id]; ok && cur.seq >= op.seq {
|
||||
return
|
||||
}
|
||||
h.pending[id] = op
|
||||
}
|
||||
|
||||
// superseded reports whether a newer op for id is already applied or waiting.
|
||||
// The caller holds writeMu, so the decision and the database call stay in order.
|
||||
func (h *healthSync) superseded(id int64, seq uint64) bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if seq > 0 && h.applied[id] >= seq {
|
||||
return true
|
||||
}
|
||||
cur, ok := h.pending[id]
|
||||
return ok && cur.seq > seq
|
||||
}
|
||||
|
||||
func (h *healthSync) apply(id int64, op healthOp) error {
|
||||
h.writeMu.Lock()
|
||||
defer h.writeMu.Unlock()
|
||||
if h.superseded(id, op.seq) {
|
||||
return nil
|
||||
}
|
||||
var err error
|
||||
if op.clear {
|
||||
if h.clear != nil {
|
||||
err = h.clear(id)
|
||||
}
|
||||
} else if h.save != nil {
|
||||
row := db.LLMHealth{ProfileID: id, Fails: op.st.Fails, Trips: op.st.Trips, LastError: op.st.LastError}
|
||||
if !op.st.OpenUntil.IsZero() {
|
||||
openUntil := op.st.OpenUntil
|
||||
row.OpenUntil = &openUntil
|
||||
}
|
||||
err = h.save(row)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.mu.Lock()
|
||||
if h.applied[id] < op.seq {
|
||||
h.applied[id] = op.seq
|
||||
}
|
||||
h.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *healthSync) write(batch map[int64]healthOp) (failed bool) {
|
||||
for id, op := range batch {
|
||||
if err := h.apply(id, op); err != nil {
|
||||
log.Printf("[llmpool] write circuit-breaker profile #%d: %v", id, err)
|
||||
h.restore(id, op)
|
||||
failed = true
|
||||
}
|
||||
}
|
||||
return failed
|
||||
}
|
||||
|
||||
func (h *healthSync) finish() {
|
||||
h.mu.Lock()
|
||||
h.stopped = true
|
||||
batch := h.pending
|
||||
h.pending = map[int64]healthOp{}
|
||||
h.mu.Unlock()
|
||||
for id, op := range batch {
|
||||
if err := h.apply(id, op); err != nil {
|
||||
log.Printf("[llmpool] write circuit-breaker profile #%d: %v", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *healthSync) loop(ctx context.Context) {
|
||||
defer close(h.done)
|
||||
for {
|
||||
failed := h.write(h.take())
|
||||
delay := h.retryAfter
|
||||
if delay <= 0 {
|
||||
delay = time.Second
|
||||
}
|
||||
var retry <-chan time.Time
|
||||
if failed {
|
||||
retry = time.After(delay)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
h.finish()
|
||||
return
|
||||
case <-h.wake:
|
||||
case <-retry:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// poolMember builds one chain member from a profile, reusing the per-profile
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/llmpool"
|
||||
)
|
||||
|
||||
func TestHealthSyncLoopWritesLatest(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
var mu sync.Mutex
|
||||
saved := map[int64]db.LLMHealth{}
|
||||
cleared := map[int64]int{}
|
||||
savedCh := make(chan struct{}, 4)
|
||||
clearedCh := make(chan struct{}, 4)
|
||||
h := newHealthSync(ctx,
|
||||
func(row db.LLMHealth) error {
|
||||
mu.Lock()
|
||||
saved[row.ProfileID] = row
|
||||
mu.Unlock()
|
||||
savedCh <- struct{}{}
|
||||
return nil
|
||||
},
|
||||
func(id int64) error {
|
||||
mu.Lock()
|
||||
cleared[id]++
|
||||
delete(saved, id)
|
||||
mu.Unlock()
|
||||
clearedCh <- struct{}{}
|
||||
return nil
|
||||
},
|
||||
)
|
||||
openUntil := time.Now().Add(time.Minute).UTC().Truncate(time.Second)
|
||||
h.enqueue(8, healthOp{st: llmpool.State{Fails: 1, LastError: "first"}})
|
||||
h.enqueue(8, healthOp{st: llmpool.State{Fails: 4, Trips: 1, OpenUntil: openUntil, LastError: "down"}})
|
||||
deadline := time.After(2 * time.Second)
|
||||
var latest db.LLMHealth
|
||||
for {
|
||||
mu.Lock()
|
||||
latest = saved[8]
|
||||
mu.Unlock()
|
||||
if latest.Fails == 4 && latest.OpenUntil != nil && latest.OpenUntil.Equal(openUntil) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-savedCh:
|
||||
case <-deadline:
|
||||
t.Fatalf("loop saved=%+v, want the latest trip", latest)
|
||||
}
|
||||
}
|
||||
h.enqueue(7, healthOp{clear: true})
|
||||
select {
|
||||
case <-clearedCh:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("loop did not clear")
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
got := saved[8]
|
||||
if got.Fails != 4 || got.Trips != 1 || got.LastError != "down" || got.OpenUntil == nil || !got.OpenUntil.Equal(openUntil) {
|
||||
t.Fatalf("saved=%+v, want the latest trip including OpenUntil", got)
|
||||
}
|
||||
if cleared[7] != 1 {
|
||||
t.Fatalf("cleared=%v", cleared)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthSyncRetriesFailedSave(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
var mu sync.Mutex
|
||||
calls := 0
|
||||
done := make(chan struct{})
|
||||
h := newHealthSync(ctx,
|
||||
func(row db.LLMHealth) error {
|
||||
mu.Lock()
|
||||
calls++
|
||||
n := calls
|
||||
mu.Unlock()
|
||||
if n == 1 {
|
||||
return errors.New("database down")
|
||||
}
|
||||
if n == 2 {
|
||||
close(done)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
func(int64) error { return nil },
|
||||
)
|
||||
h.retryAfter = 20 * time.Millisecond
|
||||
h.enqueue(3, healthOp{st: llmpool.State{Fails: 2, LastError: "blip"}})
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("failed save was not retried")
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if calls < 2 {
|
||||
t.Fatalf("save calls=%d, want a retry", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthSyncEnqueueDoesNotWaitOnSave(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
later := make(chan struct{})
|
||||
h := newHealthSync(ctx,
|
||||
func(row db.LLMHealth) error {
|
||||
if row.Fails == 1 {
|
||||
close(started)
|
||||
<-release
|
||||
return nil
|
||||
}
|
||||
if row.Fails == 9 {
|
||||
close(later)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
func(int64) error { return nil },
|
||||
)
|
||||
h.enqueue(4, healthOp{st: llmpool.State{Fails: 1}})
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("save did not start")
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
h.enqueue(4, healthOp{st: llmpool.State{Fails: 9, LastError: "later"}})
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
t.Fatal("enqueue blocked while a save was in flight")
|
||||
}
|
||||
close(release)
|
||||
select {
|
||||
case <-later:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the later state was not written")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthSyncDropsStaleWrite(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
writes := 0
|
||||
h := newHealthSync(ctx,
|
||||
func(db.LLMHealth) error {
|
||||
writes++
|
||||
return nil
|
||||
},
|
||||
func(int64) error { return nil },
|
||||
)
|
||||
h.mu.Lock()
|
||||
h.seq = 4
|
||||
h.applied[5] = 2
|
||||
h.pending[6] = healthOp{seq: 4, clear: true}
|
||||
h.mu.Unlock()
|
||||
if err := h.apply(5, healthOp{seq: 1, st: llmpool.State{Fails: 1}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := h.apply(6, healthOp{seq: 3, st: llmpool.State{Fails: 3}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if writes != 0 {
|
||||
t.Fatalf("stale writes=%d", writes)
|
||||
}
|
||||
}
|
||||
+6
-3
@@ -117,7 +117,9 @@ type Server struct {
|
||||
// It deliberately lives OUTSIDE the provider caches: rebuilding the chain
|
||||
// (saving an unrelated profile, flipping a setting) must not erase what we
|
||||
// learned about which backends are out of credit / rate-limited.
|
||||
llmHealth *llmpool.Registry
|
||||
// llmHealthSync flushes that state when ctx is cancelled, before the database closes.
|
||||
llmHealth *llmpool.Registry
|
||||
llmHealthSync *healthSync
|
||||
|
||||
// Explicit task chains use one task-scoped dynamic provider shared by goals,
|
||||
// planner, workers, and the main agent. Bundles are stable; their runtime reads
|
||||
@@ -163,9 +165,10 @@ func New(ctx context.Context, m *Manager, skillDir string, dataDir string, keyDi
|
||||
chatCancel: map[string]context.CancelCauseFunc{}, triggerQ: map[string][]triggeredRun{},
|
||||
triggerActive: map[string]int{}, triggerCfg: map[string]triggerBehavior{},
|
||||
profChatAgents: map[int64]*agent.ChatAgent{},
|
||||
provByProfile: map[int64]*provEntry{}, llmHealth: newLLMHealthRegistry(m.pg),
|
||||
taskAgents: map[string]*taskAgentBundle{}, archiveWake: make(chan struct{}, 1),
|
||||
provByProfile: map[int64]*provEntry{},
|
||||
taskAgents: map[string]*taskAgentBundle{}, archiveWake: make(chan struct{}, 1),
|
||||
loginLimit: newLoginLimiter(time.Now)}
|
||||
s.llmHealth, s.llmHealthSync = newLLMHealthRegistry(ctx, m.pg)
|
||||
for _, o := range opts {
|
||||
o(s)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user