diff --git a/cmd/artex/main.go b/cmd/artex/main.go index ebf5e5b..6595a28 100644 --- a/cmd/artex/main.go +++ b/cmd/artex/main.go @@ -127,19 +127,22 @@ func run() int { <-ctx.Done() log.Println("shutting down...") + // Close the listener before draining. Otherwise a request accepted during + // the drain can start a detached run, and mgr.Close runs while that run + // is still writing. + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + httpErr := httpSrv.Shutdown(shutdownCtx) + cancel() + if httpErr != nil { + log.Printf("http shutdown: %v", httpErr) + } drainCtx, drainCancel := context.WithTimeout(context.Background(), 15*time.Second) drainErr := srv.Wait(drainCtx) drainCancel() if drainErr != nil { log.Printf("task drain: %v", drainErr) } - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if err := httpSrv.Shutdown(shutdownCtx); err != nil { - log.Printf("http shutdown: %v", err) - return 1 - } - if drainErr != nil { + if httpErr != nil || drainErr != nil { return 1 } return 0 diff --git a/db/activity_outbox.go b/db/activity_outbox.go index c00c775..b49f234 100644 --- a/db/activity_outbox.go +++ b/db/activity_outbox.go @@ -3,8 +3,14 @@ package db import ( "encoding/json" "fmt" + "log" ) +// activityOutboxMaxAttempts is how many times a row may fail AppendActivity +// before it is dropped. One stuck row must not sit at the head of the queue +// and keep every later row from being replayed. +const activityOutboxMaxAttempts = 5 + // activityOutboxPayload is Activity plus Detail. Activity.Detail is omitted // from ordinary JSON (the list payload stays small); the outbox has to keep it // so a replayed row is the same step that failed to store. @@ -41,26 +47,29 @@ func (s *ExplorationStore) EnqueueActivity(a Activity, cause string) error { } // FlushActivityOutbox inserts queued activities in order and deletes each row -// that lands. A row that still cannot be inserted stays, with last_error -// updated, and the flush stops so later rows keep their order. A payload that -// cannot be decoded is discarded. +// that lands. The insert and the delete commit together, so a crash cannot +// leave the activity stored and the outbox row still queued. A row that still +// cannot be inserted stays, with its attempt count increased, and the flush +// continues with the next row. After activityOutboxMaxAttempts the row is +// dropped and logged. A payload that cannot be decoded is discarded. func (s *ExplorationStore) FlushActivityOutbox() (int, error) { if s == nil || s.db == nil { return 0, fmt.Errorf("activity outbox: no store") } - rows, err := s.db.Query(`SELECT id, payload FROM activity_outbox WHERE exploration_id=$1 ORDER BY id`, s.expID) + rows, err := s.db.Query(`SELECT id, payload, attempts FROM activity_outbox WHERE exploration_id=$1 ORDER BY id`, s.expID) if err != nil { return 0, err } defer rows.Close() type item struct { - id int64 - payload []byte + id int64 + payload []byte + attempts int } var items []item for rows.Next() { var it item - if err := rows.Scan(&it.id, &it.payload); err != nil { + if err := rows.Scan(&it.id, &it.payload, &it.attempts); err != nil { return 0, err } items = append(items, it) @@ -77,11 +86,30 @@ func (s *ExplorationStore) FlushActivityOutbox() (int, error) { } continue } - if _, err := s.AppendActivity(a); err != nil { - _, _ = s.db.Exec(`UPDATE activity_outbox SET last_error=$2 WHERE id=$1`, it.id, err.Error()) + tx, err := s.db.Begin() + if err != nil { return stored, err } - if _, err := s.db.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); err != nil { + if _, err := s.appendActivity(tx, a); err != nil { + _ = tx.Rollback() + next := it.attempts + 1 + if next >= activityOutboxMaxAttempts { + log.Printf("[activity] outbox %d for exploration %d dropped after %d attempts: %v", it.id, s.expID, next, err) + if _, delErr := s.db.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); delErr != nil { + return stored, delErr + } + continue + } + if _, updErr := s.db.Exec(`UPDATE activity_outbox SET attempts=$2, last_error=$3 WHERE id=$1`, it.id, next, err.Error()); updErr != nil { + return stored, updErr + } + continue + } + if _, err := tx.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); err != nil { + _ = tx.Rollback() + return stored, err + } + if err := tx.Commit(); err != nil { return stored, err } stored++ diff --git a/db/activity_outbox_test.go b/db/activity_outbox_test.go index ae64b69..a27deb3 100644 --- a/db/activity_outbox_test.go +++ b/db/activity_outbox_test.go @@ -16,3 +16,84 @@ func TestActivityOutboxKeepsDetail(t *testing.T) { t.Fatalf("round trip = %+v", out) } } + +func TestFlushActivityOutboxReplaysOnce(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + t.Skipf("postgres unavailable (%v) — skipping", err) + } + defer d.Close() + expID, err := d.CreateExploration("outbox", "replay") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _, _ = d.Exec(`DELETE FROM activity_outbox WHERE exploration_id=$1`, expID) + _, _ = d.Exec(`DELETE FROM explorations WHERE id=$1`, expID) + }) + es := d.Exploration(expID) + if err := es.EnqueueActivity(Activity{Worker: "worker", Kind: "text", Summary: "kept", Detail: "body"}, "insert failed"); err != nil { + t.Fatal(err) + } + n, err := es.FlushActivityOutbox() + if err != nil || n != 1 { + t.Fatalf("flush n=%d err=%v", n, err) + } + var activities, queued int + if err := d.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='kept'`, expID).Scan(&activities); err != nil { + t.Fatal(err) + } + if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, expID).Scan(&queued); err != nil { + t.Fatal(err) + } + if activities != 1 || queued != 0 { + t.Fatalf("activities=%d outbox=%d, want 1 and 0", activities, queued) + } + if n, err = es.FlushActivityOutbox(); err != nil || n != 0 { + t.Fatalf("second flush n=%d err=%v", n, err) + } + if err := d.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='kept'`, expID).Scan(&activities); err != nil { + t.Fatal(err) + } + if activities != 1 { + t.Fatalf("activities=%d after a second flush, want 1", activities) + } +} + +func TestFlushActivityOutboxDropsStuckRow(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + t.Skipf("postgres unavailable (%v) — skipping", err) + } + defer d.Close() + const missing = int64(9_000_000_000_001) + t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM activity_outbox WHERE exploration_id=$1`, missing) }) + es := d.Exploration(missing) + if err := es.EnqueueActivity(Activity{Worker: "worker", Kind: "text", Summary: "stuck"}, "no parent"); err != nil { + t.Fatal(err) + } + for attempt := 1; attempt < activityOutboxMaxAttempts; attempt++ { + n, err := es.FlushActivityOutbox() + if err != nil || n != 0 { + t.Fatalf("attempt %d: n=%d err=%v", attempt, n, err) + } + var queued int + if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, missing).Scan(&queued); err != nil { + t.Fatal(err) + } + if queued != 1 { + t.Fatalf("attempt %d: outbox rows=%d, want the stuck row kept", attempt, queued) + } + } + n, err := es.FlushActivityOutbox() + if err != nil || n != 0 { + t.Fatalf("final flush n=%d err=%v", n, err) + } + var queued int + if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, missing).Scan(&queued); err != nil { + t.Fatal(err) + } + if queued != 0 { + t.Fatalf("outbox rows=%d, want the stuck row dropped", queued) + } +} diff --git a/db/exploration.go b/db/exploration.go index e55424b..ee2c919 100644 --- a/db/exploration.go +++ b/db/exploration.go @@ -1382,14 +1382,24 @@ func (s *ExplorationStore) Stats() (map[string]int, error) { // --- activity (poll by global id cursor) --- +// queryRower is the subset of *sql.DB and *sql.Tx that AppendActivity needs, +// so a replay can insert the activity and delete the outbox row together. +type queryRower interface { + QueryRow(query string, args ...any) *sql.Row +} + // AppendActivity records one worker step and returns its id. func (s *ExplorationStore) AppendActivity(a Activity) (int64, error) { + return s.appendActivity(s.db, a) +} + +func (s *ExplorationStore) appendActivity(q queryRower, a Activity) (int64, error) { metadata := a.Metadata if len(metadata) == 0 { metadata = json.RawMessage(`{}`) } var id int64 - err := s.db.QueryRow(` + err := q.QueryRow(` INSERT INTO activity(exploration_id, node_id, worker, kind, tool, tool_use_id, is_error, summary, detail, metadata, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, main_seg) VALUES ($1,$2,NULLIF($3,''),NULLIF($4,''),NULLIF($5,''),NULLIF($6,''),$7,NULLIF($8,''),NULLIF($9,''),$10,$11,$12,$13,$14,$15) RETURNING id`, s.expID, a.NodeID, utf8Clean(a.Worker), utf8Clean(a.Kind), utf8Clean(a.Tool), utf8Clean(a.ToolUseID), a.IsError, utf8Clean(a.Summary), utf8Clean(a.Detail), diff --git a/db/migrations/0006_activity_outbox.sql b/db/migrations/0006_activity_outbox.sql index ff1396c..c162b2e 100644 --- a/db/migrations/0006_activity_outbox.sql +++ b/db/migrations/0006_activity_outbox.sql @@ -7,6 +7,7 @@ CREATE TABLE IF NOT EXISTS activity_outbox ( exploration_id BIGINT NOT NULL, payload JSONB NOT NULL, last_error TEXT, + attempts INT NOT NULL DEFAULT 0, created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); diff --git a/server/engine.go b/server/engine.go index b268860..6ea21c7 100644 --- a/server/engine.go +++ b/server/engine.go @@ -252,11 +252,13 @@ func runTaskRoutine(rt *taskRuntime, fn func(context.Context)) { }() } -// Wait blocks until the task goroutines registered at the moment of the call -// have finished, or until ctx ends. It refuses new registrations first so Add -// cannot race Wait. Goroutines that were already running are cancelled with -// AbortShutdown. That cancel does not write a terminal task status, and an -// in-flight intent claim stays running so the next start can reset it to open. +// Wait blocks until the task goroutines registered at the moment of the call, +// and every admitted task operation, have finished, or until ctx ends. It +// refuses new registrations and new beginTaskOperation calls first. Goroutines +// that were already running are cancelled with AbortShutdown. That cancel does +// not write a terminal task status, and an in-flight intent claim stays running +// so the next start can reset it to open. Detached chat runs are not in a task +// WaitGroup; they are counted in inflight and Wait stays until they return. func (e *Engine) Wait(ctx context.Context) error { if e == nil { return nil @@ -275,20 +277,36 @@ func (e *Engine) Wait(ctx context.Context) error { } e.runtimeMu.Unlock() for _, wg := range groups { - done := make(chan struct{}) - go func(wg *sync.WaitGroup) { - wg.Wait() - close(done) - }(wg) + if err := waitGroup(ctx, wg); err != nil { + return err + } + } + for e.inflightTotal() > 0 { + timer := time.NewTimer(20 * time.Millisecond) select { - case <-done: case <-ctx.Done(): + timer.Stop() return ctx.Err() + case <-timer.C: } } return nil } +func waitGroup(ctx context.Context, wg *sync.WaitGroup) error { + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + // StopTask permanently stops every long-lived goroutine and removes all Engine // state for a successfully deleted task. The delete barrier remains installed // until cleanup finishes, so no new task operation can race the teardown. @@ -1393,7 +1411,7 @@ func (e *Engine) runIntent(ctx context.Context, t *Task, name string, worker *ag // be started; the intent is left untouched in that case. func (e *Engine) runDetachedIntent(ctx context.Context, t *Task, intentID int64, requestID, message, agentMessage string) error { if !e.beginTaskOperation(t.ID) { - return fmt.Errorf("task is being deleted") + return fmt.Errorf("task is not accepting work") } release := true defer func() { diff --git a/server/engine_cancelcause_test.go b/server/engine_cancelcause_test.go index 44554b2..9863842 100644 --- a/server/engine_cancelcause_test.go +++ b/server/engine_cancelcause_test.go @@ -9,6 +9,40 @@ import ( "github.com/Autumn-27/artex/agent" ) +func TestWaitBlocksUntilDetachedInflightEnds(t *testing.T) { + e := NewEngine(nil) + if !e.beginTaskOperation("t1") { + t.Fatal("detached work was not admitted") + } + release := make(chan struct{}) + finished := make(chan struct{}) + go func() { + <-release + e.decInflight("t1") + close(finished) + }() + done := make(chan error, 1) + go func() { done <- e.Wait(context.Background()) }() + select { + case err := <-done: + t.Fatalf("Wait returned while the detached operation was still running: %v", err) + case <-time.After(80 * time.Millisecond): + } + if e.beginTaskOperation("t1") { + t.Fatal("Wait admitted new work while draining") + } + close(release) + <-finished + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("Wait did not return after the detached operation ended") + } +} + func TestWaitCancelsRuntimeWithShutdown(t *testing.T) { e := NewEngine(nil) rt := e.registerTaskRoutines(context.Background(), "t1", 1) diff --git a/server/engine_timeout.go b/server/engine_timeout.go index cb596d0..c8869c3 100644 --- a/server/engine_timeout.go +++ b/server/engine_timeout.go @@ -46,14 +46,20 @@ func (e *Engine) inflightCounter(taskID string) *int64 { } // beginTaskOperation atomically registers a task-owned operation unless deletion -// has already installed its barrier. The delete handler can therefore wait for -// inflight==0 without a check-then-start race recreating files after cleanup. +// has already installed its barrier, or the engine is draining for shutdown. +// The delete handler and Wait can therefore wait for inflight==0 without a +// check-then-start race admitting work that outlives the drain. func (e *Engine) beginTaskOperation(taskID string) bool { e.deleteMu.RLock() defer e.deleteMu.RUnlock() if e.IsDeleting(taskID) { return false } + e.runtimeMu.Lock() + defer e.runtimeMu.Unlock() + if e.draining { + return false + } atomic.AddInt64(e.inflightCounter(taskID), 1) return true } @@ -63,6 +69,17 @@ func (e *Engine) inflightCount(taskID string) int64 { return atomic.LoadInt64(e.inflightCounter(taskID)) } +// inflightTotal is the sum of every task's admitted operations, including +// detached chat runs that are not members of a task WaitGroup. +func (e *Engine) inflightTotal() int64 { + var n int64 + e.inflight.Range(func(_, v any) bool { + n += atomic.LoadInt64(v.(*int64)) + return true + }) + return n +} + // ---------- deadline ---------- // taskDeadline returns the task's absolute deadline (unix). Prefers the in-process