diff --git a/cmd/artex/main.go b/cmd/artex/main.go index 99e1180..6595a28 100644 --- a/cmd/artex/main.go +++ b/cmd/artex/main.go @@ -127,9 +127,24 @@ func run() int { <-ctx.Done() log.Println("shutting down...") - shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _ = httpSrv.Shutdown(shutdownCtx) + // Close the listener before draining. Otherwise a request accepted during + // the drain can start a detached run, and mgr.Close runs while that run + // is still writing. + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + httpErr := httpSrv.Shutdown(shutdownCtx) + cancel() + if httpErr != nil { + log.Printf("http shutdown: %v", httpErr) + } + drainCtx, drainCancel := context.WithTimeout(context.Background(), 15*time.Second) + drainErr := srv.Wait(drainCtx) + drainCancel() + if drainErr != nil { + log.Printf("task drain: %v", drainErr) + } + if httpErr != nil || drainErr != nil { + return 1 + } return 0 } diff --git a/db/activity_outbox.go b/db/activity_outbox.go new file mode 100644 index 0000000..b49f234 --- /dev/null +++ b/db/activity_outbox.go @@ -0,0 +1,118 @@ +package db + +import ( + "encoding/json" + "fmt" + "log" +) + +// activityOutboxMaxAttempts is how many times a row may fail AppendActivity +// before it is dropped. One stuck row must not sit at the head of the queue +// and keep every later row from being replayed. +const activityOutboxMaxAttempts = 5 + +// activityOutboxPayload is Activity plus Detail. Activity.Detail is omitted +// from ordinary JSON (the list payload stays small); the outbox has to keep it +// so a replayed row is the same step that failed to store. +type activityOutboxPayload struct { + Activity + Detail string `json:"detail,omitempty"` +} + +func marshalActivity(a Activity) ([]byte, error) { + return json.Marshal(activityOutboxPayload{Activity: a, Detail: a.Detail}) +} + +func unmarshalActivity(raw []byte) (Activity, error) { + var payload activityOutboxPayload + if err := json.Unmarshal(raw, &payload); err != nil { + return Activity{}, err + } + a := payload.Activity + a.Detail = payload.Detail + return a, nil +} + +// EnqueueActivity keeps one activity that AppendActivity could not store. +func (s *ExplorationStore) EnqueueActivity(a Activity, cause string) error { + if s == nil || s.db == nil { + return fmt.Errorf("activity outbox: no store") + } + raw, err := marshalActivity(a) + if err != nil { + return err + } + _, err = s.db.Exec(`INSERT INTO activity_outbox (exploration_id, payload, last_error) VALUES ($1, $2::jsonb, $3)`, s.expID, string(raw), cause) + return err +} + +// FlushActivityOutbox inserts queued activities in order and deletes each row +// that lands. The insert and the delete commit together, so a crash cannot +// leave the activity stored and the outbox row still queued. A row that still +// cannot be inserted stays, with its attempt count increased, and the flush +// continues with the next row. After activityOutboxMaxAttempts the row is +// dropped and logged. A payload that cannot be decoded is discarded. +func (s *ExplorationStore) FlushActivityOutbox() (int, error) { + if s == nil || s.db == nil { + return 0, fmt.Errorf("activity outbox: no store") + } + rows, err := s.db.Query(`SELECT id, payload, attempts FROM activity_outbox WHERE exploration_id=$1 ORDER BY id`, s.expID) + if err != nil { + return 0, err + } + defer rows.Close() + type item struct { + id int64 + payload []byte + attempts int + } + var items []item + for rows.Next() { + var it item + if err := rows.Scan(&it.id, &it.payload, &it.attempts); err != nil { + return 0, err + } + items = append(items, it) + } + if err := rows.Err(); err != nil { + return 0, err + } + stored := 0 + for _, it := range items { + a, err := unmarshalActivity(it.payload) + if err != nil { + if _, delErr := s.db.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); delErr != nil { + return stored, delErr + } + continue + } + tx, err := s.db.Begin() + if err != nil { + return stored, err + } + if _, err := s.appendActivity(tx, a); err != nil { + _ = tx.Rollback() + next := it.attempts + 1 + if next >= activityOutboxMaxAttempts { + log.Printf("[activity] outbox %d for exploration %d dropped after %d attempts: %v", it.id, s.expID, next, err) + if _, delErr := s.db.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); delErr != nil { + return stored, delErr + } + continue + } + if _, updErr := s.db.Exec(`UPDATE activity_outbox SET attempts=$2, last_error=$3 WHERE id=$1`, it.id, next, err.Error()); updErr != nil { + return stored, updErr + } + continue + } + if _, err := tx.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); err != nil { + _ = tx.Rollback() + return stored, err + } + if err := tx.Commit(); err != nil { + return stored, err + } + stored++ + } + return stored, nil +} diff --git a/db/activity_outbox_test.go b/db/activity_outbox_test.go new file mode 100644 index 0000000..a27deb3 --- /dev/null +++ b/db/activity_outbox_test.go @@ -0,0 +1,99 @@ +package db + +import "testing" + +func TestActivityOutboxKeepsDetail(t *testing.T) { + in := Activity{Worker: "worker", Kind: "tool_result", Summary: "done", Detail: "full body", IsError: true} + raw, err := marshalActivity(in) + if err != nil { + t.Fatal(err) + } + out, err := unmarshalActivity(raw) + if err != nil { + t.Fatal(err) + } + if out.Detail != in.Detail || out.Summary != in.Summary || out.Worker != in.Worker || out.Kind != in.Kind || !out.IsError { + t.Fatalf("round trip = %+v", out) + } +} + +func TestFlushActivityOutboxReplaysOnce(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + t.Skipf("postgres unavailable (%v) — skipping", err) + } + defer d.Close() + expID, err := d.CreateExploration("outbox", "replay") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _, _ = d.Exec(`DELETE FROM activity_outbox WHERE exploration_id=$1`, expID) + _, _ = d.Exec(`DELETE FROM explorations WHERE id=$1`, expID) + }) + es := d.Exploration(expID) + if err := es.EnqueueActivity(Activity{Worker: "worker", Kind: "text", Summary: "kept", Detail: "body"}, "insert failed"); err != nil { + t.Fatal(err) + } + n, err := es.FlushActivityOutbox() + if err != nil || n != 1 { + t.Fatalf("flush n=%d err=%v", n, err) + } + var activities, queued int + if err := d.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='kept'`, expID).Scan(&activities); err != nil { + t.Fatal(err) + } + if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, expID).Scan(&queued); err != nil { + t.Fatal(err) + } + if activities != 1 || queued != 0 { + t.Fatalf("activities=%d outbox=%d, want 1 and 0", activities, queued) + } + if n, err = es.FlushActivityOutbox(); err != nil || n != 0 { + t.Fatalf("second flush n=%d err=%v", n, err) + } + if err := d.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='kept'`, expID).Scan(&activities); err != nil { + t.Fatal(err) + } + if activities != 1 { + t.Fatalf("activities=%d after a second flush, want 1", activities) + } +} + +func TestFlushActivityOutboxDropsStuckRow(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + t.Skipf("postgres unavailable (%v) — skipping", err) + } + defer d.Close() + const missing = int64(9_000_000_000_001) + t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM activity_outbox WHERE exploration_id=$1`, missing) }) + es := d.Exploration(missing) + if err := es.EnqueueActivity(Activity{Worker: "worker", Kind: "text", Summary: "stuck"}, "no parent"); err != nil { + t.Fatal(err) + } + for attempt := 1; attempt < activityOutboxMaxAttempts; attempt++ { + n, err := es.FlushActivityOutbox() + if err != nil || n != 0 { + t.Fatalf("attempt %d: n=%d err=%v", attempt, n, err) + } + var queued int + if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, missing).Scan(&queued); err != nil { + t.Fatal(err) + } + if queued != 1 { + t.Fatalf("attempt %d: outbox rows=%d, want the stuck row kept", attempt, queued) + } + } + n, err := es.FlushActivityOutbox() + if err != nil || n != 0 { + t.Fatalf("final flush n=%d err=%v", n, err) + } + var queued int + if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, missing).Scan(&queued); err != nil { + t.Fatal(err) + } + if queued != 0 { + t.Fatalf("outbox rows=%d, want the stuck row dropped", queued) + } +} diff --git a/db/exploration.go b/db/exploration.go index ea8a55b..ee2c919 100644 --- a/db/exploration.go +++ b/db/exploration.go @@ -354,7 +354,10 @@ WHERE exploration_id=$1 AND kind='intent' AND state='running' if err != nil { return 0, err } - n, _ := res.RowsAffected() + n, err := res.RowsAffected() + if err != nil { + return 0, err + } return n, nil } @@ -1379,14 +1382,24 @@ func (s *ExplorationStore) Stats() (map[string]int, error) { // --- activity (poll by global id cursor) --- +// queryRower is the subset of *sql.DB and *sql.Tx that AppendActivity needs, +// so a replay can insert the activity and delete the outbox row together. +type queryRower interface { + QueryRow(query string, args ...any) *sql.Row +} + // AppendActivity records one worker step and returns its id. func (s *ExplorationStore) AppendActivity(a Activity) (int64, error) { + return s.appendActivity(s.db, a) +} + +func (s *ExplorationStore) appendActivity(q queryRower, a Activity) (int64, error) { metadata := a.Metadata if len(metadata) == 0 { metadata = json.RawMessage(`{}`) } var id int64 - err := s.db.QueryRow(` + err := q.QueryRow(` INSERT INTO activity(exploration_id, node_id, worker, kind, tool, tool_use_id, is_error, summary, detail, metadata, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, main_seg) VALUES ($1,$2,NULLIF($3,''),NULLIF($4,''),NULLIF($5,''),NULLIF($6,''),$7,NULLIF($8,''),NULLIF($9,''),$10,$11,$12,$13,$14,$15) RETURNING id`, s.expID, a.NodeID, utf8Clean(a.Worker), utf8Clean(a.Kind), utf8Clean(a.Tool), utf8Clean(a.ToolUseID), a.IsError, utf8Clean(a.Summary), utf8Clean(a.Detail), diff --git a/db/migrate_test.go b/db/migrate_test.go index 2ca9d54..fc6d756 100644 --- a/db/migrate_test.go +++ b/db/migrate_test.go @@ -137,9 +137,9 @@ func assertEmbeddedApplied(t *testing.T, d *DB) []appliedMigration { version int name string }{{1, llmRecordsMigration}, {2, llmUsageMigration}, {3, "0003_task_scope_icp_index.sql"}, {4, "0004_llm_usage_ts_index.sql"}, - {5, "0005_traffic_rewrite_rules.sql"}} + {5, "0005_traffic_rewrite_rules.sql"}, {6, "0006_activity_outbox.sql"}} if len(applied) != len(want) { - t.Fatalf("schema_migrations = %+v, want versions 1 to 5", applied) + t.Fatalf("schema_migrations = %+v, want versions 1 to 6", applied) } for i, w := range want { content, err := fs.ReadFile(migrationsFS(), w.name) diff --git a/db/migrations/0006_activity_outbox.sql b/db/migrations/0006_activity_outbox.sql new file mode 100644 index 0000000..c162b2e --- /dev/null +++ b/db/migrations/0006_activity_outbox.sql @@ -0,0 +1,15 @@ +-- Activities that could not be inserted (the exploration row was missing, or +-- the write failed after retries) are kept here and replayed into activity +-- once the parent row accepts them. No foreign key: the row has to survive +-- the missing exploration that caused the original failure. +CREATE TABLE IF NOT EXISTS activity_outbox ( + id BIGSERIAL PRIMARY KEY, + exploration_id BIGINT NOT NULL, + payload JSONB NOT NULL, + last_error TEXT, + attempts INT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS activity_outbox_exploration_idx + ON activity_outbox (exploration_id, id); diff --git a/server/engine.go b/server/engine.go index 4e4ebad..6ea21c7 100644 --- a/server/engine.go +++ b/server/engine.go @@ -118,6 +118,7 @@ type Engine struct { // runs under one task-scoped context. Successful deletion cancels that context, // waits for all goroutines, then releases every task-level Engine reference. runtimeMu sync.Mutex + draining bool runtimes map[string]*taskRuntime // per-task execution context: each planner.Plan / worker.Execute runs under it, @@ -162,7 +163,7 @@ type Engine struct { type taskRuntime struct { ctx context.Context - cancel context.CancelFunc + cancel context.CancelCauseFunc wg sync.WaitGroup } @@ -231,9 +232,12 @@ func (e *Engine) IsDeleting(taskID string) bool { func (e *Engine) registerTaskRoutines(parent context.Context, taskID string, count int) *taskRuntime { e.runtimeMu.Lock() defer e.runtimeMu.Unlock() + if e.draining { + return nil + } rt := e.runtimes[taskID] if rt == nil { - ctx, cancel := context.WithCancel(parent) + ctx, cancel := context.WithCancelCause(parent) rt = &taskRuntime{ctx: ctx, cancel: cancel} e.runtimes[taskID] = rt } @@ -248,6 +252,61 @@ func runTaskRoutine(rt *taskRuntime, fn func(context.Context)) { }() } +// Wait blocks until the task goroutines registered at the moment of the call, +// and every admitted task operation, have finished, or until ctx ends. It +// refuses new registrations and new beginTaskOperation calls first. Goroutines +// that were already running are cancelled with AbortShutdown. That cancel does +// not write a terminal task status, and an in-flight intent claim stays running +// so the next start can reset it to open. Detached chat runs are not in a task +// WaitGroup; they are counted in inflight and Wait stays until they return. +func (e *Engine) Wait(ctx context.Context) error { + if e == nil { + return nil + } + e.runtimeMu.Lock() + e.draining = true + groups := make([]*sync.WaitGroup, 0, len(e.runtimes)) + for _, rt := range e.runtimes { + if rt == nil { + continue + } + if rt.cancel != nil { + rt.cancel(agent.AbortShutdown) + } + groups = append(groups, &rt.wg) + } + e.runtimeMu.Unlock() + for _, wg := range groups { + if err := waitGroup(ctx, wg); err != nil { + return err + } + } + for e.inflightTotal() > 0 { + timer := time.NewTimer(20 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } + return nil +} + +func waitGroup(ctx context.Context, wg *sync.WaitGroup) error { + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + // StopTask permanently stops every long-lived goroutine and removes all Engine // state for a successfully deleted task. The delete barrier remains installed // until cleanup finishes, so no new task operation can race the teardown. @@ -260,7 +319,7 @@ func (e *Engine) StopTask(taskID string) { e.runtimeMu.Lock() rt := e.runtimes[taskID] if rt != nil { - rt.cancel() + rt.cancel(agent.AbortTaskDeleted) } e.runtimeMu.Unlock() if rt != nil { @@ -664,6 +723,23 @@ func (e *Engine) emitActivity(t *Task, r db.Activity) db.Activity { } log.Printf("[activity] task %s dropping activity record (%d dropped for this task so far) worker=%s kind=%s tool=%s tuid=%s reason=%s summary=%q: %v%s", t.ID, n, r.Worker, r.Kind, r.Tool, r.ToolUseID, dropReason(err), preview(r.Summary, 80), err, diag) + if t.Store != nil { + if qerr := t.Store.EnqueueActivity(r, err.Error()); qerr != nil { + log.Printf("[activity] task %s outbox: %v", t.ID, qerr) + } + } + // The row has no database id. Publish an error copy so a live session + // does not stay on "Running" while the original return value is unchanged. + notice := r + notice.IsError = true + if notice.Summary == "" { + notice.Summary = "activity was not stored" + } else { + notice.Summary += " (not stored)" + } + if e.bc != nil { + e.bc.Publish(t.ID, notice) + } e.touch(t.ID) return r } @@ -762,6 +838,10 @@ func (e *Engine) Run(ctx context.Context, t *Task) { } rt := e.registerTaskRoutines(ctx, t.ID, 2+workers) e.deleteMu.RUnlock() + if rt == nil { + e.started.Delete(t.ID) + return + } e.touch(t.ID) runTaskRoutine(rt, func(loopCtx context.Context) { e.plannerLoop(loopCtx, t) }) runTaskRoutine(rt, func(loopCtx context.Context) { e.claimSweepLoop(loopCtx, t) }) @@ -1266,6 +1346,13 @@ func (e *Engine) runIntent(ctx context.Context, t *Task, name string, worker *ag e.touch(t.ID) return true } + // Process shutdown cancels the task context. Leave the claim running: AbortShutdown + // says restart resets leftover running intents to open. Settling done or blocked + // here would make that reset miss the intent, so the task would resume without it. + if ectx.Err() != nil && errors.Is(context.Cause(ectx), agent.AbortShutdown) { + log.Printf("[worker %s] task %s intent #%d shutdown: leaving the claim running", name, t.ID, intent.ID) + return true + } // killed by the planner: mark stopped (don't write back results, don't auto-reclaim). if killed { if err := t.Store.SettleIntentClaim(intent.ID, token, "stopped"); err != nil { @@ -1324,7 +1411,7 @@ func (e *Engine) runIntent(ctx context.Context, t *Task, name string, worker *ag // be started; the intent is left untouched in that case. func (e *Engine) runDetachedIntent(ctx context.Context, t *Task, intentID int64, requestID, message, agentMessage string) error { if !e.beginTaskOperation(t.ID) { - return fmt.Errorf("task is being deleted") + return fmt.Errorf("task is not accepting work") } release := true defer func() { diff --git a/server/engine_cancelcause_test.go b/server/engine_cancelcause_test.go index d1f4f7a..9863842 100644 --- a/server/engine_cancelcause_test.go +++ b/server/engine_cancelcause_test.go @@ -9,6 +9,60 @@ import ( "github.com/Autumn-27/artex/agent" ) +func TestWaitBlocksUntilDetachedInflightEnds(t *testing.T) { + e := NewEngine(nil) + if !e.beginTaskOperation("t1") { + t.Fatal("detached work was not admitted") + } + release := make(chan struct{}) + finished := make(chan struct{}) + go func() { + <-release + e.decInflight("t1") + close(finished) + }() + done := make(chan error, 1) + go func() { done <- e.Wait(context.Background()) }() + select { + case err := <-done: + t.Fatalf("Wait returned while the detached operation was still running: %v", err) + case <-time.After(80 * time.Millisecond): + } + if e.beginTaskOperation("t1") { + t.Fatal("Wait admitted new work while draining") + } + close(release) + <-finished + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("Wait did not return after the detached operation ended") + } +} + +func TestWaitCancelsRuntimeWithShutdown(t *testing.T) { + e := NewEngine(nil) + rt := e.registerTaskRoutines(context.Background(), "t1", 1) + if rt == nil { + t.Fatal("runtime was not registered") + } + started := make(chan struct{}) + runTaskRoutine(rt, func(ctx context.Context) { + close(started) + <-ctx.Done() + }) + <-started + if err := e.Wait(context.Background()); err != nil { + t.Fatal(err) + } + if !errors.Is(context.Cause(rt.ctx), agent.AbortShutdown) { + t.Fatalf("cause=%v, want shutdown", context.Cause(rt.ctx)) + } +} + func TestPauseCancelsWithNamedCause(t *testing.T) { e := NewEngine(nil) execCtx := e.execContextFor(context.Background(), "t1") diff --git a/server/engine_timeout.go b/server/engine_timeout.go index 37f5469..c8869c3 100644 --- a/server/engine_timeout.go +++ b/server/engine_timeout.go @@ -46,14 +46,20 @@ func (e *Engine) inflightCounter(taskID string) *int64 { } // beginTaskOperation atomically registers a task-owned operation unless deletion -// has already installed its barrier. The delete handler can therefore wait for -// inflight==0 without a check-then-start race recreating files after cleanup. +// has already installed its barrier, or the engine is draining for shutdown. +// The delete handler and Wait can therefore wait for inflight==0 without a +// check-then-start race admitting work that outlives the drain. func (e *Engine) beginTaskOperation(taskID string) bool { e.deleteMu.RLock() defer e.deleteMu.RUnlock() if e.IsDeleting(taskID) { return false } + e.runtimeMu.Lock() + defer e.runtimeMu.Unlock() + if e.draining { + return false + } atomic.AddInt64(e.inflightCounter(taskID), 1) return true } @@ -63,6 +69,17 @@ func (e *Engine) inflightCount(taskID string) int64 { return atomic.LoadInt64(e.inflightCounter(taskID)) } +// inflightTotal is the sum of every task's admitted operations, including +// detached chat runs that are not members of a task WaitGroup. +func (e *Engine) inflightTotal() int64 { + var n int64 + e.inflight.Range(func(_, v any) bool { + n += atomic.LoadInt64(v.(*int64)) + return true + }) + return n +} + // ---------- deadline ---------- // taskDeadline returns the task's absolute deadline (unix). Prefers the in-process @@ -161,6 +178,10 @@ func (e *Engine) startDeadlineCoordinator(ctx context.Context, t *Task) { } rt := e.registerTaskRoutines(ctx, t.ID, 1) e.deleteMu.RUnlock() + if rt == nil { + e.coordStarted.Delete(t.ID) + return + } runTaskRoutine(rt, func(loopCtx context.Context) { e.deadlineCoordinator(loopCtx, t) }) } diff --git a/server/server.go b/server/server.go index 7676d93..aa88728 100644 --- a/server/server.go +++ b/server/server.go @@ -296,9 +296,16 @@ func (s *Server) restoreTaskRuntimes() { lifecycle := t.lifecycleSnapshot() // clear stale 'running' intents from a prior crash/restart (no live worker // owns them) so they re-claim instead of spinning forever in the UI. - if n, _ := t.Store.ResetRunningIntents(); n > 0 { + if n, err := t.Store.ResetRunningIntents(); err != nil { + log.Printf("[engine] task %s reset leftover running intents: %v", t.ID, err) + } else if n > 0 { log.Printf("[engine] task %s reset %d leftover running intents to open", t.ID, n) } + if n, err := t.Store.FlushActivityOutbox(); err != nil { + log.Printf("[activity] task %s outbox: %v", t.ID, err) + } else if n > 0 { + log.Printf("[activity] task %s stored %d activities recovered from the outbox", t.ID, n) + } if lifecycle.Paused { s.engine.Pause(t.ID, agent.AbortPausedOnReload) } @@ -320,6 +327,30 @@ func (s *Server) restoreTaskRuntimes() { } } +// Wait blocks until the task goroutines snapshotted now, and the archive +// worker, have finished, or until ctx ends. +func (s *Server) Wait(ctx context.Context) error { + if s == nil { + return nil + } + if s.engine != nil { + if err := s.engine.Wait(ctx); err != nil { + return err + } + } + done := make(chan struct{}) + go func() { + s.archiveWG.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + // agentMaxTurns returns the configured max_turns for an agent key (0 = unlimited, // also the fallback when no DB or no row). func (s *Server) agentMaxTurns(key string) int {