Stop admitting work before the drain and replay each outbox row once.

The listener closes before Wait, and a shutdown refuses new task operations. Wait also stays until detached runs finish. The activity insert and the outbox delete commit together, and a row that keeps failing is dropped after five attempts so it cannot block the rest.
This commit is contained in:
protonmatter
2026-11-17 15:41:56 -05:00
parent 0234025d28
commit a1c6103cbf
8 changed files with 224 additions and 32 deletions
+10 -7
View File
@@ -127,19 +127,22 @@ func run() int {
<-ctx.Done()
log.Println("shutting down...")
// Close the listener before draining. Otherwise a request accepted during
// the drain can start a detached run, and mgr.Close runs while that run
// is still writing.
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
httpErr := httpSrv.Shutdown(shutdownCtx)
cancel()
if httpErr != nil {
log.Printf("http shutdown: %v", httpErr)
}
drainCtx, drainCancel := context.WithTimeout(context.Background(), 15*time.Second)
drainErr := srv.Wait(drainCtx)
drainCancel()
if drainErr != nil {
log.Printf("task drain: %v", drainErr)
}
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := httpSrv.Shutdown(shutdownCtx); err != nil {
log.Printf("http shutdown: %v", err)
return 1
}
if drainErr != nil {
if httpErr != nil || drainErr != nil {
return 1
}
return 0
+38 -10
View File
@@ -3,8 +3,14 @@ package db
import (
"encoding/json"
"fmt"
"log"
)
// activityOutboxMaxAttempts is how many times a row may fail AppendActivity
// before it is dropped. One stuck row must not sit at the head of the queue
// and keep every later row from being replayed.
const activityOutboxMaxAttempts = 5
// activityOutboxPayload is Activity plus Detail. Activity.Detail is omitted
// from ordinary JSON (the list payload stays small); the outbox has to keep it
// so a replayed row is the same step that failed to store.
@@ -41,26 +47,29 @@ func (s *ExplorationStore) EnqueueActivity(a Activity, cause string) error {
}
// FlushActivityOutbox inserts queued activities in order and deletes each row
// that lands. A row that still cannot be inserted stays, with last_error
// updated, and the flush stops so later rows keep their order. A payload that
// cannot be decoded is discarded.
// that lands. The insert and the delete commit together, so a crash cannot
// leave the activity stored and the outbox row still queued. A row that still
// cannot be inserted stays, with its attempt count increased, and the flush
// continues with the next row. After activityOutboxMaxAttempts the row is
// dropped and logged. A payload that cannot be decoded is discarded.
func (s *ExplorationStore) FlushActivityOutbox() (int, error) {
if s == nil || s.db == nil {
return 0, fmt.Errorf("activity outbox: no store")
}
rows, err := s.db.Query(`SELECT id, payload FROM activity_outbox WHERE exploration_id=$1 ORDER BY id`, s.expID)
rows, err := s.db.Query(`SELECT id, payload, attempts FROM activity_outbox WHERE exploration_id=$1 ORDER BY id`, s.expID)
if err != nil {
return 0, err
}
defer rows.Close()
type item struct {
id int64
payload []byte
id int64
payload []byte
attempts int
}
var items []item
for rows.Next() {
var it item
if err := rows.Scan(&it.id, &it.payload); err != nil {
if err := rows.Scan(&it.id, &it.payload, &it.attempts); err != nil {
return 0, err
}
items = append(items, it)
@@ -77,11 +86,30 @@ func (s *ExplorationStore) FlushActivityOutbox() (int, error) {
}
continue
}
if _, err := s.AppendActivity(a); err != nil {
_, _ = s.db.Exec(`UPDATE activity_outbox SET last_error=$2 WHERE id=$1`, it.id, err.Error())
tx, err := s.db.Begin()
if err != nil {
return stored, err
}
if _, err := s.db.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); err != nil {
if _, err := s.appendActivity(tx, a); err != nil {
_ = tx.Rollback()
next := it.attempts + 1
if next >= activityOutboxMaxAttempts {
log.Printf("[activity] outbox %d for exploration %d dropped after %d attempts: %v", it.id, s.expID, next, err)
if _, delErr := s.db.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); delErr != nil {
return stored, delErr
}
continue
}
if _, updErr := s.db.Exec(`UPDATE activity_outbox SET attempts=$2, last_error=$3 WHERE id=$1`, it.id, next, err.Error()); updErr != nil {
return stored, updErr
}
continue
}
if _, err := tx.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); err != nil {
_ = tx.Rollback()
return stored, err
}
if err := tx.Commit(); err != nil {
return stored, err
}
stored++
+81
View File
@@ -16,3 +16,84 @@ func TestActivityOutboxKeepsDetail(t *testing.T) {
t.Fatalf("round trip = %+v", out)
}
}
func TestFlushActivityOutboxReplaysOnce(t *testing.T) {
d, err := Open(testDSN(t))
if err != nil {
t.Skipf("postgres unavailable (%v) — skipping", err)
}
defer d.Close()
expID, err := d.CreateExploration("outbox", "replay")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_, _ = d.Exec(`DELETE FROM activity_outbox WHERE exploration_id=$1`, expID)
_, _ = d.Exec(`DELETE FROM explorations WHERE id=$1`, expID)
})
es := d.Exploration(expID)
if err := es.EnqueueActivity(Activity{Worker: "worker", Kind: "text", Summary: "kept", Detail: "body"}, "insert failed"); err != nil {
t.Fatal(err)
}
n, err := es.FlushActivityOutbox()
if err != nil || n != 1 {
t.Fatalf("flush n=%d err=%v", n, err)
}
var activities, queued int
if err := d.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='kept'`, expID).Scan(&activities); err != nil {
t.Fatal(err)
}
if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, expID).Scan(&queued); err != nil {
t.Fatal(err)
}
if activities != 1 || queued != 0 {
t.Fatalf("activities=%d outbox=%d, want 1 and 0", activities, queued)
}
if n, err = es.FlushActivityOutbox(); err != nil || n != 0 {
t.Fatalf("second flush n=%d err=%v", n, err)
}
if err := d.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='kept'`, expID).Scan(&activities); err != nil {
t.Fatal(err)
}
if activities != 1 {
t.Fatalf("activities=%d after a second flush, want 1", activities)
}
}
func TestFlushActivityOutboxDropsStuckRow(t *testing.T) {
d, err := Open(testDSN(t))
if err != nil {
t.Skipf("postgres unavailable (%v) — skipping", err)
}
defer d.Close()
const missing = int64(9_000_000_000_001)
t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM activity_outbox WHERE exploration_id=$1`, missing) })
es := d.Exploration(missing)
if err := es.EnqueueActivity(Activity{Worker: "worker", Kind: "text", Summary: "stuck"}, "no parent"); err != nil {
t.Fatal(err)
}
for attempt := 1; attempt < activityOutboxMaxAttempts; attempt++ {
n, err := es.FlushActivityOutbox()
if err != nil || n != 0 {
t.Fatalf("attempt %d: n=%d err=%v", attempt, n, err)
}
var queued int
if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, missing).Scan(&queued); err != nil {
t.Fatal(err)
}
if queued != 1 {
t.Fatalf("attempt %d: outbox rows=%d, want the stuck row kept", attempt, queued)
}
}
n, err := es.FlushActivityOutbox()
if err != nil || n != 0 {
t.Fatalf("final flush n=%d err=%v", n, err)
}
var queued int
if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, missing).Scan(&queued); err != nil {
t.Fatal(err)
}
if queued != 0 {
t.Fatalf("outbox rows=%d, want the stuck row dropped", queued)
}
}
+11 -1
View File
@@ -1382,14 +1382,24 @@ func (s *ExplorationStore) Stats() (map[string]int, error) {
// --- activity (poll by global id cursor) ---
// queryRower is the subset of *sql.DB and *sql.Tx that AppendActivity needs,
// so a replay can insert the activity and delete the outbox row together.
type queryRower interface {
QueryRow(query string, args ...any) *sql.Row
}
// AppendActivity records one worker step and returns its id.
func (s *ExplorationStore) AppendActivity(a Activity) (int64, error) {
return s.appendActivity(s.db, a)
}
func (s *ExplorationStore) appendActivity(q queryRower, a Activity) (int64, error) {
metadata := a.Metadata
if len(metadata) == 0 {
metadata = json.RawMessage(`{}`)
}
var id int64
err := s.db.QueryRow(`
err := q.QueryRow(`
INSERT INTO activity(exploration_id, node_id, worker, kind, tool, tool_use_id, is_error, summary, detail, metadata, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, main_seg)
VALUES ($1,$2,NULLIF($3,''),NULLIF($4,''),NULLIF($5,''),NULLIF($6,''),$7,NULLIF($8,''),NULLIF($9,''),$10,$11,$12,$13,$14,$15)
RETURNING id`, s.expID, a.NodeID, utf8Clean(a.Worker), utf8Clean(a.Kind), utf8Clean(a.Tool), utf8Clean(a.ToolUseID), a.IsError, utf8Clean(a.Summary), utf8Clean(a.Detail),
+1
View File
@@ -7,6 +7,7 @@ CREATE TABLE IF NOT EXISTS activity_outbox (
exploration_id BIGINT NOT NULL,
payload JSONB NOT NULL,
last_error TEXT,
attempts INT NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
+30 -12
View File
@@ -252,11 +252,13 @@ func runTaskRoutine(rt *taskRuntime, fn func(context.Context)) {
}()
}
// Wait blocks until the task goroutines registered at the moment of the call
// have finished, or until ctx ends. It refuses new registrations first so Add
// cannot race Wait. Goroutines that were already running are cancelled with
// AbortShutdown. That cancel does not write a terminal task status, and an
// in-flight intent claim stays running so the next start can reset it to open.
// Wait blocks until the task goroutines registered at the moment of the call,
// and every admitted task operation, have finished, or until ctx ends. It
// refuses new registrations and new beginTaskOperation calls first. Goroutines
// that were already running are cancelled with AbortShutdown. That cancel does
// not write a terminal task status, and an in-flight intent claim stays running
// so the next start can reset it to open. Detached chat runs are not in a task
// WaitGroup; they are counted in inflight and Wait stays until they return.
func (e *Engine) Wait(ctx context.Context) error {
if e == nil {
return nil
@@ -275,20 +277,36 @@ func (e *Engine) Wait(ctx context.Context) error {
}
e.runtimeMu.Unlock()
for _, wg := range groups {
done := make(chan struct{})
go func(wg *sync.WaitGroup) {
wg.Wait()
close(done)
}(wg)
if err := waitGroup(ctx, wg); err != nil {
return err
}
}
for e.inflightTotal() > 0 {
timer := time.NewTimer(20 * time.Millisecond)
select {
case <-done:
case <-ctx.Done():
timer.Stop()
return ctx.Err()
case <-timer.C:
}
}
return nil
}
func waitGroup(ctx context.Context, wg *sync.WaitGroup) error {
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
// StopTask permanently stops every long-lived goroutine and removes all Engine
// state for a successfully deleted task. The delete barrier remains installed
// until cleanup finishes, so no new task operation can race the teardown.
@@ -1393,7 +1411,7 @@ func (e *Engine) runIntent(ctx context.Context, t *Task, name string, worker *ag
// be started; the intent is left untouched in that case.
func (e *Engine) runDetachedIntent(ctx context.Context, t *Task, intentID int64, requestID, message, agentMessage string) error {
if !e.beginTaskOperation(t.ID) {
return fmt.Errorf("task is being deleted")
return fmt.Errorf("task is not accepting work")
}
release := true
defer func() {
+34
View File
@@ -9,6 +9,40 @@ import (
"github.com/Autumn-27/artex/agent"
)
func TestWaitBlocksUntilDetachedInflightEnds(t *testing.T) {
e := NewEngine(nil)
if !e.beginTaskOperation("t1") {
t.Fatal("detached work was not admitted")
}
release := make(chan struct{})
finished := make(chan struct{})
go func() {
<-release
e.decInflight("t1")
close(finished)
}()
done := make(chan error, 1)
go func() { done <- e.Wait(context.Background()) }()
select {
case err := <-done:
t.Fatalf("Wait returned while the detached operation was still running: %v", err)
case <-time.After(80 * time.Millisecond):
}
if e.beginTaskOperation("t1") {
t.Fatal("Wait admitted new work while draining")
}
close(release)
<-finished
select {
case err := <-done:
if err != nil {
t.Fatal(err)
}
case <-time.After(time.Second):
t.Fatal("Wait did not return after the detached operation ended")
}
}
func TestWaitCancelsRuntimeWithShutdown(t *testing.T) {
e := NewEngine(nil)
rt := e.registerTaskRoutines(context.Background(), "t1", 1)
+19 -2
View File
@@ -46,14 +46,20 @@ func (e *Engine) inflightCounter(taskID string) *int64 {
}
// beginTaskOperation atomically registers a task-owned operation unless deletion
// has already installed its barrier. The delete handler can therefore wait for
// inflight==0 without a check-then-start race recreating files after cleanup.
// has already installed its barrier, or the engine is draining for shutdown.
// The delete handler and Wait can therefore wait for inflight==0 without a
// check-then-start race admitting work that outlives the drain.
func (e *Engine) beginTaskOperation(taskID string) bool {
e.deleteMu.RLock()
defer e.deleteMu.RUnlock()
if e.IsDeleting(taskID) {
return false
}
e.runtimeMu.Lock()
defer e.runtimeMu.Unlock()
if e.draining {
return false
}
atomic.AddInt64(e.inflightCounter(taskID), 1)
return true
}
@@ -63,6 +69,17 @@ func (e *Engine) inflightCount(taskID string) int64 {
return atomic.LoadInt64(e.inflightCounter(taskID))
}
// inflightTotal is the sum of every task's admitted operations, including
// detached chat runs that are not members of a task WaitGroup.
func (e *Engine) inflightTotal() int64 {
var n int64
e.inflight.Range(func(_, v any) bool {
n += atomic.LoadInt64(v.(*int64))
return true
})
return n
}
// ---------- deadline ----------
// taskDeadline returns the task's absolute deadline (unix). Prefers the in-process