mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-18 14:10:54 +08:00
Merge pull request #300 from Protonmatter/protonmatter/shutdown-outbox
Drain task goroutines on shutdown and replay unsaved activities
This commit is contained in:
+18
-3
@@ -127,9 +127,24 @@ func run() int {
|
||||
<-ctx.Done()
|
||||
|
||||
log.Println("shutting down...")
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = httpSrv.Shutdown(shutdownCtx)
|
||||
// Close the listener before draining. Otherwise a request accepted during
|
||||
// the drain can start a detached run, and mgr.Close runs while that run
|
||||
// is still writing.
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
httpErr := httpSrv.Shutdown(shutdownCtx)
|
||||
cancel()
|
||||
if httpErr != nil {
|
||||
log.Printf("http shutdown: %v", httpErr)
|
||||
}
|
||||
drainCtx, drainCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
drainErr := srv.Wait(drainCtx)
|
||||
drainCancel()
|
||||
if drainErr != nil {
|
||||
log.Printf("task drain: %v", drainErr)
|
||||
}
|
||||
if httpErr != nil || drainErr != nil {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
)
|
||||
|
||||
// activityOutboxMaxAttempts is how many times a row may fail AppendActivity
|
||||
// before it is dropped. One stuck row must not sit at the head of the queue
|
||||
// and keep every later row from being replayed.
|
||||
const activityOutboxMaxAttempts = 5
|
||||
|
||||
// activityOutboxPayload is Activity plus Detail. Activity.Detail is omitted
|
||||
// from ordinary JSON (the list payload stays small); the outbox has to keep it
|
||||
// so a replayed row is the same step that failed to store.
|
||||
type activityOutboxPayload struct {
|
||||
Activity
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
func marshalActivity(a Activity) ([]byte, error) {
|
||||
return json.Marshal(activityOutboxPayload{Activity: a, Detail: a.Detail})
|
||||
}
|
||||
|
||||
func unmarshalActivity(raw []byte) (Activity, error) {
|
||||
var payload activityOutboxPayload
|
||||
if err := json.Unmarshal(raw, &payload); err != nil {
|
||||
return Activity{}, err
|
||||
}
|
||||
a := payload.Activity
|
||||
a.Detail = payload.Detail
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// EnqueueActivity keeps one activity that AppendActivity could not store.
|
||||
func (s *ExplorationStore) EnqueueActivity(a Activity, cause string) error {
|
||||
if s == nil || s.db == nil {
|
||||
return fmt.Errorf("activity outbox: no store")
|
||||
}
|
||||
raw, err := marshalActivity(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.Exec(`INSERT INTO activity_outbox (exploration_id, payload, last_error) VALUES ($1, $2::jsonb, $3)`, s.expID, string(raw), cause)
|
||||
return err
|
||||
}
|
||||
|
||||
// FlushActivityOutbox inserts queued activities in order and deletes each row
|
||||
// that lands. The insert and the delete commit together, so a crash cannot
|
||||
// leave the activity stored and the outbox row still queued. A row that still
|
||||
// cannot be inserted stays, with its attempt count increased, and the flush
|
||||
// continues with the next row. After activityOutboxMaxAttempts the row is
|
||||
// dropped and logged. A payload that cannot be decoded is discarded.
|
||||
func (s *ExplorationStore) FlushActivityOutbox() (int, error) {
|
||||
if s == nil || s.db == nil {
|
||||
return 0, fmt.Errorf("activity outbox: no store")
|
||||
}
|
||||
rows, err := s.db.Query(`SELECT id, payload, attempts FROM activity_outbox WHERE exploration_id=$1 ORDER BY id`, s.expID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
type item struct {
|
||||
id int64
|
||||
payload []byte
|
||||
attempts int
|
||||
}
|
||||
var items []item
|
||||
for rows.Next() {
|
||||
var it item
|
||||
if err := rows.Scan(&it.id, &it.payload, &it.attempts); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
items = append(items, it)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
stored := 0
|
||||
for _, it := range items {
|
||||
a, err := unmarshalActivity(it.payload)
|
||||
if err != nil {
|
||||
if _, delErr := s.db.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); delErr != nil {
|
||||
return stored, delErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return stored, err
|
||||
}
|
||||
if _, err := s.appendActivity(tx, a); err != nil {
|
||||
_ = tx.Rollback()
|
||||
next := it.attempts + 1
|
||||
if next >= activityOutboxMaxAttempts {
|
||||
log.Printf("[activity] outbox %d for exploration %d dropped after %d attempts: %v", it.id, s.expID, next, err)
|
||||
if _, delErr := s.db.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); delErr != nil {
|
||||
return stored, delErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, updErr := s.db.Exec(`UPDATE activity_outbox SET attempts=$2, last_error=$3 WHERE id=$1`, it.id, next, err.Error()); updErr != nil {
|
||||
return stored, updErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return stored, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return stored, err
|
||||
}
|
||||
stored++
|
||||
}
|
||||
return stored, nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package db
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestActivityOutboxKeepsDetail(t *testing.T) {
|
||||
in := Activity{Worker: "worker", Kind: "tool_result", Summary: "done", Detail: "full body", IsError: true}
|
||||
raw, err := marshalActivity(in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := unmarshalActivity(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Detail != in.Detail || out.Summary != in.Summary || out.Worker != in.Worker || out.Kind != in.Kind || !out.IsError {
|
||||
t.Fatalf("round trip = %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlushActivityOutboxReplaysOnce(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
defer d.Close()
|
||||
expID, err := d.CreateExploration("outbox", "replay")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = d.Exec(`DELETE FROM activity_outbox WHERE exploration_id=$1`, expID)
|
||||
_, _ = d.Exec(`DELETE FROM explorations WHERE id=$1`, expID)
|
||||
})
|
||||
es := d.Exploration(expID)
|
||||
if err := es.EnqueueActivity(Activity{Worker: "worker", Kind: "text", Summary: "kept", Detail: "body"}, "insert failed"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, err := es.FlushActivityOutbox()
|
||||
if err != nil || n != 1 {
|
||||
t.Fatalf("flush n=%d err=%v", n, err)
|
||||
}
|
||||
var activities, queued int
|
||||
if err := d.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='kept'`, expID).Scan(&activities); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, expID).Scan(&queued); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if activities != 1 || queued != 0 {
|
||||
t.Fatalf("activities=%d outbox=%d, want 1 and 0", activities, queued)
|
||||
}
|
||||
if n, err = es.FlushActivityOutbox(); err != nil || n != 0 {
|
||||
t.Fatalf("second flush n=%d err=%v", n, err)
|
||||
}
|
||||
if err := d.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='kept'`, expID).Scan(&activities); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if activities != 1 {
|
||||
t.Fatalf("activities=%d after a second flush, want 1", activities)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlushActivityOutboxDropsStuckRow(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
defer d.Close()
|
||||
const missing = int64(9_000_000_000_001)
|
||||
t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM activity_outbox WHERE exploration_id=$1`, missing) })
|
||||
es := d.Exploration(missing)
|
||||
if err := es.EnqueueActivity(Activity{Worker: "worker", Kind: "text", Summary: "stuck"}, "no parent"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for attempt := 1; attempt < activityOutboxMaxAttempts; attempt++ {
|
||||
n, err := es.FlushActivityOutbox()
|
||||
if err != nil || n != 0 {
|
||||
t.Fatalf("attempt %d: n=%d err=%v", attempt, n, err)
|
||||
}
|
||||
var queued int
|
||||
if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, missing).Scan(&queued); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if queued != 1 {
|
||||
t.Fatalf("attempt %d: outbox rows=%d, want the stuck row kept", attempt, queued)
|
||||
}
|
||||
}
|
||||
n, err := es.FlushActivityOutbox()
|
||||
if err != nil || n != 0 {
|
||||
t.Fatalf("final flush n=%d err=%v", n, err)
|
||||
}
|
||||
var queued int
|
||||
if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, missing).Scan(&queued); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if queued != 0 {
|
||||
t.Fatalf("outbox rows=%d, want the stuck row dropped", queued)
|
||||
}
|
||||
}
|
||||
+15
-2
@@ -354,7 +354,10 @@ WHERE exploration_id=$1 AND kind='intent' AND state='running'
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
@@ -1379,14 +1382,24 @@ func (s *ExplorationStore) Stats() (map[string]int, error) {
|
||||
|
||||
// --- activity (poll by global id cursor) ---
|
||||
|
||||
// queryRower is the subset of *sql.DB and *sql.Tx that AppendActivity needs,
|
||||
// so a replay can insert the activity and delete the outbox row together.
|
||||
type queryRower interface {
|
||||
QueryRow(query string, args ...any) *sql.Row
|
||||
}
|
||||
|
||||
// AppendActivity records one worker step and returns its id.
|
||||
func (s *ExplorationStore) AppendActivity(a Activity) (int64, error) {
|
||||
return s.appendActivity(s.db, a)
|
||||
}
|
||||
|
||||
func (s *ExplorationStore) appendActivity(q queryRower, a Activity) (int64, error) {
|
||||
metadata := a.Metadata
|
||||
if len(metadata) == 0 {
|
||||
metadata = json.RawMessage(`{}`)
|
||||
}
|
||||
var id int64
|
||||
err := s.db.QueryRow(`
|
||||
err := q.QueryRow(`
|
||||
INSERT INTO activity(exploration_id, node_id, worker, kind, tool, tool_use_id, is_error, summary, detail, metadata, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, main_seg)
|
||||
VALUES ($1,$2,NULLIF($3,''),NULLIF($4,''),NULLIF($5,''),NULLIF($6,''),$7,NULLIF($8,''),NULLIF($9,''),$10,$11,$12,$13,$14,$15)
|
||||
RETURNING id`, s.expID, a.NodeID, utf8Clean(a.Worker), utf8Clean(a.Kind), utf8Clean(a.Tool), utf8Clean(a.ToolUseID), a.IsError, utf8Clean(a.Summary), utf8Clean(a.Detail),
|
||||
|
||||
+2
-2
@@ -137,9 +137,9 @@ func assertEmbeddedApplied(t *testing.T, d *DB) []appliedMigration {
|
||||
version int
|
||||
name string
|
||||
}{{1, llmRecordsMigration}, {2, llmUsageMigration}, {3, "0003_task_scope_icp_index.sql"}, {4, "0004_llm_usage_ts_index.sql"},
|
||||
{5, "0005_traffic_rewrite_rules.sql"}}
|
||||
{5, "0005_traffic_rewrite_rules.sql"}, {6, "0006_activity_outbox.sql"}}
|
||||
if len(applied) != len(want) {
|
||||
t.Fatalf("schema_migrations = %+v, want versions 1 to 5", applied)
|
||||
t.Fatalf("schema_migrations = %+v, want versions 1 to 6", applied)
|
||||
}
|
||||
for i, w := range want {
|
||||
content, err := fs.ReadFile(migrationsFS(), w.name)
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
-- Activities that could not be inserted (the exploration row was missing, or
|
||||
-- the write failed after retries) are kept here and replayed into activity
|
||||
-- once the parent row accepts them. No foreign key: the row has to survive
|
||||
-- the missing exploration that caused the original failure.
|
||||
CREATE TABLE IF NOT EXISTS activity_outbox (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
exploration_id BIGINT NOT NULL,
|
||||
payload JSONB NOT NULL,
|
||||
last_error TEXT,
|
||||
attempts INT NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS activity_outbox_exploration_idx
|
||||
ON activity_outbox (exploration_id, id);
|
||||
+91
-4
@@ -118,6 +118,7 @@ type Engine struct {
|
||||
// runs under one task-scoped context. Successful deletion cancels that context,
|
||||
// waits for all goroutines, then releases every task-level Engine reference.
|
||||
runtimeMu sync.Mutex
|
||||
draining bool
|
||||
runtimes map[string]*taskRuntime
|
||||
|
||||
// per-task execution context: each planner.Plan / worker.Execute runs under it,
|
||||
@@ -162,7 +163,7 @@ type Engine struct {
|
||||
|
||||
type taskRuntime struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
cancel context.CancelCauseFunc
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
@@ -231,9 +232,12 @@ func (e *Engine) IsDeleting(taskID string) bool {
|
||||
func (e *Engine) registerTaskRoutines(parent context.Context, taskID string, count int) *taskRuntime {
|
||||
e.runtimeMu.Lock()
|
||||
defer e.runtimeMu.Unlock()
|
||||
if e.draining {
|
||||
return nil
|
||||
}
|
||||
rt := e.runtimes[taskID]
|
||||
if rt == nil {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
ctx, cancel := context.WithCancelCause(parent)
|
||||
rt = &taskRuntime{ctx: ctx, cancel: cancel}
|
||||
e.runtimes[taskID] = rt
|
||||
}
|
||||
@@ -248,6 +252,61 @@ func runTaskRoutine(rt *taskRuntime, fn func(context.Context)) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait blocks until the task goroutines registered at the moment of the call,
|
||||
// and every admitted task operation, have finished, or until ctx ends. It
|
||||
// refuses new registrations and new beginTaskOperation calls first. Goroutines
|
||||
// that were already running are cancelled with AbortShutdown. That cancel does
|
||||
// not write a terminal task status, and an in-flight intent claim stays running
|
||||
// so the next start can reset it to open. Detached chat runs are not in a task
|
||||
// WaitGroup; they are counted in inflight and Wait stays until they return.
|
||||
func (e *Engine) Wait(ctx context.Context) error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
e.runtimeMu.Lock()
|
||||
e.draining = true
|
||||
groups := make([]*sync.WaitGroup, 0, len(e.runtimes))
|
||||
for _, rt := range e.runtimes {
|
||||
if rt == nil {
|
||||
continue
|
||||
}
|
||||
if rt.cancel != nil {
|
||||
rt.cancel(agent.AbortShutdown)
|
||||
}
|
||||
groups = append(groups, &rt.wg)
|
||||
}
|
||||
e.runtimeMu.Unlock()
|
||||
for _, wg := range groups {
|
||||
if err := waitGroup(ctx, wg); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for e.inflightTotal() > 0 {
|
||||
timer := time.NewTimer(20 * time.Millisecond)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func waitGroup(ctx context.Context, wg *sync.WaitGroup) error {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// StopTask permanently stops every long-lived goroutine and removes all Engine
|
||||
// state for a successfully deleted task. The delete barrier remains installed
|
||||
// until cleanup finishes, so no new task operation can race the teardown.
|
||||
@@ -260,7 +319,7 @@ func (e *Engine) StopTask(taskID string) {
|
||||
e.runtimeMu.Lock()
|
||||
rt := e.runtimes[taskID]
|
||||
if rt != nil {
|
||||
rt.cancel()
|
||||
rt.cancel(agent.AbortTaskDeleted)
|
||||
}
|
||||
e.runtimeMu.Unlock()
|
||||
if rt != nil {
|
||||
@@ -664,6 +723,23 @@ func (e *Engine) emitActivity(t *Task, r db.Activity) db.Activity {
|
||||
}
|
||||
log.Printf("[activity] task %s dropping activity record (%d dropped for this task so far) worker=%s kind=%s tool=%s tuid=%s reason=%s summary=%q: %v%s",
|
||||
t.ID, n, r.Worker, r.Kind, r.Tool, r.ToolUseID, dropReason(err), preview(r.Summary, 80), err, diag)
|
||||
if t.Store != nil {
|
||||
if qerr := t.Store.EnqueueActivity(r, err.Error()); qerr != nil {
|
||||
log.Printf("[activity] task %s outbox: %v", t.ID, qerr)
|
||||
}
|
||||
}
|
||||
// The row has no database id. Publish an error copy so a live session
|
||||
// does not stay on "Running" while the original return value is unchanged.
|
||||
notice := r
|
||||
notice.IsError = true
|
||||
if notice.Summary == "" {
|
||||
notice.Summary = "activity was not stored"
|
||||
} else {
|
||||
notice.Summary += " (not stored)"
|
||||
}
|
||||
if e.bc != nil {
|
||||
e.bc.Publish(t.ID, notice)
|
||||
}
|
||||
e.touch(t.ID)
|
||||
return r
|
||||
}
|
||||
@@ -762,6 +838,10 @@ func (e *Engine) Run(ctx context.Context, t *Task) {
|
||||
}
|
||||
rt := e.registerTaskRoutines(ctx, t.ID, 2+workers)
|
||||
e.deleteMu.RUnlock()
|
||||
if rt == nil {
|
||||
e.started.Delete(t.ID)
|
||||
return
|
||||
}
|
||||
e.touch(t.ID)
|
||||
runTaskRoutine(rt, func(loopCtx context.Context) { e.plannerLoop(loopCtx, t) })
|
||||
runTaskRoutine(rt, func(loopCtx context.Context) { e.claimSweepLoop(loopCtx, t) })
|
||||
@@ -1266,6 +1346,13 @@ func (e *Engine) runIntent(ctx context.Context, t *Task, name string, worker *ag
|
||||
e.touch(t.ID)
|
||||
return true
|
||||
}
|
||||
// Process shutdown cancels the task context. Leave the claim running: AbortShutdown
|
||||
// says restart resets leftover running intents to open. Settling done or blocked
|
||||
// here would make that reset miss the intent, so the task would resume without it.
|
||||
if ectx.Err() != nil && errors.Is(context.Cause(ectx), agent.AbortShutdown) {
|
||||
log.Printf("[worker %s] task %s intent #%d shutdown: leaving the claim running", name, t.ID, intent.ID)
|
||||
return true
|
||||
}
|
||||
// killed by the planner: mark stopped (don't write back results, don't auto-reclaim).
|
||||
if killed {
|
||||
if err := t.Store.SettleIntentClaim(intent.ID, token, "stopped"); err != nil {
|
||||
@@ -1324,7 +1411,7 @@ func (e *Engine) runIntent(ctx context.Context, t *Task, name string, worker *ag
|
||||
// be started; the intent is left untouched in that case.
|
||||
func (e *Engine) runDetachedIntent(ctx context.Context, t *Task, intentID int64, requestID, message, agentMessage string) error {
|
||||
if !e.beginTaskOperation(t.ID) {
|
||||
return fmt.Errorf("task is being deleted")
|
||||
return fmt.Errorf("task is not accepting work")
|
||||
}
|
||||
release := true
|
||||
defer func() {
|
||||
|
||||
@@ -9,6 +9,60 @@ import (
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
)
|
||||
|
||||
func TestWaitBlocksUntilDetachedInflightEnds(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
if !e.beginTaskOperation("t1") {
|
||||
t.Fatal("detached work was not admitted")
|
||||
}
|
||||
release := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
go func() {
|
||||
<-release
|
||||
e.decInflight("t1")
|
||||
close(finished)
|
||||
}()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.Wait(context.Background()) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("Wait returned while the detached operation was still running: %v", err)
|
||||
case <-time.After(80 * time.Millisecond):
|
||||
}
|
||||
if e.beginTaskOperation("t1") {
|
||||
t.Fatal("Wait admitted new work while draining")
|
||||
}
|
||||
close(release)
|
||||
<-finished
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Wait did not return after the detached operation ended")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitCancelsRuntimeWithShutdown(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
rt := e.registerTaskRoutines(context.Background(), "t1", 1)
|
||||
if rt == nil {
|
||||
t.Fatal("runtime was not registered")
|
||||
}
|
||||
started := make(chan struct{})
|
||||
runTaskRoutine(rt, func(ctx context.Context) {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
})
|
||||
<-started
|
||||
if err := e.Wait(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !errors.Is(context.Cause(rt.ctx), agent.AbortShutdown) {
|
||||
t.Fatalf("cause=%v, want shutdown", context.Cause(rt.ctx))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPauseCancelsWithNamedCause(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
execCtx := e.execContextFor(context.Background(), "t1")
|
||||
|
||||
@@ -46,14 +46,20 @@ func (e *Engine) inflightCounter(taskID string) *int64 {
|
||||
}
|
||||
|
||||
// beginTaskOperation atomically registers a task-owned operation unless deletion
|
||||
// has already installed its barrier. The delete handler can therefore wait for
|
||||
// inflight==0 without a check-then-start race recreating files after cleanup.
|
||||
// has already installed its barrier, or the engine is draining for shutdown.
|
||||
// The delete handler and Wait can therefore wait for inflight==0 without a
|
||||
// check-then-start race admitting work that outlives the drain.
|
||||
func (e *Engine) beginTaskOperation(taskID string) bool {
|
||||
e.deleteMu.RLock()
|
||||
defer e.deleteMu.RUnlock()
|
||||
if e.IsDeleting(taskID) {
|
||||
return false
|
||||
}
|
||||
e.runtimeMu.Lock()
|
||||
defer e.runtimeMu.Unlock()
|
||||
if e.draining {
|
||||
return false
|
||||
}
|
||||
atomic.AddInt64(e.inflightCounter(taskID), 1)
|
||||
return true
|
||||
}
|
||||
@@ -63,6 +69,17 @@ func (e *Engine) inflightCount(taskID string) int64 {
|
||||
return atomic.LoadInt64(e.inflightCounter(taskID))
|
||||
}
|
||||
|
||||
// inflightTotal is the sum of every task's admitted operations, including
|
||||
// detached chat runs that are not members of a task WaitGroup.
|
||||
func (e *Engine) inflightTotal() int64 {
|
||||
var n int64
|
||||
e.inflight.Range(func(_, v any) bool {
|
||||
n += atomic.LoadInt64(v.(*int64))
|
||||
return true
|
||||
})
|
||||
return n
|
||||
}
|
||||
|
||||
// ---------- deadline ----------
|
||||
|
||||
// taskDeadline returns the task's absolute deadline (unix). Prefers the in-process
|
||||
@@ -161,6 +178,10 @@ func (e *Engine) startDeadlineCoordinator(ctx context.Context, t *Task) {
|
||||
}
|
||||
rt := e.registerTaskRoutines(ctx, t.ID, 1)
|
||||
e.deleteMu.RUnlock()
|
||||
if rt == nil {
|
||||
e.coordStarted.Delete(t.ID)
|
||||
return
|
||||
}
|
||||
runTaskRoutine(rt, func(loopCtx context.Context) { e.deadlineCoordinator(loopCtx, t) })
|
||||
}
|
||||
|
||||
|
||||
+32
-1
@@ -296,9 +296,16 @@ func (s *Server) restoreTaskRuntimes() {
|
||||
lifecycle := t.lifecycleSnapshot()
|
||||
// clear stale 'running' intents from a prior crash/restart (no live worker
|
||||
// owns them) so they re-claim instead of spinning forever in the UI.
|
||||
if n, _ := t.Store.ResetRunningIntents(); n > 0 {
|
||||
if n, err := t.Store.ResetRunningIntents(); err != nil {
|
||||
log.Printf("[engine] task %s reset leftover running intents: %v", t.ID, err)
|
||||
} else if n > 0 {
|
||||
log.Printf("[engine] task %s reset %d leftover running intents to open", t.ID, n)
|
||||
}
|
||||
if n, err := t.Store.FlushActivityOutbox(); err != nil {
|
||||
log.Printf("[activity] task %s outbox: %v", t.ID, err)
|
||||
} else if n > 0 {
|
||||
log.Printf("[activity] task %s stored %d activities recovered from the outbox", t.ID, n)
|
||||
}
|
||||
if lifecycle.Paused {
|
||||
s.engine.Pause(t.ID, agent.AbortPausedOnReload)
|
||||
}
|
||||
@@ -320,6 +327,30 @@ func (s *Server) restoreTaskRuntimes() {
|
||||
}
|
||||
}
|
||||
|
||||
// Wait blocks until the task goroutines snapshotted now, and the archive
|
||||
// worker, have finished, or until ctx ends.
|
||||
func (s *Server) Wait(ctx context.Context) error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
if s.engine != nil {
|
||||
if err := s.engine.Wait(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
s.archiveWG.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// agentMaxTurns returns the configured max_turns for an agent key (0 = unlimited,
|
||||
// also the fallback when no DB or no row).
|
||||
func (s *Server) agentMaxTurns(key string) int {
|
||||
|
||||
Reference in New Issue
Block a user