Stop admitting work before the drain and replay each outbox row once.

The listener closes before Wait, and a shutdown refuses new task operations. Wait also stays until detached runs finish. The activity insert and the outbox delete commit together, and a row that keeps failing is dropped after five attempts so it cannot block the rest.
This commit is contained in:
protonmatter
2026-11-17 15:41:56 -05:00
parent 0234025d28
commit a1c6103cbf
8 changed files with 224 additions and 32 deletions
+30 -12
View File
@@ -252,11 +252,13 @@ func runTaskRoutine(rt *taskRuntime, fn func(context.Context)) {
}()
}
// Wait blocks until the task goroutines registered at the moment of the call
// have finished, or until ctx ends. It refuses new registrations first so Add
// cannot race Wait. Goroutines that were already running are cancelled with
// AbortShutdown. That cancel does not write a terminal task status, and an
// in-flight intent claim stays running so the next start can reset it to open.
// Wait blocks until the task goroutines registered at the moment of the call,
// and every admitted task operation, have finished, or until ctx ends. It
// refuses new registrations and new beginTaskOperation calls first. Goroutines
// that were already running are cancelled with AbortShutdown. That cancel does
// not write a terminal task status, and an in-flight intent claim stays running
// so the next start can reset it to open. Detached chat runs are not in a task
// WaitGroup; they are counted in inflight and Wait stays until they return.
func (e *Engine) Wait(ctx context.Context) error {
if e == nil {
return nil
@@ -275,20 +277,36 @@ func (e *Engine) Wait(ctx context.Context) error {
}
e.runtimeMu.Unlock()
for _, wg := range groups {
done := make(chan struct{})
go func(wg *sync.WaitGroup) {
wg.Wait()
close(done)
}(wg)
if err := waitGroup(ctx, wg); err != nil {
return err
}
}
for e.inflightTotal() > 0 {
timer := time.NewTimer(20 * time.Millisecond)
select {
case <-done:
case <-ctx.Done():
timer.Stop()
return ctx.Err()
case <-timer.C:
}
}
return nil
}
func waitGroup(ctx context.Context, wg *sync.WaitGroup) error {
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
// StopTask permanently stops every long-lived goroutine and removes all Engine
// state for a successfully deleted task. The delete barrier remains installed
// until cleanup finishes, so no new task operation can race the teardown.
@@ -1393,7 +1411,7 @@ func (e *Engine) runIntent(ctx context.Context, t *Task, name string, worker *ag
// be started; the intent is left untouched in that case.
func (e *Engine) runDetachedIntent(ctx context.Context, t *Task, intentID int64, requestID, message, agentMessage string) error {
if !e.beginTaskOperation(t.ID) {
return fmt.Errorf("task is being deleted")
return fmt.Errorf("task is not accepting work")
}
release := true
defer func() {
+34
View File
@@ -9,6 +9,40 @@ import (
"github.com/Autumn-27/artex/agent"
)
func TestWaitBlocksUntilDetachedInflightEnds(t *testing.T) {
e := NewEngine(nil)
if !e.beginTaskOperation("t1") {
t.Fatal("detached work was not admitted")
}
release := make(chan struct{})
finished := make(chan struct{})
go func() {
<-release
e.decInflight("t1")
close(finished)
}()
done := make(chan error, 1)
go func() { done <- e.Wait(context.Background()) }()
select {
case err := <-done:
t.Fatalf("Wait returned while the detached operation was still running: %v", err)
case <-time.After(80 * time.Millisecond):
}
if e.beginTaskOperation("t1") {
t.Fatal("Wait admitted new work while draining")
}
close(release)
<-finished
select {
case err := <-done:
if err != nil {
t.Fatal(err)
}
case <-time.After(time.Second):
t.Fatal("Wait did not return after the detached operation ended")
}
}
func TestWaitCancelsRuntimeWithShutdown(t *testing.T) {
e := NewEngine(nil)
rt := e.registerTaskRoutines(context.Background(), "t1", 1)
+19 -2
View File
@@ -46,14 +46,20 @@ func (e *Engine) inflightCounter(taskID string) *int64 {
}
// beginTaskOperation atomically registers a task-owned operation unless deletion
// has already installed its barrier. The delete handler can therefore wait for
// inflight==0 without a check-then-start race recreating files after cleanup.
// has already installed its barrier, or the engine is draining for shutdown.
// The delete handler and Wait can therefore wait for inflight==0 without a
// check-then-start race admitting work that outlives the drain.
func (e *Engine) beginTaskOperation(taskID string) bool {
e.deleteMu.RLock()
defer e.deleteMu.RUnlock()
if e.IsDeleting(taskID) {
return false
}
e.runtimeMu.Lock()
defer e.runtimeMu.Unlock()
if e.draining {
return false
}
atomic.AddInt64(e.inflightCounter(taskID), 1)
return true
}
@@ -63,6 +69,17 @@ func (e *Engine) inflightCount(taskID string) int64 {
return atomic.LoadInt64(e.inflightCounter(taskID))
}
// inflightTotal is the sum of every task's admitted operations, including
// detached chat runs that are not members of a task WaitGroup.
func (e *Engine) inflightTotal() int64 {
var n int64
e.inflight.Range(func(_, v any) bool {
n += atomic.LoadInt64(v.(*int64))
return true
})
return n
}
// ---------- deadline ----------
// taskDeadline returns the task's absolute deadline (unix). Prefers the in-process