mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-20 07:00:58 +08:00
Stop admitting work before the drain and replay each outbox row once.
The listener closes before Wait, and a shutdown refuses new task operations. Wait also stays until detached runs finish. The activity insert and the outbox delete commit together, and a row that keeps failing is dropped after five attempts so it cannot block the rest.
This commit is contained in:
+30
-12
@@ -252,11 +252,13 @@ func runTaskRoutine(rt *taskRuntime, fn func(context.Context)) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait blocks until the task goroutines registered at the moment of the call
|
||||
// have finished, or until ctx ends. It refuses new registrations first so Add
|
||||
// cannot race Wait. Goroutines that were already running are cancelled with
|
||||
// AbortShutdown. That cancel does not write a terminal task status, and an
|
||||
// in-flight intent claim stays running so the next start can reset it to open.
|
||||
// Wait blocks until the task goroutines registered at the moment of the call,
|
||||
// and every admitted task operation, have finished, or until ctx ends. It
|
||||
// refuses new registrations and new beginTaskOperation calls first. Goroutines
|
||||
// that were already running are cancelled with AbortShutdown. That cancel does
|
||||
// not write a terminal task status, and an in-flight intent claim stays running
|
||||
// so the next start can reset it to open. Detached chat runs are not in a task
|
||||
// WaitGroup; they are counted in inflight and Wait stays until they return.
|
||||
func (e *Engine) Wait(ctx context.Context) error {
|
||||
if e == nil {
|
||||
return nil
|
||||
@@ -275,20 +277,36 @@ func (e *Engine) Wait(ctx context.Context) error {
|
||||
}
|
||||
e.runtimeMu.Unlock()
|
||||
for _, wg := range groups {
|
||||
done := make(chan struct{})
|
||||
go func(wg *sync.WaitGroup) {
|
||||
wg.Wait()
|
||||
close(done)
|
||||
}(wg)
|
||||
if err := waitGroup(ctx, wg); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for e.inflightTotal() > 0 {
|
||||
timer := time.NewTimer(20 * time.Millisecond)
|
||||
select {
|
||||
case <-done:
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func waitGroup(ctx context.Context, wg *sync.WaitGroup) error {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// StopTask permanently stops every long-lived goroutine and removes all Engine
|
||||
// state for a successfully deleted task. The delete barrier remains installed
|
||||
// until cleanup finishes, so no new task operation can race the teardown.
|
||||
@@ -1393,7 +1411,7 @@ func (e *Engine) runIntent(ctx context.Context, t *Task, name string, worker *ag
|
||||
// be started; the intent is left untouched in that case.
|
||||
func (e *Engine) runDetachedIntent(ctx context.Context, t *Task, intentID int64, requestID, message, agentMessage string) error {
|
||||
if !e.beginTaskOperation(t.ID) {
|
||||
return fmt.Errorf("task is being deleted")
|
||||
return fmt.Errorf("task is not accepting work")
|
||||
}
|
||||
release := true
|
||||
defer func() {
|
||||
|
||||
@@ -9,6 +9,40 @@ import (
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
)
|
||||
|
||||
func TestWaitBlocksUntilDetachedInflightEnds(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
if !e.beginTaskOperation("t1") {
|
||||
t.Fatal("detached work was not admitted")
|
||||
}
|
||||
release := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
go func() {
|
||||
<-release
|
||||
e.decInflight("t1")
|
||||
close(finished)
|
||||
}()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.Wait(context.Background()) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("Wait returned while the detached operation was still running: %v", err)
|
||||
case <-time.After(80 * time.Millisecond):
|
||||
}
|
||||
if e.beginTaskOperation("t1") {
|
||||
t.Fatal("Wait admitted new work while draining")
|
||||
}
|
||||
close(release)
|
||||
<-finished
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Wait did not return after the detached operation ended")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitCancelsRuntimeWithShutdown(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
rt := e.registerTaskRoutines(context.Background(), "t1", 1)
|
||||
|
||||
@@ -46,14 +46,20 @@ func (e *Engine) inflightCounter(taskID string) *int64 {
|
||||
}
|
||||
|
||||
// beginTaskOperation atomically registers a task-owned operation unless deletion
|
||||
// has already installed its barrier. The delete handler can therefore wait for
|
||||
// inflight==0 without a check-then-start race recreating files after cleanup.
|
||||
// has already installed its barrier, or the engine is draining for shutdown.
|
||||
// The delete handler and Wait can therefore wait for inflight==0 without a
|
||||
// check-then-start race admitting work that outlives the drain.
|
||||
func (e *Engine) beginTaskOperation(taskID string) bool {
|
||||
e.deleteMu.RLock()
|
||||
defer e.deleteMu.RUnlock()
|
||||
if e.IsDeleting(taskID) {
|
||||
return false
|
||||
}
|
||||
e.runtimeMu.Lock()
|
||||
defer e.runtimeMu.Unlock()
|
||||
if e.draining {
|
||||
return false
|
||||
}
|
||||
atomic.AddInt64(e.inflightCounter(taskID), 1)
|
||||
return true
|
||||
}
|
||||
@@ -63,6 +69,17 @@ func (e *Engine) inflightCount(taskID string) int64 {
|
||||
return atomic.LoadInt64(e.inflightCounter(taskID))
|
||||
}
|
||||
|
||||
// inflightTotal is the sum of every task's admitted operations, including
|
||||
// detached chat runs that are not members of a task WaitGroup.
|
||||
func (e *Engine) inflightTotal() int64 {
|
||||
var n int64
|
||||
e.inflight.Range(func(_, v any) bool {
|
||||
n += atomic.LoadInt64(v.(*int64))
|
||||
return true
|
||||
})
|
||||
return n
|
||||
}
|
||||
|
||||
// ---------- deadline ----------
|
||||
|
||||
// taskDeadline returns the task's absolute deadline (unix). Prefers the in-process
|
||||
|
||||
Reference in New Issue
Block a user