mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-13 11:40:53 +08:00
Implements the per-intent rotation shape from #85: a task created with llm_rotate=true distributes worker intents round-robin over its LLM chain heads, so lane A can hunt on one model while lane B hunts on another (different lenses per intent, no extra tokens). Default stays the classic ordered failover from the active head; planner/goals/mainagent keep the active head always. Mechanics: the engine pins the next chain head into the run context when a worker claims an intent; the pin travels through the provider resolver (hooks.current now takes ctx). A pin is only a dispatch preference - it must belong to the task chain and never bypasses failover state (an exhausted chain stays exhausted). Schema: tasks.llm_rotate boolean, default false, idempotent ALTER; exposed on the create payload and the task DTO. Closes #85 (the ensemble/P4 shape stays parked as before)
1291 lines
52 KiB
Go
1291 lines
52 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/Autumn-27/artex/agent"
|
|
"github.com/Autumn-27/artex/db"
|
|
"github.com/Autumn-27/artex/intercept"
|
|
"github.com/Autumn-27/norma/harness"
|
|
"github.com/Autumn-27/norma/llm"
|
|
"github.com/jackc/pgx/v5/pgconn"
|
|
)
|
|
|
|
// isFKViolation reports whether err is a Postgres foreign-key violation (SQLSTATE
|
|
// 23503) — e.g. an activity insert whose exploration_id has no parent row.
|
|
func isFKViolation(err error) bool {
|
|
var pgErr *pgconn.PgError
|
|
return errors.As(err, &pgErr) && pgErr.Code == "23503"
|
|
}
|
|
|
|
// dropReason classifies why an activity write was dropped, so the log can be
|
|
// grouped/analysed by cause rather than by raw error text.
|
|
func dropReason(err error) string {
|
|
var pgErr *pgconn.PgError
|
|
if errors.As(err, &pgErr) {
|
|
switch pgErr.Code {
|
|
case "23503":
|
|
return "fk_violation(23503, parent exploration does not exist)"
|
|
case "23505":
|
|
return "unique_violation(23505)"
|
|
default:
|
|
return "pg_error(" + pgErr.Code + ")"
|
|
}
|
|
}
|
|
return "write_error"
|
|
}
|
|
|
|
// bumpDrop increments and returns the running count of dropped (unpersistable)
|
|
// activity records for a task. Concurrent planner + worker emits race here, so the
|
|
// counter is an atomic behind sync.Map. The count in the log shows loss scale at a
|
|
// glance instead of forcing a grep-and-count.
|
|
func (e *Engine) bumpDrop(taskID string) int64 {
|
|
v, _ := e.dropCnt.LoadOrStore(taskID, new(int64))
|
|
return atomic.AddInt64(v.(*int64), 1)
|
|
}
|
|
|
|
// preview collapses newlines and trims s to a short rune-safe snippet for one-line
|
|
// log output (avoids dumping a multi-KB summary/detail into the log).
|
|
func preview(s string, n int) string {
|
|
s = strings.ReplaceAll(s, "\r", " ")
|
|
s = strings.ReplaceAll(s, "\n", " ")
|
|
r := []rune(s)
|
|
if len(r) > n {
|
|
return string(r[:n]) + "…"
|
|
}
|
|
return string(r)
|
|
}
|
|
|
|
// model_error (provider/API failure: LLM-layer transient retries exhausted, or the
|
|
// stream broke midway after starting). A work that ends this way is neither
|
|
// "tried but unfinished" nor a real failure — it is external jitter, so permanently blocking it
|
|
// would waste an intent; this terminal state gets a few extra reruns with backoff, and a pause/kill/cancel arriving mid-retry yields to its branch at once.
|
|
const (
|
|
modelErrorRetries = 2 // extra reruns after a model_error finish
|
|
modelErrorRetryBackoff = 3 * time.Second // backoff before each retry
|
|
workControlWaitTimeout = 30 * time.Second
|
|
)
|
|
|
|
var errWorkControlConflict = errors.New("work control conflict")
|
|
|
|
// retryableWorkerModelError excludes errors already handled by the task router.
|
|
// In particular, a quota error after partial streaming advances the task cursor
|
|
// for the next LLM call but must not replay this whole intent on the backup.
|
|
func retryableWorkerModelError(reason harness.TerminalReason, err error) bool {
|
|
return reason == harness.ReasonModelError && !isTaskLLMRuntimeError(err)
|
|
}
|
|
|
|
// Engine drives the event-driven exploration loop with real LLM agents
|
|
// (docs §4.3/§4.4): on asset/exploration-graph change (debounced) it wakes the
|
|
// planner, which reads the route, queries assets, judges goals and emits intents;
|
|
// N concurrent work agents claim intents and execute them. There is no
|
|
// simulation mode — an LLM provider is required. The planner/worker can be
|
|
// (re)installed at runtime (LLM configured from the UI); the loops always run
|
|
// but idle until an LLM is set.
|
|
type Engine struct {
|
|
m *Manager
|
|
debounce time.Duration
|
|
|
|
bc *Broadcaster // live activity pub/sub (SSE)
|
|
|
|
started sync.Map // taskID -> bool, so Run is idempotent per task
|
|
lastAct sync.Map // taskID -> int64 unix, last planner/worker activity (heartbeat)
|
|
llmCalls sync.Map // taskID -> *int64, actual planner/worker/main-agent LLM calls
|
|
// quiet-mode (#79): open-goal fingerprint recorded at the end of the last
|
|
// completed planning round, per task. Edge-triggered rounds are skipped
|
|
// while it is unchanged and work is still in flight.
|
|
goalFP sync.Map // taskID -> string (sorted open-goal ids joined)
|
|
paused sync.Map // taskID -> bool, user-paused (planner + workers idle but loops alive)
|
|
deleting sync.Map // taskID -> bool, delete barrier (no new task-owned writes)
|
|
dropCnt sync.Map // taskID -> *int64, running count of dropped (unpersistable) activity records
|
|
|
|
// deleteMu makes installing the delete barrier atomic with registering a new
|
|
// task operation. Once BeginDelete returns, every admitted writer is reflected
|
|
// in inflight and every later writer is rejected.
|
|
deleteMu sync.RWMutex
|
|
|
|
// Every long-lived task goroutine (planner, workers and deadline coordinator)
|
|
// runs under one task-scoped context. Successful deletion cancels that context,
|
|
// waits for all goroutines, then releases every task-level Engine reference.
|
|
runtimeMu sync.Mutex
|
|
runtimes map[string]*taskRuntime
|
|
|
|
// per-task execution context: each planner.Plan / worker.Execute runs under it,
|
|
// so pausing can CANCEL an in-flight run (not just skip the next one). Recreated
|
|
// on resume since cancelling is one-shot. Every cancellation carries a named
|
|
// cause so the activity trace can identify the initiating control path.
|
|
execMu sync.Mutex
|
|
execCancel map[string]context.CancelCauseFunc
|
|
execCtx map[string]context.Context
|
|
|
|
// Per-work control lets the planner kill a worker and lets the UI pause/cancel
|
|
// one intent without pausing the whole task. The done channel closes only after
|
|
// runWorkerStep has stopped writing and committed its final state.
|
|
workMu sync.Mutex
|
|
work map[int64]*workExecution
|
|
|
|
// steerBox queues planner course-corrections for a running work (keyed by intent
|
|
// id). The worker's PreToolUse hook drains it before its next tool call and hands
|
|
// the message to the model (blocking that call) so it re-plans — no kill needed.
|
|
steerMu sync.Mutex
|
|
steerBox map[int64][]string
|
|
|
|
plannerRound sync.Map // taskID -> int, planner round counter (for UI round separators)
|
|
|
|
// Task-level timeout (see docs/task-timeout-and-settle-design.md):
|
|
settling sync.Map // taskID -> bool, the task has entered the settlement sequence (stop dispatching/claiming intents)
|
|
deadline sync.Map // taskID -> int64 unix, absolute deadline (stamped on first run; 0/missing = no limit)
|
|
stamped sync.Map // taskID -> bool, whether first_run_at has been stamped (once per process)
|
|
inflight sync.Map // taskID -> *int64, count of in-flight planner.Plan + worker.Execute (for drain)
|
|
coordStarted sync.Map // taskID -> bool, whether the deadline coordinator has started (deduplicated across Run/reload)
|
|
|
|
// resolve returns a task's dedicated planner/worker (wired by the server as the
|
|
// authoritative task-router). nil,nil means this task is deliberately unavailable
|
|
// (for example an exhausted failover chain) — there is no global-pair fallback.
|
|
resolve func(t *Task) (*agent.Planner, *agent.Worker)
|
|
resolveAuthoritative bool
|
|
// readiness reports whether a global LLM provider is configured — the signal behind
|
|
// Ready()/the llm_configured indicator. Wired once at startup; nil → not ready.
|
|
readiness func() bool
|
|
}
|
|
|
|
type taskRuntime struct {
|
|
ctx context.Context
|
|
cancel context.CancelFunc
|
|
wg sync.WaitGroup
|
|
}
|
|
|
|
type workExecution struct {
|
|
cancel context.CancelCauseFunc
|
|
done chan error
|
|
action string // user action: pause | cancel
|
|
}
|
|
|
|
// nextPlannerRound returns the next planner round number for a task (1-based).
|
|
func (e *Engine) nextPlannerRound(taskID string) int {
|
|
v, _ := e.plannerRound.LoadOrStore(taskID, 0)
|
|
n := v.(int) + 1
|
|
e.plannerRound.Store(taskID, n)
|
|
return n
|
|
}
|
|
|
|
// Pause stops a task: marks it paused AND cancels any in-flight planner/worker run
|
|
// for it (a long worker.Execute would otherwise keep going until it finishes).
|
|
func (e *Engine) Pause(taskID string, cause error) {
|
|
e.paused.Store(taskID, true)
|
|
e.cancelExec(taskID, cause)
|
|
}
|
|
|
|
// BeginDelete installs an execution barrier before task data/files are removed.
|
|
// The temporary pause is not a user pause. The server serializes this transition
|
|
// with lifecycle admission and tells AbortDelete whether the persisted task is
|
|
// paused/queued if cleanup fails.
|
|
func (e *Engine) BeginDelete(taskID string) bool {
|
|
e.deleteMu.Lock()
|
|
if _, loaded := e.deleting.LoadOrStore(taskID, true); loaded {
|
|
e.deleteMu.Unlock()
|
|
return false
|
|
}
|
|
e.paused.Store(taskID, true)
|
|
e.deleteMu.Unlock()
|
|
e.cancelExec(taskID, agent.AbortTaskDeleted)
|
|
return true
|
|
}
|
|
|
|
func (e *Engine) AbortDelete(taskID string, keepPaused bool) {
|
|
e.deleteMu.Lock()
|
|
if !e.IsDeleting(taskID) {
|
|
e.deleteMu.Unlock()
|
|
return
|
|
}
|
|
e.deleting.Delete(taskID)
|
|
if !keepPaused {
|
|
e.paused.Delete(taskID)
|
|
}
|
|
e.deleteMu.Unlock()
|
|
if !keepPaused && e.m != nil {
|
|
if t, ok := e.m.Task(taskID); ok {
|
|
t.Notify()
|
|
}
|
|
}
|
|
}
|
|
|
|
func (e *Engine) IsDeleting(taskID string) bool {
|
|
_, ok := e.deleting.Load(taskID)
|
|
return ok
|
|
}
|
|
|
|
// registerTaskRoutines reserves count goroutines in the task runtime. Callers
|
|
// hold deleteMu for reading so StopTask cannot race WaitGroup.Add with Wait.
|
|
func (e *Engine) registerTaskRoutines(parent context.Context, taskID string, count int) *taskRuntime {
|
|
e.runtimeMu.Lock()
|
|
defer e.runtimeMu.Unlock()
|
|
rt := e.runtimes[taskID]
|
|
if rt == nil {
|
|
ctx, cancel := context.WithCancel(parent)
|
|
rt = &taskRuntime{ctx: ctx, cancel: cancel}
|
|
e.runtimes[taskID] = rt
|
|
}
|
|
rt.wg.Add(count)
|
|
return rt
|
|
}
|
|
|
|
func runTaskRoutine(rt *taskRuntime, fn func(context.Context)) {
|
|
go func() {
|
|
defer rt.wg.Done()
|
|
fn(rt.ctx)
|
|
}()
|
|
}
|
|
|
|
// StopTask permanently stops every long-lived goroutine and removes all Engine
|
|
// state for a successfully deleted task. The delete barrier remains installed
|
|
// until cleanup finishes, so no new task operation can race the teardown.
|
|
func (e *Engine) StopTask(taskID string) {
|
|
e.deleteMu.Lock()
|
|
e.deleting.Store(taskID, true)
|
|
e.deleteMu.Unlock()
|
|
|
|
e.cancelExec(taskID, agent.AbortTaskDeleted)
|
|
e.runtimeMu.Lock()
|
|
rt := e.runtimes[taskID]
|
|
if rt != nil {
|
|
rt.cancel()
|
|
}
|
|
e.runtimeMu.Unlock()
|
|
if rt != nil {
|
|
rt.wg.Wait()
|
|
}
|
|
|
|
e.execMu.Lock()
|
|
if cancel := e.execCancel[taskID]; cancel != nil {
|
|
cancel(agent.AbortTaskDeleted)
|
|
}
|
|
delete(e.execCancel, taskID)
|
|
delete(e.execCtx, taskID)
|
|
e.execMu.Unlock()
|
|
|
|
e.runtimeMu.Lock()
|
|
if e.runtimes[taskID] == rt {
|
|
delete(e.runtimes, taskID)
|
|
}
|
|
e.runtimeMu.Unlock()
|
|
|
|
e.started.Delete(taskID)
|
|
e.lastAct.Delete(taskID)
|
|
e.llmCalls.Delete(taskID)
|
|
e.paused.Delete(taskID)
|
|
e.dropCnt.Delete(taskID)
|
|
e.plannerRound.Delete(taskID)
|
|
e.settling.Delete(taskID)
|
|
e.deadline.Delete(taskID)
|
|
e.stamped.Delete(taskID)
|
|
e.inflight.Delete(taskID)
|
|
e.coordStarted.Delete(taskID)
|
|
e.deleteMu.Lock()
|
|
e.deleting.Delete(taskID)
|
|
e.deleteMu.Unlock()
|
|
}
|
|
|
|
// cancelExec cancels a task's current per-task exec context (any in-flight
|
|
// planner.Plan / worker.Execute), if present. Shared by Pause and the settle
|
|
// sequence's hard-drain backstop.
|
|
func (e *Engine) cancelExec(taskID string, cause error) {
|
|
e.execMu.Lock()
|
|
if cancel := e.execCancel[taskID]; cancel != nil {
|
|
cancel(cause)
|
|
}
|
|
e.execMu.Unlock()
|
|
}
|
|
|
|
// Resume un-pauses a task and nudges a fresh planning round. The next exec under
|
|
// it gets a fresh (uncancelled) context.
|
|
func (e *Engine) Resume(t *Task) {
|
|
// BeginDelete owns the pause barrier once deletion starts. A concurrent
|
|
// resume must never clear it and let a planner/worker re-enter while cleanup
|
|
// is waiting for task operations to drain.
|
|
if t == nil {
|
|
return
|
|
}
|
|
e.deleteMu.RLock()
|
|
defer e.deleteMu.RUnlock()
|
|
if e.IsDeleting(t.ID) {
|
|
return
|
|
}
|
|
e.paused.Delete(t.ID)
|
|
t.Notify()
|
|
}
|
|
|
|
// execContextFor returns a live per-task context derived from parent, recreating
|
|
// it if a prior pause cancelled it.
|
|
func (e *Engine) execContextFor(parent context.Context, taskID string) context.Context {
|
|
e.execMu.Lock()
|
|
defer e.execMu.Unlock()
|
|
if e.IsPaused(taskID) {
|
|
// never hand out a live context while paused (guards the claim→Execute race)
|
|
c, cancel := context.WithCancelCause(parent)
|
|
cancel(agent.AbortPausedRaceGuard)
|
|
return c
|
|
}
|
|
if c := e.execCtx[taskID]; c != nil && c.Err() == nil {
|
|
return c
|
|
}
|
|
c, cancel := context.WithCancelCause(parent)
|
|
e.execCtx[taskID] = c
|
|
e.execCancel[taskID] = cancel
|
|
return c
|
|
}
|
|
|
|
// IsPaused reports whether a task is user-paused.
|
|
func (e *Engine) IsPaused(taskID string) bool {
|
|
v, ok := e.paused.Load(taskID)
|
|
return ok && v.(bool)
|
|
}
|
|
|
|
// Started reports whether the engine loops are running for a task.
|
|
func (e *Engine) Started(taskID string) bool {
|
|
_, ok := e.started.Load(taskID)
|
|
return ok
|
|
}
|
|
|
|
// LastActivity returns the unix time of the last planner/worker activity for a
|
|
// task (0 if none yet).
|
|
func (e *Engine) LastActivity(taskID string) int64 {
|
|
if v, ok := e.lastAct.Load(taskID); ok {
|
|
return v.(int64)
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// BeginLLMCall/EndLLMCall track actual provider calls separately from the
|
|
// scheduler's task-operation counter. A task can have live loops while all of
|
|
// them are waiting for a trigger; that state must remain idle in the UI.
|
|
func (e *Engine) BeginLLMCall(taskID string) {
|
|
v, _ := e.llmCalls.LoadOrStore(taskID, new(int64))
|
|
atomic.AddInt64(v.(*int64), 1)
|
|
}
|
|
|
|
func (e *Engine) EndLLMCall(taskID string) {
|
|
if v, ok := e.llmCalls.Load(taskID); ok {
|
|
p := v.(*int64)
|
|
if atomic.AddInt64(p, -1) <= 0 {
|
|
atomic.StoreInt64(p, 0)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (e *Engine) ActiveLLMCalls(taskID string) int64 {
|
|
if v, ok := e.llmCalls.Load(taskID); ok {
|
|
return atomic.LoadInt64(v.(*int64))
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func (e *Engine) touch(taskID string) { e.lastAct.Store(taskID, time.Now().Unix()) }
|
|
|
|
func NewEngine(m *Manager) *Engine {
|
|
return &Engine{m: m, debounce: 800 * time.Millisecond, bc: NewBroadcaster(),
|
|
execCancel: map[string]context.CancelCauseFunc{}, execCtx: map[string]context.Context{},
|
|
work: map[int64]*workExecution{}, steerBox: map[int64][]string{},
|
|
runtimes: map[string]*taskRuntime{}}
|
|
}
|
|
|
|
// registerWork records the cancel for the work currently running intentID.
|
|
func (e *Engine) registerWork(intentID int64, cancel context.CancelCauseFunc) {
|
|
e.workMu.Lock()
|
|
e.work[intentID] = &workExecution{cancel: cancel, done: make(chan error, 1)}
|
|
e.workMu.Unlock()
|
|
}
|
|
|
|
// detachWork removes the live control handle once Execute has returned. complete
|
|
// must be called after the final intent state write so a waiting cancel handler can
|
|
// safely delete the worker's blackboard output without racing a late write.
|
|
func (e *Engine) detachWork(intentID int64) (action string, complete func(error)) {
|
|
e.workMu.Lock()
|
|
run := e.work[intentID]
|
|
if run != nil {
|
|
delete(e.work, intentID)
|
|
action = run.action
|
|
run.cancel(agent.AbortWorkFinished) // release resources (no-op if already cancelled)
|
|
}
|
|
e.workMu.Unlock()
|
|
e.steerMu.Lock()
|
|
delete(e.steerBox, intentID) // drop any undelivered steering for a finished work
|
|
e.steerMu.Unlock()
|
|
if run == nil {
|
|
return action, func(error) {}
|
|
}
|
|
return action, func(err error) { run.done <- err }
|
|
}
|
|
|
|
// ControlWork requests a user-visible pause or cancellation and waits until the
|
|
// worker has fully stopped writing. Cancellation cleanup is performed by the API
|
|
// handler after this returns; pause state is committed by runWorkerStep itself.
|
|
func (e *Engine) ControlWork(ctx context.Context, intentID int64, action string) error {
|
|
if action != "pause" && action != "cancel" {
|
|
return fmt.Errorf("unsupported work action %q", action)
|
|
}
|
|
if ctx == nil {
|
|
ctx = context.Background()
|
|
}
|
|
e.workMu.Lock()
|
|
run := e.work[intentID]
|
|
if run == nil {
|
|
e.workMu.Unlock()
|
|
return fmt.Errorf("%w: intent %d currently has no running work (it may have finished or was never claimed)", errWorkControlConflict, intentID)
|
|
}
|
|
if run.action != "" {
|
|
e.workMu.Unlock()
|
|
return fmt.Errorf("%w: intent %d is executing the %s operation", errWorkControlConflict, intentID, run.action)
|
|
}
|
|
run.action = action
|
|
done := run.done
|
|
cause := error(agent.AbortWorkPausedByUser)
|
|
if action == "cancel" {
|
|
cause = agent.AbortWorkCancelledByUser
|
|
}
|
|
run.cancel(cause)
|
|
e.workMu.Unlock()
|
|
|
|
timer := time.NewTimer(workControlWaitTimeout)
|
|
defer timer.Stop()
|
|
select {
|
|
case err := <-done:
|
|
return err
|
|
case <-ctx.Done():
|
|
e.releaseWorkControl(intentID, run, action)
|
|
return fmt.Errorf("waiting for intent %d %s to settle: %w", intentID, action, ctx.Err())
|
|
case <-timer.C:
|
|
e.releaseWorkControl(intentID, run, action)
|
|
return fmt.Errorf("waiting for intent %d %s to settle: %w", intentID, action, context.DeadlineExceeded)
|
|
}
|
|
}
|
|
|
|
// releaseWorkControl drops only this caller's reservation after its wait is
|
|
// cancelled. The work context stays cancelled; runWorkerStep recognizes the
|
|
// named cancellation cause and settles the intent into the recoverable paused
|
|
// state even if the HTTP caller has gone away.
|
|
func (e *Engine) releaseWorkControl(intentID int64, run *workExecution, action string) {
|
|
e.workMu.Lock()
|
|
if current := e.work[intentID]; current == run && current.action == action {
|
|
current.action = ""
|
|
}
|
|
e.workMu.Unlock()
|
|
}
|
|
|
|
func transitionIntentState(store *db.ExplorationStore, intentID int64, expected, state string) error {
|
|
changed, err := store.CompareAndSetIntentState(intentID, expected, state)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !changed {
|
|
return fmt.Errorf("%w: intent %d is no longer in the %s state", db.ErrIntentStateConflict, intentID, expected)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SteerWork queues a mid-run course-correction for the work running intentID (the
|
|
// planner's steer_work tool). The worker delivers it before its next tool call and
|
|
// re-plans — no kill. Errors if no work is currently running that intent.
|
|
func (e *Engine) SteerWork(intentID int64, msg string) error {
|
|
if strings.TrimSpace(msg) == "" {
|
|
return fmt.Errorf("steering message must not be empty")
|
|
}
|
|
e.workMu.Lock()
|
|
running := e.work[intentID] != nil
|
|
e.workMu.Unlock()
|
|
if !running {
|
|
return fmt.Errorf("intent %d currently has no running work (it may have finished or was never claimed)", intentID)
|
|
}
|
|
e.steerMu.Lock()
|
|
e.steerBox[intentID] = append(e.steerBox[intentID], msg)
|
|
e.steerMu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
// drainSteer pops the oldest queued steering message for intentID (FIFO), if any.
|
|
func (e *Engine) drainSteer(intentID int64) (string, bool) {
|
|
e.steerMu.Lock()
|
|
defer e.steerMu.Unlock()
|
|
q := e.steerBox[intentID]
|
|
if len(q) == 0 {
|
|
return "", false
|
|
}
|
|
msg := q[0]
|
|
if len(q) == 1 {
|
|
delete(e.steerBox, intentID)
|
|
} else {
|
|
e.steerBox[intentID] = q[1:]
|
|
}
|
|
return msg, true
|
|
}
|
|
|
|
// steerHooks wraps the guard's hook runner so the planner can steer a running work:
|
|
// before each tool call it drains a queued course-correction (if any) and blocks the
|
|
// call, handing the message back to the model — which re-plans its next step instead
|
|
// of running the tool. No queued message → the guard behaves exactly as before.
|
|
// It also handles rerunning "idle turns"; see Stop.
|
|
type steerHooks struct {
|
|
inner harness.HookRunner
|
|
drain func() (string, bool)
|
|
// nudges counts the idle-turn reruns already injected for this intent, capped by
|
|
// limit. It is a pointer: the harness holds a copy of steerHooks, and the count must be shared.
|
|
nudges *atomic.Int64
|
|
// limit caps the number of idle-turn reruns; Engine.emptyTurnNudgeLimit() parses it
|
|
// from the "empty response retry count". <=0 = do not intervene (the user explicitly disabled this layer).
|
|
limit int
|
|
// label looks like "worker-1 · #42"; used only for logging.
|
|
label string
|
|
}
|
|
|
|
// Default number of reruns for idle turns (thinking only, neither body text nor tool
|
|
// calls), kept aligned with the SDK's built-in empty-response retry default
|
|
// (emptyResponseRetries in norma/llm/openai.go) — the two layers share one knob, so unconfigured behavior should match too. See Engine.emptyTurnNudgeLimit.
|
|
//
|
|
// Note this number is a per-intent total, not "how many in a row": the harness's own
|
|
// stopHookActive already limits consecutive idling to a single nudge — if the turn after
|
|
// that nudge is still idle, the Stop hook is not called again and the run just ends;
|
|
// only a real tool turn refreshes the quota (norma/harness/query.go:534). So this gate blocks the pathological "tool → idle → nudge → tool → idle" loop from burning the intent budget.
|
|
const defaultEmptyTurnNudges = 2
|
|
|
|
// emptyTurnNudge is the rerun instruction injected on an idle turn.
|
|
//
|
|
// Such a turn looks like a natural end to the harness (stop_reason=end_turn with no
|
|
// tool_use), and none of the five LLM retry layers applies — it is not an error, the
|
|
// model "finished thinking but never acted". The SDK's empty-response retry cannot
|
|
// reach it either: it judges emptiness by "did any event get yielded", and thinking
|
|
// deltas are themselves events (SEThinkingDelta in norma/llm/openai.go), so
|
|
// thinking-only is not empty. That layer also resends the whole prompt verbatim; for an idle turn dictated by the shape of the context, resending only makes the model think again. Here we append an instruction instead, letting it continue with the thinking it already produced — only a changed input gives reason for different behavior.
|
|
const emptyTurnNudge = "[Idle-turn reminder] Your last turn produced only thinking — no body reply and no tool calls — " +
|
|
"so this turn yielded nothing. Execute the next step you just planned: either call a tool or write your conclusion. Do not repeat the thinking."
|
|
|
|
// isThinkingOnlyTurn reports whether the latest assistant turn produced neither
|
|
// text nor a tool call — i.e. the model spent the whole round thinking.
|
|
func isThinkingOnlyTurn(messages []llm.Message) bool {
|
|
for i := len(messages) - 1; i >= 0; i-- {
|
|
m := messages[i]
|
|
if m.Role != llm.RoleAssistant {
|
|
continue
|
|
}
|
|
return strings.TrimSpace(m.Text()) == "" && len(m.ToolUses()) == 0
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (h steerHooks) PreToolUse(ctx context.Context, name string, input []byte) (bool, string, []byte) {
|
|
if msg, ok := h.drain(); ok {
|
|
return true, "[Planner real-time steering] " + msg +
|
|
"\n(This is the planner's immediate instruction for this intent; the tool call was not executed — adjust your next step accordingly. If it conflicts with your current plan, this takes precedence.)", nil
|
|
}
|
|
if h.inner != nil {
|
|
return h.inner.PreToolUse(ctx, name, input)
|
|
}
|
|
return false, "", nil
|
|
}
|
|
|
|
func (h steerHooks) PostToolUse(ctx context.Context, name string, input, result []byte, isErr bool) {
|
|
if h.inner != nil {
|
|
h.inner.PostToolUse(ctx, name, input, result, isErr)
|
|
}
|
|
}
|
|
|
|
// Stop adds an "idle turn" rerun layer on top of guard's existing semantics: when
|
|
// the model emitted only thinking — neither body text nor tool calls — the harness
|
|
// treats it as a natural end and finishes with an empty summary (ReasonCompleted +
|
|
// asst.Text() in query.go), leaving an unfinished intent stranded mid-way. Inject a rerun instruction here so the model continues with the thinking it already produced.
|
|
func (h steerHooks) Stop(ctx context.Context, messages []llm.Message) (bool, []string, string) {
|
|
var (
|
|
prevent bool
|
|
blocking []string
|
|
msg string
|
|
)
|
|
if h.inner != nil {
|
|
prevent, blocking, msg = h.inner.Stop(ctx, messages)
|
|
}
|
|
// inner has already decided to hard-stop, or is already injecting its own rerun message → respect it and add nothing.
|
|
// limit<=0 = the user configured the "empty response retry count" as -1, i.e. explicitly disabled this layer.
|
|
if prevent || len(blocking) > 0 || h.nudges == nil || h.limit <= 0 || !isThinkingOnlyTurn(messages) {
|
|
return prevent, blocking, msg
|
|
}
|
|
n := h.nudges.Add(1)
|
|
if n > int64(h.limit) {
|
|
log.Printf("[work %s] idle turns (thinking only, no body text, no tools) reached the rerun cap %d, letting it finish", h.label, h.limit)
|
|
return prevent, blocking, msg
|
|
}
|
|
log.Printf("[work %s] idle turn (thinking only, no body text, no tools), injecting rerun instruction (%d/%d)", h.label, n, h.limit)
|
|
return false, []string{emptyTurnNudge}, ""
|
|
}
|
|
|
|
// KillWork cancels the in-flight work running intentID (planner's kill_work tool).
|
|
// The work's agent-core session honors ctx cancellation and aborts promptly.
|
|
func (e *Engine) KillWork(intentID int64) error {
|
|
e.workMu.Lock()
|
|
run := e.work[intentID]
|
|
e.workMu.Unlock()
|
|
if run == nil {
|
|
return fmt.Errorf("intent %d currently has no running work (it may have finished or was never claimed)", intentID)
|
|
}
|
|
run.cancel(agent.AbortKilledByPlanner)
|
|
return nil
|
|
}
|
|
|
|
// Broadcaster exposes the engine's live activity pub/sub (used by the SSE handler).
|
|
func (e *Engine) Broadcaster() *Broadcaster { return e.bc }
|
|
|
|
// emitActivity persists one captured step AND fans it out to live subscribers,
|
|
// from a single point so storage and the SSE stream never diverge.
|
|
func (e *Engine) emitActivity(t *Task, r db.Activity) db.Activity {
|
|
id, err := e.appendActivity(t, r)
|
|
if err != nil {
|
|
// NO LONGER SILENT: dropping a record breaks command↔result pairing in the
|
|
// trace — a tool_use whose tool_result was lost shows as "Running" forever, and
|
|
// a lost 'result'/'round' record leaves the session with no summary ("no summary").
|
|
// Everything needed to analyze the root cause goes into ONE error-level line: reason class,
|
|
// summary preview, running drop count for this task, and — on the FK case — a
|
|
// live probe of WHY the parent exploration is unreachable.
|
|
n := e.bumpDrop(t.ID)
|
|
diag := ""
|
|
// On the FK-parent failure (23503) probe the live DB so the log records WHY the
|
|
// exploration is unreachable (row gone / wrong expID) instead of just that it is.
|
|
if isFKViolation(err) {
|
|
storeID := t.Store.ID()
|
|
if exists, refs, maxID, dErr := e.m.pg.ExplorationDiag(storeID); dErr != nil {
|
|
diag = fmt.Sprintf(" | FK diagnostic query failed (store.expID=%d task.ExpID=%d): %v", storeID, t.ExpID, dErr)
|
|
} else {
|
|
diag = fmt.Sprintf(" | FK diagnosis: store.expID=%d task.ExpID=%d exploration exists=%v tasks referencing it=%d MAX(exploration.id)=%d",
|
|
storeID, t.ExpID, exists, refs, maxID)
|
|
}
|
|
}
|
|
log.Printf("[activity] task %s dropping activity record (%d dropped for this task so far) worker=%s kind=%s tool=%s tuid=%s reason=%s summary=%q: %v%s",
|
|
t.ID, n, r.Worker, r.Kind, r.Tool, r.ToolUseID, dropReason(err), preview(r.Summary, 80), err, diag)
|
|
e.touch(t.ID)
|
|
return r
|
|
}
|
|
r.ID = id
|
|
if r.CreatedAt.IsZero() {
|
|
r.CreatedAt = time.Now()
|
|
}
|
|
e.bc.Publish(t.ID, r)
|
|
e.touch(t.ID)
|
|
return r
|
|
}
|
|
|
|
// appendActivity persists one activity row, retrying briefly on write failure.
|
|
// Concurrent planner + worker inserts into the same exploration's activity log
|
|
// occasionally fail; a couple of quick retries recover most. Crucially, every
|
|
// failure is now LOGGED (it used to be swallowed by an `if err == nil`), so the
|
|
// underlying DB error is finally visible for diagnosis.
|
|
func (e *Engine) appendActivity(t *Task, r db.Activity) (int64, error) {
|
|
var id int64
|
|
var err error
|
|
for attempt := 1; attempt <= 3; attempt++ {
|
|
if id, err = t.Store.AppendActivity(r); err == nil {
|
|
if attempt > 1 {
|
|
log.Printf("[activity] task %s write retry %d succeeded (worker=%s kind=%s tool=%s)",
|
|
t.ID, attempt, r.Worker, r.Kind, r.Tool)
|
|
}
|
|
return id, nil
|
|
}
|
|
log.Printf("[activity] task %s write failed (attempt %d/3, worker=%s kind=%s tool=%s expID=%d): %v",
|
|
t.ID, attempt, r.Worker, r.Kind, r.Tool, t.Store.ID(), err)
|
|
time.Sleep(time.Duration(attempt) * 25 * time.Millisecond)
|
|
}
|
|
return 0, err
|
|
}
|
|
|
|
// SetReadiness wires the global "an LLM provider is configured" predicate (read by
|
|
// Ready() / the llm_configured indicator). Called once at startup.
|
|
func (e *Engine) SetReadiness(fn func() bool) { e.readiness = fn }
|
|
|
|
// SetAgentResolver installs a per-task planner/worker resolver (wired by the server).
|
|
// Called once at startup before any task loop runs, so no lock is needed on reads.
|
|
func (e *Engine) SetAgentResolver(fn func(t *Task) (*agent.Planner, *agent.Worker)) {
|
|
e.resolve = fn
|
|
e.resolveAuthoritative = false
|
|
}
|
|
|
|
// SetAuthoritativeAgentResolver installs a resolver whose nil result must not
|
|
// fall through to the global provider. Task-level failover chains use this so a
|
|
// fully exhausted chain cannot silently bypass its configured boundary.
|
|
func (e *Engine) SetAuthoritativeAgentResolver(fn func(t *Task) (*agent.Planner, *agent.Worker)) {
|
|
e.resolve = fn
|
|
e.resolveAuthoritative = true
|
|
}
|
|
|
|
// snapshotFor returns the planner/worker a task should run on, from the task-router
|
|
// resolver. nil,nil means the task is deliberately unavailable (e.g. an exhausted
|
|
// failover chain); there is no global-pair fallback.
|
|
func (e *Engine) snapshotFor(t *Task) (*agent.Planner, *agent.Worker) {
|
|
if e.resolve != nil {
|
|
p, w := e.resolve(t)
|
|
if (p != nil && w != nil) || e.resolveAuthoritative {
|
|
return p, w
|
|
}
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
// Ready reports whether a global LLM provider is configured (via the readiness
|
|
// predicate wired at startup).
|
|
func (e *Engine) Ready() bool {
|
|
return e.readiness != nil && e.readiness()
|
|
}
|
|
|
|
// ReadyFor reports whether a specific task can resolve a planner/worker pair.
|
|
// An explicit task profile chain can be runnable even when no global default
|
|
// provider is configured, so task status must not rely on Ready alone.
|
|
func (e *Engine) ReadyFor(t *Task) bool {
|
|
p, w := e.snapshotFor(t)
|
|
return p != nil && w != nil
|
|
}
|
|
|
|
// Run starts the planner loop + N worker loops for a task. The loops always run
|
|
// but no-op until an LLM is configured (so a task created while idle picks up
|
|
// automatically once LLM is set from the UI).
|
|
func (e *Engine) Run(ctx context.Context, t *Task) {
|
|
workers := e.m.Workers()
|
|
e.deleteMu.RLock()
|
|
if e.IsDeleting(t.ID) {
|
|
e.deleteMu.RUnlock()
|
|
return
|
|
}
|
|
if _, loaded := e.started.LoadOrStore(t.ID, true); loaded {
|
|
e.deleteMu.RUnlock()
|
|
t.Notify() // already running — just nudge a planning round
|
|
return
|
|
}
|
|
rt := e.registerTaskRoutines(ctx, t.ID, 1+workers)
|
|
e.deleteMu.RUnlock()
|
|
e.touch(t.ID)
|
|
runTaskRoutine(rt, func(loopCtx context.Context) { e.plannerLoop(loopCtx, t) })
|
|
for i := 0; i < workers; i++ {
|
|
name := fmt.Sprintf("work#%d", i+1)
|
|
runTaskRoutine(rt, func(loopCtx context.Context) { e.workerLoop(loopCtx, t, name) })
|
|
}
|
|
e.startDeadlineCoordinator(ctx, t) // task-level timeout timer (only when timeout>0; deduplicated)
|
|
// Kick the first planning round only when there are no active intents at all
|
|
// (open+running). For a task with seeded intents: the seed is already open, or the
|
|
// worker just started above has claimed it as running first — both count as "work
|
|
// exists", so skip the first planner round: the worker runs the seed intent directly and NotifyDone/heartbeat wakes the planner when done.
|
|
// ⚠️ Do not use Frontier (counts only open): a worker claiming (open→running) races with this check and would kick spuriously. After a restart-driven auto-recovery only running intents may remain — skip then too.
|
|
if has, _ := t.Store.HasActiveIntent(); !has {
|
|
t.Notify() // kick the first planning round (acted on once LLM is ready)
|
|
}
|
|
}
|
|
|
|
// plannerHeartbeatInterval parses a task's planner heartbeat interval.
|
|
// db.CreateTask already normalizes it (anything below 600 is raised to 600); this is one more safety net against abnormal in-memory values.
|
|
func plannerHeartbeatInterval(t *Task) time.Duration {
|
|
sec := t.PlanHeartbeatSeconds
|
|
if sec < db.MinPlanHeartbeatSeconds { // floor = default = 600 (10min)
|
|
sec = db.MinPlanHeartbeatSeconds
|
|
}
|
|
return time.Duration(sec) * time.Second
|
|
}
|
|
|
|
// resetPlannerTimer safely re-arms a Timer that may already have fired (the standard Stop→drain→Reset pattern).
|
|
func resetPlannerTimer(timer *time.Timer, d time.Duration) {
|
|
if !timer.Stop() {
|
|
select {
|
|
case <-timer.C:
|
|
default:
|
|
}
|
|
}
|
|
timer.Reset(d)
|
|
}
|
|
|
|
// quietSkipEdge reports whether an edge-triggered planning round should be
|
|
// dropped (#79): the open-goal set is unchanged since the last completed round
|
|
// AND active work exists (open or running intents). Fails open (false) on any
|
|
// store error: when in doubt, plan, preserving the pre-quiet-mode behavior.
|
|
func (e *Engine) quietSkipEdge(t *Task) bool {
|
|
fp, err := t.Store.OpenGoalFingerprint()
|
|
if err != nil {
|
|
return false
|
|
}
|
|
if last, ok := e.goalFP.Load(t.ID); !ok || last != fp {
|
|
return false
|
|
}
|
|
active, err := t.Store.HasActiveIntent()
|
|
return err == nil && active
|
|
}
|
|
|
|
func (e *Engine) plannerLoop(ctx context.Context, t *Task) {
|
|
interval := plannerHeartbeatInterval(t)
|
|
// Arming the heartbeat timer at loop entry = counting from the task start: even a
|
|
// seed task that skips the first planner round (Run does not kick the first round
|
|
// when the frontier is non-empty) and blocks here gets its first planning round at "task start + interval". Every later wake-up (edge/heartbeat) re-arms it = time since the last planning trigger of any kind.
|
|
heartbeat := time.NewTimer(interval)
|
|
defer heartbeat.Stop()
|
|
|
|
// runRound runs one planning round (with debounce coalescing + the guards). src is only used to tell the trigger source apart in logs.
|
|
runRound := func(src string) {
|
|
// debounce: coalesce a burst of changes into one planning round
|
|
timer := time.NewTimer(e.debounce)
|
|
drain:
|
|
for {
|
|
select {
|
|
case <-t.notify:
|
|
case <-timer.C:
|
|
break drain
|
|
}
|
|
}
|
|
planner, _ := e.snapshotFor(t)
|
|
if planner == nil {
|
|
return // idle until LLM configured
|
|
}
|
|
if e.IsPaused(t.ID) {
|
|
return // user-paused: don't plan
|
|
}
|
|
if e.IsDeleting(t.ID) {
|
|
return
|
|
}
|
|
// terminal task (goals all met → done, or failed): the run is over. A
|
|
// resume/nudge — e.g. auto-resume of the active task on restart — must NOT
|
|
// re-plan (it would burn an LLM round and re-confirm a settled result).
|
|
if isTerminalStatus(t.lifecycleSnapshot().Status) {
|
|
return
|
|
}
|
|
// Task-level timeout settlement in progress: drop ordinary wake-ups — neither
|
|
// worker settlement write-backs nor Resume's Notify trigger regular planning rounds; the final round is driven directly by the coordinator (settleTask), not here.
|
|
if e.isSettling(t.ID) {
|
|
return
|
|
}
|
|
// goalless (human direct-inject) branch: when the task has no open goals the
|
|
// planner does not run — running it would re-judge met→cancelExec and kill intents
|
|
// the user injected directly via the main agent. Whether to end is decided by the
|
|
// frontier instead: open/running intents remain → stay running and wait silently; intents all drained → land in done. The whole branch is pure Go, triggers no LLM calls, and writes no planning-round marker.
|
|
if open, err := t.Store.HasOpenGoal(); err == nil && !open {
|
|
t.drainTriggers() // discard accumulated done/finding triggers to avoid unbounded growth in long goalless sessions
|
|
if active, err := t.Store.HasActiveIntent(); err == nil && !active {
|
|
// Frontier drained and no running intents → settle. Use the Guarded variant for
|
|
// the CAS, to avoid stepping on concurrent pause/delete/timeout-settlement state transitions.
|
|
if won, err := e.m.SetTaskStatusGuarded(t.ID, "done"); err != nil {
|
|
log.Printf("[goalless] task %s failed to settle into done: %v", t.ID, err)
|
|
} else if won {
|
|
e.emitActivity(t, db.Activity{Worker: "system", Kind: "text",
|
|
Summary: "All goals have been met, the direct-injected intents have finished executing, and the task is over"})
|
|
}
|
|
}
|
|
return // the goalless branch never enters planner.Plan
|
|
}
|
|
// quiet-mode (#79): an edge round is redundant while work is in flight
|
|
// and the open-goal set is unchanged since the last completed round —
|
|
// there is no new decision for the plan to make. Drain the triggers so
|
|
// they do not accumulate and let the heartbeat supervise; the next edge
|
|
// after a goal opens or closes, or once the frontier drains and all
|
|
// workers are idle, plans for real.
|
|
if src == "edge" && t.PlanQuietMode && e.quietSkipEdge(t) {
|
|
t.drainTriggers()
|
|
log.Printf("[planner] task %s quiet: work in flight, goals unchanged, edge round skipped", t.ID)
|
|
return
|
|
}
|
|
if !e.beginTaskOperation(t.ID) {
|
|
return
|
|
}
|
|
defer e.decInflight(t.ID)
|
|
e.stampFirstRun(t) // first real planning round → stamp first_run_at + compute the deadline (only for tasks with a timeout)
|
|
e.touch(t.ID)
|
|
emit := func(r db.Activity) { e.emitActivity(t, r) }
|
|
ectx := e.clockCtx(e.execContextFor(ctx, t.ID), t, false) // cancellable by Pause; carries the task deadline
|
|
if ectx.Err() != nil || e.IsDeleting(t.ID) {
|
|
return
|
|
}
|
|
log.Printf("[planner] task %s planning… (trigger: %s)", t.ID, src)
|
|
// round marker: each Plan() is one planner round; emit a boundary so the
|
|
// UI can separate rounds in the transcript (kind='round').
|
|
e.emitActivity(t, db.Activity{Worker: "planner", Kind: "round",
|
|
Summary: fmt.Sprintf("Planning round %d", e.nextPlannerRound(t.ID))})
|
|
// what fired this round (worker done / finding; may be several — debounce
|
|
// coalesces a burst; empty for time/heartbeat wakes).
|
|
triggers := t.drainTriggers()
|
|
taskIDInt, _ := strconv.ParseInt(t.ID, 10, 64)
|
|
e.BeginLLMCall(t.ID)
|
|
met, reason, err := planner.Plan(ectx, taskIDInt, e.m.assets, t.Store, t.Goal, triggers, emit)
|
|
e.EndLLMCall(t.ID)
|
|
switch {
|
|
case err != nil && ectx.Err() == nil:
|
|
log.Printf("[planner] task %s planning error: %v", t.ID, err)
|
|
case met:
|
|
log.Printf("[planner] task %s judged all goals met: %s", t.ID, reason)
|
|
// All goals met → persist the task state as done (the frontend DTO shows this terminal state first).
|
|
if err := e.m.SetTaskStatus(t.ID, "done"); err != nil {
|
|
log.Printf("[planner] task %s failed to persist completion: %v", t.ID, err)
|
|
}
|
|
// Task judged complete → cancel the running workers at once: whatever their
|
|
// in-flight intents produce no longer matters. The next worker-loop pass claims
|
|
// nothing new at the terminal gate; the cancelled ones take the "task completed" branch below and end up stopped (not blocked).
|
|
e.cancelExec(t.ID, agent.AbortGoalMet)
|
|
default:
|
|
log.Printf("[planner] task %s planning finished", t.ID)
|
|
}
|
|
// Record the post-round open-goal fingerprint for quiet-mode (#79):
|
|
// goals opened or closed by this round change it, so a later edge can
|
|
// tell "nothing changed" apart from "the plan input changed".
|
|
if fp, err := t.Store.OpenGoalFingerprint(); err == nil {
|
|
e.goalFP.Store(t.ID, fp)
|
|
}
|
|
e.touch(t.ID)
|
|
}
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.notify:
|
|
runRound("edge") // worker finish / finding / kill / resume / seed first round
|
|
case <-heartbeat.C:
|
|
// Periodic safety net: deadlock fallback + wake up to supervise in-flight workers (steer/kill) + periodic recheck.
|
|
runRound("heartbeat")
|
|
}
|
|
// Re-arm the heartbeat after every wake-up (edge or heartbeat): any planning trigger restarts this idle timer.
|
|
resetPlannerTimer(heartbeat, interval)
|
|
}
|
|
}
|
|
|
|
func (e *Engine) workerLoop(ctx context.Context, t *Task, name string) {
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
default:
|
|
}
|
|
_, worker := e.snapshotFor(t)
|
|
if worker == nil {
|
|
if sleepCtx(ctx, 1500*time.Millisecond) {
|
|
return
|
|
}
|
|
continue
|
|
}
|
|
if e.IsPaused(t.ID) {
|
|
if sleepCtx(ctx, 1000*time.Millisecond) {
|
|
return
|
|
}
|
|
continue // user-paused: don't claim/execute intents
|
|
}
|
|
if e.IsDeleting(t.ID) {
|
|
return
|
|
}
|
|
if e.isSettling(t.ID) {
|
|
if sleepCtx(ctx, 1000*time.Millisecond) {
|
|
return
|
|
}
|
|
continue // task timeout settlement in progress: claim no new intents (running ones settle themselves; the coordinator waits for them to drain)
|
|
}
|
|
if isTerminalStatus(e.m.TaskStatus(t.ID)) {
|
|
if sleepCtx(ctx, 1000*time.Millisecond) {
|
|
return
|
|
}
|
|
continue // task already terminal (done/failed/timeout): stop claiming leftover intents; do not churn the frontier after completion
|
|
}
|
|
if !e.beginTaskOperation(t.ID) {
|
|
return
|
|
}
|
|
claimed := e.runWorkerStep(ctx, t, name, worker)
|
|
e.decInflight(t.ID)
|
|
if !claimed && sleepCtx(ctx, 800*time.Millisecond) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// runWorkerStep claims one intent from the frontier and fully settles it via
|
|
// runIntent. Returns false when nothing was claimable. The pool worker loop is its
|
|
// only caller.
|
|
func (e *Engine) runWorkerStep(ctx context.Context, t *Task, name string, worker *agent.Worker) bool {
|
|
intent := e.claimNext(t, name)
|
|
if intent == nil {
|
|
return false
|
|
}
|
|
log.Printf("[worker %s] task %s claimed intent #%d", name, t.ID, intent.ID)
|
|
return e.runIntent(ctx, t, name, worker, intent, "", "")
|
|
}
|
|
|
|
// runIntent executes and fully settles one already-claimed (state=running) intent.
|
|
// Both the pool worker loop (via runWorkerStep) and the human-message handler (via
|
|
// runDetachedIntent, a dedicated goroutine outside the worker pool) call it, so the
|
|
// execute/retry/state-write logic lives in exactly one place. A non-empty message
|
|
// is injected as this turn's input through ExecuteWithMessage; requestID keys the
|
|
// transcript marker that dedups re-injection across model_error retries. The caller
|
|
// must already hold one task-operation admission for the whole sequence so a delete
|
|
// cannot observe quiescence between the LLM return and the final DB writes.
|
|
func (e *Engine) runIntent(ctx context.Context, t *Task, name string, worker *agent.Worker, intent *db.Node, requestID, message string) bool {
|
|
// Per-intent LLM rotation (#85): each claimed worker intent runs on the
|
|
// next chain head. The pin travels through the run context into the
|
|
// provider resolver; non-worker roles and failover keep the active head.
|
|
if t.LLMRotate {
|
|
if head, ok := t.nextRotatedProfile(); ok {
|
|
ctx = withLLMProfilePin(ctx, head)
|
|
}
|
|
}
|
|
hasChatMessage := message != ""
|
|
e.stampFirstRun(t) // first real execution → stamp first_run_at + compute the deadline (only for tasks with a timeout)
|
|
e.touch(t.ID)
|
|
emit := func(r db.Activity) { e.emitActivity(t, r) }
|
|
ectx := e.clockCtx(e.execContextFor(ctx, t.ID), t, false) // cancellable by Pause; carries the task deadline
|
|
if ectx.Err() != nil || e.IsDeleting(t.ID) {
|
|
if err := transitionIntentState(t.Store, intent.ID, "running", "open"); err != nil {
|
|
log.Printf("[worker %s] task %s intent #%d claim rollback failed: %v", name, t.ID, intent.ID, err)
|
|
}
|
|
return true
|
|
}
|
|
// per-work child context so the planner's kill_work can stop just this work.
|
|
workCtx, workCancel := context.WithCancelCause(ectx)
|
|
e.registerWork(intent.ID, workCancel)
|
|
// wrap the guard hooks so steer_work can inject a mid-run course-correction
|
|
// for THIS intent (drained before the worker's next tool call).
|
|
iid := intent.ID
|
|
taskEmit := func(a db.Activity) {
|
|
nid := iid
|
|
a.NodeID, a.Worker = &nid, name
|
|
emit(a)
|
|
}
|
|
label := fmt.Sprintf("%s · #%d", name, iid)
|
|
workCtx = intercept.WithTaskContext(workCtx, t.ID, label, taskEmit)
|
|
// nudges deliberately sits outside the model_error rerun loop: the idle-turn rerun
|
|
// cap is a per-intent total, and one rerun round must not reset it back to zero.
|
|
hooks := steerHooks{
|
|
inner: t.Guard.Hooks(),
|
|
drain: func() (string, bool) { return e.drainSteer(iid) },
|
|
nudges: &atomic.Int64{},
|
|
limit: e.emptyTurnNudgeLimit(),
|
|
label: label,
|
|
}
|
|
wTaskID, _ := strconv.ParseInt(t.ID, 10, 64)
|
|
e.BeginLLMCall(t.ID)
|
|
var reason harness.TerminalReason
|
|
var wrote agent.WriteCounts
|
|
var err error
|
|
if hasChatMessage {
|
|
reason, wrote, err = worker.ExecuteWithMessage(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding, requestID, message)
|
|
} else {
|
|
reason, wrote, err = worker.Execute(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding)
|
|
}
|
|
e.EndLLMCall(t.ID)
|
|
// model_error finish → a few extra reruns (retry after backoff). Retry only while
|
|
// the intent still belongs to this work and the task is not paused/killed/cancelled
|
|
// [and has not entered settlement]; otherwise yield to the matching branch (no retries during settlement, so backoff does not eat into other workers' graceful-settlement window).
|
|
maxRetries, retryBackoff := e.modelErrorRetryPolicy()
|
|
for attempt := 1; attempt <= maxRetries &&
|
|
retryableWorkerModelError(reason, err) &&
|
|
workCtx.Err() == nil && ectx.Err() == nil && !e.IsPaused(t.ID) && !e.isSettling(t.ID); attempt++ {
|
|
log.Printf("[worker %s] task %s intent #%d finished with model_error, retrying after %v (%d/%d)",
|
|
name, t.ID, intent.ID, retryBackoff, attempt, maxRetries)
|
|
if sleepCtx(workCtx, retryBackoff) {
|
|
break // cancelled during backoff (kill/pause) → handled by the branches below
|
|
}
|
|
e.BeginLLMCall(t.ID)
|
|
if hasChatMessage {
|
|
reason, wrote, err = worker.ExecuteWithMessage(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding, requestID, message)
|
|
} else {
|
|
reason, wrote, err = worker.Execute(workCtx, name, wTaskID, e.m.assets, t.Store, intent, hooks, emit, e.m.enrich, t.NotifyFinding)
|
|
}
|
|
e.EndLLMCall(t.ID)
|
|
}
|
|
// Capture kill state before detachWork cancels workCtx. kill = this work's
|
|
// ctx was cancelled (planner kill_work) while the TASK ctx kept running; a
|
|
// pause cancels the task ctx (ectx) instead. Checking workCtx.Err() AFTER
|
|
// unregister would always be true (unregister cancels it) → every completed
|
|
// work would be wrongly marked stopped.
|
|
workCause := context.Cause(workCtx)
|
|
killed := workCtx.Err() != nil && ectx.Err() == nil
|
|
action, completeWork := e.detachWork(intent.ID)
|
|
// A caller may stop waiting and release its in-memory reservation before the
|
|
// agent honors cancellation. The named context cause remains authoritative and
|
|
// still settles the stopped run into a recoverable state.
|
|
if action == "" {
|
|
switch {
|
|
case errors.Is(workCause, agent.AbortWorkPausedByUser):
|
|
action = "pause"
|
|
case errors.Is(workCause, agent.AbortWorkCancelledByUser):
|
|
action = "cancel"
|
|
}
|
|
}
|
|
var controlErr error
|
|
defer func() { completeWork(controlErr) }()
|
|
if action == "pause" {
|
|
controlErr = transitionIntentState(t.Store, intent.ID, "running", "paused")
|
|
if controlErr != nil {
|
|
log.Printf("[worker %s] task %s intent #%d failed to persist the paused state: %v", name, t.ID, intent.ID, controlErr)
|
|
return true
|
|
}
|
|
log.Printf("[worker %s] task %s intent #%d paused", name, t.ID, intent.ID)
|
|
e.touch(t.ID)
|
|
return true
|
|
}
|
|
if action == "cancel" {
|
|
// Park the stopped run in paused before handing cleanup to the API. If the
|
|
// request disconnects after cancellation, the intent remains recoverable and
|
|
// a later cancel can finish cleanup instead of leaving a phantom running row.
|
|
controlErr = transitionIntentState(t.Store, intent.ID, "running", "paused")
|
|
if controlErr != nil {
|
|
log.Printf("[worker %s] task %s intent #%d failed to persist the cancel barrier: %v", name, t.ID, intent.ID, controlErr)
|
|
return true
|
|
}
|
|
log.Printf("[worker %s] task %s intent #%d stopped, waiting for cancel cleanup", name, t.ID, intent.ID)
|
|
e.touch(t.ID)
|
|
return true
|
|
}
|
|
// if a pause cancelled this run mid-flight, return the intent to the frontier
|
|
// so it is re-claimed on resume — the worker will resume the prior LLM
|
|
// conversation from its transcript instead of restarting from scratch.
|
|
if ectx.Err() != nil && taskExecutionPaused(context.Cause(ectx)) {
|
|
if err := transitionIntentState(t.Store, intent.ID, "running", "open"); err != nil {
|
|
log.Printf("[worker %s] task %s intent #%d task-pause rollback failed: %v", name, t.ID, intent.ID, err)
|
|
}
|
|
return true
|
|
}
|
|
// The hard-fallback cancel of task-timeout settlement (not pause, not kill) cancelled
|
|
// this run → classify it as exhausted (settled), not mislabeled blocked. By then the worker has usually written its result back during the settlement phase.
|
|
if ectx.Err() != nil && e.isSettling(t.ID) {
|
|
if err := transitionIntentState(t.Store, intent.ID, "running", "exhausted"); err != nil {
|
|
log.Printf("[worker %s] task %s intent #%d failed to persist the timeout-settlement state: %v", name, t.ID, intent.ID, err)
|
|
}
|
|
log.Printf("[worker %s] task %s intent #%d ended by task-timeout settlement (exhausted), wrote back %s", name, t.ID, intent.ID, wrote)
|
|
e.touch(t.ID)
|
|
return true
|
|
}
|
|
// Task judged complete (done via the regular path) → cancelExec above cancelled this
|
|
// run. The intent's result is meaningless now; mark it stopped (not blocked) and do not pollute the intent states of a completed task.
|
|
if ectx.Err() != nil && isTerminalStatus(e.m.TaskStatus(t.ID)) {
|
|
if err := transitionIntentState(t.Store, intent.ID, "running", "stopped"); err != nil {
|
|
log.Printf("[worker %s] task %s intent #%d failed to persist the terminal stop: %v", name, t.ID, intent.ID, err)
|
|
}
|
|
log.Printf("[worker %s] task %s intent #%d cancelled because the task completed (stopped)", name, t.ID, intent.ID)
|
|
e.touch(t.ID)
|
|
return true
|
|
}
|
|
// killed by the planner: mark stopped (don't write back results, don't auto-reclaim).
|
|
if killed {
|
|
if err := transitionIntentState(t.Store, intent.ID, "running", "stopped"); err != nil {
|
|
log.Printf("[worker %s] task %s intent #%d failed to persist the planner stop: %v", name, t.ID, intent.ID, err)
|
|
}
|
|
log.Printf("[worker %s] task %s intent #%d killed (stopped)", name, t.ID, intent.ID)
|
|
e.touch(t.ID)
|
|
t.Notify()
|
|
return true
|
|
}
|
|
if err != nil {
|
|
log.Printf("[worker %s] intent %d: %v", name, intent.ID, err)
|
|
}
|
|
// Terminal routing: hitting the turn cap is not completion. max_turns → exhausted
|
|
// (so the planner knows this direction was tried but not truly finished and needs a different angle, instead of skipping it forever as covered); error → blocked; normal → done.
|
|
state := "done"
|
|
switch {
|
|
case err != nil:
|
|
state = "blocked"
|
|
case reason == harness.ReasonMaxTurns:
|
|
state = "exhausted"
|
|
log.Printf("[worker %s] intent %d hit the turn cap (exhausted), wrote back %s", name, intent.ID, wrote)
|
|
case reason == harness.ReasonTimeout:
|
|
state = "exhausted"
|
|
log.Printf("[worker %s] intent %d ran out of time (exhausted), wrote back %s after settling", name, intent.ID, wrote)
|
|
}
|
|
if state == "blocked" && isTaskLLMChainExhausted(err) {
|
|
_ = t.Store.SetIntentBlockedReason(intent.ID, db.IntentBlockedLLMQuota)
|
|
} else {
|
|
if stateErr := transitionIntentState(t.Store, intent.ID, "running", state); stateErr != nil {
|
|
log.Printf("[worker %s] task %s intent #%d failed to persist terminal state %s: %v", name, t.ID, intent.ID, state, stateErr)
|
|
}
|
|
}
|
|
log.Printf("[worker %s] task %s intent #%d finished: %s (wrote back %s)", name, t.ID, intent.ID, state, wrote)
|
|
e.touch(t.ID)
|
|
t.NotifyDone(intent.ID) // results changed the graph -> wake the planner (with the just-finished intent id)
|
|
return true
|
|
}
|
|
|
|
// runDetachedIntent runs one paused intent OUTSIDE the worker pool in its own
|
|
// goroutine — the human-message path. It transitions the intent paused->running
|
|
// itself (never through 'open'), so the pool, which only claims 'open', can never
|
|
// race it; the "at most one run per intent" invariant still holds because winning
|
|
// the CAS is the sole entry and work[intentID] was cleared when the pause settled.
|
|
// Because it does not compete for a frontier slot, a user message continues the
|
|
// worker immediately even when all pool slots are busy (mirroring how the
|
|
// main-agent chat handler starts its run directly). The spawned goroutine owns one
|
|
// task-operation admission for the whole run and roots its context at ctx (pass the
|
|
// server root, never the HTTP request, so a disconnect cannot strand the run while
|
|
// task pause/delete/shutdown still stops it). Returns an error if the run could not
|
|
// be started; the intent is left untouched in that case.
|
|
func (e *Engine) runDetachedIntent(ctx context.Context, t *Task, intentID int64, requestID, message, agentMessage string) error {
|
|
if !e.beginTaskOperation(t.ID) {
|
|
return fmt.Errorf("task is being deleted")
|
|
}
|
|
release := true
|
|
defer func() {
|
|
if release {
|
|
e.decInflight(t.ID)
|
|
}
|
|
}()
|
|
_, worker := e.snapshotFor(t)
|
|
if worker == nil {
|
|
return fmt.Errorf("worker not ready yet")
|
|
}
|
|
node, err := t.Store.GetNode(intentID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if node == nil || node.Kind != db.KindIntent {
|
|
return fmt.Errorf("intent not found")
|
|
}
|
|
changed, err := t.Store.CompareAndSetIntentState(intentID, "paused", "running")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !changed {
|
|
return fmt.Errorf("%w: intent is no longer in the paused state", db.ErrIntentStateConflict)
|
|
}
|
|
node.State, node.Owner = "running", "chat"
|
|
// Record the human turn as a visible activity BEFORE the run starts, so it is
|
|
// ordered ahead of any worker step and never appears without the run happening.
|
|
// Keep the UI copy concise; ExecuteWithMessage writes the server-resolved
|
|
// reference snapshot into the intent transcript as the LLM input.
|
|
uid := intentID
|
|
e.emitActivity(t, db.Activity{NodeID: &uid, Worker: "user", Kind: "user", Summary: message, Detail: message})
|
|
release = false // ownership of the admission passes to the goroutine
|
|
go func() {
|
|
defer e.decInflight(t.ID)
|
|
e.runIntent(ctx, t, "chat", worker, node, requestID, agentMessage)
|
|
}()
|
|
return nil
|
|
}
|
|
|
|
func taskExecutionPaused(cause error) bool {
|
|
var abort *agent.AbortCause
|
|
if !errors.As(cause, &abort) {
|
|
return false
|
|
}
|
|
switch abort.Code {
|
|
case "paused_by_user", "paused_by_orchestrator", "paused_on_reload", "paused_race_guard",
|
|
"queued_for_admission", "llm_unavailable_queued", "task_deleted":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func sleepCtx(ctx context.Context, d time.Duration) (done bool) {
|
|
select {
|
|
case <-ctx.Done():
|
|
return true
|
|
case <-time.After(d):
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (e *Engine) claimNext(t *Task, name string) *db.Node {
|
|
fr, _ := t.Store.Frontier(20)
|
|
for _, in := range fr {
|
|
if ok, _ := t.Store.ClaimIntent(in.ID, name); ok {
|
|
return in
|
|
}
|
|
}
|
|
return nil
|
|
}
|