Compare commits

...
Author SHA1 Message Date
BaaRai 0caa9bece4 fix(llm-rotation): precise exhaustion wording and binding check state
Follow-up to #270 by @Toys0125:

- README: an intent pinned to an exhausted profile falls back to the task's
  current profile instead of retrying it, and saving the chain does not reset
  the rotation cursor.
- api.ts: the update response carries llm_rotate.
- TaskLLMProfileChain: keep a failed worker-binding lookup separate from the
  initial load, which flashed the "could not be checked" warning.
2026-11-16 23:53:22 +01:00
BaaRai 4df6ed2678 Merge pull request #270 from Toys0125/feat/multi-llm-worker-rotation
Enable concurrent multi-LLM worker rotation in task controls
2026-11-16 23:53:08 +01:00
Toys0125 d18b841a70 feat: expose concurrent multi-LLM worker rotation 2026-11-16 15:31:47 -06:00
BaaRai 9f99b7172a Merge pull request #267 from BaaRai/test/posture-check-corpus
test(posture_check): pin the refusal corpus and use ASCII ellipsis
2026-11-16 18:36:58 +01:00
16 changed files with 398 additions and 30 deletions
+25
View File
@@ -62,6 +62,31 @@ docker compose logs artex | grep "Setup token"
---
## Multi-provider workers
In **System → LLM**, configure the profiles you want to use. In **System → Settings**,
set the concurrent worker count to at least **2** before starting a task (the count
applies to subsequently started tasks). In **System → Agents → Worker**, leave the
model set to follow the task; a fixed Worker model takes precedence over rotation.
When creating a task, select both profiles in **LLM config chain**, arrange their
order, and enable **Enable multi-LLM worker rotation**. For an existing task, open
its LLM config control, change the switch or profile order, and save.
Rotation assigns successive worker intents A, B, A, B, regardless of completion
order. Separate intents can run concurrently on different providers; each intent
keeps its selected provider unless it fails over. Planner and other unbound roles
continue using the task's current profile. Turning rotation off restores ordered
task-level failover. An excluded provider is skipped, so an intent pinned to an
exhausted profile falls back to the task's current profile instead of retrying it;
rotation keeps cycling the whole list and does not rebalance around a dead provider.
Saving the chain resets their failure state. Changes do not interrupt in-flight
calls. The rotation cursor starts again after a server restart and saving the chain
does not reset it; the profile order and switch remain persisted.
The task creation and `PUT /api/tasks/{id}/llm` APIs accept `llm_rotate: true` alongside
the ordered `llm_profile_ids`. Omitting `llm_rotate` on an update preserves its value.
## Screenshots
| Tasks | Findings |
+16 -6
View File
@@ -76,12 +76,16 @@ func applyTaskLLMContext(
t.ActiveLLMProfileID = activeProfileID
t.LLMChainRevision = revision
t.LLMProfileIDs = make([]int64, 0, len(chain))
t.ReadyLLMProfileIDs = nil
t.LLMFailoverState = "default"
t.LLMFailoverReason = ""
var latest *TaskLLMProfile
activeReady := false
for i := range chain {
entry := chain[i]
if entry.Status == "ready" {
t.ReadyLLMProfileIDs = append(t.ReadyLLMProfileIDs, entry.ProfileID)
}
t.LLMProfileIDs = append(t.LLMProfileIDs, entry.ProfileID)
if t.ActiveLLMProfileID != nil && entry.ProfileID == *t.ActiveLLMProfileID && entry.Status == "ready" {
activeReady = true
@@ -405,7 +409,7 @@ FROM task_llm_profiles WHERE task_id=$1 ORDER BY position`, taskID)
// Terminal (done/failed/timeout) tasks may also edit the chain: after a task ends the
// main agent conversation still uses this chain, so it must be replaceable when the
// models on it become unavailable — otherwise a completed task could never be interacted with again.
func (d *DB) ReplaceTaskLLMProfiles(taskID int64, profileIDs []int64, activeProfileID int64) error {
func (d *DB) ReplaceTaskLLMProfiles(taskID int64, profileIDs []int64, activeProfileID int64, rotate ...*bool) error {
tx, err := d.Begin()
if err != nil {
return err
@@ -445,9 +449,14 @@ func (d *DB) ReplaceTaskLLMProfiles(taskID int64, profileIDs []int64, activeProf
if activeProfileID > 0 {
active = activeProfileID
}
var rotation any
if len(rotate) > 0 && rotate[0] != nil {
rotation = *rotate[0]
}
if _, err := tx.Exec(`UPDATE tasks
SET active_llm_profile_id=$2, llm_profile_id=$2, llm_chain_revision=llm_chain_revision+1
WHERE id=$1`, taskID, active); err != nil {
SET active_llm_profile_id=$2, llm_profile_id=$2, llm_chain_revision=llm_chain_revision+1,
llm_rotate=COALESCE($3::boolean,llm_rotate)
WHERE id=$1`, taskID, active, rotation); err != nil {
return err
}
return tx.Commit()
@@ -476,8 +485,9 @@ func (d *DB) markTaskLLMProfileQuotaExhausted(taskID, profileID, revision int64,
var (
active sql.NullInt64
currentRevision int64
rotate bool
)
if err := tx.QueryRow(`SELECT active_llm_profile_id, llm_chain_revision FROM tasks WHERE id=$1 FOR UPDATE`, taskID).Scan(&active, &currentRevision); err != nil {
if err := tx.QueryRow(`SELECT active_llm_profile_id, llm_chain_revision, llm_rotate FROM tasks WHERE id=$1 FOR UPDATE`, taskID).Scan(&active, &currentRevision, &rotate); err != nil {
return out, err
}
out.PreviousProfileID = profileID
@@ -520,8 +530,8 @@ WHERE task_id=$1 AND profile_id=$2`, taskID, profileID, reason); err != nil {
}
var next int64
err = tx.QueryRow(`SELECT profile_id FROM task_llm_profiles
WHERE task_id=$1 AND position>$2 AND status='ready'
ORDER BY position LIMIT 1`, taskID, position).Scan(&next)
WHERE task_id=$1 AND ($3 OR position>$2) AND status='ready'
ORDER BY position LIMIT 1`, taskID, position, rotate).Scan(&next)
switch err {
case nil:
out.Advanced = true
+70
View File
@@ -0,0 +1,70 @@
package db
import (
"fmt"
"testing"
"time"
)
func TestTaskRotationPersistenceAndFailover(t *testing.T) {
d, err := Open(testDSN(t))
if err != nil {
t.Skipf("postgres unavailable: %v", err)
}
defer d.Close()
ids := []int64{}
for i := 0; i < 2; i++ {
id, err := d.SaveProfile(&LLMProfile{Name: fmt.Sprintf("rotation-%d-%d", time.Now().UnixNano(), i), Format: "openai", Model: "test", APIKey: "test-key"})
if err != nil {
t.Fatal(err)
}
ids = append(ids, id)
defer cleanupProfile(t, d, id)
}
on := true
task, err := d.CreateTaskWithOptions("rotation", "test", TaskCreateOptions{LLMProfileIDs: ids, LLMRotate: &on})
if err != nil {
t.Fatal(err)
}
defer d.DeleteTask(task.ID)
if !task.LLMRotate {
t.Fatal("create response lost rotation")
}
// Starting at the last profile still permits an earlier healthy provider in rotation mode.
if err := d.ReplaceTaskLLMProfiles(task.ID, ids, ids[1]); err != nil {
t.Fatal(err)
}
got, err := d.GetTask(task.ID)
if err != nil {
t.Fatal(err)
}
if !got.LLMRotate {
t.Fatal("legacy update lost persisted rotation")
}
tr, err := d.MarkTaskLLMProfileQuotaExhausted(task.ID, ids[1], "insufficient_quota")
if err != nil {
t.Fatal(err)
}
if tr.NextProfileID == nil || *tr.NextProfileID != ids[0] || tr.ChainExhausted {
t.Fatalf("healthy earlier provider skipped: %+v", tr)
}
got, err = d.GetTask(task.ID)
if err != nil {
t.Fatal(err)
}
if len(got.ReadyLLMProfileIDs) != 1 || got.ReadyLLMProfileIDs[0] != ids[0] {
t.Fatalf("ready profiles: %v", got.ReadyLLMProfileIDs)
}
off := false
reordered := []int64{ids[1], ids[0]}
if err := d.ReplaceTaskLLMProfiles(task.ID, reordered, 0, &off); err != nil {
t.Fatal(err)
}
got, err = d.GetTask(task.ID)
if err != nil {
t.Fatal(err)
}
if got.LLMRotate || fmt.Sprint(got.LLMProfileIDs) != fmt.Sprint(reordered) || len(got.ReadyLLMProfileIDs) != 2 {
t.Fatalf("update/reload lost settings: %+v", got)
}
}
+2
View File
@@ -29,6 +29,7 @@ type Task struct {
// Task-level ordered LLM chain. LLMProfileID remains the compatibility alias
// for ActiveLLMProfileID while older API clients still send one profile id.
LLMProfileIDs []int64 `json:"llm_profile_ids,omitempty"`
ReadyLLMProfileIDs []int64 `json:"-"`
ActiveLLMProfileID *int64 `json:"active_llm_profile_id,omitempty"`
LLMChainRevision int64 `json:"-"`
LLMFailoverState string `json:"llm_failover_state,omitempty"`
@@ -256,6 +257,7 @@ VALUES ($1, 'fact', $2, 0, 'origin', 'system')`, expID, string(originPayload));
if opts.LLMRotate != nil {
rotate = *opts.LLMRotate
}
t.LLMRotate = rotate
t.TargetGate = opts.TargetGate != nil && *opts.TargetGate
if err := tx.QueryRow(`
INSERT INTO tasks(name, category_id, description, goal, exploration_id, llm_profile_id, active_llm_profile_id, timeout_seconds, plan_heartbeat_seconds, plan_quiet_mode, coverage_enabled, llm_rotate, target_gate)
+1 -1
View File
@@ -139,7 +139,7 @@ func taskDTO(t *Task, status string) TaskDTO {
LLMProfileID: llmState.ProfileID,
LLMProfileIDs: profileIDs,
ActiveLLMProfileID: llmState.ActiveID,
LLMRotate: t.LLMRotate,
LLMRotate: llmState.Rotate,
LLMFailoverState: llmState.FailoverState,
LLMFailoverReason: llmState.FailoverReason,
SourceTaskIDs: sourceIDs,
+2 -4
View File
@@ -1109,10 +1109,8 @@ func (e *Engine) runIntent(ctx context.Context, t *Task, name string, worker *ag
// Per-intent LLM rotation (#85): each claimed worker intent runs on the
// next chain head. The pin travels through the run context into the
// provider resolver; non-worker roles and failover keep the active head.
if t.LLMRotate {
if head, ok := t.nextRotatedProfile(); ok {
ctx = withLLMProfilePin(ctx, head)
}
if head, ok := t.nextRotatedProfile(); ok {
ctx = withLLMProfilePin(ctx, head)
}
hasChatMessage := message != ""
e.stampFirstRun(t) // first real execution → stamp first_run_at + compute the deadline (only for tasks with a timeout)
+12 -5
View File
@@ -161,6 +161,7 @@ func (t *Task) updateLifecycle(update func(*taskLifecycleState)) {
}
type taskLLMState struct {
Rotate bool
ProfileID *int64
ProfileIDs []int64
ActiveID *int64
@@ -181,6 +182,7 @@ func (t *Task) llmStateSnapshot() taskLLMState {
t.llmMu.RLock()
defer t.llmMu.RUnlock()
return taskLLMState{
Rotate: t.LLMRotate,
ProfileID: cloneInt64Ptr(t.LLMProfileID),
ProfileIDs: append(make([]int64, 0, len(t.LLMProfileIDs)), t.LLMProfileIDs...),
ActiveID: cloneInt64Ptr(t.ActiveLLMProfileID),
@@ -195,10 +197,12 @@ func (t *Task) llmStateSnapshot() taskLLMState {
// chain. The active head stays authoritative for every non-worker role and for
// failover ordering; rotation only distributes worker lanes across heads.
func (t *Task) nextRotatedProfile() (int64, bool) {
t.llmMu.RLock()
defer t.llmMu.RUnlock()
t.rotateMu.Lock()
defer t.rotateMu.Unlock()
ids := t.LLMProfileIDs
if len(ids) == 0 {
if !t.LLMRotate || len(ids) == 0 {
return 0, false
}
pick := ids[t.rotateNext%len(ids)]
@@ -206,12 +210,15 @@ func (t *Task) nextRotatedProfile() (int64, bool) {
return pick, true
}
func (t *Task) setLLMState(profileID, activeID *int64, profileIDs []int64, revision int64, state, reason string) bool {
func (t *Task) setLLMState(profileID, activeID *int64, profileIDs []int64, revision int64, state, reason string, rotate ...bool) bool {
t.llmMu.Lock()
defer t.llmMu.Unlock()
if revision < t.LLMChainRevision {
return false
}
if len(rotate) > 0 {
t.LLMRotate = rotate[0]
}
t.LLMProfileID = cloneInt64Ptr(profileID)
t.ActiveLLMProfileID = cloneInt64Ptr(activeID)
t.LLMProfileIDs = append(t.LLMProfileIDs[:0], profileIDs...)
@@ -1132,12 +1139,12 @@ func (m *Manager) DeleteCompanyWithAssets(id int64, deleteAssets bool) (int64, e
// ReplaceTaskLLMProfiles resets a task's ordered provider chain and mirrors the
// committed state onto the live task handle. Terminal tasks are editable too —
// their main-agent conversation keeps running on the chain after the task finishes.
func (m *Manager) ReplaceTaskLLMProfiles(id string, profileIDs []int64, activeProfileID int64) (int64, error) {
func (m *Manager) ReplaceTaskLLMProfiles(id string, profileIDs []int64, activeProfileID int64, rotate ...*bool) (int64, error) {
n, err := strconv.ParseInt(id, 10, 64)
if err != nil {
return 0, err
}
if err := m.pg.ReplaceTaskLLMProfiles(n, profileIDs, activeProfileID); err != nil {
if err := m.pg.ReplaceTaskLLMProfiles(n, profileIDs, activeProfileID, rotate...); err != nil {
return 0, err
}
pt, err := m.pg.GetTask(n)
@@ -1146,7 +1153,7 @@ func (m *Manager) ReplaceTaskLLMProfiles(id string, profileIDs []int64, activePr
}
m.mu.Lock()
if task := m.tasks[id]; task != nil {
task.setLLMState(pt.LLMProfileID, pt.ActiveLLMProfileID, pt.LLMProfileIDs, pt.LLMChainRevision, pt.LLMFailoverState, pt.LLMFailoverReason)
task.setLLMState(pt.LLMProfileID, pt.ActiveLLMProfileID, pt.LLMProfileIDs, pt.LLMChainRevision, pt.LLMFailoverState, pt.LLMFailoverReason, pt.LLMRotate)
}
m.mu.Unlock()
// Terminal tasks do not reopen budget-blocked intents: no worker is running for
+4 -2
View File
@@ -1617,7 +1617,7 @@ type createTaskReq struct {
SourceTaskIDs []string `json:"source_task_ids,omitempty"` // direct, read-only inherited source tasks
CompanyIDs []int64 `json:"company_ids,omitempty"` // links the company scope and snapshots its current assets; does not copy assets or force intent generation
TimeoutSeconds int `json:"timeout_seconds"`
LLMRotate *bool `json:"llm_rotate"` // task-level timeout (seconds); 0/omitted = no limit
LLMRotate *bool `json:"llm_rotate"` // distribute worker intents across the ordered profiles
PlanHeartbeatSeconds int `json:"plan_heartbeat_seconds"` // planner heartbeat interval (seconds); 0/omitted = default 600 (10min); floor = default = 600, lower values are raised to 600
PlanQuietMode *bool `json:"plan_quiet_mode,omitempty"` // planner quiet-mode switch (#86); omitted = default on; false = edge rounds always plan
TargetGate *bool `json:"target_gate,omitempty"` // approval gate for asset targets, opt-in; omitted/null = off
@@ -1758,6 +1758,7 @@ func (s *Server) updateTaskLLMProfiles(w http.ResponseWriter, r *http.Request) {
var req struct {
LLMProfileIDs []int64 `json:"llm_profile_ids"`
ActiveLLMProfileID *int64 `json:"active_llm_profile_id"`
LLMRotate *bool `json:"llm_rotate"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeErr(w, 400, "bad json: "+err.Error())
@@ -1771,7 +1772,7 @@ func (s *Server) updateTaskLLMProfiles(w http.ResponseWriter, r *http.Request) {
if req.ActiveLLMProfileID != nil {
active = *req.ActiveLLMProfileID
}
reopened, err := s.m.ReplaceTaskLLMProfiles(t.ID, req.LLMProfileIDs, active)
reopened, err := s.m.ReplaceTaskLLMProfiles(t.ID, req.LLMProfileIDs, active, req.LLMRotate)
if err != nil {
writeErr(w, 400, err.Error())
return
@@ -1785,6 +1786,7 @@ func (s *Server) updateTaskLLMProfiles(w http.ResponseWriter, r *http.Request) {
llmState := t.llmStateSnapshot()
result := map[string]any{
"id": t.ID, "llm_profile_ids": llmState.ProfileIDs,
"llm_rotate": llmState.Rotate,
"active_llm_profile_id": llmState.ActiveID,
"llm_failover_state": llmState.FailoverState, "reopened_intents": reopened,
}
+2 -2
View File
@@ -206,7 +206,7 @@ func (r *taskLLMRuntime) currentFor(ctx context.Context) (taskLLMSelection, erro
}
sel.profileID = *pt.ActiveLLMProfileID
if pin, ok := llmProfilePinFrom(ctx); ok && pin != sel.profileID {
for _, id := range pt.LLMProfileIDs {
for _, id := range pt.ReadyLLMProfileIDs {
if id == pin {
sel.profileID = pin
break
@@ -794,7 +794,7 @@ func (s *Server) syncTaskLLMState(pt *db.Task) {
id := fmt.Sprintf("%d", pt.ID)
s.m.mu.Lock()
if task := s.m.tasks[id]; task != nil {
task.setLLMState(pt.LLMProfileID, pt.ActiveLLMProfileID, pt.LLMProfileIDs, pt.LLMChainRevision, pt.LLMFailoverState, pt.LLMFailoverReason)
task.setLLMState(pt.LLMProfileID, pt.ActiveLLMProfileID, pt.LLMProfileIDs, pt.LLMChainRevision, pt.LLMFailoverState, pt.LLMFailoverReason, pt.LLMRotate)
}
s.m.mu.Unlock()
}
+182 -1
View File
@@ -2,7 +2,15 @@ package server
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/norma/llm"
)
func TestLLMProfilePinRoundTrip(t *testing.T) {
@@ -17,8 +25,177 @@ func TestLLMProfilePinRoundTrip(t *testing.T) {
}
}
// Use two real HTTP provider adapters and a barrier: provider B must finish and
// another intent must start while provider A's first request is still blocked.
func TestRotationConcurrentProvidersAndPrecedence(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable: %v", err)
}
defer m.Close()
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
started := make(chan int, 3)
release := []chan struct{}{make(chan struct{}), make(chan struct{})}
ids := []int64{}
for i := 0; i < 2; i++ {
i := i
endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
started <- i
select {
case <-release[i]:
case <-r.Context().Done():
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"choices":[{"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}`)
}))
defer endpoint.Close()
id, err := m.pg.SaveProfile(&db.LLMProfile{Name: fmt.Sprintf("rotation-http-%d-%d", time.Now().UnixNano(), i), Format: "openai", BaseURL: endpoint.URL + "/v1", Model: "test", APIKey: "test-key"})
if err != nil {
t.Fatal(err)
}
ids = append(ids, id)
defer m.pg.DeleteProfile(id)
}
// Release blocked handlers even when an assertion fails.
defer func() {
for _, ch := range release {
select {
case <-ch:
default:
close(ch)
}
}
}()
on := true
task, err := m.CreateTaskWithOptions("rotation HTTP", "test", db.TaskCreateOptions{LLMProfileIDs: ids, LLMRotate: &on})
if err != nil {
t.Fatal(err)
}
defer m.DeleteTask(task.ID, DeleteTaskOptions{})
s := &Server{m: m, provByProfile: map[int64]*provEntry{}}
runtime := &taskLLMRuntime{s: s, taskID: task.ID, agentKey: "worker"}
done := make(chan error, 3)
dispatch := func(want int) {
t.Helper()
head, _ := task.nextRotatedProfile()
go func() {
_, _, _, err := runtime.Complete(withLLMProfilePin(ctx, head), llm.CompletionRequest{})
done <- err
}()
select {
case got := <-started:
if got != want {
t.Fatalf("provider=%d want=%d", got, want)
}
case <-ctx.Done():
t.Fatal("provider request did not start concurrently")
}
}
dispatch(0)
dispatch(1)
close(release[1])
select {
case err := <-done:
if err != nil {
t.Fatal(err)
}
case <-ctx.Done():
t.Fatal(ctx.Err())
}
dispatch(0)
close(release[0])
for i := 0; i < 2; i++ {
select {
case err := <-done:
if err != nil {
t.Fatal(err)
}
case <-ctx.Done():
t.Fatal(ctx.Err())
}
}
// An exhausted pin must fall back rather than retry the failed provider forever.
n, _ := parseTaskID(task.ID)
if _, err := m.pg.MarkTaskLLMProfileQuotaExhausted(n, ids[1], "insufficient_quota"); err != nil {
t.Fatal(err)
}
sel, err := runtime.currentFor(withLLMProfilePin(ctx, ids[1]))
if err != nil || sel.profileID != ids[0] {
t.Fatalf("exhausted pin selected: %+v, %v", sel, err)
}
if err := m.pg.SetAgentLLMProfile("worker", &ids[1]); err != nil {
t.Fatal(err)
}
defer m.pg.SetAgentLLMProfile("worker", nil)
sel, err = runtime.currentFor(withLLMProfilePin(ctx, ids[0]))
if err != nil || sel.profileID != 0 {
t.Fatalf("fixed binding did not outrank rotation: %+v, %v", sel, err)
}
off := false
if _, err := m.ReplaceTaskLLMProfiles(task.ID, []int64{ids[1], ids[0]}, 0, &off); err != nil {
t.Fatal(err)
}
if task.llmStateSnapshot().Rotate || taskDTO(task, "idle").LLMRotate {
t.Fatal("live task or API response retained disabled rotation")
}
if _, err := m.ReplaceTaskLLMProfiles(task.ID, ids, 0, &on); err != nil {
t.Fatal(err)
}
pt, err := m.pg.GetTask(n)
if err != nil {
t.Fatal(err)
}
reloaded := taskFromPG(pt, task.Store, m.interceptor)
if !reloaded.llmStateSnapshot().Rotate {
t.Fatal("task reload lost enabled rotation")
}
}
func TestRotationConcurrentDispatchAndUpdates(t *testing.T) {
task := &Task{LLMProfileIDs: []int64{1, 2}, LLMRotate: true}
const workers = 100
counts := make(chan int64, workers)
var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
id, ok := task.nextRotatedProfile()
if !ok {
t.Error("missing assignment")
return
}
counts <- id
}()
}
wg.Wait()
close(counts)
got := map[int64]int{}
for id := range counts {
got[id]++
}
if got[1] != workers/2 || got[2] != workers/2 {
t.Fatalf("unbalanced assignments: %v", got)
}
// Replace the chain while dispatching, exercising the shared list lock.
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < workers; i++ {
task.setLLMState(nil, nil, []int64{3, 4}, int64(i+1), "ready", "", i%2 == 0)
}
}()
for i := 0; i < workers; i++ {
task.nextRotatedProfile()
task.llmStateSnapshot()
}
wg.Wait()
}
func TestNextRotatedProfileRoundRobin(t *testing.T) {
tk := &Task{LLMProfileIDs: []int64{3, 1, 2}}
tk := &Task{LLMProfileIDs: []int64{3, 1, 2}, LLMRotate: true}
want := []int64{3, 1, 2, 3, 1}
for i, w := range want {
got, ok := tk.nextRotatedProfile()
@@ -29,4 +206,8 @@ func TestNextRotatedProfileRoundRobin(t *testing.T) {
if _, ok := (&Task{}).nextRotatedProfile(); ok {
t.Fatal("empty chain must not rotate")
}
tk.setLLMState(nil, nil, []int64{3, 1, 2}, 1, "ready", "", false)
if _, ok := tk.nextRotatedProfile(); ok {
t.Fatal("disabled rotation must not assign a pin")
}
}
@@ -76,6 +76,7 @@ function taskProfileIDs(task: Task): string[] {
function TaskLLMControl({ task, profiles, onUpdated }: { task: Task; profiles: LLMProfile[]; onUpdated: () => void }) {
const [open, setOpen] = React.useState(false);
const [profileIDs, setProfileIDs] = React.useState<string[]>(() => taskProfileIDs(task));
const [llmRotate, setLLMRotate] = React.useState(task.llm_rotate ?? false);
const [activeProfileID, setActiveProfileID] = React.useState(
task.active_llm_profile_id ? String(task.active_llm_profile_id) : (taskProfileIDs(task)[0] ?? ""),
);
@@ -96,13 +97,15 @@ function TaskLLMControl({ task, profiles, onUpdated }: { task: Task; profiles: L
const activeProfile = profiles.find((profile) => profile.id === activeID);
const currentLabel = exhausted
? "Configuration chain exhausted"
: (activeProfile?.name ?? (activeID ? `Config #${activeID}` : "Follow default config"));
: task.llm_rotate
? `Worker rotation (${chain.length})`
: (activeProfile?.name ?? (activeID ? `Config #${activeID}` : "Follow default config"));
const activeIndex = chain.indexOf(activeID);
const backupCount = activeIndex >= 0 ? Math.max(0, chain.length - activeIndex - 1) : 0;
const currentTitle = [currentLabel, activeProfile?.model, backupCount > 0 ? `${backupCount} backups` : ""]
.filter(Boolean)
.join(" · ");
let editorDescription = "Changes to the order or the current config take effect from the next LLM call.";
let editorDescription = "Rotation changes apply to new worker intents. Profile changes apply to subsequent LLM calls; in-flight calls keep their provider. Saving resets exhausted profiles.";
if (terminal) editorDescription = "The task has ended; changes only affect subsequent main agent conversations.";
let saveLabel = "Save";
if (exhausted) saveLabel = "Save and reset";
@@ -110,6 +113,7 @@ function TaskLLMControl({ task, profiles, onUpdated }: { task: Task; profiles: L
const syncDraft = React.useCallback(() => {
const next = taskProfileIDs(task);
setLLMRotate(task.llm_rotate ?? false);
setProfileIDs(next);
setActiveProfileID(task.active_llm_profile_id ? String(task.active_llm_profile_id) : (next[0] ?? ""));
}, [task]);
@@ -131,6 +135,7 @@ function TaskLLMControl({ task, profiles, onUpdated }: { task: Task; profiles: L
task.id,
profileIDs.map(Number),
activeProfileID ? Number(activeProfileID) : undefined,
llmRotate,
);
if (result.switch_event) {
toast.info(result.switch_event.summary, { id: `task-${task.id}-llm-${result.switch_event.seq}` });
@@ -161,12 +166,12 @@ function TaskLLMControl({ task, profiles, onUpdated }: { task: Task; profiles: L
>
<BrainIcon data-icon="inline-start" />
<span className="hidden max-w-36 truncate lg:inline">{currentLabel}</span>
{backupCount > 0 && <span className="hidden text-muted-foreground xl:inline">+{backupCount}</span>}
{!task.llm_rotate && backupCount > 0 && <span className="hidden text-muted-foreground xl:inline">+{backupCount}</span>}
</Button>
</PopoverTrigger>
<PopoverContent ref={popoverContentRef} align="start" className="w-[min(28rem,calc(100vw-2rem))] gap-4 p-4">
<PopoverHeader>
<PopoverTitle>Task LLM config chain</PopoverTitle>
<PopoverTitle>Task LLM config chain{task.llm_rotate ? " · Worker rotation on" : ""}</PopoverTitle>
<PopoverDescription>{editorDescription}</PopoverDescription>
</PopoverHeader>
@@ -183,6 +188,8 @@ function TaskLLMControl({ task, profiles, onUpdated }: { task: Task; profiles: L
<TaskLLMProfileChain
profiles={profiles}
value={profileIDs}
rotate={llmRotate}
onRotateChange={setLLMRotate}
onValueChange={handleProfileIDsChange}
activeProfileId={activeProfileID}
onActiveProfileChange={setActiveProfileID}
+6 -1
View File
@@ -3085,6 +3085,7 @@ function CreateTaskSheet({
const [sourceTaskIDs, setSourceTaskIDs] = React.useState<string[]>([]);
const [companyIDs, setCompanyIDs] = React.useState<number[]>([]);
const [llmProfileIDs, setLLMProfileIDs] = React.useState<string[]>([]);
const [llmRotate, setLLMRotate] = React.useState(false);
const [creating, setCreating] = React.useState(false);
const [timeoutMin, setTimeoutMin] = React.useState(""); // task-level timeout (minutes); empty/0 = no limit
const [heartbeatMin, setHeartbeatMin] = React.useState("10"); // planner heartbeat (minutes); default 10, minimum 10 (matching the backend)
@@ -3152,6 +3153,7 @@ function CreateTaskSheet({
description: description.trim(),
goal: goal.trim(),
llmProfileIds: llmProfileIDs.map(Number),
llmRotate,
sourceTaskIds: sourceTaskIDs,
companyIds: companyIDs,
timeoutSeconds: timeoutSec,
@@ -3174,6 +3176,7 @@ function CreateTaskSheet({
setSourceTaskIDs([]);
setCompanyIDs([]);
setLLMProfileIDs([]);
setLLMRotate(false);
setTimeoutMin("");
setHeartbeatMin("10");
setSeedFirstIntent(false);
@@ -3330,11 +3333,13 @@ function CreateTaskSheet({
<TaskLLMProfileChain
profiles={profiles}
value={llmProfileIDs}
rotate={llmRotate}
onRotateChange={setLLMRotate}
onValueChange={setLLMProfileIDs}
inputId="llm-profiles"
portalContainer={sheetContentRef}
/>
<FieldDescription>Failover follows the list order; the first entry is the current config, and the next one is used only when the quota is clearly insufficient.</FieldDescription>
<FieldDescription>Reorder profiles to choose worker rotation order or failover priority. Saving the chain resets exhausted profiles.</FieldDescription>
</Field>
<Field>
<FieldLabel htmlFor="task-credentials">Test accounts (optional)</FieldLabel>
+57 -3
View File
@@ -18,9 +18,13 @@ import {
ComboboxValue,
} from "@/components/ui/combobox";
import { Select, SelectContent, SelectGroup, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { Switch } from "@/components/ui/switch";
import { api } from "@/lib/api";
import type { LLMProfile } from "@/lib/types";
interface TaskLLMProfileChainProps {
rotate?: boolean;
onRotateChange?: (rotate: boolean) => void;
profiles: LLMProfile[];
value: string[];
onValueChange: (value: string[]) => void;
@@ -44,6 +48,8 @@ function ProfileRoleBadge({ index, currentIndex }: { index: number; currentIndex
}
export function TaskLLMProfileChain({
rotate = false,
onRotateChange,
profiles,
value,
onValueChange,
@@ -53,6 +59,15 @@ export function TaskLLMProfileChain({
inputId,
portalContainer,
}: TaskLLMProfileChainProps) {
const rotationID = React.useId();
const [workerBinding, setWorkerBinding] = React.useState<number | null>(null);
const [bindingCheckFailed, setBindingCheckFailed] = React.useState(false);
React.useEffect(() => {
api
.agents()
.then((agents) => setWorkerBinding(agents.find((a) => a.key === "worker")?.llm_profile_id ?? null))
.catch(() => setBindingCheckFailed(true));
}, []);
const profilesByID = React.useMemo(() => new Map(profiles.map((profile) => [profile.id, profile])), [profiles]);
const itemIDs = React.useMemo(() => profiles.map((profile) => profile.id), [profiles]);
const profilesUnavailable = profiles.length === 0;
@@ -92,6 +107,39 @@ export function TaskLLMProfileChain({
return (
<div className="flex flex-col gap-3">
{onRotateChange && (
<div className="flex flex-col gap-2">
<label htmlFor={rotationID} className="flex items-center justify-between gap-3 font-medium text-sm">
Enable multi-LLM worker rotation
<Switch id={rotationID} checked={rotate} onCheckedChange={onRotateChange} disabled={disabled} />
</label>
<p className="text-muted-foreground text-xs">
{rotate
? "New worker intents use the selected profiles in round-robin order. Independent workers can run on different providers simultaneously. Other roles keep the current profile."
: "Ordered failover: workers share the current profile and switch after quota exhaustion or repeated provider failures."}
</p>
{rotate && value.length < 2 && (
<p className="text-amber-600 text-xs">Select at least two profiles to distribute work across providers.</p>
)}
{rotate && workerBinding != null && (
<p role="alert" className="text-amber-600 text-xs">
Worker has a fixed model binding, which overrides rotation. Set Worker’s model to follow the task in Agent
settings to use these profiles.
</p>
)}
{rotate && bindingCheckFailed && (
<p className="text-amber-600 text-xs">
Worker binding could not be checked. A fixed Worker model overrides rotation; verify Agent settings.
</p>
)}
{rotate && (
<p className="text-muted-foreground text-xs">
Set concurrent worker count to at least 2 in System settings before starting the task. In-flight calls
keep their provider. Exhausted profiles are skipped until the chain is reset.
</p>
)}
</div>
)}
<Combobox
items={itemIDs}
itemToStringValue={profileLabel}
@@ -136,7 +184,9 @@ export function TaskLLMProfileChain({
</Combobox>
{value.length === 0 ? (
<p className="text-muted-foreground text-xs">When no profile is set, the task follows the agent or the global active profile.</p>
<p className="text-muted-foreground text-xs">
When no profile is set, the task follows the agent or the global active profile.
</p>
) : (
<div className="flex flex-col divide-y rounded-lg border">
{value.map((id, index) => {
@@ -153,7 +203,11 @@ export function TaskLLMProfileChain({
<p className="truncate text-muted-foreground text-xs">{profile?.model ?? "Profile unavailable"}</p>
</div>
</div>
<ProfileRoleBadge index={index} currentIndex={currentIndex} />
{rotate ? (
<Badge variant="secondary">Rotation {index + 1}</Badge>
) : (
<ProfileRoleBadge index={index} currentIndex={currentIndex} />
)}
<div className="col-span-2 col-start-2 flex shrink-0 items-center gap-1 justify-self-end sm:col-auto sm:justify-self-auto">
<Button
type="button"
@@ -194,7 +248,7 @@ export function TaskLLMProfileChain({
{onActiveProfileChange && value.length > 0 && (
<div className="flex flex-col items-stretch gap-2 sm:flex-row sm:items-center sm:justify-between sm:gap-3">
<span className="font-medium text-sm">Current profile</span>
<span className="font-medium text-sm">{rotate ? "Planner / failover profile" : "Current profile"}</span>
<Select
value={activeProfileId && value.includes(activeProfileId) ? activeProfileId : value[0]}
onValueChange={onActiveProfileChange}
+5 -1
View File
@@ -258,6 +258,7 @@ export const api = {
description: string;
goal: string;
llmProfileIds?: number[];
llmRotate?: boolean;
sourceTaskIds?: string[];
companyIds?: number[];
timeoutSeconds?: number;
@@ -273,6 +274,7 @@ export const api = {
description: input.description,
goal: input.goal,
llm_profile_ids: input.llmProfileIds ?? [],
llm_rotate: input.llmRotate ?? false,
source_task_ids: input.sourceTaskIds ?? [],
company_ids: input.companyIds ?? [],
timeout_seconds: input.timeoutSeconds ?? 0,
@@ -310,10 +312,11 @@ export const api = {
input: Partial<Pick<TaskTemplate, "name" | "description" | "goal" | "category_id" | "intercept_rules">>,
) => patch<TaskTemplate>(`/task-templates/${id}`, input),
deleteTaskTemplate: (id: number) => del<{ deleted: number }>(`/task-templates/${id}`),
updateTaskLLMProfiles: (id: string, llmProfileIds: number[], activeLLMProfileId?: number) =>
updateTaskLLMProfiles: (id: string, llmProfileIds: number[], activeLLMProfileId?: number, llmRotate?: boolean) =>
put<{
id: string;
llm_profile_ids: number[];
llm_rotate: boolean;
active_llm_profile_id?: number;
llm_failover_state: string;
reopened_intents: number;
@@ -321,6 +324,7 @@ export const api = {
}>(`/tasks/${id}/llm`, {
llm_profile_ids: llmProfileIds,
active_llm_profile_id: activeLLMProfileId ?? null,
llm_rotate: llmRotate,
}),
deleteTask: (id: string, options: DeleteTaskOptions) => del<DeleteTaskResult>(`/tasks/${id}`, options),
controlTask: (id: string, action: "pause" | "resume") =>
+2
View File
@@ -1200,6 +1200,7 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re
tokens: { input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_write_tokens: 0 },
llm_profile_id: profileIDs[0],
llm_profile_ids: profileIDs,
llm_rotate: b.llm_rotate === true,
active_llm_profile_id: profileIDs[0],
llm_failover_state: profileIDs.length ? "ready" : "default",
source_task_ids: sourceTaskIDs,
@@ -1422,6 +1423,7 @@ function route(m: string, path: string, seg: string[], q: URLSearchParams, b: Re
const target = mockTasks.find((item) => item.id === seg[1]);
if (target) {
target.llm_profile_ids = ids;
if (typeof b.llm_rotate === "boolean") target.llm_rotate = b.llm_rotate;
target.llm_profile_id = ids[0];
target.active_llm_profile_id = activeID;
target.llm_failover_state = ids.length ? "ready" : "default";
+1
View File
@@ -32,6 +32,7 @@ export interface Task {
tokens?: TokenTotal; // whole-task token consumption
llm_profile_id?: number; // LLM profile used; absent = default profile
llm_profile_ids?: number[]; // ordered task-level failover chain
llm_rotate?: boolean;
active_llm_profile_id?: number; // profile used by the next LLM call
llm_failover_state?: "default" | "ready" | "chain_exhausted" | string;
llm_failover_reason?: string;