mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-18 22:20:56 +08:00
Merge pull request #302 from Protonmatter/protonmatter/budget-restore
Restore task spend from the lanes the live ledger counts
This commit is contained in:
@@ -28,6 +28,7 @@ All notable changes to this project are documented in this file. The format is b
|
||||
|
||||
### Changed
|
||||
|
||||
- **Task budgets survive a restart**: planner, worker, judge, and compactor rows in `llm_usage` are counted back into `ARTEX_TASK_BUDGET_*`. A restored task already inside the wrap-up reserve only has that reserve left, so the run wraps up instead of starting another worker. The per-worker estimate is not restored and starts from the defaults (50,000 tokens and 10 turns) until a worker finishes in this process. Chat, goal decomposition, and side questions stay outside the cap.
|
||||
- **Untrusted-output boundary is now on by default**: Norma frames tool results for the worker, planner, chat and main agent unless `ARTEX_UNTRUSTED_BOUNDARY` is set to `0`, `false`, `off` or `no`. It was opt-in via `ARTEX_UNTRUSTED_BOUNDARY=1` in 0.6.0 (#166); the default flips here (#203).
|
||||
- **Versioned migrations**: the `db` package applies numbered `db/migrations/NNNN_slug.sql` files once, in order, under the schema advisory lock, and records a SHA-256 checksum in `schema_migrations`; missing, renamed, changed or out-of-order migrations fail closed (#226).
|
||||
- **Coverage graph performance**: the coverage graph batches its connector lookups (three queries instead of one per root and per company) and adds the partial index `idx_ts_icp`; on a task of about 3,000 assets the p50 dropped from 126 ms to 27 ms and the queries per call from 221 to 3 (#230).
|
||||
|
||||
@@ -255,7 +255,10 @@ worker remains confined to the intent it claimed.
|
||||
- **Budgets**: `ARTEX_TASK_BUDGET_TOKENS` / `ARTEX_TASK_BUDGET_TURNS` cap a whole task (planner and
|
||||
workers); `ARTEX_INTENT_BUDGET_TOKENS` / `ARTEX_INTENT_BUDGET_TURNS` cap one worker run. Near the
|
||||
limit the task enters a forced wrap-up; a worker is not started when the remaining budget cannot
|
||||
cover it. Budgets are in memory: a restart starts fresh.
|
||||
cover it. Spend already in `llm_usage` is restored after a restart. A task that is already inside
|
||||
the wrap-up reserve only has that reserve left, so the revived run wraps up instead of starting
|
||||
another worker. The per-worker cost estimate starts again from the defaults (50,000 tokens and
|
||||
10 turns) until a worker finishes in this process.
|
||||
- **Noise governor**: `ARTEX_NOISE_PROFILE=quiet|balanced|aggressive` paces requests ARTEX sends to
|
||||
targets (per-host rate, burst, in-flight limit, jitter, backoff on 429/503, short GET caching).
|
||||
With a profile set, agents also receive low-and-slow flags for the installed scanners.
|
||||
@@ -455,7 +458,13 @@ workers, else 50,000 tokens and 10 turns). Near the wrap-up reserve (the larger
|
||||
tokens, and 2 turns) the task enters a forced wrap-up: the planner writes a summary and final goal
|
||||
judgment, and workers may only write back. The task then ends `done` if its goals are met, else
|
||||
`timeout`. Practical floor: below 50,000 tokens no worker starts; 50,000-70,000 typically gives one
|
||||
worker then wrap-up; pick well above 70,000 for real exploration.
|
||||
worker then wrap-up; pick well above 70,000 for real exploration. Planner, worker, judge, and
|
||||
compactor rows in `llm_usage` are counted back into the cap when a ledger is created, including
|
||||
after a restart and after a revived run drops its ledger. A restored task already inside the
|
||||
wrap-up reserve does not get another worker: only that reserve remains, and the run wraps up.
|
||||
Chat, goal decomposition, and side questions are stored on the same task and are not counted.
|
||||
The per-worker average is not restored (usage rows have no run boundary), so the estimate starts
|
||||
from the defaults until a worker finishes in this process.
|
||||
|
||||
**Common flags**: `./start.sh -addr :8787 -proxy :8788` (`-addr` frontend/API, `-proxy` traffic
|
||||
recording proxy).
|
||||
|
||||
+17
-1
@@ -92,7 +92,8 @@ func BudgetsFromEnv() (task, intent Budget) {
|
||||
NewBudget(env("ARTEX_INTENT_BUDGET_TOKENS"), env("ARTEX_INTENT_BUDGET_TURNS"))
|
||||
}
|
||||
|
||||
// BudgetLedger meters one task, in memory (a process restart starts afresh).
|
||||
// BudgetLedger meters one task. The total is restored from llm_usage when the
|
||||
// ledger is created, including after a restart. Per-worker averages are not.
|
||||
type BudgetLedger struct {
|
||||
task, intent Budget
|
||||
mu sync.Mutex
|
||||
@@ -110,6 +111,21 @@ func NewBudgetLedger(task, intent Budget) *BudgetLedger {
|
||||
return &BudgetLedger{task: task, intent: intent}
|
||||
}
|
||||
|
||||
// RestoreSpend adds usage that was already recorded before this ledger existed.
|
||||
// It only adds to the task total. Per-worker averages stay at the defaults
|
||||
// until a worker finishes against this ledger, because llm_usage rows do not
|
||||
// mark where one worker run ends and the next begins. A nil ledger (no budget
|
||||
// configured) is a no-op.
|
||||
func (l *BudgetLedger) RestoreSpend(spent Spend) {
|
||||
if l == nil || (spent.Tokens == 0 && spent.Turns == 0) {
|
||||
return
|
||||
}
|
||||
l.mu.Lock()
|
||||
l.spent.Tokens += spent.Tokens
|
||||
l.spent.Turns += spent.Turns
|
||||
l.mu.Unlock()
|
||||
}
|
||||
|
||||
// Record meters one task-level call (the planner's own rounds).
|
||||
func (l *BudgetLedger) Record(u llm.Usage) {
|
||||
l.mu.Lock()
|
||||
|
||||
@@ -144,3 +144,14 @@ func TestBudgetCountsMatchUsageFixture(t *testing.T) {
|
||||
t.Fatalf("spent %+v remaining %+v estimate %+v, want {%d 5}, remaining, {10400 3}", st.Spent, st.Remaining, st.Estimate, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreSpendCountsTowardCap(t *testing.T) {
|
||||
l := NewBudgetLedger(NewBudget(100, 10), NewBudget(0, 0))
|
||||
l.RestoreSpend(Spend{Tokens: 40, Turns: 4})
|
||||
st := l.State()
|
||||
if st.Spent != (Spend{40, 4}) || st.Remaining != (Budget{60, 6}) {
|
||||
t.Fatalf("spent=%+v remaining=%+v", st.Spent, st.Remaining)
|
||||
}
|
||||
var none *BudgetLedger
|
||||
none.RestoreSpend(Spend{Tokens: 1, Turns: 1})
|
||||
}
|
||||
|
||||
@@ -33,6 +33,24 @@ func (d *DB) EnsureLLMUsageTable() error {
|
||||
return d.execEmbeddedMigration(llmUsageMigration)
|
||||
}
|
||||
|
||||
// TaskLLMSpend is the token and turn total the in-memory task ledger would
|
||||
// already have counted. One row is one turn. The ledger sees planner and worker
|
||||
// completions, plus judge and compactor calls made on those contexts. Main-agent
|
||||
// chat, goal decomposition, and side questions write llm_usage rows for the same
|
||||
// task and never enter the ledger, so they are left out: counting them would
|
||||
// tighten the turn cap after a restart. Cache read and write are not added:
|
||||
// input_tokens already includes them. Every status counts, because a failed call
|
||||
// is metered too.
|
||||
func (d *DB) TaskLLMSpend(taskID string) (tokens, turns int64, err error) {
|
||||
if d == nil || taskID == "" {
|
||||
return 0, 0, nil
|
||||
}
|
||||
err = d.QueryRow(`SELECT COALESCE(SUM(input_tokens + output_tokens), 0), COUNT(*)
|
||||
FROM llm_usage
|
||||
WHERE task_id=$1 AND worker IN ('planner', 'worker', 'judge', 'compactor')`, taskID).Scan(&tokens, &turns)
|
||||
return tokens, turns, err
|
||||
}
|
||||
|
||||
// InsertLLMUsage appends one metering row. Best-effort: callers log and continue on
|
||||
// error (a lost metering row must never break the LLM call).
|
||||
func (d *DB) InsertLLMUsage(u *LLMUsage) error {
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Judge calls share the planner or worker sink and count toward the task cap.
|
||||
// Chat rows for the same task do not, or a restart would tighten the cap.
|
||||
func TestTaskLLMSpendCountsJudgeNotChat(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v)", err)
|
||||
}
|
||||
defer d.Close()
|
||||
|
||||
taskID := fmt.Sprintf("spend-%d", time.Now().UnixNano())
|
||||
t.Cleanup(func() {
|
||||
_, _ = d.Exec(`DELETE FROM llm_usage WHERE task_id=$1`, taskID)
|
||||
})
|
||||
insert := func(worker string, in, out int) {
|
||||
t.Helper()
|
||||
if err := d.InsertLLMUsage(&LLMUsage{
|
||||
TaskID: taskID, Worker: worker, InputTokens: in, OutputTokens: out, Status: "ok",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
insert("judge", 30, 10)
|
||||
insert("chat", 1000, 1000)
|
||||
|
||||
tokens, turns, err := d.TaskLLMSpend(taskID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tokens != 40 || turns != 1 {
|
||||
t.Fatalf("tokens=%d turns=%d, want the judge row only (40 tokens, 1 turn)", tokens, turns)
|
||||
}
|
||||
}
|
||||
+30
-2
@@ -17,8 +17,36 @@ func (e *Engine) budgetFor(taskID string) *agent.BudgetLedger {
|
||||
if v, ok := e.budgets.Load(taskID); ok {
|
||||
return v.(*agent.BudgetLedger)
|
||||
}
|
||||
v, _ := e.budgets.LoadOrStore(taskID, agent.NewBudgetLedger(agent.BudgetsFromEnv()))
|
||||
return v.(*agent.BudgetLedger)
|
||||
created := agent.NewBudgetLedger(agent.BudgetsFromEnv())
|
||||
e.seedBudget(taskID, created)
|
||||
v, loaded := e.budgets.LoadOrStore(taskID, created)
|
||||
if loaded {
|
||||
return v.(*agent.BudgetLedger)
|
||||
}
|
||||
return created
|
||||
}
|
||||
|
||||
// seedBudget rebuilds spent tokens and turns from llm_usage for every new
|
||||
// ledger. budgetFor calls it on first use, and again after resetSettledRun or
|
||||
// StopTask deletes the ledger in this same process. The in-memory sink runs
|
||||
// before the row is inserted, so a call that happens after this seed is
|
||||
// counted once. A nil ledger means no budget is configured and nothing is read.
|
||||
// The restore does not rebuild per-worker averages, so the next worker estimate
|
||||
// starts from the defaults until a worker finishes against this ledger.
|
||||
func (e *Engine) seedBudget(taskID string, ledger *agent.BudgetLedger) {
|
||||
if ledger == nil || e == nil || e.m == nil || e.m.pg == nil || taskID == "" {
|
||||
return
|
||||
}
|
||||
tokens, turns, err := e.m.pg.TaskLLMSpend(taskID)
|
||||
if err != nil {
|
||||
log.Printf("[budget] task %s: cannot restore spend from llm_usage: %v", taskID, err)
|
||||
return
|
||||
}
|
||||
if tokens == 0 && turns == 0 {
|
||||
return
|
||||
}
|
||||
ledger.RestoreSpend(agent.Spend{Tokens: tokens, Turns: turns})
|
||||
log.Printf("[budget] task %s restored %d tokens / %d turns from llm_usage", taskID, tokens, turns)
|
||||
}
|
||||
|
||||
// budgetStop is the enforcement point of a planner round: it logs the remaining
|
||||
|
||||
@@ -2,9 +2,12 @@ package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
pgdb "github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
)
|
||||
|
||||
@@ -67,3 +70,64 @@ func TestResetSettledRunRestartsBudget(t *testing.T) {
|
||||
t.Fatalf("revived run: ledger not fresh, spent %d tokens", got)
|
||||
}
|
||||
}
|
||||
|
||||
// With Postgres, resetSettledRun drops the ledger and the next one is seeded
|
||||
// again from llm_usage. In-memory spend that was never written does not come
|
||||
// back. A restore that lands on the wrap-up reserve leaves only that reserve,
|
||||
// and the worker estimate starts from the defaults.
|
||||
func TestResetSettledRunReseedsBudgetFromUsage(t *testing.T) {
|
||||
for _, k := range []string{"ARTEX_TASK_BUDGET_TURNS", "ARTEX_INTENT_BUDGET_TOKENS", "ARTEX_INTENT_BUDGET_TURNS"} {
|
||||
t.Setenv(k, "")
|
||||
}
|
||||
t.Setenv("ARTEX_TASK_BUDGET_TOKENS", "100000")
|
||||
m, err := NewManager(t.TempDir(), "")
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v)", err)
|
||||
}
|
||||
defer m.Close()
|
||||
|
||||
taskID := strconv.FormatInt(time.Now().UnixNano(), 10)
|
||||
t.Cleanup(func() {
|
||||
_, _ = m.pg.Exec(`DELETE FROM llm_usage WHERE task_id=$1`, taskID)
|
||||
})
|
||||
// 100000 - wrap-up reserve (max(5%, 20000)) = 80000. One planner row.
|
||||
if err := m.pg.InsertLLMUsage(&pgdb.LLMUsage{
|
||||
TaskID: taskID, Worker: "planner", InputTokens: 80_000, Status: "ok",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := NewEngine(m)
|
||||
task := &Task{ID: taskID}
|
||||
l := e.budgetFor(task.ID)
|
||||
st := l.State()
|
||||
if st.Spent.Tokens != 80_000 || st.Spent.Turns != 1 {
|
||||
t.Fatalf("seed spent=%+v, want 80000 tokens and 1 turn", st.Spent)
|
||||
}
|
||||
if st.Remaining.Tokens != st.Margin.Tokens || !st.AtMargin || st.Dispatch {
|
||||
t.Fatalf("restored task near the cap: remaining=%d margin=%d at=%v dispatch=%v",
|
||||
st.Remaining.Tokens, st.Margin.Tokens, st.AtMargin, st.Dispatch)
|
||||
}
|
||||
if st.Estimate.Tokens != agent.DefaultWorkerEstimateTokens || st.Estimate.Turns != agent.DefaultWorkerEstimateTurns {
|
||||
t.Fatalf("estimate=%+v, want the defaults (worker history is not restored)", st.Estimate)
|
||||
}
|
||||
|
||||
l.Record(llm.Usage{InputTokens: 1000})
|
||||
l.BeginWrapup()
|
||||
e.settling.Store(task.ID, true)
|
||||
e.resetSettledRun(task.ID)
|
||||
if e.isSettling(task.ID) {
|
||||
t.Fatal("revived run is still marked settling")
|
||||
}
|
||||
again := e.budgetFor(task.ID)
|
||||
if again == l {
|
||||
t.Fatal("revive reused the deleted ledger")
|
||||
}
|
||||
got := again.State()
|
||||
if got.Spent.Tokens != 80_000 || got.Spent.Turns != 1 {
|
||||
t.Fatalf("reseed spent=%+v, want the usage row and not the in-memory extra", got.Spent)
|
||||
}
|
||||
if got.Remaining.Tokens != got.Margin.Tokens || got.Dispatch {
|
||||
t.Fatalf("reseed still past the wrap-up margin: remaining=%d margin=%d dispatch=%v",
|
||||
got.Remaining.Tokens, got.Margin.Tokens, got.Dispatch)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,7 +110,8 @@ func (e *Engine) resetTimeoutRevival(taskID string) {
|
||||
}
|
||||
|
||||
// resetSettledRun clears the settle state of a finished run (settling and the
|
||||
// budget ledger) so a revived task can plan and claim again with a fresh budget.
|
||||
// budget ledger) so a revived task can plan and claim again. The next ledger
|
||||
// is seeded from llm_usage, so the spend already recorded still counts.
|
||||
// It also clears the persisted end_reason of the previous run (ApplyTaskAdmission
|
||||
// already did in the same UPDATE as the revive; this is idempotent and never
|
||||
// touches a task that is terminal again).
|
||||
|
||||
Reference in New Issue
Block a user