Compare commits

...
Author SHA1 Message Date
BaaRai 5463ea4375 ci: build a dedicated database for the online index scripts
The job-level test database has no schema and cached go test results do not guarantee any suffixed test database exists; create one from db/schema.sql and db/migrations, then apply and roll back the scripts.
2026-11-15 23:26:24 +01:00
BaaRai 5a8498bb4f fix(db): online indexes for tool-result lookup and per-task token aggregation
Adds scripts/sql/20261115-task-query-indexes.sql (CREATE INDEX CONCURRENTLY on activity(tool_use_id) WHERE kind = tool_result and on llm_usage((COALESCE(task_id, "")), model) with covering columns, then VACUUM (ANALYZE) llm_usage), its rollback and a README. CI applies and rolls back both scripts. Backported from #209 (contributed by @renaesop).
2026-11-15 23:08:24 +01:00
BaaRai 2ad8ba16c3 Merge pull request #247 from BaaRai/fix/noa-budget-no-retry
fix(server): do not rerun intents refused by the NOA budget
2026-11-15 22:54:09 +01:00
BaaRai 09c9e6cfc2 fix(server): do not rerun intents refused by the NOA budget
The NOA budget refusal now wraps an exported ErrNoaBudget sentinel, which survives the Norma harness unchanged; retryableWorkerModelError excludes it, so a deterministic refusal is no longer replayed twice by the engine.
2026-11-15 22:26:21 +01:00
BaaRai 59c3002bcc Merge pull request #246 from BaaRai/chore/norma-v0.4.10
chore(deps): bump Norma to v0.4.10
2026-11-15 22:15:15 +01:00
BaaRai 68f2db1d0b chore(deps): bump Norma to v0.4.10
Brings the English shell tool, task notice and NOA archive strings and the corrected dispatch session note (BaaRai/Norma v0.4.10). No other module moves.
2026-11-15 20:39:53 +01:00
BaaRai f0be3ae51f Merge pull request #245 from BaaRai/feat/noa-budget-wiring
feat(agent): size NOA to the configured window and refuse irreducible requests
2026-11-15 20:06:59 +01:00
BaaRai 3b755228d6 feat(agent): size NOA to the configured window and refuse irreducible requests
Bump Norma to v0.4.9 (BaaRai/Norma v0.4.9). enableNoa passes the configured context window to noaadapter as ModelContextLimit and, once NOA is enabled, wraps the provider in a budget check that refuses a request whose estimate exceeds that window before any transport.
2026-11-15 19:08:49 +01:00
BaaRai d147dce344 Merge pull request #244 from BaaRai/fix/cache-write-cost
fix(db): stop billing cache writes twice in profile cost
2026-11-15 18:37:12 +01:00
11 changed files with 340 additions and 9 deletions
+13
View File
@@ -48,6 +48,19 @@ jobs:
# as rotating failures in the agent package). Serial package execution
# keeps the database state deterministic.
run: go test -p 1 ./...
- name: online index scripts
# Apply and roll back scripts/sql on a database built here from
# db/schema.sql and db/migrations. The job-level artextest database
# never gets the schema (packages use suffixed databases), and cached
# test results mean no test database is guaranteed to exist.
env:
SQL_DSN: postgres://artex:testpass@localhost:5432/artextest_sqlscripts?sslmode=disable
run: |
psql "$ARTEX_PG_DSN" -X -v ON_ERROR_STOP=1 -c 'CREATE DATABASE artextest_sqlscripts'
psql "$SQL_DSN" -X -q -v ON_ERROR_STOP=1 -f db/schema.sql
for f in db/migrations/*.sql; do psql "$SQL_DSN" -X -q -v ON_ERROR_STOP=1 -f "$f"; done
psql "$SQL_DSN" -X -v ON_ERROR_STOP=1 -f scripts/sql/20261115-task-query-indexes.sql
psql "$SQL_DSN" -X -v ON_ERROR_STOP=1 -f scripts/sql/20261115-task-query-indexes.rollback.sql
- name: go test -race
# Race detector on the packages that pass under -race today. ./server
# joined once its suite was checked with -race against postgres and its
+17 -4
View File
@@ -39,10 +39,17 @@ func enableNoa(opts *agentcore.Options, enabled func() bool, archiveRoot, sessio
if opts.OnWarn == nil {
opts.OnWarn = onWarn
}
if err := noaadapter.Enable(opts, noaadapter.Options{
ArchiveBaseDir: filepath.Join(archiveRoot, "noa"),
SessionID: sessionID,
OnWarn: onWarn,
// noa sizes its thresholds to the window this agent is configured for, read
// before Compaction is cleared below; without it noa assumes its own default.
window := defaultWindowK * 1000
if opts.Compaction != nil && opts.Compaction.ContextWindow > 0 {
window = opts.Compaction.ContextWindow
}
if err := noaEnable(opts, noaadapter.Options{
ArchiveBaseDir: filepath.Join(archiveRoot, "noa"),
SessionID: sessionID,
OnWarn: onWarn,
ModelContextLimit: window,
}); err != nil {
if onWarn != nil {
onWarn("failed to enable noa compression, falling back to built-in compaction: " + err.Error())
@@ -52,4 +59,10 @@ func enableNoa(opts *agentcore.Options, enabled func() bool, archiveRoot, sessio
// Compactor overrides Compaction, but agentcore warns on every run when both are
// set; clear it explicitly.
opts.Compaction = nil
// noa can only trim what it may drop; a request whose protected part alone
// exceeds the window is refused here instead of by the provider.
opts.Provider = &noaBudgetProvider{Provider: opts.Provider, window: window}
}
// noaEnable is noaadapter.Enable; tests replace it to observe the options.
var noaEnable = noaadapter.Enable
+50
View File
@@ -0,0 +1,50 @@
package agent
import (
"context"
"errors"
"fmt"
"iter"
"github.com/Autumn-27/artex/sidequestion"
"github.com/Autumn-27/norma/llm"
)
// noaRequestOverhead covers request framing the input estimate does not count.
const noaRequestOverhead = 512
// ErrNoaBudget marks a request refused by the NOA budget check. The refusal is
// deterministic (the same history fails the same way), so callers must not
// retry it.
var ErrNoaBudget = errors.New("noa request budget exceeded")
// noaBudgetProvider refuses, before any transport, a request that cannot fit the
// window noa was configured with. noa trims what it is allowed to drop; when
// the protected part (system prompt, recent turns, the output reservation)
// alone is over the window, sending it only buys a provider rejection.
type noaBudgetProvider struct {
llm.Provider
window int
}
func (p *noaBudgetProvider) check(req llm.CompletionRequest) error {
estimate := sidequestion.EstimateInputTokens(req) + max(req.MaxTokens, 0) + noaRequestOverhead
if estimate > p.window {
return fmt.Errorf("%w: estimate %d exceeds configured budget %d; protected context cannot fit, increase the configured window or start a new session", ErrNoaBudget, estimate, p.window)
}
return nil
}
func (p *noaBudgetProvider) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
if err := p.check(req); err != nil {
return llm.Message{}, "", llm.Usage{}, err
}
return p.Provider.Complete(ctx, req)
}
func (p *noaBudgetProvider) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
if err := p.check(req); err != nil {
return func(yield func(llm.StreamEvent, error) bool) { yield(llm.StreamEvent{}, err) }
}
return p.Provider.Stream(ctx, req)
}
+155
View File
@@ -0,0 +1,155 @@
package agent
import (
"context"
"errors"
"iter"
"strings"
"testing"
"github.com/Autumn-27/norma/agentcore"
"github.com/Autumn-27/norma/harness"
"github.com/Autumn-27/norma/llm"
"github.com/Autumn-27/norma/noaadapter"
)
// A worker whose protected context exceeds a tiny NOA window ends with
// model_error and an error that still wraps ErrNoaBudget after the Norma harness
// and the capture layer: the engine's retry check relies on that.
func TestWorkerNoaBudgetRefusalKeepsSentinelThroughHarness(t *testing.T) {
f := newSessionFixture(t)
inner := &countingProvider{}
w := NewWorker(inner, "test-model", t.TempDir(), nil, 2000, 3)
w.SetNoaEnabled(func() bool { return true })
reason, _, err := w.Execute(t.Context(), "work#1", f.expID, nil, f.ts, f.intent, passHooks{}, nil, nil, nil)
if reason != harness.ReasonModelError || !errors.Is(err, ErrNoaBudget) || inner.calls != 0 {
t.Fatalf("reason=%q err=%v provider calls=%d; want model_error wrapping ErrNoaBudget and no call", reason, err, inner.calls)
}
}
// countingProvider answers with one text turn and counts the calls it receives.
type countingProvider struct{ calls int }
func (p *countingProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
p.calls++
return func(yield func(llm.StreamEvent, error) bool) {
if yield(llm.StreamEvent{Type: llm.SETextDelta, Text: "ok"}, nil) {
yield(llm.StreamEvent{Type: llm.SEMessageStop}, nil)
}
}
}
func (p *countingProvider) Complete(context.Context, llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
p.calls++
return llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{llm.TextBlock("ok")}}, "end_turn", llm.Usage{}, nil
}
// withNoaEnable swaps the noaadapter.Enable seam for the duration of a test.
func withNoaEnable(t *testing.T, fn func(*agentcore.Options, noaadapter.Options) error) {
t.Helper()
prev := noaEnable
noaEnable = fn
t.Cleanup(func() { noaEnable = prev })
}
func TestEnableNoaUsesConfiguredWindow(t *testing.T) {
for _, tc := range []struct {
name string
compaction func() *agentcore.Options
want int
}{
{"configured", func() *agentcore.Options { return &agentcore.Options{Compaction: compactionConfig(32000)} }, 32000},
{"default", func() *agentcore.Options { return &agentcore.Options{} }, defaultWindowK * 1000},
} {
t.Run(tc.name, func(t *testing.T) {
var got noaadapter.Options
withNoaEnable(t, func(opts *agentcore.Options, o noaadapter.Options) error {
got = o
return noaadapter.Enable(opts, o)
})
inner := &countingProvider{}
opts := tc.compaction()
opts.Provider = inner
enableNoa(opts, func() bool { return true }, t.TempDir(), "noa-window", nil)
if got.ModelContextLimit != tc.want {
t.Fatalf("noaadapter.Options.ModelContextLimit=%d, want %d", got.ModelContextLimit, tc.want)
}
bp, ok := opts.Provider.(*noaBudgetProvider)
if !ok || bp.window != tc.want || bp.Provider != inner {
t.Fatalf("provider %T %+v, want *noaBudgetProvider{window %d} around the original", opts.Provider, opts.Provider, tc.want)
}
if opts.Compaction != nil || opts.Compactor == nil {
t.Fatalf("Compaction=%v Compactor=%v; noa must replace the built-in compaction", opts.Compaction, opts.Compactor)
}
})
}
}
func irreducibleRequest() llm.CompletionRequest {
return llm.CompletionRequest{System: []string{strings.Repeat("protected system prompt ", 8000)}, MaxTokens: 8000}
}
func TestNoaBudgetRefusesIrreducibleRequestBeforeTransport(t *testing.T) {
t.Run("atomic", func(t *testing.T) {
inner := &countingProvider{}
p := &noaBudgetProvider{Provider: inner, window: 32000}
_, _, _, err := p.Complete(t.Context(), irreducibleRequest())
if err == nil || !strings.Contains(err.Error(), "exceeds configured budget 32000") || inner.calls != 0 {
t.Fatalf("err=%v provider calls=%d", err, inner.calls)
}
if !errors.Is(err, ErrNoaBudget) {
t.Fatalf("atomic refusal does not wrap ErrNoaBudget: %v", err)
}
if _, _, _, err := p.Complete(t.Context(), llm.CompletionRequest{System: []string{"short"}, MaxTokens: 1000}); err != nil || inner.calls != 1 {
t.Fatalf("a request that fits was refused or not sent: err=%v calls=%d", err, inner.calls)
}
})
t.Run("stream", func(t *testing.T) {
inner := &countingProvider{}
p := &noaBudgetProvider{Provider: inner, window: 32000}
var events int
var err error
for ev, e := range p.Stream(t.Context(), irreducibleRequest()) {
if e != nil {
err = e
break
}
_ = ev
events++
}
if err == nil || !strings.Contains(err.Error(), "exceeds configured budget 32000") || events != 0 || inner.calls != 0 {
t.Fatalf("err=%v events=%d provider calls=%d", err, events, inner.calls)
}
if !errors.Is(err, ErrNoaBudget) {
t.Fatalf("stream refusal does not wrap ErrNoaBudget: %v", err)
}
for _, e := range p.Stream(t.Context(), llm.CompletionRequest{System: []string{"short"}, MaxTokens: 1000}) {
if e != nil {
t.Fatal(e)
}
}
if inner.calls != 1 {
t.Fatalf("a request that fits was not streamed: calls=%d", inner.calls)
}
})
}
func TestEnableNoaFailureKeepsBuiltInCompaction(t *testing.T) {
withNoaEnable(t, func(*agentcore.Options, noaadapter.Options) error { return errors.New("archive dir unusable") })
inner := &countingProvider{}
compaction := compactionConfig(32000)
opts := &agentcore.Options{Compaction: compaction, Provider: inner}
var warns []string
enableNoa(opts, func() bool { return true }, t.TempDir(), "noa-fail", func(m string) { warns = append(warns, m) })
if len(warns) != 1 || !strings.Contains(warns[0], "falling back to built-in compaction") || !strings.Contains(warns[0], "archive dir unusable") {
t.Fatalf("warnings %q", warns)
}
if opts.Compaction != compaction || opts.Compactor != nil {
t.Fatalf("Compaction=%v Compactor=%v; the built-in compaction must stay", opts.Compaction, opts.Compactor)
}
if opts.Provider != llm.Provider(inner) {
t.Fatalf("provider %T was wrapped although noa is off", opts.Provider)
}
}
+2 -2
View File
@@ -3,7 +3,7 @@ module github.com/Autumn-27/artex
go 1.26.10
require (
github.com/Autumn-27/norma v0.4.3
github.com/Autumn-27/norma v0.4.10
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.10.0
@@ -61,4 +61,4 @@ require (
modernc.org/memory v1.11.0 // indirect
)
replace github.com/Autumn-27/norma => github.com/BaaRai/Norma v0.4.8
replace github.com/Autumn-27/norma => github.com/BaaRai/Norma v0.4.10
+2 -2
View File
@@ -1,5 +1,5 @@
github.com/BaaRai/Norma v0.4.8 h1:k3at2aYSJ6aIDjEo7tC4VDSLBHXw2HobYjpKChVe6hI=
github.com/BaaRai/Norma v0.4.8/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw=
github.com/BaaRai/Norma v0.4.10 h1:+Sj6m8gZQIcVoOr5x9b3y7T1jI+95u43M07MLuvdaZk=
github.com/BaaRai/Norma v0.4.10/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw=
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
+6
View File
@@ -0,0 +1,6 @@
\set ON_ERROR_STOP on
-- Removes only the two indexes introduced by this change; retains table data.
SET lock_timeout = '5s';
SET statement_timeout = '10min';
DROP INDEX CONCURRENTLY IF EXISTS public.idx_activity_tool_result_lookup;
DROP INDEX CONCURRENTLY IF EXISTS public.idx_llm_usage_task_model_tokens;
+22
View File
@@ -0,0 +1,22 @@
\set ON_ERROR_STOP on
-- Run with psql outside a transaction. Adds indexes without blocking normal DML.
SET lock_timeout = '5s';
SET statement_timeout = '10min';
SET maintenance_work_mem = '128MB';
SET max_parallel_maintenance_workers = 1;
-- db/commands.go joins by tool_use_id without an exploration_id constraint.
-- The existing (exploration_id, tool_use_id, id) index has the wrong leading key.
CREATE INDEX CONCURRENTLY idx_activity_tool_result_lookup
ON public.activity (tool_use_id) INCLUDE (is_error)
WHERE kind = 'tool_result';
-- Match TokenByModel's actual COALESCE(task_id, '') predicate and cover its sums.
CREATE INDEX CONCURRENTLY idx_llm_usage_task_model_tokens
ON public.llm_usage ((COALESCE(task_id, '')), model)
INCLUDE (task_id, input_tokens, output_tokens, cache_read, cache_write);
-- Mark committed append-only pages visible so the covering index can avoid heap reads.
SET vacuum_cost_delay = '2ms';
SET vacuum_cost_limit = 1000;
VACUUM (ANALYZE) public.llm_usage;
+38
View File
@@ -0,0 +1,38 @@
# Online task-query indexes
`20261115-task-query-indexes.sql` adds two PostgreSQL indexes for existing queries:
- `activity(tool_use_id) INCLUDE (is_error) WHERE kind = 'tool_result'` supports the tool-result join in `db/commands.go` (`LEFT JOIN activity r ON r.tool_use_id = u.tool_use_id AND r.kind = 'tool_result'`). That join does not constrain the result row's exploration ID, so the existing `idx_act_tool_call (exploration_id, tool_use_id, id)` has the wrong leading key for it.
- `llm_usage((COALESCE(task_id, '')), model) INCLUDE (task_id, input_tokens, output_tokens, cache_read, cache_write)` covers the per-task model aggregation in `db/llm_usage.go` (`TokenByModel`). That query filters on `COALESCE(task_id,'') = $1` and groups by `model`. The expression matches that predicate; the existing `idx_llm_usage_model (task_id, model)` cannot serve it.
## Apply
```sh
psql -X -v ON_ERROR_STOP=1 -d <database> -f scripts/sql/20261115-task-query-indexes.sql
```
Run it outside a transaction: `CREATE INDEX CONCURRENTLY` cannot run inside one, so do not wrap the file in `BEGIN`/`COMMIT` or use `psql -1`. Use the deployment's normal PostgreSQL authentication.
Both indexes are created concurrently, so normal reads and writes continue. The script sets a 5-second lock timeout, a 10-minute statement timeout and bounded maintenance resources. Its final `VACUUM (ANALYZE)` is an ordinary vacuum, not `VACUUM FULL`; it updates visibility and statistics so the covering index can serve index-only scans.
This is an explicit, one-time maintenance step, not a numbered migration. `db/migrate.go` runs each migration file inside a transaction, where `CONCURRENTLY` is not allowed. A plain `CREATE INDEX` at startup would lock `activity` against writes for the whole build. No application or database restart is required.
## After an interruption
The script intentionally fails if an index with the same name already exists. A partial run may have created the first index before the second failed. An interrupted concurrent build can also leave an invalid index behind. Before retrying, check the state of both indexes:
```sql
SELECT c.relname, i.indisvalid, i.indisready
FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid
WHERE c.relname IN ('idx_activity_tool_result_lookup', 'idx_llm_usage_task_model_tokens');
```
Both must show `indisvalid = true` and `indisready = true`. Drop an index that is invalid or not ready with `DROP INDEX CONCURRENTLY`, or run the rollback below, then rerun the script.
## Rollback
```sh
psql -X -v ON_ERROR_STOP=1 -d <database> -f scripts/sql/20261115-task-query-indexes.rollback.sql
```
The rollback drops only these two indexes, concurrently and only if they exist. Table data is kept. Review the current index definitions before applying it. No application restart is required.
+3 -1
View File
@@ -79,8 +79,10 @@ var errWorkControlConflict = errors.New("work control conflict")
// retryableWorkerModelError excludes errors already handled by the task router.
// In particular, a quota error after partial streaming advances the task cursor
// for the next LLM call but must not replay this whole intent on the backup.
// A NOA budget refusal is deterministic and never reached the provider, so a
// rerun would only fail the same way.
func retryableWorkerModelError(reason harness.TerminalReason, err error) bool {
return reason == harness.ReasonModelError && !isTaskLLMRuntimeError(err)
return reason == harness.ReasonModelError && !isTaskLLMRuntimeError(err) && !errors.Is(err, agent.ErrNoaBudget)
}
// Engine drives the event-driven exploration loop with real LLM agents
+32
View File
@@ -0,0 +1,32 @@
package server
import (
"errors"
"fmt"
"testing"
"github.com/Autumn-27/artex/agent"
"github.com/Autumn-27/norma/harness"
)
// A NOA budget refusal is deterministic: the engine must not rerun the intent,
// whether the sentinel arrives bare or wrapped.
func TestWorkerModelErrorRetrySkipsNoaBudget(t *testing.T) {
for _, tc := range []struct {
name string
err error
want bool
}{
{"noa budget", agent.ErrNoaBudget, false},
{"noa budget wrapped", fmt.Errorf("%w: estimate 64000 exceeds configured budget 32000", agent.ErrNoaBudget), false},
{"noa budget wrapped twice", fmt.Errorf("worker: %w", fmt.Errorf("%w: estimate", agent.ErrNoaBudget)), false},
{"generic model error", errors.New("connection reset by peer"), true},
{"task router error", &taskLLMError{taskID: "7", cause: errors.New("insufficient_quota")}, false},
} {
t.Run(tc.name, func(t *testing.T) {
if got := retryableWorkerModelError(harness.ReasonModelError, tc.err); got != tc.want {
t.Fatalf("retryableWorkerModelError(model_error, %v) = %v, want %v", tc.err, got, tc.want)
}
})
}
}