mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-16 21:20:53 +08:00
Compare commits
34
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dcdc0652ad | ||
|
|
d9df84d2e5 | ||
|
|
959ec538ad | ||
|
|
2c514a4cd9 | ||
|
|
d005bfe5ba | ||
|
|
0374f32aeb | ||
|
|
be317a1337 | ||
|
|
feaaf9e416 | ||
|
|
5463ea4375 | ||
|
|
73aa56c16f | ||
|
|
5a8498bb4f | ||
|
|
2ad8ba16c3 | ||
|
|
09c9e6cfc2 | ||
|
|
59c3002bcc | ||
|
|
68f2db1d0b | ||
|
|
f0be3ae51f | ||
|
|
3b755228d6 | ||
|
|
d147dce344 | ||
|
|
60728092e1 | ||
|
|
ff599c65d3 | ||
|
|
ce096cdba1 | ||
|
|
50f6d07640 | ||
|
|
e6ab0f34a7 | ||
|
|
5ed28cde9f | ||
|
|
773fa9183a | ||
|
|
1cbab7cc39 | ||
|
|
bc75b5dc43 | ||
|
|
cddcd98f3b | ||
|
|
d42988b6a8 | ||
|
|
3975e58ce0 | ||
|
|
fdba29f1a9 | ||
|
|
a59d430175 | ||
|
|
c2d1f557c3 | ||
|
|
e636408f66 |
@@ -29,6 +29,10 @@ jobs:
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
- name: gofmt
|
||||
run: |
|
||||
files=$(gofmt -l $(git ls-files '*.go'))
|
||||
if [ -n "$files" ]; then echo "gofmt required:"; echo "$files"; exit 1; fi
|
||||
- run: go build ./...
|
||||
- run: go vet ./...
|
||||
# Skipped tests, tracked in issue #9:
|
||||
@@ -44,6 +48,19 @@ jobs:
|
||||
# as rotating failures in the agent package). Serial package execution
|
||||
# keeps the database state deterministic.
|
||||
run: go test -p 1 ./...
|
||||
- name: online index scripts
|
||||
# Apply and roll back scripts/sql on a database built here from
|
||||
# db/schema.sql and db/migrations. The job-level artextest database
|
||||
# never gets the schema (packages use suffixed databases), and cached
|
||||
# test results mean no test database is guaranteed to exist.
|
||||
env:
|
||||
SQL_DSN: postgres://artex:testpass@localhost:5432/artextest_sqlscripts?sslmode=disable
|
||||
run: |
|
||||
psql "$ARTEX_PG_DSN" -X -v ON_ERROR_STOP=1 -c 'CREATE DATABASE artextest_sqlscripts'
|
||||
psql "$SQL_DSN" -X -q -v ON_ERROR_STOP=1 -f db/schema.sql
|
||||
for f in db/migrations/*.sql; do psql "$SQL_DSN" -X -q -v ON_ERROR_STOP=1 -f "$f"; done
|
||||
psql "$SQL_DSN" -X -v ON_ERROR_STOP=1 -f scripts/sql/20261115-task-query-indexes.sql
|
||||
psql "$SQL_DSN" -X -v ON_ERROR_STOP=1 -f scripts/sql/20261115-task-query-indexes.rollback.sql
|
||||
- name: go test -race
|
||||
# Race detector on the packages that pass under -race today. ./server
|
||||
# joined once its suite was checked with -race against postgres and its
|
||||
@@ -82,4 +99,5 @@ jobs:
|
||||
cache-dependency-path: web/package-lock.json
|
||||
- run: npm ci
|
||||
- run: npx tsc --noEmit
|
||||
- run: npm run test:polling
|
||||
- run: npm run build:static
|
||||
|
||||
+8
-2
@@ -169,9 +169,12 @@ func (c *ChatAgent) Chat(ctx context.Context, agentKey, sessionID, message strin
|
||||
if c.guard != nil {
|
||||
opts.Hooks = c.guard.Hooks()
|
||||
}
|
||||
// The session id is the provider-side identity (session header, usage metering)
|
||||
// with or without persistence; Norma only puts it on ctx when it writes a transcript.
|
||||
opts.SessionID = sessionID
|
||||
ctx = transcript.WithSessionID(ctx, opts.SessionID)
|
||||
if c.tx != nil { // persist raw human↔AI conversation; one accumulating file per thread
|
||||
opts.Transcript = c.tx
|
||||
opts.SessionID = sessionID
|
||||
}
|
||||
// experimental feature: when enabled, noa takes over context compression (archive centralized under <workDir>/noa/<SessionID>, persistent).
|
||||
enableNoa(&opts, c.noaEnabledFn, c.workDir, "chat-"+sessionID, noaWarn("chat-"+sessionID))
|
||||
@@ -180,8 +183,11 @@ func (c *ChatAgent) Chat(ctx context.Context, agentKey, sessionID, message strin
|
||||
defer s.Close()
|
||||
// reload prior conversation so the agent has context across turns (each Chat is
|
||||
// a fresh session). First turn: no file yet → Resume loads nothing and proceeds.
|
||||
// A corrupt or unreadable transcript stops the turn before any model or tool call.
|
||||
if c.tx != nil {
|
||||
_ = s.Resume(sessionID)
|
||||
if err := resumeSession(s, opts.SessionID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
// re-unlock skill-gated MCPs from prior Skill() calls in the reloaded history so
|
||||
// revealed tools stay callable across the fresh session.
|
||||
|
||||
@@ -49,7 +49,7 @@ func findingWorkflowTools(agentKey string, tools []actool.CoreTool) ([]actool.Co
|
||||
if tool.Name() == "bind_finding_traffic" {
|
||||
continue
|
||||
}
|
||||
if agentKey == "reporter" && (tool.Name() == "traffic_search" || tool.Name() == "traffic_get" || tool.Name() == "traffic_blob") {
|
||||
if agentKey == "reporter" && (tool.Name() == "traffic_search" || tool.Name() == "traffic_get" || tool.Name() == "traffic_blob" || tool.Name() == "traffic_messages") {
|
||||
continue
|
||||
}
|
||||
switch tool.Name() {
|
||||
|
||||
+13
-7
@@ -169,14 +169,17 @@ func (m *MainAgent) Chat(ctx context.Context, taskID int64, mainSeg int, as *db.
|
||||
if hooks != nil { // typed-nil guard: only set when concrete (avoids harness panic)
|
||||
opts.Hooks = hooks
|
||||
}
|
||||
// Segment 0 keeps the legacy "exp%d-main" name so existing transcripts still
|
||||
// load; each new session (seg>=1) gets its own file for a clean context. The id
|
||||
// is the provider-side identity (session header, usage metering) with or without
|
||||
// persistence; Norma only puts it on ctx when it writes a transcript.
|
||||
opts.SessionID = fmt.Sprintf("exp%d-main", ts.ID())
|
||||
if mainSeg > 0 {
|
||||
opts.SessionID = fmt.Sprintf("exp%d-main-s%d", ts.ID(), mainSeg)
|
||||
}
|
||||
ctx = transcript.WithSessionID(ctx, opts.SessionID)
|
||||
if m.tx != nil { // persist raw human↔AI conversation; one accumulating file per segment
|
||||
opts.Transcript = m.tx
|
||||
// Segment 0 keeps the legacy "exp%d-main" name so existing transcripts still
|
||||
// load; each new session (seg>=1) gets its own file for a clean context.
|
||||
opts.SessionID = fmt.Sprintf("exp%d-main", ts.ID())
|
||||
if mainSeg > 0 {
|
||||
opts.SessionID = fmt.Sprintf("exp%d-main-s%d", ts.ID(), mainSeg)
|
||||
}
|
||||
}
|
||||
// experimental feature: when enabled, noa takes over context compression (archive centralized under <workDir>/noa/<SessionID>, persistent).
|
||||
// the session id follows the same rule as the transcript (segment-aware), keeping archive and resume aligned.
|
||||
@@ -191,8 +194,11 @@ func (m *MainAgent) Chat(ctx context.Context, taskID int64, mainSeg int, as *db.
|
||||
// reload the prior conversation from the transcript so the agent has context
|
||||
// across turns (each Chat is a fresh session; without this it can't see earlier
|
||||
// messages). First turn: no file yet → Resume loads nothing and proceeds.
|
||||
// A corrupt or unreadable transcript stops the turn before any model or tool call.
|
||||
if m.tx != nil {
|
||||
_ = s.Resume(opts.SessionID)
|
||||
if err := resumeSession(s, opts.SessionID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
// C2: this session is fresh each turn; re-unlock skill-gated MCPs from prior
|
||||
// Skill() calls in the reloaded history so revealed tools stay callable.
|
||||
|
||||
+17
-4
@@ -39,10 +39,17 @@ func enableNoa(opts *agentcore.Options, enabled func() bool, archiveRoot, sessio
|
||||
if opts.OnWarn == nil {
|
||||
opts.OnWarn = onWarn
|
||||
}
|
||||
if err := noaadapter.Enable(opts, noaadapter.Options{
|
||||
ArchiveBaseDir: filepath.Join(archiveRoot, "noa"),
|
||||
SessionID: sessionID,
|
||||
OnWarn: onWarn,
|
||||
// noa sizes its thresholds to the window this agent is configured for, read
|
||||
// before Compaction is cleared below; without it noa assumes its own default.
|
||||
window := defaultWindowK * 1000
|
||||
if opts.Compaction != nil && opts.Compaction.ContextWindow > 0 {
|
||||
window = opts.Compaction.ContextWindow
|
||||
}
|
||||
if err := noaEnable(opts, noaadapter.Options{
|
||||
ArchiveBaseDir: filepath.Join(archiveRoot, "noa"),
|
||||
SessionID: sessionID,
|
||||
OnWarn: onWarn,
|
||||
ModelContextLimit: window,
|
||||
}); err != nil {
|
||||
if onWarn != nil {
|
||||
onWarn("failed to enable noa compression, falling back to built-in compaction: " + err.Error())
|
||||
@@ -52,4 +59,10 @@ func enableNoa(opts *agentcore.Options, enabled func() bool, archiveRoot, sessio
|
||||
// Compactor overrides Compaction, but agentcore warns on every run when both are
|
||||
// set; clear it explicitly.
|
||||
opts.Compaction = nil
|
||||
// noa can only trim what it may drop; a request whose protected part alone
|
||||
// exceeds the window is refused here instead of by the provider.
|
||||
opts.Provider = &noaBudgetProvider{Provider: opts.Provider, window: window}
|
||||
}
|
||||
|
||||
// noaEnable is noaadapter.Enable; tests replace it to observe the options.
|
||||
var noaEnable = noaadapter.Enable
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"iter"
|
||||
|
||||
"github.com/Autumn-27/artex/sidequestion"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
)
|
||||
|
||||
// noaRequestOverhead covers request framing the input estimate does not count.
|
||||
const noaRequestOverhead = 512
|
||||
|
||||
// ErrNoaBudget marks a request refused by the NOA budget check. The refusal is
|
||||
// deterministic (the same history fails the same way), so callers must not
|
||||
// retry it.
|
||||
var ErrNoaBudget = errors.New("noa request budget exceeded")
|
||||
|
||||
// noaBudgetProvider refuses, before any transport, a request that cannot fit the
|
||||
// window noa was configured with. noa trims what it is allowed to drop; when
|
||||
// the protected part (system prompt, recent turns, the output reservation)
|
||||
// alone is over the window, sending it only buys a provider rejection.
|
||||
type noaBudgetProvider struct {
|
||||
llm.Provider
|
||||
window int
|
||||
}
|
||||
|
||||
func (p *noaBudgetProvider) check(req llm.CompletionRequest) error {
|
||||
estimate := sidequestion.EstimateInputTokens(req) + max(req.MaxTokens, 0) + noaRequestOverhead
|
||||
if estimate > p.window {
|
||||
return fmt.Errorf("%w: estimate %d exceeds configured budget %d; protected context cannot fit, increase the configured window or start a new session", ErrNoaBudget, estimate, p.window)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *noaBudgetProvider) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
|
||||
if err := p.check(req); err != nil {
|
||||
return llm.Message{}, "", llm.Usage{}, err
|
||||
}
|
||||
return p.Provider.Complete(ctx, req)
|
||||
}
|
||||
|
||||
func (p *noaBudgetProvider) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
||||
if err := p.check(req); err != nil {
|
||||
return func(yield func(llm.StreamEvent, error) bool) { yield(llm.StreamEvent{}, err) }
|
||||
}
|
||||
return p.Provider.Stream(ctx, req)
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"iter"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/norma/agentcore"
|
||||
"github.com/Autumn-27/norma/harness"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
"github.com/Autumn-27/norma/noaadapter"
|
||||
)
|
||||
|
||||
// A worker whose protected context exceeds a tiny NOA window ends with
|
||||
// model_error and an error that still wraps ErrNoaBudget after the Norma harness
|
||||
// and the capture layer: the engine's retry check relies on that.
|
||||
func TestWorkerNoaBudgetRefusalKeepsSentinelThroughHarness(t *testing.T) {
|
||||
f := newSessionFixture(t)
|
||||
inner := &countingProvider{}
|
||||
w := NewWorker(inner, "test-model", t.TempDir(), nil, 2000, 3)
|
||||
w.SetNoaEnabled(func() bool { return true })
|
||||
reason, _, err := w.Execute(t.Context(), "work#1", f.expID, nil, f.ts, f.intent, passHooks{}, nil, nil, nil)
|
||||
if reason != harness.ReasonModelError || !errors.Is(err, ErrNoaBudget) || inner.calls != 0 {
|
||||
t.Fatalf("reason=%q err=%v provider calls=%d; want model_error wrapping ErrNoaBudget and no call", reason, err, inner.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// countingProvider answers with one text turn and counts the calls it receives.
|
||||
type countingProvider struct{ calls int }
|
||||
|
||||
func (p *countingProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
||||
p.calls++
|
||||
return func(yield func(llm.StreamEvent, error) bool) {
|
||||
if yield(llm.StreamEvent{Type: llm.SETextDelta, Text: "ok"}, nil) {
|
||||
yield(llm.StreamEvent{Type: llm.SEMessageStop}, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *countingProvider) Complete(context.Context, llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
|
||||
p.calls++
|
||||
return llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{llm.TextBlock("ok")}}, "end_turn", llm.Usage{}, nil
|
||||
}
|
||||
|
||||
// withNoaEnable swaps the noaadapter.Enable seam for the duration of a test.
|
||||
func withNoaEnable(t *testing.T, fn func(*agentcore.Options, noaadapter.Options) error) {
|
||||
t.Helper()
|
||||
prev := noaEnable
|
||||
noaEnable = fn
|
||||
t.Cleanup(func() { noaEnable = prev })
|
||||
}
|
||||
|
||||
func TestEnableNoaUsesConfiguredWindow(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
compaction func() *agentcore.Options
|
||||
want int
|
||||
}{
|
||||
{"configured", func() *agentcore.Options { return &agentcore.Options{Compaction: compactionConfig(32000)} }, 32000},
|
||||
{"default", func() *agentcore.Options { return &agentcore.Options{} }, defaultWindowK * 1000},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var got noaadapter.Options
|
||||
withNoaEnable(t, func(opts *agentcore.Options, o noaadapter.Options) error {
|
||||
got = o
|
||||
return noaadapter.Enable(opts, o)
|
||||
})
|
||||
inner := &countingProvider{}
|
||||
opts := tc.compaction()
|
||||
opts.Provider = inner
|
||||
enableNoa(opts, func() bool { return true }, t.TempDir(), "noa-window", nil)
|
||||
|
||||
if got.ModelContextLimit != tc.want {
|
||||
t.Fatalf("noaadapter.Options.ModelContextLimit=%d, want %d", got.ModelContextLimit, tc.want)
|
||||
}
|
||||
bp, ok := opts.Provider.(*noaBudgetProvider)
|
||||
if !ok || bp.window != tc.want || bp.Provider != inner {
|
||||
t.Fatalf("provider %T %+v, want *noaBudgetProvider{window %d} around the original", opts.Provider, opts.Provider, tc.want)
|
||||
}
|
||||
if opts.Compaction != nil || opts.Compactor == nil {
|
||||
t.Fatalf("Compaction=%v Compactor=%v; noa must replace the built-in compaction", opts.Compaction, opts.Compactor)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func irreducibleRequest() llm.CompletionRequest {
|
||||
return llm.CompletionRequest{System: []string{strings.Repeat("protected system prompt ", 8000)}, MaxTokens: 8000}
|
||||
}
|
||||
|
||||
func TestNoaBudgetRefusesIrreducibleRequestBeforeTransport(t *testing.T) {
|
||||
t.Run("atomic", func(t *testing.T) {
|
||||
inner := &countingProvider{}
|
||||
p := &noaBudgetProvider{Provider: inner, window: 32000}
|
||||
_, _, _, err := p.Complete(t.Context(), irreducibleRequest())
|
||||
if err == nil || !strings.Contains(err.Error(), "exceeds configured budget 32000") || inner.calls != 0 {
|
||||
t.Fatalf("err=%v provider calls=%d", err, inner.calls)
|
||||
}
|
||||
if !errors.Is(err, ErrNoaBudget) {
|
||||
t.Fatalf("atomic refusal does not wrap ErrNoaBudget: %v", err)
|
||||
}
|
||||
if _, _, _, err := p.Complete(t.Context(), llm.CompletionRequest{System: []string{"short"}, MaxTokens: 1000}); err != nil || inner.calls != 1 {
|
||||
t.Fatalf("a request that fits was refused or not sent: err=%v calls=%d", err, inner.calls)
|
||||
}
|
||||
})
|
||||
t.Run("stream", func(t *testing.T) {
|
||||
inner := &countingProvider{}
|
||||
p := &noaBudgetProvider{Provider: inner, window: 32000}
|
||||
var events int
|
||||
var err error
|
||||
for ev, e := range p.Stream(t.Context(), irreducibleRequest()) {
|
||||
if e != nil {
|
||||
err = e
|
||||
break
|
||||
}
|
||||
_ = ev
|
||||
events++
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), "exceeds configured budget 32000") || events != 0 || inner.calls != 0 {
|
||||
t.Fatalf("err=%v events=%d provider calls=%d", err, events, inner.calls)
|
||||
}
|
||||
if !errors.Is(err, ErrNoaBudget) {
|
||||
t.Fatalf("stream refusal does not wrap ErrNoaBudget: %v", err)
|
||||
}
|
||||
for _, e := range p.Stream(t.Context(), llm.CompletionRequest{System: []string{"short"}, MaxTokens: 1000}) {
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
if inner.calls != 1 {
|
||||
t.Fatalf("a request that fits was not streamed: calls=%d", inner.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestEnableNoaFailureKeepsBuiltInCompaction(t *testing.T) {
|
||||
withNoaEnable(t, func(*agentcore.Options, noaadapter.Options) error { return errors.New("archive dir unusable") })
|
||||
inner := &countingProvider{}
|
||||
compaction := compactionConfig(32000)
|
||||
opts := &agentcore.Options{Compaction: compaction, Provider: inner}
|
||||
var warns []string
|
||||
enableNoa(opts, func() bool { return true }, t.TempDir(), "noa-fail", func(m string) { warns = append(warns, m) })
|
||||
|
||||
if len(warns) != 1 || !strings.Contains(warns[0], "falling back to built-in compaction") || !strings.Contains(warns[0], "archive dir unusable") {
|
||||
t.Fatalf("warnings %q", warns)
|
||||
}
|
||||
if opts.Compaction != compaction || opts.Compactor != nil {
|
||||
t.Fatalf("Compaction=%v Compactor=%v; the built-in compaction must stay", opts.Compaction, opts.Compactor)
|
||||
}
|
||||
if opts.Provider != llm.Provider(inner) {
|
||||
t.Fatalf("provider %T was wrapped although noa is off", opts.Provider)
|
||||
}
|
||||
}
|
||||
+4
-1
@@ -567,9 +567,12 @@ func (p *Planner) Plan(ctx context.Context, taskID int64, as *db.AssetStore, ts
|
||||
if hooks != nil { // typed-nil guard: only set when concrete (avoids harness panic)
|
||||
opts.Hooks = hooks
|
||||
}
|
||||
// The session id is the provider-side identity (session header, usage metering)
|
||||
// with or without persistence; Norma only puts it on ctx when it writes a transcript.
|
||||
opts.SessionID = fmt.Sprintf("exp%d-planner", ts.ID())
|
||||
ctx = transcript.WithSessionID(ctx, opts.SessionID)
|
||||
if p.tx != nil { // persist raw LLM conversation; one accumulating file per task's planner
|
||||
opts.Transcript = p.tx
|
||||
opts.SessionID = fmt.Sprintf("exp%d-planner", ts.ID())
|
||||
}
|
||||
// Experimental: when on, noa takes over context compression (archives persist under <workDir>/noa/<SessionID>).
|
||||
noaSession := fmt.Sprintf("exp%d-planner", ts.ID())
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/Autumn-27/norma/agentcore"
|
||||
)
|
||||
|
||||
// resumeSession reloads the persisted conversation of session id into s. Norma's
|
||||
// transcript.Store.Load treats a missing file as a fresh session (no records, no
|
||||
// error); any other failure, such as a corrupt line or an unreadable path, is
|
||||
// returned so the caller stops before the first model or tool call instead of
|
||||
// appending a new conversation to the damaged file.
|
||||
func resumeSession(s *agentcore.Session, id string) error {
|
||||
if err := s.Resume(id); err != nil {
|
||||
return fmt.Errorf("restore Norma session %q: %w", id, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"iter"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
"github.com/Autumn-27/norma/transcript"
|
||||
)
|
||||
|
||||
// sessionProbe is a provider that answers every call with one text turn and
|
||||
// records the session id it saw on ctx plus the request messages.
|
||||
type sessionProbe struct {
|
||||
mu sync.Mutex
|
||||
ids []string
|
||||
msgs [][]llm.Message
|
||||
}
|
||||
|
||||
func (p *sessionProbe) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
||||
p.mu.Lock()
|
||||
p.ids = append(p.ids, transcript.SessionIDFrom(ctx))
|
||||
p.msgs = append(p.msgs, req.Messages)
|
||||
p.mu.Unlock()
|
||||
return func(yield func(llm.StreamEvent, error) bool) {
|
||||
if yield(llm.StreamEvent{Type: llm.SETextDelta, Text: "done"}, nil) {
|
||||
yield(llm.StreamEvent{Type: llm.SEMessageDelta, StopReason: "end_turn"}, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *sessionProbe) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
|
||||
acc := llm.NewAccumulator()
|
||||
for ev, err := range p.Stream(ctx, req) {
|
||||
if err != nil {
|
||||
return llm.Message{}, "", llm.Usage{}, err
|
||||
}
|
||||
acc.Add(ev)
|
||||
}
|
||||
return acc.Message(), acc.StopReason, acc.Usage, nil
|
||||
}
|
||||
|
||||
func (p *sessionProbe) seen() []string {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return append([]string(nil), p.ids...)
|
||||
}
|
||||
|
||||
// jsonlFiles lists every .jsonl file under dir.
|
||||
func jsonlFiles(t *testing.T, dir string) []string {
|
||||
t.Helper()
|
||||
var out []string
|
||||
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() && strings.HasSuffix(path, ".jsonl") {
|
||||
out = append(out, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// damageTranscript makes the main transcript of id corrupt (a non-JSON line) or
|
||||
// unreadable (a directory). It returns the bytes a corrupt file must keep.
|
||||
func damageTranscript(t *testing.T, store *transcript.Store, id string, unreadable bool) string {
|
||||
t.Helper()
|
||||
path := store.MainPath(id)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if unreadable {
|
||||
if err := os.Mkdir(path, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
const original = "{corrupt transcript\n"
|
||||
if err := os.WriteFile(path, []byte(original), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return original
|
||||
}
|
||||
|
||||
// assertTranscriptUntouched checks that a damaged transcript was neither
|
||||
// rewritten nor appended to.
|
||||
func assertTranscriptUntouched(t *testing.T, store *transcript.Store, id, original string, unreadable bool) {
|
||||
t.Helper()
|
||||
path := store.MainPath(id)
|
||||
if unreadable {
|
||||
entries, err := os.ReadDir(path)
|
||||
if err != nil || len(entries) != 0 {
|
||||
t.Fatalf("unreadable transcript changed: entries=%v err=%v", entries, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil || string(b) != original {
|
||||
t.Fatalf("damaged transcript changed: %q err=%v", b, err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertRestoreError(t *testing.T, err error, calls int) {
|
||||
t.Helper()
|
||||
if err == nil || !strings.Contains(err.Error(), "restore Norma session") || calls != 0 {
|
||||
t.Fatalf("err=%v provider calls=%d", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
var damageCases = []struct {
|
||||
name string
|
||||
unreadable bool
|
||||
}{{"corrupt", false}, {"unreadable-directory", true}}
|
||||
|
||||
func TestChatSessionIdentityWithoutTranscript(t *testing.T) {
|
||||
p := &sessionProbe{}
|
||||
workDir := t.TempDir()
|
||||
c := NewChatAgent(p, "test-model", workDir, nil, 100000)
|
||||
for range 2 {
|
||||
if _, err := c.Chat(t.Context(), "assistant", "conv-stable", "hello", 1, 0, false, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if ids := p.seen(); len(ids) != 2 || ids[0] != "conv-stable" || ids[1] != "conv-stable" {
|
||||
t.Fatalf("provider session ids: %v", ids)
|
||||
}
|
||||
if files := jsonlFiles(t, workDir); len(files) != 0 {
|
||||
t.Fatalf("transcript written without persistence: %v", files)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatMissingTranscriptStartsFresh(t *testing.T) {
|
||||
store := transcript.NewStore(t.TempDir())
|
||||
p := &sessionProbe{}
|
||||
c := NewChatAgent(p, "test-model", t.TempDir(), store, 100000)
|
||||
if _, err := c.Chat(t.Context(), "assistant", "conv-new", "hello", 1, 0, false, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ids := p.seen(); len(ids) != 1 || ids[0] != "conv-new" {
|
||||
t.Fatalf("provider session ids: %v", ids)
|
||||
}
|
||||
recs, err := store.Load(store.MainPath("conv-new"))
|
||||
if err != nil || len(recs) == 0 {
|
||||
t.Fatalf("fresh transcript not written: %d records, err=%v", len(recs), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatRefusesDamagedTranscript(t *testing.T) {
|
||||
for _, tc := range damageCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
store := transcript.NewStore(t.TempDir())
|
||||
original := damageTranscript(t, store, "conv-broken", tc.unreadable)
|
||||
p := &sessionProbe{}
|
||||
c := NewChatAgent(p, "test-model", t.TempDir(), store, 100000)
|
||||
_, err := c.Chat(t.Context(), "assistant", "conv-broken", "hello", 1, 0, false, nil)
|
||||
assertRestoreError(t, err, len(p.seen()))
|
||||
assertTranscriptUntouched(t, store, "conv-broken", original, tc.unreadable)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatResumesSignedThinkingHistory(t *testing.T) {
|
||||
store := transcript.NewStore(t.TempDir())
|
||||
w := store.NewWriter("conv-signed", "")
|
||||
w.RecordMessage(llm.UserText("earlier question"), llm.Usage{})
|
||||
w.RecordMessage(llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{
|
||||
{Type: llm.BlockThinking, Thinking: "reasoning", Signature: "signed"},
|
||||
{Type: llm.BlockRedactedThinking, Data: "opaque"},
|
||||
{Type: llm.BlockText, Text: "earlier answer"},
|
||||
}}, llm.Usage{})
|
||||
if err := w.Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := &sessionProbe{}
|
||||
c := NewChatAgent(p, "test-model", t.TempDir(), store, 100000)
|
||||
if _, err := c.Chat(t.Context(), "assistant", "conv-signed", "next question", 1, 0, false, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if len(p.msgs) != 1 {
|
||||
t.Fatalf("provider calls: %d", len(p.msgs))
|
||||
}
|
||||
var thinking, redacted bool
|
||||
for _, m := range p.msgs[0] {
|
||||
for _, b := range m.Content {
|
||||
thinking = thinking || (b.Type == llm.BlockThinking && b.Thinking == "reasoning" && b.Signature == "signed")
|
||||
redacted = redacted || (b.Type == llm.BlockRedactedThinking && b.Data == "opaque")
|
||||
}
|
||||
}
|
||||
if !thinking || !redacted {
|
||||
t.Fatalf("resumed history lost signed thinking=%v redacted=%v: %+v", thinking, redacted, p.msgs[0])
|
||||
}
|
||||
}
|
||||
|
||||
// sessionFixture is one exploration with one intent on the test database.
|
||||
type sessionFixture struct {
|
||||
expID int64
|
||||
intentID int64
|
||||
ts *db.ExplorationStore
|
||||
intent *db.Node
|
||||
}
|
||||
|
||||
func newSessionFixture(t *testing.T) sessionFixture {
|
||||
t.Helper()
|
||||
d := testDB(t)
|
||||
t.Cleanup(func() { _ = d.Close() })
|
||||
expID, err := d.CreateExploration("session identity probe", "keep the session id stable")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM explorations WHERE id=$1`, expID) })
|
||||
ts := d.Exploration(expID)
|
||||
payload := map[string]any{"summary": "session probe"}
|
||||
intentID, err := ts.AddNode("intent", payload, 0, "running", "planner", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(payload)
|
||||
return sessionFixture{expID: expID, intentID: intentID, ts: ts, intent: &db.Node{ID: intentID, Payload: raw}}
|
||||
}
|
||||
|
||||
// run drives one agent kind once and returns the session id it should use.
|
||||
func (f sessionFixture) run(t *testing.T, kind string, p *sessionProbe, tx *transcript.Store) (string, error) {
|
||||
t.Helper()
|
||||
workDir := t.TempDir()
|
||||
switch kind {
|
||||
case "worker":
|
||||
_, _, err := NewWorker(p, "test-model", workDir, tx, 0, 3).Execute(t.Context(), "work#1", f.expID, nil, f.ts, f.intent, passHooks{}, nil, nil, nil)
|
||||
return WorkerSessionID(f.expID, f.intentID), err
|
||||
case "planner":
|
||||
_, _, err := NewPlanner(p, "test-model", workDir, tx, 0, 3).Plan(t.Context(), f.expID, nil, f.ts, "probe goal", nil, passHooks{}, nil)
|
||||
return fmt.Sprintf("exp%d-planner", f.expID), err
|
||||
case "mainagent":
|
||||
_, err := NewMainAgent(p, "test-model", workDir, tx, 0, 3).Chat(t.Context(), f.expID, 0, nil, f.ts, "probe goal", "status?", passHooks{}, nil, nil, nil, nil, nil)
|
||||
return fmt.Sprintf("exp%d-main", f.expID), err
|
||||
case "mainagent-seg":
|
||||
_, err := NewMainAgent(p, "test-model", workDir, tx, 0, 3).Chat(t.Context(), f.expID, 2, nil, f.ts, "probe goal", "status?", passHooks{}, nil, nil, nil, nil, nil)
|
||||
return fmt.Sprintf("exp%d-main-s2", f.expID), err
|
||||
}
|
||||
t.Fatalf("unknown agent kind %q", kind)
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func TestAgentSessionIdentityWithAndWithoutTranscript(t *testing.T) {
|
||||
f := newSessionFixture(t)
|
||||
for _, kind := range []string{"worker", "planner", "mainagent", "mainagent-seg"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
bare := &sessionProbe{}
|
||||
want, err := f.run(t, kind, bare, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := transcript.NewStore(t.TempDir())
|
||||
persisted := &sessionProbe{}
|
||||
if _, err := f.run(t, kind, persisted, store); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, ids := range [][]string{bare.seen(), persisted.seen()} {
|
||||
if len(ids) == 0 {
|
||||
t.Fatal("provider never called")
|
||||
}
|
||||
for _, id := range ids {
|
||||
if id != want {
|
||||
t.Fatalf("provider session ids %v, want %q", ids, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
if recs, err := store.Load(store.MainPath(want)); err != nil || len(recs) == 0 {
|
||||
t.Fatalf("transcript %q not written under the session id: %d records, err=%v", want, len(recs), err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentRefusesDamagedTranscript(t *testing.T) {
|
||||
f := newSessionFixture(t)
|
||||
for _, kind := range []string{"worker", "mainagent"} {
|
||||
for _, tc := range damageCases {
|
||||
t.Run(kind+"/"+tc.name, func(t *testing.T) {
|
||||
store := transcript.NewStore(t.TempDir())
|
||||
id := WorkerSessionID(f.expID, f.intentID)
|
||||
if kind == "mainagent" {
|
||||
id = fmt.Sprintf("exp%d-main", f.expID)
|
||||
}
|
||||
original := damageTranscript(t, store, id, tc.unreadable)
|
||||
p := &sessionProbe{}
|
||||
_, err := f.run(t, kind, p, store)
|
||||
assertRestoreError(t, err, len(p.seen()))
|
||||
assertTranscriptUntouched(t, store, id, original, tc.unreadable)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -46,9 +46,9 @@ const stagnationMinRepeats = 3
|
||||
|
||||
func detectStagnation(done, findings []*db.Node) []map[string]any {
|
||||
type dir struct {
|
||||
firstID int64
|
||||
repeats int
|
||||
summary string
|
||||
firstID int64
|
||||
repeats int
|
||||
summary string
|
||||
}
|
||||
dirs := map[string]*dir{}
|
||||
order := []string{}
|
||||
@@ -551,7 +551,7 @@ func (t *ToolSet) graphOverviewData() map[string]any {
|
||||
// with the data so no prompt-template change is needed on deployed installs.
|
||||
if stale := detectStagnation(recentDone, vulnNodes); len(stale) > 0 {
|
||||
out["stagnation_warning"] = map[string]any{
|
||||
"directions": stale,
|
||||
"directions": stale,
|
||||
"instruction": "these directions were dispatched repeatedly with zero findings produced since their first attempt — do NOT re-dispatch the same approach again; open a materially different direction, or judge the goal on the existing evidence instead of grinding",
|
||||
}
|
||||
}
|
||||
|
||||
+11
-3
@@ -292,7 +292,7 @@ func workerTrafficBlock(recording bool) string {
|
||||
if !recording {
|
||||
return ""
|
||||
}
|
||||
return "\n\n**Traffic tools**:\n- traffic_search / traffic_get / traffic_blob: review responses, find already-visited resources — **check the traffic first, don't curl the same URL twice**. traffic_search **requires a host** and by default returns only 3 ultra-light index entries (id/method/url/status/resp_len, no response content) — explicitly raise limit when you need more; body_contains does full-text search in request/response bodies (at least 3 characters; supports substrings and CJK text, e.g. for passwords/keys/errors/internal addresses); to see one entry's raw content use traffic_get(id) — oversized bodies show as @blob sha256:<hash>, fetched in segments with traffic_blob(hash)."
|
||||
return "\n\n**Traffic tools**:\n- traffic_search / traffic_get / traffic_blob / traffic_messages: review responses, find already-visited resources — **check the traffic first, don't curl the same URL twice**. traffic_search **requires a host** and by default returns only 3 ultra-light index entries (id/method/url/status/resp_len, no response content) — explicitly raise limit when you need more; body_contains does full-text search in request/response bodies (at least 3 characters; supports substrings and CJK text, e.g. for passwords/keys/errors/internal addresses); to see one entry's raw content use traffic_get(id) — oversized bodies show as @blob sha256:<hash>, fetched in segments with traffic_blob(hash). WebSocket (status 101) and SSE (text/event-stream) records are captured too: traffic_get shows the handshake, and traffic_messages(id) lists their messages in order with direction and type."
|
||||
}
|
||||
|
||||
// artifactSpec is section [C]: the code-owned, non-editable tail appended to every
|
||||
@@ -540,9 +540,12 @@ func (w *Worker) execute(ctx context.Context, name string, taskID int64, as *db.
|
||||
if hooks != nil { // typed-nil guard: only set when concrete (avoids harness panic)
|
||||
opts.Hooks = hooks
|
||||
}
|
||||
// The session id is the provider-side identity (session header, usage metering)
|
||||
// with or without persistence; Norma only puts it on ctx when it writes a transcript.
|
||||
opts.SessionID = WorkerSessionID(ts.ID(), intent.ID)
|
||||
ctx = transcript.WithSessionID(ctx, opts.SessionID)
|
||||
if w.tx != nil { // persist raw LLM conversation; one file per worked intent
|
||||
opts.Transcript = w.tx
|
||||
opts.SessionID = WorkerSessionID(ts.ID(), intent.ID)
|
||||
}
|
||||
intentID := intent.ID
|
||||
emitWrap := func(r db.Activity) {
|
||||
@@ -574,7 +577,12 @@ func (w *Worker) execute(ctx context.Context, name string, taskID int64, as *db.
|
||||
// restarting from scratch.
|
||||
alreadyRecorded := false
|
||||
if w.tx != nil {
|
||||
_ = s.Resume(opts.SessionID)
|
||||
// A corrupt or unreadable transcript stops the run before any model or tool
|
||||
// call. No terminal reason: the engine marks the intent blocked without the
|
||||
// model_error reruns, which would only fail the same way again.
|
||||
if err := resumeSession(s, opts.SessionID); err != nil {
|
||||
return "", tsx.Writes(), err
|
||||
}
|
||||
alreadyRecorded = requestID != "" && hasWorkerChatMessage(s.Messages(), requestID)
|
||||
if len(s.Messages()) > 0 && message == "" {
|
||||
seedUnlockFromHistory(s.Messages(), def.UnlockSkill)
|
||||
|
||||
@@ -7,12 +7,12 @@ import "time"
|
||||
// these match the *target asset*: an exact/fuzzy domain·ip·url, or a CIDR range.
|
||||
// This layer only stores rules; the matching/enforcement logic lives elsewhere.
|
||||
type AssetInterceptRule struct {
|
||||
ID int64 `json:"id"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Kind string `json:"kind"` // exact_domain|exact_ip|exact_url|fuzzy_domain|fuzzy_ip|fuzzy_url|cidr
|
||||
Pattern string `json:"pattern"`
|
||||
Note string `json:"note"`
|
||||
Builtin bool `json:"builtin"`
|
||||
ID int64 `json:"id"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Kind string `json:"kind"` // exact_domain|exact_ip|exact_url|fuzzy_domain|fuzzy_ip|fuzzy_url|cidr
|
||||
Pattern string `json:"pattern"`
|
||||
Note string `json:"note"`
|
||||
Builtin bool `json:"builtin"`
|
||||
// Action is only used by task-level rules: 'block'=intercept, 'allow'=allow (whitelist).
|
||||
// Global rules (asset_intercept_rules) do not have this column; it is always empty and treated as intercept.
|
||||
Action string `json:"action,omitempty"`
|
||||
|
||||
+6
-6
@@ -1168,11 +1168,11 @@ func TestQueryDSLInScopeMembership(t *testing.T) {
|
||||
}
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
root := fmt.Sprintf("sc%d.invalid", stamp) // in-scope root domain (task)
|
||||
srcRoot := fmt.Sprintf("src%d.invalid", stamp) // in-scope via source task
|
||||
out := fmt.Sprintf("out%d.invalid", stamp) // out of every scope
|
||||
marker := fmt.Sprintf("mk%d", stamp) // bare-text token present in all rows
|
||||
foreignTask := stamp + 777 // asset produced by an unrelated task
|
||||
root := fmt.Sprintf("sc%d.invalid", stamp) // in-scope root domain (task)
|
||||
srcRoot := fmt.Sprintf("src%d.invalid", stamp) // in-scope via source task
|
||||
out := fmt.Sprintf("out%d.invalid", stamp) // out of every scope
|
||||
marker := fmt.Sprintf("mk%d", stamp) // bare-text token present in all rows
|
||||
foreignTask := stamp + 777 // asset produced by an unrelated task
|
||||
|
||||
// Scope: task owns root; source task owns srcRoot; task owns an IP /24.
|
||||
for _, sc := range []struct {
|
||||
@@ -1200,7 +1200,7 @@ func TestQueryDSLInScopeMembership(t *testing.T) {
|
||||
{"subdomain", "api." + root, root, "", "", "", marker}, // under task root
|
||||
{"service", "www." + root, root, "https://www." + root + "/", "", "", marker}, // service under task root
|
||||
{"endpoint", "www." + root, root, "https://www." + root + "/a?" + marker + "=1", "GET", "", ""},
|
||||
{"subdomain", "dev." + srcRoot, srcRoot, "", "", "", marker}, // under source-task root
|
||||
{"subdomain", "dev." + srcRoot, srcRoot, "", "", "", marker}, // under source-task root
|
||||
{"endpoint", "198.51.100.9", "198.51.100.9", "http://198.51.100.9:8080/" + marker, "GET", "", ""}, // IP-literal host, ip col empty
|
||||
{"subdomain", "x." + out, out, "", "", "", marker}, // out of scope
|
||||
}
|
||||
|
||||
+50
-26
@@ -1486,11 +1486,13 @@ type TaskListMetrics struct {
|
||||
Findings FindingSeverityCounts // this task's vulnerability count in the findings table (bucketed by severity)
|
||||
}
|
||||
|
||||
// TaskListMetricsAll returns list aggregates for every live task in one query.
|
||||
// The lateral lookups use the per-exploration indexes instead of grouping the
|
||||
// complete activity history on every poll.
|
||||
func (d *DB) TaskListMetricsAll() (map[int64]TaskListMetrics, error) {
|
||||
rows, err := d.Query(`
|
||||
// taskListMetricsQuery is the select shared by TaskListMetricsAll and
|
||||
// TaskListMetrics; each appends its own WHERE on task. The lateral lookups use
|
||||
// the per-exploration indexes instead of grouping the complete activity history
|
||||
// on every poll. latest_activity orders by the column activity.created_at, so
|
||||
// idx_act_latest (exploration_id, created_at DESC) serves the LIMIT 1; ordering
|
||||
// by the output alias would sort every activity row of the task.
|
||||
const taskListMetricsQuery = `
|
||||
SELECT task.exploration_id,
|
||||
COALESCE(token_metrics.input_tokens,0),
|
||||
COALESCE(token_metrics.output_tokens,0),
|
||||
@@ -1514,10 +1516,10 @@ func (d *DB) TaskListMetricsAll() (map[int64]TaskListMetrics, error) {
|
||||
WHERE exploration_id=task.exploration_id AND kind='result'
|
||||
) token_metrics ON true
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT EXTRACT(EPOCH FROM created_at)::bigint AS created_at
|
||||
SELECT EXTRACT(EPOCH FROM activity.created_at)::bigint AS created_at
|
||||
FROM activity
|
||||
WHERE exploration_id=task.exploration_id
|
||||
ORDER BY created_at DESC
|
||||
WHERE activity.exploration_id=task.exploration_id
|
||||
ORDER BY activity.created_at DESC
|
||||
LIMIT 1
|
||||
) latest_activity ON true
|
||||
LEFT JOIN LATERAL (
|
||||
@@ -1539,30 +1541,41 @@ func (d *DB) TaskListMetricsAll() (map[int64]TaskListMetrics, error) {
|
||||
FROM findings
|
||||
WHERE task_id=task.id
|
||||
) finding_metrics ON true
|
||||
WHERE task.deleted_at IS NULL`)
|
||||
WHERE task.deleted_at IS NULL`
|
||||
|
||||
// scanTaskListMetrics reads one row of taskListMetricsQuery.
|
||||
func scanTaskListMetrics(row interface{ Scan(...any) error }) (int64, TaskListMetrics, error) {
|
||||
var explorationID int64
|
||||
var metrics TaskListMetrics
|
||||
err := row.Scan(
|
||||
&explorationID,
|
||||
&metrics.Tokens.InputTokens,
|
||||
&metrics.Tokens.OutputTokens,
|
||||
&metrics.Tokens.CacheReadTokens,
|
||||
&metrics.Tokens.CacheWriteTokens,
|
||||
&metrics.LastActivity,
|
||||
&metrics.Goals.Total,
|
||||
&metrics.Goals.Met,
|
||||
&metrics.RunningIntents,
|
||||
&metrics.Findings.Critical,
|
||||
&metrics.Findings.High,
|
||||
&metrics.Findings.Medium,
|
||||
&metrics.Findings.Low,
|
||||
)
|
||||
return explorationID, metrics, err
|
||||
}
|
||||
|
||||
// TaskListMetricsAll returns list aggregates for every live task in one query.
|
||||
func (d *DB) TaskListMetricsAll() (map[int64]TaskListMetrics, error) {
|
||||
rows, err := d.Query(taskListMetricsQuery)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[int64]TaskListMetrics{}
|
||||
for rows.Next() {
|
||||
var explorationID int64
|
||||
var metrics TaskListMetrics
|
||||
if err := rows.Scan(
|
||||
&explorationID,
|
||||
&metrics.Tokens.InputTokens,
|
||||
&metrics.Tokens.OutputTokens,
|
||||
&metrics.Tokens.CacheReadTokens,
|
||||
&metrics.Tokens.CacheWriteTokens,
|
||||
&metrics.LastActivity,
|
||||
&metrics.Goals.Total,
|
||||
&metrics.Goals.Met,
|
||||
&metrics.RunningIntents,
|
||||
&metrics.Findings.Critical,
|
||||
&metrics.Findings.High,
|
||||
&metrics.Findings.Medium,
|
||||
&metrics.Findings.Low,
|
||||
); err != nil {
|
||||
explorationID, metrics, err := scanTaskListMetrics(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[explorationID] = metrics
|
||||
@@ -1570,6 +1583,17 @@ func (d *DB) TaskListMetricsAll() (map[int64]TaskListMetrics, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// TaskListMetrics returns the same aggregates for the one live task of
|
||||
// exploration expID, with the same laterals as TaskListMetricsAll. A missing or
|
||||
// deleted task gives zero metrics, as a missing key of TaskListMetricsAll does.
|
||||
func (d *DB) TaskListMetrics(expID int64) (TaskListMetrics, error) {
|
||||
_, metrics, err := scanTaskListMetrics(d.QueryRow(taskListMetricsQuery+` AND task.exploration_id=$1`, expID))
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return TaskListMetrics{}, nil
|
||||
}
|
||||
return metrics, err
|
||||
}
|
||||
|
||||
// GoalCountsAll returns goal totals (total / met) per exploration id, one query
|
||||
// for all tasks — used by listTasks so the task list shows progress for every task.
|
||||
func (d *DB) GoalCountsAll() (map[int64]GoalCounts, error) {
|
||||
|
||||
+6
-5
@@ -341,11 +341,12 @@ func applyCostEur(d *DB, out []ProfileUsage) error {
|
||||
if !ok || (pr.in == 0 && pr.outp == 0 && pr.cr == 0 && pr.cw == 0) {
|
||||
continue
|
||||
}
|
||||
// InputTokens includes the cached subset (Anthropic and OpenAI usage
|
||||
// semantics), so the fresh share is input minus cache reads; charging
|
||||
// both buckets at full input price double-counts the cache. Clamped at
|
||||
// zero for gateways that report the buckets disjointly.
|
||||
fresh := out[i].InputTokens - out[i].CacheReadTokens
|
||||
// InputTokens already includes both cached subsets: Norma normalizes it to
|
||||
// input + cache read + cache write (OpenAI prompt_tokens semantics). The
|
||||
// fresh share is therefore input minus cache reads minus cache writes;
|
||||
// charging either subset at full input price as well bills it twice.
|
||||
// Clamped at zero for gateways that report the buckets disjointly.
|
||||
fresh := out[i].InputTokens - out[i].CacheReadTokens - out[i].CacheWriteTokens
|
||||
if fresh < 0 {
|
||||
fresh = 0
|
||||
}
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// costProfiles saves a priced profile (EUR per million tokens: input 2.00,
|
||||
// output 4.00, cache read 0.20, cache write 2.50) and an unpriced one.
|
||||
func costProfiles(t *testing.T, d *DB) (priced, unpriced string) {
|
||||
t.Helper()
|
||||
suffix := time.Now().UnixNano()
|
||||
priced, unpriced = fmt.Sprintf("t-cost-priced-%d", suffix), fmt.Sprintf("t-cost-unpriced-%d", suffix)
|
||||
for _, p := range []*LLMProfile{
|
||||
{Name: priced, Format: "anthropic", Model: "m", APIKey: "k",
|
||||
PriceInputPerM: 2, PriceOutputPerM: 4, PriceCacheReadPerM: 0.2, PriceCacheWritePerM: 2.5},
|
||||
{Name: unpriced, Format: "anthropic", Model: "m", APIKey: "k"},
|
||||
} {
|
||||
id, err := d.SaveProfile(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { d.Exec(`DELETE FROM llm_profiles WHERE id=$1`, id) })
|
||||
}
|
||||
return priced, unpriced
|
||||
}
|
||||
|
||||
func assertCost(t *testing.T, got *float64, want float64) {
|
||||
t.Helper()
|
||||
if got == nil {
|
||||
t.Fatalf("cost=nil, want %.2f", want)
|
||||
}
|
||||
if math.Abs(*got-want) > 1e-9 {
|
||||
t.Fatalf("cost=%.4f, want %.2f", *got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// InputTokens already includes the cache read and cache write subsets, so only
|
||||
// the remainder is billed at the input price.
|
||||
func TestCostSeparatesFreshReadAndWriteBuckets(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
priced, unpriced := costProfiles(t, d)
|
||||
// Every case adds 100k output tokens: 0.1 × 4.00 = 0.40.
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
in, read, write int
|
||||
want float64
|
||||
}{
|
||||
{"fresh", 1_000_000, 0, 0, 2.40}, // 1M×2.00 + 0.40
|
||||
{"read_only", 1_000_000, 1_000_000, 0, 0.60}, // 0 fresh + 1M×0.20 + 0.40
|
||||
{"write_only", 1_000_000, 0, 1_000_000, 2.90}, // 0 fresh + 1M×2.50 + 0.40
|
||||
{"mixed", 1_000_000, 200_000, 300_000, 2.19}, // 500k×2.00 + 200k×0.20 + 300k×2.50 + 0.40
|
||||
{"inconsistent_gateway", 100_000, 200_000, 300_000, 1.19}, // fresh clamped to 0 + 0.04 + 0.75 + 0.40
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
out := []ProfileUsage{
|
||||
{ProfileName: priced, InputTokens: tc.in, CacheReadTokens: tc.read, CacheWriteTokens: tc.write, OutputTokens: 100_000},
|
||||
{ProfileName: unpriced, InputTokens: tc.in, CacheReadTokens: tc.read, CacheWriteTokens: tc.write, OutputTokens: 100_000},
|
||||
{ProfileName: "t-cost-unknown-profile", InputTokens: tc.in, OutputTokens: 100_000},
|
||||
}
|
||||
if err := applyCostEur(d, out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertCost(t, out[0].CostEur, tc.want)
|
||||
if out[1].CostEur != nil || out[2].CostEur != nil {
|
||||
t.Fatalf("unpriced usage got a cost: %v %v", out[1].CostEur, out[2].CostEur)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Several ledger rows of one profile are summed before pricing; with no clamp
|
||||
// involved the profile cost equals the sum of the per-row costs.
|
||||
func TestUsageByProfileCostAggregatesRows(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
priced, unpriced := costProfiles(t, d)
|
||||
t.Cleanup(func() { d.Exec(`DELETE FROM llm_usage WHERE profile_name IN ($1,$2)`, priced, unpriced) })
|
||||
rows := []LLMUsage{
|
||||
// fresh 400k×2.00 + 200k×0.20 + 40k×4.00 = 0.80 + 0.04 + 0.16 = 1.00
|
||||
{InputTokens: 600_000, CacheRead: 200_000, OutputTokens: 40_000},
|
||||
// fresh 100k×2.00 + 300k×2.50 + 60k×4.00 = 0.20 + 0.75 + 0.24 = 1.19
|
||||
{InputTokens: 400_000, CacheWrite: 300_000, OutputTokens: 60_000},
|
||||
}
|
||||
for _, profile := range []string{priced, unpriced} {
|
||||
for _, r := range rows {
|
||||
r.ProfileName, r.Model, r.Status = profile, "m", "ok"
|
||||
if err := d.InsertLLMUsage(&r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
usage, err := d.UsageByProfile()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := 0
|
||||
for _, p := range usage {
|
||||
switch p.ProfileName {
|
||||
case priced:
|
||||
found++
|
||||
if p.Calls != 2 || p.InputTokens != 1_000_000 || p.CacheReadTokens != 200_000 || p.CacheWriteTokens != 300_000 || p.OutputTokens != 100_000 {
|
||||
t.Fatalf("aggregate %+v", p)
|
||||
}
|
||||
assertCost(t, p.CostEur, 2.19) // = 1.00 + 1.19, the mixed case
|
||||
case unpriced:
|
||||
found++
|
||||
if p.Calls != 2 || p.CostEur != nil {
|
||||
t.Fatalf("unpriced aggregate %+v", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
if found != 2 {
|
||||
t.Fatalf("profiles missing from UsageByProfile: found %d of 2", found)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Migration 0004 leaves a valid, ready btree index on llm_usage(ts) for the
|
||||
// time-window queries (UsageDaily, JudgeUsageStats).
|
||||
func TestLLMUsageTsIndex(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
var valid, ready bool
|
||||
var def string
|
||||
err = d.QueryRow(`
|
||||
SELECT i.indisvalid, i.indisready, pg_get_indexdef(i.indexrelid)
|
||||
FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname='public' AND c.relname='idx_llm_usage_ts'`).Scan(&valid, &ready, &def)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatal("index public.idx_llm_usage_ts does not exist after Open")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !valid || !ready {
|
||||
t.Fatalf("idx_llm_usage_ts indisvalid=%v indisready=%v, want both true (%s)", valid, ready, def)
|
||||
}
|
||||
if !strings.Contains(def, "USING btree (ts)") {
|
||||
t.Fatalf("idx_llm_usage_ts definition %q does not index ts with a btree", def)
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -136,9 +136,9 @@ func assertEmbeddedApplied(t *testing.T, d *DB) []appliedMigration {
|
||||
want := []struct {
|
||||
version int
|
||||
name string
|
||||
}{{1, llmRecordsMigration}, {2, llmUsageMigration}, {3, "0003_task_scope_icp_index.sql"}}
|
||||
}{{1, llmRecordsMigration}, {2, llmUsageMigration}, {3, "0003_task_scope_icp_index.sql"}, {4, "0004_llm_usage_ts_index.sql"}}
|
||||
if len(applied) != len(want) {
|
||||
t.Fatalf("schema_migrations = %+v, want versions 1 to 3", applied)
|
||||
t.Fatalf("schema_migrations = %+v, want versions 1 to 4", applied)
|
||||
}
|
||||
for i, w := range want {
|
||||
content, err := fs.ReadFile(migrationsFS(), w.name)
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
-- UsageDaily and JudgeUsageStats (db/llm_usage.go) filter llm_usage on
|
||||
-- ts >= now() - interval; the dashboard polls UsageDaily every 15 seconds.
|
||||
-- llm_usage is an append-only ledger that gains a row per LLM call and is never
|
||||
-- purged, so without an index on ts those queries scan the whole table.
|
||||
--
|
||||
-- This is a plain (non-concurrent) CREATE INDEX: migrations run in a
|
||||
-- transaction, where CONCURRENTLY is not allowed. It is safe here because
|
||||
-- db.Open applies migrations at startup, before the HTTP server listens, so the
|
||||
-- SHARE lock it takes blocks no agent call or request in flight.
|
||||
--
|
||||
-- For an exceptionally large ledger, build the same index online before the
|
||||
-- upgrade, outside a transaction:
|
||||
-- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_llm_usage_ts ON llm_usage(ts);
|
||||
-- This migration then finds it and does nothing.
|
||||
CREATE INDEX IF NOT EXISTS idx_llm_usage_ts ON llm_usage(ts);
|
||||
+22
-5
@@ -79,15 +79,20 @@ type TaskArchive struct {
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// archiveBlockerScope selects the live direct dependents that block archiving
|
||||
// their source task. TaskArchiveBlockers and TaskArchiveBlocker share it so the
|
||||
// list and the detail view cannot disagree.
|
||||
const archiveBlockerScope = `
|
||||
FROM task_relations relation
|
||||
JOIN tasks child ON child.id=relation.task_id AND child.deleted_at IS NULL
|
||||
LEFT JOIN task_archives pending ON pending.task_id=child.id
|
||||
WHERE (pending.id IS NULL OR pending.state NOT IN ('archive_queued','archiving'))`
|
||||
|
||||
// TaskArchiveBlockers returns one live direct dependent for every source task
|
||||
// that cannot currently be archived. Dependents already queued for archiving do
|
||||
// not block their source because the FIFO worker will compact them first.
|
||||
func (d *DB) TaskArchiveBlockers() (map[int64]int64, error) {
|
||||
rows, err := d.Query(`SELECT relation.source_task_id, MIN(child.id)
|
||||
FROM task_relations relation
|
||||
JOIN tasks child ON child.id=relation.task_id AND child.deleted_at IS NULL
|
||||
LEFT JOIN task_archives pending ON pending.task_id=child.id
|
||||
WHERE pending.id IS NULL OR pending.state NOT IN ('archive_queued','archiving')
|
||||
rows, err := d.Query(`SELECT relation.source_task_id, MIN(child.id)` + archiveBlockerScope + `
|
||||
GROUP BY relation.source_task_id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -104,6 +109,18 @@ GROUP BY relation.source_task_id`)
|
||||
return blockers, rows.Err()
|
||||
}
|
||||
|
||||
// TaskArchiveBlocker returns the live direct dependent that blocks taskID, or
|
||||
// 0 when nothing blocks it. Same rules as TaskArchiveBlockers, scoped to one
|
||||
// source task so getTask does not aggregate the whole relation table per poll.
|
||||
func (d *DB) TaskArchiveBlocker(taskID int64) (int64, error) {
|
||||
var dependent sql.NullInt64
|
||||
if err := d.QueryRow(`SELECT MIN(child.id)`+archiveBlockerScope+`
|
||||
AND relation.source_task_id=$1`, taskID).Scan(&dependent); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return dependent.Int64, nil
|
||||
}
|
||||
|
||||
type TaskArchivePage struct {
|
||||
Items []TaskArchive `json:"items"`
|
||||
Total int `json:"total"`
|
||||
|
||||
@@ -362,6 +362,7 @@ func TestTaskArchiveBlockersIgnoreQueuedDependents(t *testing.T) {
|
||||
if blockers[source.ID] != dependent.ID {
|
||||
t.Fatalf("source blocker=%d, want dependent %d", blockers[source.ID], dependent.ID)
|
||||
}
|
||||
assertPerTaskBlockers(t, d, blockers, map[int64]int64{source.ID: dependent.ID, dependent.ID: 0})
|
||||
if err := d.SetPaused(dependent.ID, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -375,6 +376,25 @@ func TestTaskArchiveBlockersIgnoreQueuedDependents(t *testing.T) {
|
||||
if blocker := blockers[source.ID]; blocker != 0 {
|
||||
t.Fatalf("queued dependent must not block source, got %d", blocker)
|
||||
}
|
||||
assertPerTaskBlockers(t, d, blockers, map[int64]int64{source.ID: 0, dependent.ID: 0})
|
||||
}
|
||||
|
||||
// assertPerTaskBlockers checks TaskArchiveBlocker against the expected blocker
|
||||
// of each task and against the global TaskArchiveBlockers map.
|
||||
func assertPerTaskBlockers(t *testing.T, d *DB, global map[int64]int64, want map[int64]int64) {
|
||||
t.Helper()
|
||||
for taskID, wantBlocker := range want {
|
||||
got, err := d.TaskArchiveBlocker(taskID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != wantBlocker {
|
||||
t.Fatalf("TaskArchiveBlocker(%d)=%d, want %d", taskID, got, wantBlocker)
|
||||
}
|
||||
if got != global[taskID] {
|
||||
t.Fatalf("TaskArchiveBlocker(%d)=%d, TaskArchiveBlockers()[%d]=%d", taskID, got, taskID, global[taskID])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverInterruptedArchiveRequiresManualRetry(t *testing.T) {
|
||||
|
||||
@@ -117,3 +117,133 @@ VALUES ($1,'goal','{}','met'), ($1,'goal','{}','open')`, task.ExplorationID); er
|
||||
t.Fatalf("unexpected goal metrics: %+v", metrics.Goals)
|
||||
}
|
||||
}
|
||||
|
||||
// TaskListMetrics(expID) is exactly the row TaskListMetricsAll returns for that
|
||||
// exploration, on every counter, and TaskListMetricsAll still lists every task.
|
||||
func TestTaskListMetricsMatchesAll(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) - skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
newest := time.Now().Add(-time.Minute).Truncate(time.Second)
|
||||
seed := func(name string, n int) *Task {
|
||||
t.Helper()
|
||||
task, err := d.CreateTask(name, "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, err := d.DeleteTaskCascade(task.ID, false, true)
|
||||
cleanupErr(t, "task "+name, err)
|
||||
})
|
||||
exp := task.ExplorationID
|
||||
// n result rows with tokens; the newest activity first, then an older row
|
||||
// with a higher id, so ordering by id instead of created_at would fail.
|
||||
for _, q := range []struct {
|
||||
sql string
|
||||
args []any
|
||||
}{
|
||||
{`INSERT INTO activity(exploration_id,kind,input_tokens,output_tokens,cache_read_tokens,cache_write_tokens,created_at)
|
||||
SELECT $1,'result',10,5,3,1,$3::timestamptz - interval '1 hour' FROM generate_series(1,$2::int)`, []any{exp, n, newest}},
|
||||
{`INSERT INTO activity(exploration_id,kind,created_at) VALUES ($1,'text',$2::timestamptz)`, []any{exp, newest}},
|
||||
{`INSERT INTO activity(exploration_id,kind,created_at) SELECT $1,'text',$3::timestamptz - interval '2 hours' FROM generate_series(1,$2::int)`, []any{exp, n, newest}},
|
||||
{`INSERT INTO exploration_nodes(exploration_id,kind,payload,state)
|
||||
SELECT $1,'goal','{}',CASE WHEN g<=$2::int THEN 'met' ELSE 'open' END FROM generate_series(1,$2::int+1) g`, []any{exp, n}},
|
||||
{`INSERT INTO exploration_nodes(exploration_id,kind,payload,state)
|
||||
SELECT $1,'intent','{}',CASE WHEN g<=$2::int THEN 'running' ELSE 'done' END FROM generate_series(1,$2::int+2) g`, []any{exp, n}},
|
||||
} {
|
||||
if _, err := d.Exec(q.sql, q.args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := d.Exec(`INSERT INTO findings(task_id,severity)
|
||||
SELECT $1, s FROM unnest(ARRAY['critical','high','high','medium','low','info']) s, generate_series(1,$2)`, task.ID, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return task
|
||||
}
|
||||
a := seed("task-list-metrics-a", 2)
|
||||
b := seed("task-list-metrics-b", 3)
|
||||
|
||||
all, err := d.TaskListMetricsAll()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
task *Task
|
||||
n int
|
||||
}{{a, 2}, {b, 3}} {
|
||||
row, ok := all[c.task.ExplorationID]
|
||||
if !ok {
|
||||
t.Fatalf("TaskListMetricsAll misses task %d", c.task.ID)
|
||||
}
|
||||
one, err := d.TaskListMetrics(c.task.ExplorationID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if one != row {
|
||||
t.Fatalf("task %d: TaskListMetrics=%+v, TaskListMetricsAll=%+v", c.task.ID, one, row)
|
||||
}
|
||||
n := c.n
|
||||
want := TaskListMetrics{
|
||||
Tokens: TokenUsage{InputTokens: 10 * n, OutputTokens: 5 * n, CacheReadTokens: 3 * n, CacheWriteTokens: n},
|
||||
LastActivity: newest.Unix(),
|
||||
Goals: GoalCounts{Total: n + 1, Met: n},
|
||||
RunningIntents: n,
|
||||
Findings: FindingSeverityCounts{Critical: n, High: 2 * n, Medium: n, Low: n},
|
||||
}
|
||||
if one != want {
|
||||
t.Fatalf("task %d metrics=%+v, want %+v", c.task.ID, one, want)
|
||||
}
|
||||
}
|
||||
if none, err := d.TaskListMetrics(-1); err != nil || none != (TaskListMetrics{}) {
|
||||
t.Fatalf("missing exploration: %+v, %v; want zero metrics", none, err)
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkTaskListMetrics compares the per-exploration metrics query with the
|
||||
// all-tasks one on three tasks of 40,000 activity rows each. No time assertion.
|
||||
func BenchmarkTaskListMetrics(b *testing.B) {
|
||||
dsn, _, err := DSN()
|
||||
if err != nil {
|
||||
b.Skipf("no database config (%v) - skipping", err)
|
||||
}
|
||||
d, err := Open(dsn)
|
||||
if err != nil {
|
||||
b.Skipf("postgres unavailable (%v) - skipping", err)
|
||||
}
|
||||
b.Cleanup(func() { d.Close() })
|
||||
var exp int64
|
||||
for i := range 3 {
|
||||
task, err := d.CreateTask(fmt.Sprintf("bench-task-list-metrics-%d", i), "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.Cleanup(func() { _, _ = d.DeleteTaskCascade(task.ID, false, true) })
|
||||
if _, err := d.Exec(`INSERT INTO activity(exploration_id,kind,input_tokens,created_at)
|
||||
SELECT $1, CASE WHEN g%20=0 THEN 'result' ELSE 'tool_use' END, 100, now() - (g||' seconds')::interval
|
||||
FROM generate_series(1,40000) g`, task.ExplorationID); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
exp = task.ExplorationID
|
||||
}
|
||||
if _, err := d.Exec(`ANALYZE activity`); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
b.Run("one", func(b *testing.B) {
|
||||
for range b.N {
|
||||
if _, err := d.TaskListMetrics(exp); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
b.Run("all", func(b *testing.B) {
|
||||
for range b.N {
|
||||
if _, err := d.TaskListMetricsAll(); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -6,14 +6,14 @@ import "testing"
|
||||
// ip/cidr scope path so a target like "10.0.188.136:3000" no longer 404s.
|
||||
func TestStripHostPort(t *testing.T) {
|
||||
cases := []struct{ in, want string }{
|
||||
{"10.0.188.136:3000", "10.0.188.136"}, // the reported IP case
|
||||
{"10.0.188.136", "10.0.188.136"}, // bare IPv4 unchanged
|
||||
{"[2001:db8::1]:8080", "2001:db8::1"}, // bracketed IPv6 + port
|
||||
{"2001:db8::1", "2001:db8::1"}, // bare IPv6 unchanged (has colons)
|
||||
{" 1.2.3.4:80 ", "1.2.3.4"}, // trims surrounding space
|
||||
{"example.com:443", "example.com"}, // domain + port → bare host
|
||||
{"api.example.com:8080", "api.example.com"}, // subdomain + port
|
||||
{"example.com", "example.com"}, // bare domain unchanged
|
||||
{"10.0.188.136:3000", "10.0.188.136"}, // the reported IP case
|
||||
{"10.0.188.136", "10.0.188.136"}, // bare IPv4 unchanged
|
||||
{"[2001:db8::1]:8080", "2001:db8::1"}, // bracketed IPv6 + port
|
||||
{"2001:db8::1", "2001:db8::1"}, // bare IPv6 unchanged (has colons)
|
||||
{" 1.2.3.4:80 ", "1.2.3.4"}, // trims surrounding space
|
||||
{"example.com:443", "example.com"}, // domain + port → bare host
|
||||
{"api.example.com:8080", "api.example.com"}, // subdomain + port
|
||||
{"example.com", "example.com"}, // bare domain unchanged
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := stripHostPort(c.in); got != c.want {
|
||||
|
||||
+11
-11
@@ -28,19 +28,19 @@ type Task struct {
|
||||
LLMProfileID *int64 `json:"llm_profile_id,omitempty"`
|
||||
// Task-level ordered LLM chain. LLMProfileID remains the compatibility alias
|
||||
// for ActiveLLMProfileID while older API clients still send one profile id.
|
||||
LLMProfileIDs []int64 `json:"llm_profile_ids,omitempty"`
|
||||
ActiveLLMProfileID *int64 `json:"active_llm_profile_id,omitempty"`
|
||||
LLMChainRevision int64 `json:"-"`
|
||||
LLMFailoverState string `json:"llm_failover_state,omitempty"`
|
||||
LLMFailoverReason string `json:"llm_failover_reason,omitempty"`
|
||||
LLMProfileIDs []int64 `json:"llm_profile_ids,omitempty"`
|
||||
ActiveLLMProfileID *int64 `json:"active_llm_profile_id,omitempty"`
|
||||
LLMChainRevision int64 `json:"-"`
|
||||
LLMFailoverState string `json:"llm_failover_state,omitempty"`
|
||||
LLMFailoverReason string `json:"llm_failover_reason,omitempty"`
|
||||
// LLMRotate: per-intent chain-head rotation switch (#85); false = classic
|
||||
// ordered failover from the active head.
|
||||
LLMRotate bool `json:"llm_rotate"`
|
||||
SourceTaskIDs []int64 `json:"source_task_ids,omitempty"`
|
||||
CompanyIDs []int64 `json:"company_ids,omitempty"`
|
||||
ParentRef string `json:"parent_ref,omitempty"` // parent task id (recorded by orchestration spawn; empty=top level)
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"` // when it entered a terminal state (done/failed/timeout); nil for non-terminal
|
||||
LLMRotate bool `json:"llm_rotate"`
|
||||
SourceTaskIDs []int64 `json:"source_task_ids,omitempty"`
|
||||
CompanyIDs []int64 `json:"company_ids,omitempty"`
|
||||
ParentRef string `json:"parent_ref,omitempty"` // parent task id (recorded by orchestration spawn; empty=top level)
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"` // when it entered a terminal state (done/failed/timeout); nil for non-terminal
|
||||
// Task-level timeout (see the task-level timeout and wrap-up design doc).
|
||||
TimeoutSeconds int `json:"timeout_seconds"` // 0=no time limit
|
||||
FirstRunAt *time.Time `json:"first_run_at,omitempty"` // when it first actually started running (not created_at); nil=not yet run
|
||||
|
||||
@@ -3,9 +3,10 @@ module github.com/Autumn-27/artex
|
||||
go 1.26.10
|
||||
|
||||
require (
|
||||
github.com/Autumn-27/norma v0.4.3
|
||||
github.com/Autumn-27/norma v0.4.10
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/klauspost/compress v1.18.6
|
||||
github.com/lqqyt2423/go-mitmproxy v1.9.2
|
||||
@@ -27,7 +28,6 @@ require (
|
||||
github.com/gaissmai/bart v0.26.0 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/gorilla/css v1.0.1 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/hinshun/vt10x v0.0.0-20220301184237-5011da428d02 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
@@ -61,4 +61,4 @@ require (
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/Autumn-27/norma => github.com/BaaRai/Norma v0.4.8
|
||||
replace github.com/Autumn-27/norma => github.com/BaaRai/Norma v0.4.10
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
github.com/BaaRai/Norma v0.4.8 h1:k3at2aYSJ6aIDjEo7tC4VDSLBHXw2HobYjpKChVe6hI=
|
||||
github.com/BaaRai/Norma v0.4.8/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw=
|
||||
github.com/BaaRai/Norma v0.4.10 h1:+Sj6m8gZQIcVoOr5x9b3y7T1jI+95u43M07MLuvdaZk=
|
||||
github.com/BaaRai/Norma v0.4.10/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw=
|
||||
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
|
||||
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
|
||||
|
||||
@@ -39,7 +39,7 @@ var noTargetTools = toolSet(
|
||||
// the operator runs the command, not the platform
|
||||
"ask_operator",
|
||||
// recorded traffic
|
||||
"traffic_search", "traffic_get", "traffic_blob",
|
||||
"traffic_search", "traffic_get", "traffic_blob", "traffic_messages",
|
||||
)
|
||||
|
||||
func toolSet(names ...string) map[string]bool {
|
||||
|
||||
@@ -191,7 +191,7 @@ func TestExtractSkipsNoTargetTools(t *testing.T) {
|
||||
"web_search", "Read", "Write", "Edit", "MultiEdit", "LS", "Glob", "Grep", "TodoWrite", "sleep",
|
||||
"TaskOutput", "TaskStop", "TaskList",
|
||||
"add_intent", "insert_assets", "report_finding", "record_fact", "add_task_scope", "graph_overview",
|
||||
"spawn_task", "ask_operator", "traffic_search", "traffic_get", "traffic_blob",
|
||||
"spawn_task", "ask_operator", "traffic_search", "traffic_get", "traffic_blob", "traffic_messages",
|
||||
} {
|
||||
wantTargets(t, tool, targetsOf(tool, input))
|
||||
}
|
||||
|
||||
+6
-6
@@ -195,11 +195,11 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S
|
||||
return
|
||||
}
|
||||
// Always track usage (cheap); accumulate text/thinking only for bodies.
|
||||
// Anthropic (and the other providers) split token usage across events:
|
||||
// message_start carries ONLY the input side (input + cache), message_delta
|
||||
// ONLY the output. They must be FOLDED with Add — overwriting on delta
|
||||
// would zero out the input already counted at start (mirrors the SDK's own
|
||||
// llm.Accumulator; see norma/llm/accumulate.go).
|
||||
// message_start carries the input side (input + cache) and is added.
|
||||
// Since Norma v0.4.7 message_delta also echoes that input side next to the
|
||||
// output, so it is folded with llm.Usage.MergeEcho (max on echoed fields,
|
||||
// sum on output), as llm.Accumulator does; Add would count the input and
|
||||
// cache twice. An output-only delta folds the same either way.
|
||||
switch ev.Type {
|
||||
case llm.SETextDelta:
|
||||
if recordBodies {
|
||||
@@ -215,7 +215,7 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S
|
||||
if ev.StopReason != "" {
|
||||
stopReason = ev.StopReason
|
||||
}
|
||||
usage.Add(ev.Usage)
|
||||
usage.MergeEcho(ev.Usage)
|
||||
}
|
||||
if !yield(ev, err) {
|
||||
return
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
package llmrec
|
||||
|
||||
import (
|
||||
"context"
|
||||
"iter"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// echoStreamProvider streams one message_start and one message_delta with the
|
||||
// given (already normalized) usage, as Norma's providers emit them.
|
||||
type echoStreamProvider struct {
|
||||
completeProvider
|
||||
start, delta llm.Usage
|
||||
}
|
||||
|
||||
func (p echoStreamProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
||||
return func(yield func(llm.StreamEvent, error) bool) {
|
||||
for _, ev := range []llm.StreamEvent{
|
||||
{Type: llm.SEMessageStart, Usage: p.start},
|
||||
{Type: llm.SETextDelta, Text: "answer"},
|
||||
{Type: llm.SEMessageDelta, StopReason: "end_turn", Usage: p.delta},
|
||||
{Type: llm.SEMessageStop},
|
||||
} {
|
||||
if !yield(ev, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Norma normalizes Anthropic usage so InputTokens includes the cache read and
|
||||
// write subsets, and since v0.4.7 message_delta echoes that input side.
|
||||
var usageEchoCases = []struct {
|
||||
name string
|
||||
start, delta llm.Usage
|
||||
want llm.Usage
|
||||
}{
|
||||
{
|
||||
name: "full_echo",
|
||||
start: llm.Usage{InputTokens: 23, CacheReadTokens: 5, CacheWriteTokens: 3},
|
||||
delta: llm.Usage{InputTokens: 23, CacheReadTokens: 5, CacheWriteTokens: 3, OutputTokens: 5},
|
||||
want: llm.Usage{InputTokens: 23, OutputTokens: 5, CacheReadTokens: 5, CacheWriteTokens: 3},
|
||||
},
|
||||
{
|
||||
// Raw start: input 500, cache read 200, cache creation 40 → normalized 740.
|
||||
// Raw delta: input 500, output 77, no cache fields → normalized 500.
|
||||
name: "partial_echo",
|
||||
start: llm.Usage{InputTokens: 740, CacheReadTokens: 200, CacheWriteTokens: 40},
|
||||
delta: llm.Usage{InputTokens: 500, OutputTokens: 77},
|
||||
want: llm.Usage{InputTokens: 740, OutputTokens: 77, CacheReadTokens: 200, CacheWriteTokens: 40},
|
||||
},
|
||||
{
|
||||
name: "no_echo",
|
||||
start: llm.Usage{InputTokens: 23, CacheReadTokens: 5, CacheWriteTokens: 3},
|
||||
delta: llm.Usage{OutputTokens: 5},
|
||||
want: llm.Usage{InputTokens: 23, OutputTokens: 5, CacheReadTokens: 5, CacheWriteTokens: 3},
|
||||
},
|
||||
}
|
||||
|
||||
func drainRecorderStream(t *testing.T, ctx context.Context, r *Recorder) {
|
||||
t.Helper()
|
||||
for _, err := range r.Stream(ctx, llm.CompletionRequest{}) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The metered usage of a streamed call counts an echoed input side once.
|
||||
func TestStreamUsageEchoCountedOnce(t *testing.T) {
|
||||
for _, tc := range usageEchoCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var got []llm.Usage
|
||||
ctx := WithUsageSink(t.Context(), func(u llm.Usage) { got = append(got, u) })
|
||||
drainRecorderStream(t, ctx, Wrap(echoStreamProvider{start: tc.start, delta: tc.delta}, nil, "fixture", "profile", "", "", func() bool { return false }))
|
||||
if len(got) != 1 || got[0] != tc.want {
|
||||
t.Fatalf("metered %+v, want one call with %+v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The llm_usage ledger gets one row per streamed call, with the echo counted once.
|
||||
func TestStreamUsageEchoLedgerRow(t *testing.T) {
|
||||
dsn, _, err := db.DSN()
|
||||
if err != nil {
|
||||
t.Skipf("no database config (%v)", err)
|
||||
}
|
||||
pg, err := db.Open(dsn)
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v)", err)
|
||||
}
|
||||
defer pg.Close()
|
||||
if err := pg.EnsureLLMUsageTable(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tc := range usageEchoCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
profile := "usage-echo-" + uuid.NewString()
|
||||
t.Cleanup(func() { _, _ = pg.Exec(`DELETE FROM llm_usage WHERE profile_name=$1`, profile) })
|
||||
drainRecorderStream(t, t.Context(), Wrap(echoStreamProvider{start: tc.start, delta: tc.delta}, pg, "fixture", profile, "", "", func() bool { return false }))
|
||||
var rows int
|
||||
var got llm.Usage
|
||||
err := pg.QueryRow(`SELECT count(*), COALESCE(sum(input_tokens),0), COALESCE(sum(output_tokens),0), COALESCE(sum(cache_read),0), COALESCE(sum(cache_write),0)
|
||||
FROM llm_usage WHERE profile_name=$1`, profile).Scan(&rows, &got.InputTokens, &got.OutputTokens, &got.CacheReadTokens, &got.CacheWriteTokens)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows != 1 || got != tc.want {
|
||||
t.Fatalf("ledger rows=%d usage=%+v, want 1 row with %+v", rows, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
\set ON_ERROR_STOP on
|
||||
-- Removes only the two indexes introduced by this change; retains table data.
|
||||
SET lock_timeout = '5s';
|
||||
SET statement_timeout = '10min';
|
||||
DROP INDEX CONCURRENTLY IF EXISTS public.idx_activity_tool_result_lookup;
|
||||
DROP INDEX CONCURRENTLY IF EXISTS public.idx_llm_usage_task_model_tokens;
|
||||
Executable
+22
@@ -0,0 +1,22 @@
|
||||
\set ON_ERROR_STOP on
|
||||
-- Run with psql outside a transaction. Adds indexes without blocking normal DML.
|
||||
SET lock_timeout = '5s';
|
||||
SET statement_timeout = '10min';
|
||||
SET maintenance_work_mem = '128MB';
|
||||
SET max_parallel_maintenance_workers = 1;
|
||||
|
||||
-- db/commands.go joins by tool_use_id without an exploration_id constraint.
|
||||
-- The existing (exploration_id, tool_use_id, id) index has the wrong leading key.
|
||||
CREATE INDEX CONCURRENTLY idx_activity_tool_result_lookup
|
||||
ON public.activity (tool_use_id) INCLUDE (is_error)
|
||||
WHERE kind = 'tool_result';
|
||||
|
||||
-- Match TokenByModel's actual COALESCE(task_id, '') predicate and cover its sums.
|
||||
CREATE INDEX CONCURRENTLY idx_llm_usage_task_model_tokens
|
||||
ON public.llm_usage ((COALESCE(task_id, '')), model)
|
||||
INCLUDE (task_id, input_tokens, output_tokens, cache_read, cache_write);
|
||||
|
||||
-- Mark committed append-only pages visible so the covering index can avoid heap reads.
|
||||
SET vacuum_cost_delay = '2ms';
|
||||
SET vacuum_cost_limit = 1000;
|
||||
VACUUM (ANALYZE) public.llm_usage;
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
# Online task-query indexes
|
||||
|
||||
`20261115-task-query-indexes.sql` adds two PostgreSQL indexes for existing queries:
|
||||
|
||||
- `activity(tool_use_id) INCLUDE (is_error) WHERE kind = 'tool_result'` supports the tool-result join in `db/commands.go` (`LEFT JOIN activity r ON r.tool_use_id = u.tool_use_id AND r.kind = 'tool_result'`). That join does not constrain the result row's exploration ID, so the existing `idx_act_tool_call (exploration_id, tool_use_id, id)` has the wrong leading key for it.
|
||||
- `llm_usage((COALESCE(task_id, '')), model) INCLUDE (task_id, input_tokens, output_tokens, cache_read, cache_write)` covers the per-task model aggregation in `db/llm_usage.go` (`TokenByModel`). That query filters on `COALESCE(task_id,'') = $1` and groups by `model`. The expression matches that predicate; the existing `idx_llm_usage_model (task_id, model)` cannot serve it.
|
||||
|
||||
## Apply
|
||||
|
||||
```sh
|
||||
psql -X -v ON_ERROR_STOP=1 -d <database> -f scripts/sql/20261115-task-query-indexes.sql
|
||||
```
|
||||
|
||||
Run it outside a transaction: `CREATE INDEX CONCURRENTLY` cannot run inside one, so do not wrap the file in `BEGIN`/`COMMIT` or use `psql -1`. Use the deployment's normal PostgreSQL authentication.
|
||||
|
||||
Both indexes are created concurrently, so normal reads and writes continue. The script sets a 5-second lock timeout, a 10-minute statement timeout and bounded maintenance resources. Its final `VACUUM (ANALYZE)` is an ordinary vacuum, not `VACUUM FULL`; it updates visibility and statistics so the covering index can serve index-only scans.
|
||||
|
||||
This is an explicit, one-time maintenance step, not a numbered migration. `db/migrate.go` runs each migration file inside a transaction, where `CONCURRENTLY` is not allowed. A plain `CREATE INDEX` at startup would lock `activity` against writes for the whole build. No application or database restart is required.
|
||||
|
||||
## After an interruption
|
||||
|
||||
The script intentionally fails if an index with the same name already exists. A partial run may have created the first index before the second failed. An interrupted concurrent build can also leave an invalid index behind. Before retrying, check the state of both indexes:
|
||||
|
||||
```sql
|
||||
SELECT c.relname, i.indisvalid, i.indisready
|
||||
FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE c.relname IN ('idx_activity_tool_result_lookup', 'idx_llm_usage_task_model_tokens');
|
||||
```
|
||||
|
||||
Both must show `indisvalid = true` and `indisready = true`. Drop an index that is invalid or not ready with `DROP INDEX CONCURRENTLY`, or run the rollback below, then rerun the script.
|
||||
|
||||
## Rollback
|
||||
|
||||
```sh
|
||||
psql -X -v ON_ERROR_STOP=1 -d <database> -f scripts/sql/20261115-task-query-indexes.rollback.sql
|
||||
```
|
||||
|
||||
The rollback drops only these two indexes, concurrently and only if they exist. Table data is kept. Review the current index definitions before applying it. No application restart is required.
|
||||
@@ -6,9 +6,9 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
actool "github.com/Autumn-27/norma/tool"
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
"github.com/Autumn-27/artex/db"
|
||||
actool "github.com/Autumn-27/norma/tool"
|
||||
)
|
||||
|
||||
// hasSkillTool reports whether the packed tool set contains the Skill meta-tool.
|
||||
|
||||
+9
-2
@@ -82,9 +82,16 @@ func applyTaskArchiveBlocker(dto *TaskDTO, blockers map[int64]int64) {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if dependentID := blockers[taskID]; dependentID > 0 {
|
||||
dto.ArchiveBlockedBy = strconv.FormatInt(dependentID, 10)
|
||||
applyTaskArchiveBlockerID(dto, blockers[taskID])
|
||||
}
|
||||
|
||||
// applyTaskArchiveBlockerID records dependentID as the task's archive blocker;
|
||||
// 0 (nothing blocks it) leaves the DTO unchanged.
|
||||
func applyTaskArchiveBlockerID(dto *TaskDTO, dependentID int64) {
|
||||
if dto == nil || dependentID <= 0 {
|
||||
return
|
||||
}
|
||||
dto.ArchiveBlockedBy = strconv.FormatInt(dependentID, 10)
|
||||
}
|
||||
|
||||
// TokenTotalDTO is a whole-task (all agents) token aggregate.
|
||||
|
||||
+3
-1
@@ -79,8 +79,10 @@ var errWorkControlConflict = errors.New("work control conflict")
|
||||
// retryableWorkerModelError excludes errors already handled by the task router.
|
||||
// In particular, a quota error after partial streaming advances the task cursor
|
||||
// for the next LLM call but must not replay this whole intent on the backup.
|
||||
// A NOA budget refusal is deterministic and never reached the provider, so a
|
||||
// rerun would only fail the same way.
|
||||
func retryableWorkerModelError(reason harness.TerminalReason, err error) bool {
|
||||
return reason == harness.ReasonModelError && !isTaskLLMRuntimeError(err)
|
||||
return reason == harness.ReasonModelError && !isTaskLLMRuntimeError(err) && !errors.Is(err, agent.ErrNoaBudget)
|
||||
}
|
||||
|
||||
// Engine drives the event-driven exploration loop with real LLM agents
|
||||
|
||||
@@ -88,7 +88,7 @@ func (s *Server) seedFindingWorkflowTools() {
|
||||
// Upgrade only the original default binding. Customized lists and enabled
|
||||
// flags survive; the one-time flag also preserves future user unbinding.
|
||||
readers := `["worker","reporter"]`
|
||||
for _, key := range []string{"traffic_search", "traffic_get", "traffic_blob"} {
|
||||
for _, key := range []string{"traffic_search", "traffic_get", "traffic_blob", "traffic_messages"} {
|
||||
if _, err := s.m.pg.Exec(`UPDATE tools SET agents=$2::jsonb WHERE key=$1 AND system AND (agents='["worker"]'::jsonb OR (agents @> '["worker","planner","mainagent","auto","pentest"]'::jsonb AND jsonb_array_length(agents)=5))`, key, readers); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -7,19 +7,19 @@ import (
|
||||
|
||||
func TestGoalGateCheck(t *testing.T) {
|
||||
good := map[string]string{
|
||||
"url target": "Audit " + strings.Repeat("x ", 60) + " at https://example.com/app every module for injection issues and record one finding per issue with evidence.",
|
||||
"bare domain": strings.Repeat("probe ", 25) + "the api at target.example.org for unauthenticated data exposure, one record proof, then stop.",
|
||||
"ipv4": strings.Repeat("recon ", 25) + "10.0.0.5 and map every listening service before any test.",
|
||||
"localhost port": strings.Repeat("verify ", 25) + "localhost:3000 health endpoints respond identically before and after config reload.",
|
||||
"path": strings.Repeat("review ", 25) + "the module at /app/data/core for hardcoded secrets and report each hit.",
|
||||
"targetless opt-out": "[targetless] " + strings.Repeat("summarize ", 20) + "the attached material into a prioritized hypothesis list.",
|
||||
"url target": "Audit " + strings.Repeat("x ", 60) + " at https://example.com/app every module for injection issues and record one finding per issue with evidence.",
|
||||
"bare domain": strings.Repeat("probe ", 25) + "the api at target.example.org for unauthenticated data exposure, one record proof, then stop.",
|
||||
"ipv4": strings.Repeat("recon ", 25) + "10.0.0.5 and map every listening service before any test.",
|
||||
"localhost port": strings.Repeat("verify ", 25) + "localhost:3000 health endpoints respond identically before and after config reload.",
|
||||
"path": strings.Repeat("review ", 25) + "the module at /app/data/core for hardcoded secrets and report each hit.",
|
||||
"targetless opt-out": "[targetless] " + strings.Repeat("summarize ", 20) + "the attached material into a prioritized hypothesis list.",
|
||||
}
|
||||
good["targetless short but marked"] = "[targetless] analyze nothing"
|
||||
|
||||
bad := map[string]string{
|
||||
"no target long": strings.Repeat("do a thorough security review of everything and report anything interesting that comes up during the exploration phase of this audit, thanks. ", 2),
|
||||
"too short": "audit the site",
|
||||
"short and bare": "find bugs",
|
||||
"no target long": strings.Repeat("do a thorough security review of everything and report anything interesting that comes up during the exploration phase of this audit, thanks. ", 2),
|
||||
"too short": "audit the site",
|
||||
"short and bare": "find bugs",
|
||||
}
|
||||
for name, g := range good {
|
||||
if name == "targetless short+:url" {
|
||||
|
||||
+7
-7
@@ -55,13 +55,13 @@ type Task struct {
|
||||
LLMChainRevision int64 `json:"-"`
|
||||
LLMFailoverState string `json:"llm_failover_state,omitempty"`
|
||||
// LLMRotate (#85): per-intent round-robin over the task's LLM chain heads.
|
||||
LLMRotate bool `json:"llm_rotate,omitempty"`
|
||||
rotateMu sync.Mutex
|
||||
rotateNext int
|
||||
LLMFailoverReason string `json:"llm_failover_reason,omitempty"`
|
||||
SourceTaskIDs []int64 `json:"source_task_ids,omitempty"`
|
||||
CompanyIDs []int64 `json:"company_ids,omitempty"`
|
||||
Status string `json:"status"` // persisted lifecycle status (done/failed/timeout are terminal; empty/other is derived from runtime state)
|
||||
LLMRotate bool `json:"llm_rotate,omitempty"`
|
||||
rotateMu sync.Mutex
|
||||
rotateNext int
|
||||
LLMFailoverReason string `json:"llm_failover_reason,omitempty"`
|
||||
SourceTaskIDs []int64 `json:"source_task_ids,omitempty"`
|
||||
CompanyIDs []int64 `json:"company_ids,omitempty"`
|
||||
Status string `json:"status"` // persisted lifecycle status (done/failed/timeout are terminal; empty/other is derived from runtime state)
|
||||
// Task-level timeout (see docs/task-timeout-and-settle-design.md). DeadlineAt/FirstRunAt are unix seconds, 0 = unset/not yet run.
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
PlanHeartbeatSeconds int `json:"plan_heartbeat_seconds"` // planner heartbeat trigger interval (seconds)
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
"github.com/Autumn-27/norma/harness"
|
||||
)
|
||||
|
||||
// A NOA budget refusal is deterministic: the engine must not rerun the intent,
|
||||
// whether the sentinel arrives bare or wrapped.
|
||||
func TestWorkerModelErrorRetrySkipsNoaBudget(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{"noa budget", agent.ErrNoaBudget, false},
|
||||
{"noa budget wrapped", fmt.Errorf("%w: estimate 64000 exceeds configured budget 32000", agent.ErrNoaBudget), false},
|
||||
{"noa budget wrapped twice", fmt.Errorf("worker: %w", fmt.Errorf("%w: estimate", agent.ErrNoaBudget)), false},
|
||||
{"generic model error", errors.New("connection reset by peer"), true},
|
||||
{"task router error", &taskLLMError{taskID: "7", cause: errors.New("insufficient_quota")}, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := retryableWorkerModelError(harness.ReasonModelError, tc.err); got != tc.want {
|
||||
t.Fatalf("retryableWorkerModelError(model_error, %v) = %v, want %v", tc.err, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+43
-10
@@ -823,6 +823,7 @@ func (s *Server) Handler() http.Handler {
|
||||
mux.HandleFunc("DELETE /api/traffic/hosts", s.deleteTrafficHosts)
|
||||
mux.HandleFunc("DELETE /api/traffic/all", s.deleteAllTraffic)
|
||||
mux.HandleFunc("GET /api/traffic/exchange", s.getTrafficExchange)
|
||||
mux.HandleFunc("GET /api/traffic/stream", s.getTrafficStream)
|
||||
mux.HandleFunc("GET /api/traffic/blob", s.getTrafficBlob)
|
||||
mux.HandleFunc("POST /api/traffic/ingest", s.ingestTraffic)
|
||||
mux.HandleFunc("GET /api/commands", s.pgListCommands)
|
||||
@@ -1609,12 +1610,12 @@ type createTaskReq struct {
|
||||
CategoryID *int64 `json:"category_id,omitempty"`
|
||||
Description string `json:"description"`
|
||||
Goal string `json:"goal"`
|
||||
LLMProfileID *int64 `json:"llm_profile_id,omitempty"` // the LLM profile to run this task on; omitted/null = use the active profile
|
||||
LLMProfileIDs []int64 `json:"llm_profile_ids,omitempty"` // ordered task-level profile chain; the first entry takes effect initially
|
||||
SourceTaskIDs []string `json:"source_task_ids,omitempty"` // direct, read-only inherited source tasks
|
||||
CompanyIDs []int64 `json:"company_ids,omitempty"` // links the company scope and snapshots its current assets; does not copy assets or force intent generation
|
||||
LLMProfileID *int64 `json:"llm_profile_id,omitempty"` // the LLM profile to run this task on; omitted/null = use the active profile
|
||||
LLMProfileIDs []int64 `json:"llm_profile_ids,omitempty"` // ordered task-level profile chain; the first entry takes effect initially
|
||||
SourceTaskIDs []string `json:"source_task_ids,omitempty"` // direct, read-only inherited source tasks
|
||||
CompanyIDs []int64 `json:"company_ids,omitempty"` // links the company scope and snapshots its current assets; does not copy assets or force intent generation
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
LLMRotate *bool `json:"llm_rotate"` // task-level timeout (seconds); 0/omitted = no limit
|
||||
LLMRotate *bool `json:"llm_rotate"` // task-level timeout (seconds); 0/omitted = no limit
|
||||
PlanHeartbeatSeconds int `json:"plan_heartbeat_seconds"` // planner heartbeat interval (seconds); 0/omitted = default 600 (10min); floor = default = 600, lower values are raised to 600
|
||||
PlanQuietMode *bool `json:"plan_quiet_mode,omitempty"` // planner quiet-mode switch (#86); omitted = default on; false = edge rounds always plan
|
||||
TargetGate *bool `json:"target_gate,omitempty"` // approval gate for asset targets, opt-in; omitted/null = off
|
||||
@@ -1927,13 +1928,15 @@ func (s *Server) getTask(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
dto := taskDTO(t, s.resolvedTaskStatus(t))
|
||||
metrics, err := s.m.PG().TaskListMetricsAll()
|
||||
metrics, err := s.m.PG().TaskListMetrics(t.ExpID)
|
||||
if err != nil {
|
||||
log.Printf("[tasks] detail metrics query failed for %s — metrics zeroed: %v", t.ID, err)
|
||||
}
|
||||
s.applyTaskMetrics(&dto, t, metrics[t.ExpID])
|
||||
archiveBlockers, _ := s.m.PG().TaskArchiveBlockers()
|
||||
applyTaskArchiveBlocker(&dto, archiveBlockers)
|
||||
s.applyTaskMetrics(&dto, t, metrics)
|
||||
// Best effort: only this task's blocker, not the whole relation table.
|
||||
taskID, _ := strconv.ParseInt(t.ID, 10, 64)
|
||||
blocker, _ := s.m.PG().TaskArchiveBlocker(taskID)
|
||||
applyTaskArchiveBlockerID(&dto, blocker)
|
||||
writeJSON(w, 200, dto)
|
||||
}
|
||||
|
||||
@@ -3449,7 +3452,37 @@ func (s *Server) getTrafficExchange(w http.ResponseWriter, r *http.Request) {
|
||||
writeErr(w, 404, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"req": req, "resp": resp})
|
||||
// A WebSocket or SSE exchange carries its messages separately
|
||||
// (GET /api/traffic/stream); stream_kind is "" for an ordinary exchange.
|
||||
count, kind, err := tr.StreamInfo(id)
|
||||
if err != nil {
|
||||
log.Printf("[traffic] stream info for %s: %v", id, err)
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"req": req, "resp": resp, "stream_count": count, "stream_kind": kind})
|
||||
}
|
||||
|
||||
// getTrafficStream returns one page of a WebSocket/SSE exchange's messages in
|
||||
// order (limit 1..200, default 50).
|
||||
func (s *Server) getTrafficStream(w http.ResponseWriter, r *http.Request) {
|
||||
tr := s.m.Traffic()
|
||||
if tr == nil {
|
||||
writeErr(w, 404, "traffic disabled")
|
||||
return
|
||||
}
|
||||
q := r.URL.Query()
|
||||
id := strings.TrimSpace(q.Get("exchange"))
|
||||
if id == "" {
|
||||
writeErr(w, 400, "missing exchange")
|
||||
return
|
||||
}
|
||||
offset, _ := strconv.Atoi(q.Get("offset"))
|
||||
limit, _ := strconv.Atoi(q.Get("limit"))
|
||||
msgs, total, err := tr.StreamMessages(id, offset, limit)
|
||||
if err != nil {
|
||||
writeErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"items": msgs, "total": total})
|
||||
}
|
||||
|
||||
// getTrafficBlob streams one oversized body by its sha256. Bodies past the inline
|
||||
|
||||
@@ -38,6 +38,21 @@ VALUES ($1,'goal','{}','met'), ($1,'goal','{}','open')`, task.ExpID); err != nil
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A second task, so the detail must pick its own row and the list must keep
|
||||
// serving every task.
|
||||
other, err := m.CreateTask("detail metrics other", "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
otherID, _ := strconv.ParseInt(other.ID, 10, 64)
|
||||
defer func() { _, _ = m.pg.DeleteTaskCascade(otherID, false, true) }()
|
||||
defer func() { _, _ = m.pg.Exec(`DELETE FROM findings WHERE task_id=$1`, taskID) }()
|
||||
if _, err := m.pg.Exec(`
|
||||
INSERT INTO findings(task_id, severity) VALUES ($1,'critical'), ($1,'high'), ($1,'high'), ($1,'low'), ($2,'medium')`,
|
||||
taskID, otherID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
|
||||
defer stopTestServer(t, s, cancel)
|
||||
@@ -45,7 +60,19 @@ VALUES ($1,'goal','{}','met'), ($1,'goal','{}','open')`, task.ExpID); err != nil
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// After New: restoring the task resets running intents to open.
|
||||
if _, err := m.pg.Exec(`
|
||||
INSERT INTO exploration_nodes(exploration_id, kind, payload, state)
|
||||
VALUES ($1,'intent','{}','running'), ($1,'intent','{}','done')`, task.ExpID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
type findingCounts struct {
|
||||
Critical int `json:"critical"`
|
||||
High int `json:"high"`
|
||||
Medium int `json:"medium"`
|
||||
Low int `json:"low"`
|
||||
}
|
||||
var probe struct {
|
||||
Tokens struct {
|
||||
InputTokens int `json:"input_tokens"`
|
||||
@@ -53,8 +80,10 @@ VALUES ($1,'goal','{}','met'), ($1,'goal','{}','open')`, task.ExpID); err != nil
|
||||
CacheReadTokens int `json:"cache_read_tokens"`
|
||||
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||
} `json:"tokens"`
|
||||
GoalsMet int `json:"goals_met"`
|
||||
GoalsTotal int `json:"goals_total"`
|
||||
GoalsMet int `json:"goals_met"`
|
||||
GoalsTotal int `json:"goals_total"`
|
||||
InFlight int `json:"in_flight"`
|
||||
Findings findingCounts `json:"findings"`
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/tasks/"+task.ID, nil)
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
@@ -73,6 +102,9 @@ VALUES ($1,'goal','{}','met'), ($1,'goal','{}','open')`, task.ExpID); err != nil
|
||||
if probe.GoalsMet != 1 || probe.GoalsTotal != 2 {
|
||||
t.Fatalf("detail goal progress not filled from metrics: %d/%d", probe.GoalsMet, probe.GoalsTotal)
|
||||
}
|
||||
if probe.InFlight != 1 || probe.Findings != (findingCounts{Critical: 1, High: 2, Low: 1}) {
|
||||
t.Fatalf("detail in-flight/findings not filled from this task's metrics: %d %+v", probe.InFlight, probe.Findings)
|
||||
}
|
||||
|
||||
// The list endpoint must report the same numbers for the same task.
|
||||
listReq := httptest.NewRequest(http.MethodGet, "/api/tasks", nil)
|
||||
@@ -89,27 +121,32 @@ VALUES ($1,'goal','{}','met'), ($1,'goal','{}','open')`, task.ExpID); err != nil
|
||||
InputTokens int `json:"input_tokens"`
|
||||
OutputTokens int `json:"output_tokens"`
|
||||
} `json:"tokens"`
|
||||
GoalsMet int `json:"goals_met"`
|
||||
GoalsTotal int `json:"goals_total"`
|
||||
GoalsMet int `json:"goals_met"`
|
||||
GoalsTotal int `json:"goals_total"`
|
||||
InFlight int `json:"in_flight"`
|
||||
Findings findingCounts `json:"findings"`
|
||||
} `json:"tasks"`
|
||||
}
|
||||
if err := json.Unmarshal(listRec.Body.Bytes(), &listed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
found, foundOther := false, false
|
||||
for _, lt := range listed.Tasks {
|
||||
if lt.ID == other.ID {
|
||||
foundOther = lt.Findings == (findingCounts{Medium: 1})
|
||||
}
|
||||
if lt.ID != task.ID {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if lt.Tokens.InputTokens != probe.Tokens.InputTokens ||
|
||||
lt.Tokens.OutputTokens != probe.Tokens.OutputTokens ||
|
||||
lt.GoalsMet != probe.GoalsMet || lt.GoalsTotal != probe.GoalsTotal {
|
||||
t.Fatalf("list and detail metrics disagree: list=%+v/%d detail=%+v/%d",
|
||||
lt.Tokens, lt.GoalsMet, probe.Tokens, probe.GoalsTotal)
|
||||
lt.GoalsMet != probe.GoalsMet || lt.GoalsTotal != probe.GoalsTotal ||
|
||||
lt.InFlight != probe.InFlight || lt.Findings != probe.Findings {
|
||||
t.Fatalf("list and detail metrics disagree: list=%+v detail=%+v", lt, probe)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("task %s missing from list response", task.ID)
|
||||
if !found || !foundOther {
|
||||
t.Fatalf("list misses a task or its metrics: task %s found=%v, task %s with its findings=%v", task.ID, found, other.ID, foundOther)
|
||||
}
|
||||
}
|
||||
|
||||
+75
-19
@@ -317,10 +317,14 @@ func (r *taskLLMRuntime) Complete(ctx context.Context, req llm.CompletionRequest
|
||||
}
|
||||
|
||||
func completeTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest, hooks taskLLMStreamHooks) (llm.Message, string, llm.Usage, error) {
|
||||
// total is the usage of the logical call: the sum of every physical attempt
|
||||
// (same-provider retries and profile failover). Each attempt is already billed
|
||||
// once by the recorder below; this sum only feeds the caller's accounting.
|
||||
var total llm.Usage
|
||||
for {
|
||||
selection, err := hooks.current(ctx)
|
||||
if err != nil {
|
||||
return llm.Message{}, "", llm.Usage{}, err
|
||||
return llm.Message{}, "", total, err
|
||||
}
|
||||
var (
|
||||
msg llm.Message
|
||||
@@ -333,13 +337,14 @@ func completeTaskLLM(ctx context.Context, taskID string, req llm.CompletionReque
|
||||
retries, backoffOf := sameProviderRetryPolicy(selection.retry)
|
||||
for attempt := 0; ; attempt++ {
|
||||
msg, sr, usage, callErr = selection.provider.Complete(ctx, req)
|
||||
total.Add(usage)
|
||||
if callErr != nil && ctx.Err() == nil &&
|
||||
attempt < retries && isRetryableStreamError(callErr) {
|
||||
backoff := backoffOf(attempt)
|
||||
log.Printf("[task-llm] task %s non-streaming call failed, retrying on the same provider after %v (%d/%d): %v",
|
||||
taskID, backoff, attempt+1, retries, callErr)
|
||||
if sleepCtx(ctx, backoff) {
|
||||
break // ctx cancelled during backoff → stop retrying
|
||||
return llm.Message{}, "", total, ctx.Err() // cancelled during backoff → stop
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -347,7 +352,7 @@ func completeTaskLLM(ctx context.Context, taskID string, req llm.CompletionReque
|
||||
}
|
||||
if callErr == nil {
|
||||
resetLLMFailStreak(taskID, selection.profileID)
|
||||
return msg, sr, usage, nil
|
||||
return msg, sr, total, nil
|
||||
}
|
||||
// Non-quota failures on a chain profile escalate after a consecutive
|
||||
// streak (#99): same advance path as quota exhaust, truthful reason.
|
||||
@@ -356,31 +361,31 @@ func completeTaskLLM(ctx context.Context, taskID string, req llm.CompletionReque
|
||||
cause := fmt.Errorf("profile #%d failed %d consecutive calls (last: %w)", selection.profileID, llmFailStreakThreshold, callErr)
|
||||
transition, markErr := hooks.exhaust(selection, cause)
|
||||
if markErr != nil {
|
||||
return llm.Message{}, "", llm.Usage{}, fmt.Errorf("mark profile exhausted after %v: %w", callErr, markErr)
|
||||
return llm.Message{}, "", total, fmt.Errorf("mark profile exhausted after %v: %w", callErr, markErr)
|
||||
}
|
||||
if transition.Advanced && !transition.Stale && hooks.transition != nil {
|
||||
hooks.transition(selection, transition, callErr)
|
||||
}
|
||||
if !transition.Stale && transition.NextProfileID == nil {
|
||||
return llm.Message{}, "", llm.Usage{}, &taskLLMError{taskID: taskID, chainExhausted: transition.ChainExhausted, cause: callErr}
|
||||
return llm.Message{}, "", total, &taskLLMError{taskID: taskID, chainExhausted: transition.ChainExhausted, cause: callErr}
|
||||
}
|
||||
continue // replay the same logical request on the next profile
|
||||
}
|
||||
return llm.Message{}, "", llm.Usage{}, callErr
|
||||
return llm.Message{}, "", total, callErr
|
||||
}
|
||||
// profileID=0: the explicit chain was cleared; non-quota error: pass through. Neither changes the task failover state.
|
||||
if selection.profileID == 0 || !isQuotaExhaustedError(callErr) {
|
||||
return llm.Message{}, "", llm.Usage{}, callErr
|
||||
return llm.Message{}, "", total, callErr
|
||||
}
|
||||
transition, markErr := hooks.exhaust(selection, callErr)
|
||||
if markErr != nil {
|
||||
return llm.Message{}, "", llm.Usage{}, fmt.Errorf("mark profile quota exhausted after %v: %w", callErr, markErr)
|
||||
return llm.Message{}, "", total, fmt.Errorf("mark profile quota exhausted after %v: %w", callErr, markErr)
|
||||
}
|
||||
if transition.Advanced && !transition.Stale && hooks.transition != nil {
|
||||
hooks.transition(selection, transition, callErr)
|
||||
}
|
||||
if !transition.Stale && transition.NextProfileID == nil {
|
||||
return llm.Message{}, "", llm.Usage{}, &taskLLMError{taskID: taskID, chainExhausted: transition.ChainExhausted, cause: callErr}
|
||||
return llm.Message{}, "", total, &taskLLMError{taskID: taskID, chainExhausted: transition.ChainExhausted, cause: callErr}
|
||||
}
|
||||
// No output has been delivered to the caller, so replaying the same logical request on the next profile is safe.
|
||||
}
|
||||
@@ -397,6 +402,35 @@ func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest
|
||||
committed := false
|
||||
var pending []llm.StreamEvent
|
||||
var streamErr error
|
||||
// The caller's llm.Accumulator sums usage across the whole logical call
|
||||
// (every attempt and profile): message_start is added, message_delta is
|
||||
// folded with MergeEcho. Folding one attempt's echo against the sum of
|
||||
// earlier attempts would lose input/cache tokens, so each attempt's echoes
|
||||
// are normalized here against that attempt alone and only the increment
|
||||
// is passed on, as a message_start. The recorder below already billed the
|
||||
// physical attempt; this only shapes what the caller adds up.
|
||||
var attemptUsage llm.Usage
|
||||
forward := func(event llm.StreamEvent) bool {
|
||||
switch event.Type {
|
||||
case llm.SEMessageStart:
|
||||
attemptUsage.Add(event.Usage)
|
||||
case llm.SEMessageDelta:
|
||||
next := attemptUsage
|
||||
next.MergeEcho(event.Usage)
|
||||
inc := llm.Usage{
|
||||
InputTokens: next.InputTokens - attemptUsage.InputTokens,
|
||||
OutputTokens: next.OutputTokens - attemptUsage.OutputTokens,
|
||||
CacheReadTokens: next.CacheReadTokens - attemptUsage.CacheReadTokens,
|
||||
CacheWriteTokens: next.CacheWriteTokens - attemptUsage.CacheWriteTokens,
|
||||
}
|
||||
attemptUsage = next
|
||||
if inc != (llm.Usage{}) && !yield(llm.StreamEvent{Type: llm.SEMessageStart, Usage: inc}, nil) {
|
||||
return false
|
||||
}
|
||||
event.Usage = llm.Usage{}
|
||||
}
|
||||
return yield(event, nil)
|
||||
}
|
||||
retries, backoffOf := sameProviderRetryPolicy(selection.retry)
|
||||
// Same-provider safe-window retry: a transient failure before committed
|
||||
// (nothing delivered to the caller yet) can be replayed verbatim without
|
||||
@@ -405,6 +439,7 @@ func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest
|
||||
committed = false
|
||||
pending = nil
|
||||
streamErr = nil
|
||||
attemptUsage = llm.Usage{}
|
||||
for event, err := range selection.provider.Stream(ctx, req) {
|
||||
if err != nil {
|
||||
streamErr = err
|
||||
@@ -416,24 +451,34 @@ func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest
|
||||
}
|
||||
if !committed {
|
||||
for _, buffered := range pending {
|
||||
if !yield(buffered, nil) {
|
||||
if !forward(buffered) {
|
||||
return
|
||||
}
|
||||
}
|
||||
pending = nil
|
||||
committed = true
|
||||
}
|
||||
if !yield(event, nil) {
|
||||
if !forward(event) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if streamErr != nil && !committed {
|
||||
// A failed uncommitted attempt delivers none of its events (they are
|
||||
// never replayed), but the usage it reported survives as one
|
||||
// message_start so the logical call still counts it.
|
||||
if u := pendingUsage(pending); u != (llm.Usage{}) && !yield(llm.StreamEvent{Type: llm.SEMessageStart, Usage: u}, nil) {
|
||||
return
|
||||
}
|
||||
pending = nil
|
||||
}
|
||||
if streamErr != nil && !committed && ctx.Err() == nil &&
|
||||
attempt < retries && isRetryableStreamError(streamErr) {
|
||||
backoff := backoffOf(attempt)
|
||||
log.Printf("[task-llm] task %s pre-commit stream failure, retrying on the same provider after %v (%d/%d): %v",
|
||||
taskID, backoff, attempt+1, retries, streamErr)
|
||||
if sleepCtx(ctx, backoff) {
|
||||
break // ctx cancelled during backoff → stop retrying
|
||||
yield(llm.StreamEvent{}, ctx.Err()) // cancelled during backoff → stop
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -442,7 +487,7 @@ func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest
|
||||
if streamErr == nil {
|
||||
resetLLMFailStreak(taskID, selection.profileID)
|
||||
for _, buffered := range pending {
|
||||
if !yield(buffered, nil) {
|
||||
if !forward(buffered) {
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -474,11 +519,6 @@ func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest
|
||||
// bundle was still in use. Agent/global fallback errors follow the legacy
|
||||
// behavior and never mutate task failover state.
|
||||
if selection.profileID == 0 || !isQuotaExhaustedError(streamErr) {
|
||||
for _, buffered := range pending {
|
||||
if !yield(buffered, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
yield(llm.StreamEvent{}, streamErr)
|
||||
return
|
||||
}
|
||||
@@ -508,9 +548,25 @@ func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest
|
||||
}
|
||||
}
|
||||
|
||||
// pendingUsage is the usage the buffered (uncommitted) events of one attempt
|
||||
// reported. Only message_start can be buffered: message_delta commits.
|
||||
func pendingUsage(pending []llm.StreamEvent) llm.Usage {
|
||||
var u llm.Usage
|
||||
for _, ev := range pending {
|
||||
if ev.Type == llm.SEMessageStart {
|
||||
u.Add(ev.Usage)
|
||||
}
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// streamEventCommitsOutput reports whether delivering event makes the attempt
|
||||
// unreplayable. Signed and redacted thinking and tool-use signatures are opaque
|
||||
// output the model bound to this exact response, so they commit like text.
|
||||
func streamEventCommitsOutput(event llm.StreamEvent) bool {
|
||||
switch event.Type {
|
||||
case llm.SETextDelta, llm.SEThinkingDelta, llm.SEToolInputJSON:
|
||||
case llm.SETextDelta, llm.SEThinkingDelta, llm.SEToolInputJSON,
|
||||
llm.SEThinkingSignature, llm.SERedactedThinking, llm.SEToolUseSignature:
|
||||
return event.Text != ""
|
||||
case llm.SEToolUseStart, llm.SEMessageDelta, llm.SEMessageStop:
|
||||
return true
|
||||
|
||||
@@ -77,4 +77,3 @@ func TestEscalationAdvancesAfterConsecutiveFailures(t *testing.T) {
|
||||
t.Fatalf("exhaust calls=%d want 2", exhaustCalls)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -184,6 +184,18 @@ func TestApplyTaskArchiveBlocker(t *testing.T) {
|
||||
if !strings.Contains(string(raw), `"archive_blocked_by_task_id":"11"`) {
|
||||
t.Fatalf("archive blocker missing from payload: %s", raw)
|
||||
}
|
||||
|
||||
byID := taskDTO(&Task{ID: "7"}, "paused")
|
||||
applyTaskArchiveBlockerID(&byID, 12)
|
||||
if byID.ArchiveBlockedBy != "12" {
|
||||
t.Fatalf("archive blocker by id=%q, want 12", byID.ArchiveBlockedBy)
|
||||
}
|
||||
none := taskDTO(&Task{ID: "7"}, "paused")
|
||||
applyTaskArchiveBlockerID(&none, 0)
|
||||
if none.ArchiveBlockedBy != "" {
|
||||
t.Fatalf("no blocker set %q", none.ArchiveBlockedBy)
|
||||
}
|
||||
applyTaskArchiveBlockerID(nil, 12) // must not panic
|
||||
}
|
||||
|
||||
func TestTaskLLMStateSnapshotIsConcurrentAndDetached(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"iter"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/llmrec"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
)
|
||||
|
||||
// Usage reported by one failed attempt, and by a successful one. A stream
|
||||
// reports the input side in message_start and echoes it, with the output, in
|
||||
// message_delta; the atomic path returns the attempt's normalized usage.
|
||||
var (
|
||||
failedAttemptUsage = llm.Usage{InputTokens: 11, OutputTokens: 2, CacheReadTokens: 3, CacheWriteTokens: 2}
|
||||
okStartUsage = llm.Usage{InputTokens: 13, CacheReadTokens: 3, CacheWriteTokens: 2}
|
||||
okDeltaUsage = llm.Usage{InputTokens: 17, OutputTokens: 5, CacheReadTokens: 4, CacheWriteTokens: 2}
|
||||
okAttemptUsage = llm.Usage{InputTokens: 17, OutputTokens: 5, CacheReadTokens: 4, CacheWriteTokens: 2}
|
||||
)
|
||||
|
||||
// usageAttemptProvider fails its first fail attempts (always, when fail < 0)
|
||||
// after reporting failedAttemptUsage, then answers "done".
|
||||
type usageAttemptProvider struct {
|
||||
fail int
|
||||
failErr error
|
||||
onFail func()
|
||||
// extra events a failed stream attempt emits before its error.
|
||||
failEvents []llm.StreamEvent
|
||||
calls int
|
||||
}
|
||||
|
||||
func (p *usageAttemptProvider) failing() bool {
|
||||
p.calls++
|
||||
failed := p.fail < 0 || p.calls <= p.fail
|
||||
if failed && p.onFail != nil {
|
||||
p.onFail()
|
||||
}
|
||||
return failed
|
||||
}
|
||||
|
||||
func (p *usageAttemptProvider) Complete(context.Context, llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
|
||||
if p.failing() {
|
||||
return llm.Message{}, "", failedAttemptUsage, p.failErr
|
||||
}
|
||||
return llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{llm.TextBlock("done")}}, "end_turn", okAttemptUsage, nil
|
||||
}
|
||||
|
||||
func (p *usageAttemptProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
||||
return func(yield func(llm.StreamEvent, error) bool) {
|
||||
if p.failing() {
|
||||
events := append([]llm.StreamEvent{{Type: llm.SEMessageStart, Usage: failedAttemptUsage}}, p.failEvents...)
|
||||
for _, ev := range events {
|
||||
if !yield(ev, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
yield(llm.StreamEvent{}, p.failErr)
|
||||
return
|
||||
}
|
||||
for _, ev := range []llm.StreamEvent{
|
||||
{Type: llm.SEMessageStart, Usage: okStartUsage},
|
||||
{Type: llm.SETextDelta, Text: "done"},
|
||||
{Type: llm.SEMessageDelta, StopReason: "end_turn", Usage: okDeltaUsage},
|
||||
{Type: llm.SEMessageStop},
|
||||
} {
|
||||
if !yield(ev, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// logicalResult is what the SDK sees for one logical call.
|
||||
type logicalResult struct {
|
||||
usage llm.Usage
|
||||
text string
|
||||
events []llm.StreamEvent
|
||||
err error
|
||||
}
|
||||
|
||||
func runLogicalCall(ctx context.Context, stream bool, taskID string, hooks taskLLMStreamHooks) logicalResult {
|
||||
if !stream {
|
||||
msg, _, usage, err := completeTaskLLM(ctx, taskID, llm.CompletionRequest{}, hooks)
|
||||
return logicalResult{usage: usage, text: msg.Text(), err: err}
|
||||
}
|
||||
var r logicalResult
|
||||
acc := llm.NewAccumulator()
|
||||
for ev, err := range streamTaskLLM(ctx, taskID, llm.CompletionRequest{}, hooks) {
|
||||
if err != nil {
|
||||
r.err = err
|
||||
break
|
||||
}
|
||||
r.events = append(r.events, ev)
|
||||
acc.Add(ev)
|
||||
}
|
||||
r.usage, r.text = acc.Usage, acc.Message().Text()
|
||||
return r
|
||||
}
|
||||
|
||||
func singleProfileHooks(p llm.Provider, retry agent.RetryConfig) taskLLMStreamHooks {
|
||||
return taskLLMStreamHooks{
|
||||
current: func(context.Context) (taskLLMSelection, error) {
|
||||
return taskLLMSelection{provider: p, retry: retry}, nil
|
||||
},
|
||||
exhaust: func(taskLLMSelection, error) (db.TaskLLMTransition, error) {
|
||||
return db.TaskLLMTransition{}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func sumUsage(us ...llm.Usage) llm.Usage {
|
||||
var t llm.Usage
|
||||
for _, u := range us {
|
||||
t.Add(u)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
var usageModes = []struct {
|
||||
name string
|
||||
stream bool
|
||||
}{{"atomic", false}, {"stream", true}}
|
||||
|
||||
func TestTaskLLMRetrySumsAttemptUsage(t *testing.T) {
|
||||
for _, m := range usageModes {
|
||||
t.Run(m.name, func(t *testing.T) {
|
||||
p := &usageAttemptProvider{fail: 1, failErr: errors.New("connection reset by peer")}
|
||||
r := runLogicalCall(t.Context(), m.stream, "p12-retry", singleProfileHooks(p, agent.RetryConfig{StreamAttempts: 1, StreamInterval: time.Millisecond}))
|
||||
if r.err != nil {
|
||||
t.Fatal(r.err)
|
||||
}
|
||||
want := sumUsage(failedAttemptUsage, okAttemptUsage) // {28 7 7 4}
|
||||
if r.usage != want || p.calls != 2 || r.text != "done" {
|
||||
t.Fatalf("usage=%+v want %+v, calls=%d, text=%q", r.usage, want, p.calls, r.text)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskLLMExhaustedRetriesReturnSummedUsage(t *testing.T) {
|
||||
for _, m := range usageModes {
|
||||
t.Run(m.name, func(t *testing.T) {
|
||||
p := &usageAttemptProvider{fail: -1, failErr: errors.New("connection reset by peer")}
|
||||
r := runLogicalCall(t.Context(), m.stream, "p12-exhausted", singleProfileHooks(p, agent.RetryConfig{StreamAttempts: 1, StreamInterval: time.Millisecond}))
|
||||
if r.err == nil || r.err.Error() != "connection reset by peer" {
|
||||
t.Fatalf("err=%v", r.err)
|
||||
}
|
||||
want := sumUsage(failedAttemptUsage, failedAttemptUsage)
|
||||
if r.usage != want || p.calls != 2 {
|
||||
t.Fatalf("usage=%+v want %+v, calls=%d", r.usage, want, p.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskLLMProfileFailoverSumsUsage(t *testing.T) {
|
||||
for _, m := range usageModes {
|
||||
t.Run(m.name, func(t *testing.T) {
|
||||
first := &usageAttemptProvider{fail: -1, failErr: errors.New("openai: status 402: insufficient_quota")}
|
||||
second := &usageAttemptProvider{}
|
||||
active := int64(11)
|
||||
hooks := taskLLMStreamHooks{
|
||||
current: func(context.Context) (taskLLMSelection, error) {
|
||||
if active == 11 {
|
||||
return taskLLMSelection{profileID: 11, provider: first}, nil
|
||||
}
|
||||
return taskLLMSelection{profileID: 22, provider: second}, nil
|
||||
},
|
||||
exhaust: func(taskLLMSelection, error) (db.TaskLLMTransition, error) {
|
||||
active = 22
|
||||
next := active
|
||||
return db.TaskLLMTransition{PreviousProfileID: 11, NextProfileID: &next, Advanced: true}, nil
|
||||
},
|
||||
}
|
||||
r := runLogicalCall(t.Context(), m.stream, "p12-failover-"+m.name, hooks)
|
||||
if r.err != nil {
|
||||
t.Fatal(r.err)
|
||||
}
|
||||
want := sumUsage(failedAttemptUsage, okAttemptUsage)
|
||||
if r.usage != want || first.calls != 1 || second.calls != 1 || r.text != "done" {
|
||||
t.Fatalf("usage=%+v want %+v, calls first=%d second=%d, text=%q", r.usage, want, first.calls, second.calls, r.text)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskLLMCancelDuringBackoffStops(t *testing.T) {
|
||||
for _, m := range usageModes {
|
||||
t.Run(m.name, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
p := &usageAttemptProvider{fail: -1, failErr: errors.New("connection reset by peer")}
|
||||
// Cancel once the retry is waiting out its (long) backoff.
|
||||
p.onFail = func() { time.AfterFunc(20*time.Millisecond, cancel) }
|
||||
r := runLogicalCall(ctx, m.stream, "p12-cancel", singleProfileHooks(p, agent.RetryConfig{StreamAttempts: 2, StreamInterval: time.Minute}))
|
||||
if !errors.Is(r.err, context.Canceled) {
|
||||
t.Fatalf("err=%v, want context.Canceled", r.err)
|
||||
}
|
||||
if p.calls != 1 || r.usage != failedAttemptUsage {
|
||||
t.Fatalf("calls=%d usage=%+v", p.calls, r.usage)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskLLMUncommittedStreamAttemptKeepsUsageWithoutReplay(t *testing.T) {
|
||||
p := &usageAttemptProvider{
|
||||
fail: 1,
|
||||
failErr: errors.New("anthropic: overloaded_error"),
|
||||
// Buffered, non-committing events of the failed attempt.
|
||||
failEvents: []llm.StreamEvent{{Type: llm.SEThinkingDelta}, {Type: llm.SEToolInputJSON}},
|
||||
}
|
||||
r := runLogicalCall(t.Context(), true, "p12-uncommitted", singleProfileHooks(p, agent.RetryConfig{StreamAttempts: 1, StreamInterval: time.Millisecond}))
|
||||
if r.err != nil {
|
||||
t.Fatal(r.err)
|
||||
}
|
||||
if want := sumUsage(failedAttemptUsage, okAttemptUsage); r.usage != want {
|
||||
t.Fatalf("usage=%+v want %+v", r.usage, want)
|
||||
}
|
||||
if r.text != "done" {
|
||||
t.Fatalf("text=%q, want the second attempt's only", r.text)
|
||||
}
|
||||
// The failed attempt contributes one usage-only message_start, nothing else.
|
||||
if len(r.events) == 0 || r.events[0].Type != llm.SEMessageStart || r.events[0].Usage != failedAttemptUsage {
|
||||
t.Fatalf("first event %+v", r.events)
|
||||
}
|
||||
for _, ev := range r.events {
|
||||
if ev.Type == llm.SEThinkingDelta || ev.Type == llm.SEToolInputJSON {
|
||||
t.Fatalf("buffered event of the failed attempt was delivered: %+v", r.events)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskLLMOpaqueThinkingCommitsStream(t *testing.T) {
|
||||
defer withZeroRetryBackoff()()
|
||||
for _, kind := range []llm.StreamEventType{llm.SEThinkingSignature, llm.SERedactedThinking, llm.SEToolUseSignature} {
|
||||
t.Run(string(kind), func(t *testing.T) {
|
||||
if !streamEventCommitsOutput(llm.StreamEvent{Type: kind, Text: "opaque"}) {
|
||||
t.Fatalf("%s does not commit", kind)
|
||||
}
|
||||
// A retryable failure after the opaque block is surfaced, not replayed.
|
||||
p := &scriptedLLMProvider{events: []llm.StreamEvent{{Type: kind, Text: "opaque"}}, err: errors.New("connection reset by peer")}
|
||||
events, err := collectTaskLLMStream(streamTaskLLM(t.Context(), "p12-opaque", llm.CompletionRequest{}, singleProfileHooks(p, agent.RetryConfig{})))
|
||||
if err == nil || p.calls != 1 || len(events) != 1 || events[0].Type != kind {
|
||||
t.Fatalf("calls=%d events=%+v err=%v", p.calls, events, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The recorder below the task router writes one llm_usage row per physical
|
||||
// attempt; summing the attempts for the logical call adds no row.
|
||||
func TestTaskLLMRetryKeepsOneLedgerRowPerPhysicalCall(t *testing.T) {
|
||||
m, err := NewManager(t.TempDir(), "")
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
defer m.Close()
|
||||
pg := m.PG()
|
||||
for _, mode := range usageModes {
|
||||
t.Run(mode.name, func(t *testing.T) {
|
||||
taskID := fmt.Sprintf("p12-ledger-%s-%d", mode.name, time.Now().UnixNano())
|
||||
t.Cleanup(func() { _, _ = pg.Exec(`DELETE FROM llm_usage WHERE task_id=$1`, taskID) })
|
||||
p := &usageAttemptProvider{fail: 1, failErr: errors.New("connection reset by peer")}
|
||||
rec := llmrec.Wrap(p, pg, "test-model", "test-profile", "", "", func() bool { return false })
|
||||
ctx := llmrec.WithTaskID(t.Context(), taskID)
|
||||
r := runLogicalCall(ctx, mode.stream, taskID, singleProfileHooks(rec, agent.RetryConfig{StreamAttempts: 1, StreamInterval: time.Millisecond}))
|
||||
if r.err != nil {
|
||||
t.Fatal(r.err)
|
||||
}
|
||||
var rows, failed int
|
||||
if err := pg.QueryRow(`SELECT count(*), count(*) FILTER (WHERE status='error') FROM llm_usage WHERE task_id=$1`, taskID).Scan(&rows, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows != 2 || failed != 1 || p.calls != 2 {
|
||||
t.Fatalf("llm_usage rows=%d (error %d), physical calls=%d; want 2 rows, 1 error", rows, failed, p.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -5,9 +5,9 @@ import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
actool "github.com/Autumn-27/norma/tool"
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
"github.com/Autumn-27/artex/db"
|
||||
actool "github.com/Autumn-27/norma/tool"
|
||||
)
|
||||
|
||||
func names(tools []actool.CoreTool) map[string]actool.CoreTool {
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTrafficStreamRoute(t *testing.T) {
|
||||
s, request := ingestServer(t)
|
||||
db := s.m.Traffic().DB()
|
||||
if _, err := db.Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES('1-0001',1,'ws.example.com','GET','/ws','wss://ws.example.com/ws',101,'websocket',0,0,'')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.Exec(`INSERT INTO exchange_bodies(id,req_head,resp_head) VALUES('1-0001','GET /ws HTTP/1.1\n','HTTP/1.1 101 Switching Protocols\n')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const total = 250
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 1; i <= total; i++ {
|
||||
kind, payload := 1, []byte("hello")
|
||||
if i%2 == 0 {
|
||||
kind, payload = 2, []byte{0, 1, 2}
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload) VALUES('1-0001',?,1,'c2s',?,?,?)`, i, kind, len(payload), payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
type page struct {
|
||||
Items []struct {
|
||||
Seq int64 `json:"seq"`
|
||||
Type string `json:"type"`
|
||||
Payload string `json:"payload"`
|
||||
Encoding string `json:"encoding"`
|
||||
} `json:"items"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
get := func(path string, wantStatus int) page {
|
||||
t.Helper()
|
||||
w := request("GET", path, "")
|
||||
if w.Code != wantStatus {
|
||||
t.Fatalf("GET %s = %d %s, want %d", path, w.Code, w.Body, wantStatus)
|
||||
}
|
||||
var p page
|
||||
if wantStatus == 200 {
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &p); err != nil {
|
||||
t.Fatalf("GET %s: %v in %s", path, err, w.Body)
|
||||
}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
get("/api/traffic/stream", 400)
|
||||
if p := get("/api/traffic/stream?exchange=nope", 200); p.Items == nil || len(p.Items) != 0 || p.Total != 0 {
|
||||
t.Fatalf("unknown exchange = %+v, want items [] and total 0", p)
|
||||
}
|
||||
p := get("/api/traffic/stream?exchange=1-0001", 200)
|
||||
if p.Total != total || len(p.Items) != 50 || p.Items[0].Seq != 1 {
|
||||
t.Fatalf("default page: total %d, %d items, first seq %d", p.Total, len(p.Items), p.Items[0].Seq)
|
||||
}
|
||||
if first, second := p.Items[0], p.Items[1]; first.Type != "text" || first.Payload != "hello" || second.Type != "binary" || second.Encoding != "base64" || second.Payload != "AAEC" {
|
||||
t.Fatalf("payloads %+v %+v", first, second)
|
||||
}
|
||||
if p := get("/api/traffic/stream?exchange=1-0001&limit=500", 200); len(p.Items) != 200 {
|
||||
t.Fatalf("limit=500 returned %d items, want the 200 cap", len(p.Items))
|
||||
}
|
||||
if p := get("/api/traffic/stream?exchange=1-0001&offset=-5&limit=1", 200); len(p.Items) != 1 || p.Items[0].Seq != 1 {
|
||||
t.Fatalf("negative offset: %+v, want seq 1", p.Items)
|
||||
}
|
||||
if p := get("/api/traffic/stream?exchange=1-0001&offset=248&limit=10", 200); len(p.Items) != 2 || p.Items[0].Seq != 249 {
|
||||
t.Fatalf("last page: %+v, want seqs 249 and 250", p.Items)
|
||||
}
|
||||
|
||||
w := request("GET", "/api/traffic/exchange?id=1-0001", "")
|
||||
var ex struct {
|
||||
StreamCount int `json:"stream_count"`
|
||||
StreamKind string `json:"stream_kind"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &ex); err != nil || w.Code != 200 || ex.StreamCount != total || ex.StreamKind != "websocket" {
|
||||
t.Fatalf("exchange = %d %s", w.Code, w.Body)
|
||||
}
|
||||
}
|
||||
@@ -32,8 +32,11 @@ func (s SideQuestionService) Answer(ctx context.Context, req llm.CompletionReque
|
||||
out.Text += ev.Text
|
||||
case llm.SEToolUseStart:
|
||||
out.ToolUse = true
|
||||
case llm.SEMessageStart, llm.SEMessageDelta:
|
||||
case llm.SEMessageStart:
|
||||
out.Usage.Add(ev.Usage)
|
||||
case llm.SEMessageDelta:
|
||||
// message_delta may echo message_start's input side: MergeEcho, not Add.
|
||||
out.Usage.MergeEcho(ev.Usage)
|
||||
case llm.SEMessageStop:
|
||||
complete = true
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package sidequestion
|
||||
|
||||
import (
|
||||
"context"
|
||||
"iter"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
)
|
||||
|
||||
// A streamed side answer counts message_delta's echo of the input side once.
|
||||
func TestAnswerStreamUsageEchoCountedOnce(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
start, delta llm.Usage
|
||||
want llm.Usage
|
||||
}{
|
||||
{"full_echo",
|
||||
llm.Usage{InputTokens: 23, CacheReadTokens: 5, CacheWriteTokens: 3},
|
||||
llm.Usage{InputTokens: 23, CacheReadTokens: 5, CacheWriteTokens: 3, OutputTokens: 5},
|
||||
llm.Usage{InputTokens: 23, OutputTokens: 5, CacheReadTokens: 5, CacheWriteTokens: 3}},
|
||||
{"partial_echo",
|
||||
llm.Usage{InputTokens: 740, CacheReadTokens: 200, CacheWriteTokens: 40},
|
||||
llm.Usage{InputTokens: 500, OutputTokens: 77},
|
||||
llm.Usage{InputTokens: 740, OutputTokens: 77, CacheReadTokens: 200, CacheWriteTokens: 40}},
|
||||
{"no_echo",
|
||||
llm.Usage{InputTokens: 23, CacheReadTokens: 5, CacheWriteTokens: 3},
|
||||
llm.Usage{OutputTokens: 5},
|
||||
llm.Usage{InputTokens: 23, OutputTokens: 5, CacheReadTokens: 5, CacheWriteTokens: 3}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
p := fakeProvider{stream: func(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
||||
return func(yield func(llm.StreamEvent, error) bool) {
|
||||
for _, ev := range []llm.StreamEvent{
|
||||
{Type: llm.SEMessageStart, Usage: tc.start},
|
||||
{Type: llm.SETextDelta, Text: "answer"},
|
||||
{Type: llm.SEMessageDelta, StopReason: "end_turn", Usage: tc.delta},
|
||||
{Type: llm.SEMessageStop},
|
||||
} {
|
||||
if !yield(ev, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}}
|
||||
out, err := (SideQuestionService{Provider: p}).Answer(t.Context(), llm.CompletionRequest{}, true, nil)
|
||||
if err != nil || out.Text != "answer" || out.Usage != tc.want {
|
||||
t.Fatalf("answer=%+v err=%v, want usage %+v", out, err, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,588 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
"unsafe"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
|
||||
)
|
||||
|
||||
// Stream recording (WebSocket and Server-Sent Events). go-mitmproxy reports a
|
||||
// stream as Start, one Message hook per message, then End; a stream never reaches
|
||||
// the Response hook, so record() never sees it. Each stream becomes one exchange
|
||||
// (status 101 "websocket", or the real text/event-stream response) whose messages
|
||||
// go to stream_messages in arrival order.
|
||||
|
||||
const (
|
||||
// wsText and wsBinary are the WebSocket data opcodes (RFC 6455 §5.2), the
|
||||
// values go-mitmproxy reports in WebSocketMessage.Type.
|
||||
wsText = 1
|
||||
wsBinary = 2
|
||||
// streamPreview caps one message's payload in a StreamMessages row; a longer
|
||||
// payload lives in the blob store and the row carries its hash.
|
||||
streamPreview = 4 * 1024
|
||||
// maxStreamInline is the inline threshold for message payloads: anything
|
||||
// longer spills to the blob store, so a page of 200 rows stays bounded.
|
||||
maxStreamInline = streamPreview
|
||||
// toolStreamPreview caps one message's preview in traffic_messages, so a page
|
||||
// of messages stays small in the agent's context.
|
||||
toolStreamPreview = 1024
|
||||
)
|
||||
|
||||
// streamState tracks one live stream. mu serializes the two WebSocket forwarding
|
||||
// goroutines, which both fire WebSocketMessage on the same flow.
|
||||
type streamState struct {
|
||||
mu sync.Mutex
|
||||
id string // exchange id; "" when the stream could not be recorded
|
||||
kind string // "websocket" | "sse"
|
||||
consumed int // library messages/events already handled
|
||||
seq int64
|
||||
stored int
|
||||
// storedBytes counts what the kept messages occupy: the payload after
|
||||
// truncation, plus the inline preview spill keeps for a spilled payload.
|
||||
storedBytes int64
|
||||
dropped bool
|
||||
droppedMsgs int64
|
||||
droppedBytes int64
|
||||
// reqBytes and respBytes are the bytes observed in each direction, dropped
|
||||
// messages included.
|
||||
reqBytes, respBytes int64
|
||||
}
|
||||
|
||||
// streamRow is one stream_messages row before it is written.
|
||||
type streamRow struct {
|
||||
ts int64 // unix seconds, like exchanges.ts; seq carries the order
|
||||
direction string
|
||||
kind int
|
||||
event string
|
||||
eventID string
|
||||
retry int
|
||||
size int
|
||||
payload []byte // kept bytes (after maxStreamPayload truncation)
|
||||
truncated bool
|
||||
ctype string // content type handed to spill for the binary check
|
||||
}
|
||||
|
||||
func (s *sink) WebSocketStart(f *mproxy.Flow) { s.startStream(f, "websocket") }
|
||||
func (s *sink) SSEStart(f *mproxy.Flow) { s.startStream(f, "sse") }
|
||||
|
||||
func (s *sink) WebSocketMessage(f *mproxy.Flow) {
|
||||
st := s.stream(f)
|
||||
if st == nil || f.WebScoket == nil {
|
||||
return
|
||||
}
|
||||
st.mu.Lock()
|
||||
defer st.mu.Unlock()
|
||||
s.t.persistStream(st, wsMessagesSince(f.WebScoket, st))
|
||||
}
|
||||
|
||||
func (s *sink) SSEMessage(f *mproxy.Flow) {
|
||||
st := s.stream(f)
|
||||
if st == nil || f.SSE == nil {
|
||||
return
|
||||
}
|
||||
st.mu.Lock()
|
||||
defer st.mu.Unlock()
|
||||
s.t.persistStream(st, sseEventsSince(f.SSE, st))
|
||||
}
|
||||
|
||||
func (s *sink) WebSocketEnd(f *mproxy.Flow) {
|
||||
v, ok := s.streams.LoadAndDelete(f)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
st := v.(*streamState)
|
||||
st.mu.Lock()
|
||||
defer st.mu.Unlock()
|
||||
if f.WebScoket != nil {
|
||||
s.t.persistStream(st, wsMessagesSince(f.WebScoket, st))
|
||||
}
|
||||
s.t.finishStream(st)
|
||||
}
|
||||
|
||||
func (s *sink) SSEEnd(f *mproxy.Flow) {
|
||||
v, ok := s.streams.LoadAndDelete(f)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
st := v.(*streamState)
|
||||
st.mu.Lock()
|
||||
defer st.mu.Unlock()
|
||||
if f.SSE != nil {
|
||||
s.t.persistStream(st, sseEventsSince(f.SSE, st))
|
||||
}
|
||||
s.t.finishStream(st)
|
||||
}
|
||||
|
||||
func (s *sink) stream(f *mproxy.Flow) *streamState {
|
||||
v, ok := s.streams.Load(f)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return v.(*streamState)
|
||||
}
|
||||
|
||||
// startStream registers the flow before recording its exchange, so a message
|
||||
// hook racing the start waits on st.mu instead of being dropped.
|
||||
func (s *sink) startStream(f *mproxy.Flow, kind string) {
|
||||
st := &streamState{kind: kind}
|
||||
st.mu.Lock()
|
||||
defer st.mu.Unlock()
|
||||
s.streams.Store(f, st)
|
||||
if f.Request == nil || f.Request.URL == nil || (kind == "sse" && f.Response == nil) {
|
||||
return
|
||||
}
|
||||
id, err := s.t.recordStreamStart(f, kind)
|
||||
if err != nil {
|
||||
log.Printf("[traffic] failed to record %s stream %s: %v", kind, f.Request.URL, err)
|
||||
return
|
||||
}
|
||||
st.id = id
|
||||
if kind == "sse" {
|
||||
st.reqBytes = int64(len(f.Request.Body))
|
||||
}
|
||||
}
|
||||
|
||||
// libMutex returns the unexported mu sync.Mutex that go-mitmproxy holds while it
|
||||
// appends to WebSocketData.Messages / SSEData.Events. The library exposes no
|
||||
// accessor, and the two WebSocket goroutines append concurrently, so reading the
|
||||
// slice without that lock is a data race. Reaching a private field ties this to
|
||||
// the pinned go-mitmproxy v1.9.2: TestLibMutexFound is the upgrade guard. Nil
|
||||
// when the field is absent, and then nothing is read (see lockLib).
|
||||
func libMutex(v any) *sync.Mutex {
|
||||
rv := reflect.ValueOf(v)
|
||||
if rv.Kind() != reflect.Pointer || rv.IsNil() {
|
||||
return nil
|
||||
}
|
||||
f := rv.Elem().FieldByName("mu")
|
||||
if !f.IsValid() || f.Type() != reflect.TypeOf(sync.Mutex{}) || !f.CanAddr() {
|
||||
return nil
|
||||
}
|
||||
return (*sync.Mutex)(unsafe.Pointer(f.UnsafeAddr()))
|
||||
}
|
||||
|
||||
var libMutexMissing sync.Once
|
||||
|
||||
// lockLib takes the library's lock on v. ok is false when the lock cannot be
|
||||
// found; callers then read nothing rather than race the library.
|
||||
func lockLib(v any) (unlock func(), ok bool) {
|
||||
mu := libMutex(v)
|
||||
if mu == nil {
|
||||
libMutexMissing.Do(func() {
|
||||
log.Printf("[traffic] go-mitmproxy stream data has no mu field; stream messages are not recorded")
|
||||
})
|
||||
return nil, false
|
||||
}
|
||||
mu.Lock()
|
||||
return mu.Unlock, true
|
||||
}
|
||||
|
||||
// wsMessagesSince returns every WebSocket message appended since the last call,
|
||||
// in the library's order, so each is handled exactly once whichever goroutine's
|
||||
// hook gets there first. The library slice itself is never modified.
|
||||
func wsMessagesSince(ws *mproxy.WebSocketData, st *streamState) []streamRow {
|
||||
unlock, ok := lockLib(ws)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
msgs := ws.Messages
|
||||
if st.consumed > len(msgs) {
|
||||
st.consumed = len(msgs)
|
||||
}
|
||||
fresh := append([]*mproxy.WebSocketMessage(nil), msgs[st.consumed:]...)
|
||||
st.consumed = len(msgs)
|
||||
unlock()
|
||||
rows := make([]streamRow, 0, len(fresh))
|
||||
for _, m := range fresh {
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
dir, ctype := "s2c", "text/plain; charset=utf-8"
|
||||
if m.FromClient {
|
||||
dir = "c2s"
|
||||
st.reqBytes += int64(len(m.Content))
|
||||
} else {
|
||||
st.respBytes += int64(len(m.Content))
|
||||
}
|
||||
if m.Type == wsBinary {
|
||||
ctype = "application/octet-stream"
|
||||
}
|
||||
rows = append(rows, streamRow{ts: m.Timestamp.Unix(), direction: dir, kind: m.Type, size: len(m.Content), payload: m.Content, ctype: ctype})
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
// sseEventsSince is wsMessagesSince for Server-Sent Events.
|
||||
func sseEventsSince(sse *mproxy.SSEData, st *streamState) []streamRow {
|
||||
unlock, ok := lockLib(sse)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
events := sse.Events
|
||||
if st.consumed > len(events) {
|
||||
st.consumed = len(events)
|
||||
}
|
||||
fresh := append([]*mproxy.SSEEvent(nil), events[st.consumed:]...)
|
||||
st.consumed = len(events)
|
||||
unlock()
|
||||
rows := make([]streamRow, 0, len(fresh))
|
||||
for _, e := range fresh {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
st.respBytes += int64(len(e.Raw))
|
||||
rows = append(rows, streamRow{ts: e.Time.Unix(), direction: "s2c", event: e.Event, eventID: e.ID,
|
||||
retry: e.Retry, size: len(e.Data), payload: []byte(e.Data), ctype: "text/event-stream"})
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
// streamDisplayURL is the URL shown for a WebSocket exchange: http→ws and
|
||||
// https→wss. A plain ws:// connection reaches the WebSocket hooks through its
|
||||
// CONNECT tunnel, where only the target authority is known, not the path.
|
||||
func streamDisplayURL(f *mproxy.Flow, kind string) (method, display, tmpl string) {
|
||||
u := *f.Request.URL
|
||||
method = f.Request.Method
|
||||
if kind == "websocket" && strings.EqualFold(method, "CONNECT") {
|
||||
host := u.Host
|
||||
if host == "" {
|
||||
host = u.Opaque
|
||||
}
|
||||
return "GET", (&url.URL{Scheme: "ws", Host: host, Path: "/"}).String(), "/"
|
||||
}
|
||||
if kind == "websocket" {
|
||||
switch strings.ToLower(u.Scheme) {
|
||||
case "http", "":
|
||||
u.Scheme = "ws"
|
||||
case "https":
|
||||
u.Scheme = "wss"
|
||||
}
|
||||
}
|
||||
return method, u.String(), db.TemplatePath(f.Request.URL.EscapedPath())
|
||||
}
|
||||
|
||||
// recordStreamStart writes the stream's exchange: metadata, request head and
|
||||
// body, the response head (synthetic 101 for WebSocket), and the header text in
|
||||
// the full-text index. Messages are never indexed.
|
||||
func (t *Traffic) recordStreamStart(f *mproxy.Flow, kind string) (string, error) {
|
||||
now := time.Now()
|
||||
id := t.newExchangeID(now)
|
||||
method, display, tmpl := streamDisplayURL(f, kind)
|
||||
host := f.Request.URL.Hostname()
|
||||
target := f.Request.URL.RequestURI()
|
||||
if strings.EqualFold(f.Request.Method, "CONNECT") {
|
||||
target = f.Request.URL.Host // authority-form, as on the wire
|
||||
}
|
||||
reqHead := fmt.Sprintf("%s %s %s\n%s", f.Request.Method, target, f.Request.Proto, requestHeaderLines(f.Request))
|
||||
status, ct, respHead := 101, "websocket", "HTTP/1.1 101 Switching Protocols\n"
|
||||
if kind == "sse" {
|
||||
status = f.Response.StatusCode
|
||||
ct = f.Response.Header.Get("Content-Type")
|
||||
respHead = fmt.Sprintf("HTTP %d\n%s", status, headerLines(f.Response.Header))
|
||||
}
|
||||
|
||||
t.wmu.Lock()
|
||||
defer t.wmu.Unlock()
|
||||
reqB := t.spill(f.Request.Body, f.Request.Header.Get("Content-Type"))
|
||||
tx, err := t.db.Begin()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("beginning transaction: %w", err)
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // no-op once committed
|
||||
res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,'')`, id, now.Unix(), host, method, tmpl, display, status, ct, len(f.Request.Body), 0)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("writing index: %w", err)
|
||||
}
|
||||
rowid, err := res.LastInsertId()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fetching rowid: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT OR REPLACE INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob)
|
||||
VALUES(?,?,?,?,?,NULL,NULL)`, id, reqHead, reqB.inline, nullIfEmpty(reqB.hash), respHead); err != nil {
|
||||
return "", fmt.Errorf("writing body: %w", err)
|
||||
}
|
||||
if reqB.hash != "" {
|
||||
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, reqB.hash, id); err != nil {
|
||||
return "", fmt.Errorf("registering blob reference: %w", err)
|
||||
}
|
||||
}
|
||||
if t.fts {
|
||||
idx := strings.Join([]string{display, reqHead, reqB.index, respHead}, "\n")
|
||||
if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowid, idx); err != nil {
|
||||
return "", fmt.Errorf("writing full-text index: %w", err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return "", fmt.Errorf("committing: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// persistStream applies the per-message and per-exchange limits to rows and
|
||||
// writes what is kept. The first message past a per-exchange limit stores one
|
||||
// 'meta' row instead; nothing is stored after it. Caller holds st.mu. Errors are
|
||||
// logged: a recording failure never interrupts the stream.
|
||||
func (t *Traffic) persistStream(st *streamState, rows []streamRow) {
|
||||
if st.id == "" || len(rows) == 0 {
|
||||
return
|
||||
}
|
||||
keep := make([]streamRow, 0, len(rows))
|
||||
addMeta := false
|
||||
for _, r := range rows {
|
||||
if len(r.payload) > maxStreamPayload {
|
||||
r.payload = r.payload[:maxStreamPayload]
|
||||
r.truncated = true
|
||||
}
|
||||
cost := streamStoredCost(r)
|
||||
if st.dropped || st.stored >= maxStreamMessages || st.storedBytes+cost > maxStreamBytes {
|
||||
if !st.dropped {
|
||||
st.dropped = true
|
||||
addMeta = true
|
||||
}
|
||||
st.droppedMsgs++
|
||||
st.droppedBytes += int64(r.size)
|
||||
continue
|
||||
}
|
||||
st.stored++
|
||||
st.storedBytes += cost
|
||||
keep = append(keep, r)
|
||||
}
|
||||
var meta []byte
|
||||
if addMeta {
|
||||
meta = streamMetaPayload(st)
|
||||
}
|
||||
if len(keep) == 0 && meta == nil {
|
||||
return
|
||||
}
|
||||
|
||||
t.wmu.Lock()
|
||||
defer t.wmu.Unlock()
|
||||
// Sequence numbers and blob writes come first, as in record(): blob writes are
|
||||
// filesystem work and must not sit inside the SQLite transaction. Holding wmu
|
||||
// keeps gcBlobs from collecting a blob before its reference is committed.
|
||||
type preparedRow struct {
|
||||
streamRow
|
||||
seq int64
|
||||
body storedBody
|
||||
}
|
||||
prepared := make([]preparedRow, 0, len(keep))
|
||||
for _, r := range keep {
|
||||
st.seq++
|
||||
prepared = append(prepared, preparedRow{streamRow: r, seq: st.seq, body: t.spillAt(r.payload, r.ctype, maxStreamInline)})
|
||||
}
|
||||
var metaSeq int64
|
||||
if meta != nil {
|
||||
st.seq++
|
||||
metaSeq = st.seq
|
||||
}
|
||||
|
||||
tx, err := t.db.Begin()
|
||||
if err != nil {
|
||||
log.Printf("[traffic] failed to record stream messages of %s (beginning transaction): %v", st.id, err)
|
||||
return
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // no-op once committed
|
||||
for _, p := range prepared {
|
||||
inline := p.body.inline
|
||||
if inline == nil {
|
||||
inline = []byte{}
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,event,event_id,retry,size,payload,payload_blob,truncated)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`, st.id, p.seq, p.ts, p.direction, p.kind, p.event, p.eventID, p.retry, p.size,
|
||||
inline, nullIfEmpty(p.body.hash), boolInt(p.truncated)); err != nil {
|
||||
log.Printf("[traffic] failed to record stream message %d of %s: %v", p.seq, st.id, err)
|
||||
return
|
||||
}
|
||||
if p.body.hash != "" {
|
||||
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, p.body.hash, st.id); err != nil {
|
||||
log.Printf("[traffic] failed to record stream message %d of %s (registering blob reference): %v", p.seq, st.id, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if meta != nil {
|
||||
if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload)
|
||||
VALUES(?,?,?,'meta',0,?,?)`, st.id, metaSeq, time.Now().Unix(), len(meta), meta); err != nil {
|
||||
log.Printf("[traffic] failed to record stream limit marker of %s: %v", st.id, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
log.Printf("[traffic] failed to record stream messages of %s (committing): %v", st.id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// streamStoredCost is what keeping r adds to the index and blob store: the
|
||||
// payload, plus the inline preview spillAt keeps when the payload spills (a text
|
||||
// prefix of at most blobPreview, or a one-line tag for binary).
|
||||
func streamStoredCost(r streamRow) int64 {
|
||||
n := int64(len(r.payload))
|
||||
if len(r.payload) <= maxStreamInline {
|
||||
return n
|
||||
}
|
||||
if isBinaryBody(r.ctype, r.payload) {
|
||||
return n + int64(len(binaryTag(r.ctype, r.payload)))
|
||||
}
|
||||
return n + int64(len(truncateUTF8(r.payload, blobPreview)))
|
||||
}
|
||||
|
||||
func streamMetaPayload(st *streamState) []byte {
|
||||
b, _ := json.Marshal(map[string]int64{"dropped_messages": st.droppedMsgs, "dropped_bytes": st.droppedBytes})
|
||||
return b
|
||||
}
|
||||
|
||||
func boolInt(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// finishStream records the bytes observed in each direction and the final
|
||||
// counts of the limit marker. Caller holds st.mu.
|
||||
func (t *Traffic) finishStream(st *streamState) {
|
||||
if st.id == "" {
|
||||
return
|
||||
}
|
||||
t.wmu.Lock()
|
||||
defer t.wmu.Unlock()
|
||||
if _, err := t.db.Exec(`UPDATE exchanges SET req_len=?, resp_len=? WHERE id=?`, st.reqBytes, st.respBytes, st.id); err != nil {
|
||||
log.Printf("[traffic] failed to record stream sizes of %s: %v", st.id, err)
|
||||
}
|
||||
if st.dropped {
|
||||
meta := streamMetaPayload(st)
|
||||
if _, err := t.db.Exec(`UPDATE stream_messages SET payload=?, size=? WHERE exchange_id=? AND direction='meta'`, meta, len(meta), st.id); err != nil {
|
||||
log.Printf("[traffic] failed to update stream limit marker of %s: %v", st.id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// StreamMessage is one recorded WebSocket message or SSE event.
|
||||
type StreamMessage struct {
|
||||
Seq int64 `json:"seq"`
|
||||
TS int64 `json:"ts"` // unix seconds, like an exchange's ts; Seq orders
|
||||
Direction string `json:"direction"`
|
||||
Kind int `json:"kind"`
|
||||
// Type is "text" or "binary" (WebSocket), "event" (SSE), or "meta" (the
|
||||
// limit marker).
|
||||
Type string `json:"type"`
|
||||
Event string `json:"event,omitempty"`
|
||||
EventID string `json:"event_id,omitempty"`
|
||||
Retry int `json:"retry,omitempty"`
|
||||
Size int `json:"size"`
|
||||
Payload string `json:"payload"`
|
||||
// Encoding is "base64" for binary payloads, empty for UTF-8 text.
|
||||
Encoding string `json:"encoding,omitempty"`
|
||||
Blob string `json:"blob,omitempty"`
|
||||
Truncated bool `json:"truncated,omitempty"`
|
||||
}
|
||||
|
||||
// StreamMessages returns one page of an exchange's stream messages in order.
|
||||
// limit is 1..200 (default 50) and offset >= 0. Every payload comes back as at
|
||||
// most streamPreview (4 KiB); a longer one lives in the blob store and the row
|
||||
// carries its hash for the rest (traffic_blob, GET /api/traffic/blob).
|
||||
func (t *Traffic) StreamMessages(exchangeID string, offset, limit int) ([]StreamMessage, int, error) {
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
if limit > 200 {
|
||||
limit = 200
|
||||
}
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
var total int
|
||||
if err := t.db.QueryRow(`SELECT COUNT(*) FROM stream_messages WHERE exchange_id=?`, exchangeID).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
rows, err := t.db.Query(`SELECT seq,ts,direction,kind,event,event_id,retry,size,payload,payload_blob,truncated
|
||||
FROM stream_messages WHERE exchange_id=? ORDER BY seq LIMIT ? OFFSET ?`, exchangeID, limit, offset)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []StreamMessage{}
|
||||
for rows.Next() {
|
||||
var m StreamMessage
|
||||
var payload []byte
|
||||
var blob sql.NullString
|
||||
var truncated int
|
||||
if err := rows.Scan(&m.Seq, &m.TS, &m.Direction, &m.Kind, &m.Event, &m.EventID, &m.Retry, &m.Size, &payload, &blob, &truncated); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
m.Truncated = truncated != 0
|
||||
m.Blob = blob.String
|
||||
switch {
|
||||
case m.Direction == "meta":
|
||||
m.Type = "meta"
|
||||
case m.Kind == wsBinary:
|
||||
m.Type = "binary"
|
||||
case m.Kind == wsText:
|
||||
m.Type = "text"
|
||||
default:
|
||||
m.Type = "event"
|
||||
}
|
||||
if m.Type == "binary" {
|
||||
if m.Blob != "" {
|
||||
// A spilled binary payload keeps only a descriptive tag inline;
|
||||
// the preview is the payload's own leading bytes.
|
||||
payload, _, _ = t.BlobRange(m.Blob, 0, streamPreview)
|
||||
}
|
||||
m.Payload, m.Encoding = base64.StdEncoding.EncodeToString(clipBytes(payload, streamPreview)), "base64"
|
||||
} else {
|
||||
m.Payload = truncateUTF8(payload, streamPreview)
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, total, rows.Err()
|
||||
}
|
||||
|
||||
// StreamInfo reports how many stream messages an exchange holds and whether it
|
||||
// is a WebSocket or SSE exchange ("" for an ordinary exchange).
|
||||
func (t *Traffic) StreamInfo(exchangeID string) (count int, kind string, err error) {
|
||||
var ct sql.NullString
|
||||
err = t.db.QueryRow(`SELECT content_type FROM exchanges WHERE id=?`, exchangeID).Scan(&ct)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return 0, "", err
|
||||
}
|
||||
switch c := strings.ToLower(ct.String); {
|
||||
case c == "websocket":
|
||||
kind = "websocket"
|
||||
case strings.Contains(c, "text/event-stream"):
|
||||
kind = "sse"
|
||||
}
|
||||
if err := t.db.QueryRow(`SELECT COUNT(*) FROM stream_messages WHERE exchange_id=?`, exchangeID).Scan(&count); err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
return count, kind, nil
|
||||
}
|
||||
|
||||
// streamToolPreview renders one message for traffic_messages: at most 1 KiB of
|
||||
// text (hex for binary), plus the blob hash when the payload spilled.
|
||||
func streamToolPreview(m StreamMessage) string {
|
||||
if m.Encoding == "base64" {
|
||||
raw, _ := base64.StdEncoding.DecodeString(m.Payload)
|
||||
return "hex:" + fmt.Sprintf("%x", clipBytes(raw, toolStreamPreview/2))
|
||||
}
|
||||
p := m.Payload
|
||||
if len(p) > toolStreamPreview {
|
||||
p = truncateUTF8([]byte(p), toolStreamPreview)
|
||||
}
|
||||
if !utf8.ValidString(p) {
|
||||
p = strings.ToValidUTF8(p, "�")
|
||||
}
|
||||
return p
|
||||
}
|
||||
@@ -0,0 +1,637 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
|
||||
)
|
||||
|
||||
// localhostURL names an httptest server by host name: the MITM leaf certificate
|
||||
// carries a DNS name, not an IP SAN (as in pace_test.go).
|
||||
func localhostURL(u string) string { return strings.Replace(u, "127.0.0.1", "localhost", 1) }
|
||||
|
||||
// waitFor polls cond until it holds or 10 seconds pass.
|
||||
func waitFor(t *testing.T, what string, cond func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for !cond() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("timed out waiting for %s", what)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// wsDialer dials through the recording proxy and trusts its CA.
|
||||
func wsDialer(t *testing.T, tr *Traffic, proxyURL string) *websocket.Dialer {
|
||||
t.Helper()
|
||||
pem, err := os.ReadFile(tr.CACertPath())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(pem) {
|
||||
t.Fatal("no CA in", tr.CACertPath())
|
||||
}
|
||||
u, _ := url.Parse(proxyURL)
|
||||
return &websocket.Dialer{Proxy: http.ProxyURL(u), TLSClientConfig: &tls.Config{RootCAs: pool}, HandshakeTimeout: 5 * time.Second}
|
||||
}
|
||||
|
||||
type streamExchange struct {
|
||||
id, url, contentType string
|
||||
status int
|
||||
reqLen, respLen int64
|
||||
}
|
||||
|
||||
// streamExchanges returns the exchanges recorded with a stream content type.
|
||||
func streamExchanges(t *testing.T, tr *Traffic, contentLike string) []streamExchange {
|
||||
t.Helper()
|
||||
rows, err := tr.DB().Query(`SELECT id,url,content_type,status,req_len,resp_len FROM exchanges WHERE content_type LIKE ? ORDER BY id`, contentLike)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []streamExchange
|
||||
for rows.Next() {
|
||||
var e streamExchange
|
||||
if err := rows.Scan(&e.id, &e.url, &e.contentType, &e.status, &e.reqLen, &e.respLen); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func allMessages(t *testing.T, tr *Traffic, id string) []StreamMessage {
|
||||
t.Helper()
|
||||
var out []StreamMessage
|
||||
for offset := 0; ; offset += 200 {
|
||||
page, total, err := tr.StreamMessages(id, offset, 200)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out = append(out, page...)
|
||||
if len(out) >= total || len(page) == 0 {
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// echoWSServer echoes every message back with the same type: text prefixed
|
||||
// with "echo:", binary reversed.
|
||||
func echoWSServer(t *testing.T) *httptest.Server {
|
||||
t.Helper()
|
||||
up := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
c, err := up.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close()
|
||||
for {
|
||||
mt, msg, err := c.ReadMessage()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if mt == websocket.BinaryMessage {
|
||||
rev := make([]byte, len(msg))
|
||||
for i := range msg {
|
||||
rev[len(msg)-1-i] = msg[i]
|
||||
}
|
||||
msg = rev
|
||||
} else {
|
||||
msg = append([]byte("echo:"), msg...)
|
||||
}
|
||||
if err := c.WriteMessage(mt, msg); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func closeWS(c *websocket.Conn) {
|
||||
_ = c.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
|
||||
_ = c.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
for {
|
||||
if _, _, err := c.ReadMessage(); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
_ = c.Close()
|
||||
}
|
||||
|
||||
func TestWebSocketMessagesRecorded(t *testing.T) {
|
||||
tr, _, proxyURL := startProxy(t)
|
||||
srv := echoWSServer(t)
|
||||
target := "wss://" + strings.TrimPrefix(localhostURL(srv.URL), "https://") + "/ws?room=1"
|
||||
c, _, err := wsDialer(t, tr, proxyURL).Dial(target, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exchange := func(mt int, msg []byte) []byte {
|
||||
if err := c.WriteMessage(mt, msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gotType, got, err := c.ReadMessage()
|
||||
if err != nil || gotType != mt {
|
||||
t.Fatalf("read type=%d err=%v", gotType, err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
before := time.Now().Unix()
|
||||
if got := exchange(websocket.TextMessage, []byte("hello")); string(got) != "echo:hello" {
|
||||
t.Fatalf("echo %q", got)
|
||||
}
|
||||
if got := exchange(websocket.BinaryMessage, []byte{0, 1, 2, 255}); !bytes.Equal(got, []byte{255, 2, 1, 0}) {
|
||||
t.Fatalf("binary echo %v", got)
|
||||
}
|
||||
closeWS(c)
|
||||
|
||||
var ex streamExchange
|
||||
waitFor(t, "websocket end", func() bool {
|
||||
exs := streamExchanges(t, tr, "websocket")
|
||||
if len(exs) != 1 || exs[0].reqLen == 0 {
|
||||
return false
|
||||
}
|
||||
ex = exs[0]
|
||||
return true
|
||||
})
|
||||
if ex.status != 101 || ex.url != target {
|
||||
t.Fatalf("exchange %+v, want status 101 and url %s", ex, target)
|
||||
}
|
||||
// c2s: "hello" (5) + 4 binary bytes; s2c: "echo:hello" (10) + 4.
|
||||
if ex.reqLen != 9 || ex.respLen != 14 {
|
||||
t.Fatalf("req_len=%d resp_len=%d, want 9 and 14", ex.reqLen, ex.respLen)
|
||||
}
|
||||
var respHead string
|
||||
if err := tr.DB().QueryRow(`SELECT resp_head FROM exchange_bodies WHERE id=?`, ex.id).Scan(&respHead); err != nil || !strings.HasPrefix(respHead, "HTTP/1.1 101 Switching Protocols") {
|
||||
t.Fatalf("resp_head %q err=%v", respHead, err)
|
||||
}
|
||||
if count, kind, err := tr.StreamInfo(ex.id); err != nil || count != 4 || kind != "websocket" {
|
||||
t.Fatalf("StreamInfo = %d %q %v", count, kind, err)
|
||||
}
|
||||
|
||||
msgs := allMessages(t, tr, ex.id)
|
||||
want := []struct {
|
||||
dir, typ, payload string
|
||||
kind int
|
||||
}{
|
||||
{"c2s", "text", "hello", 1},
|
||||
{"s2c", "text", "echo:hello", 1},
|
||||
{"c2s", "binary", base64.StdEncoding.EncodeToString([]byte{0, 1, 2, 255}), 2},
|
||||
{"s2c", "binary", base64.StdEncoding.EncodeToString([]byte{255, 2, 1, 0}), 2},
|
||||
}
|
||||
if len(msgs) != len(want) {
|
||||
t.Fatalf("got %d messages: %+v", len(msgs), msgs)
|
||||
}
|
||||
last := before
|
||||
for i, w := range want {
|
||||
m := msgs[i]
|
||||
if m.Seq != int64(i+1) || m.Direction != w.dir || m.Type != w.typ || m.Kind != w.kind || m.Payload != w.payload {
|
||||
t.Fatalf("message %d = %+v, want %+v", i, m, w)
|
||||
}
|
||||
if (w.typ == "binary") != (m.Encoding == "base64") {
|
||||
t.Fatalf("message %d encoding %q", i, m.Encoding)
|
||||
}
|
||||
if m.TS < last {
|
||||
t.Fatalf("message %d ts %d before %d", i, m.TS, last)
|
||||
}
|
||||
last = m.TS
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSEEventsRecorded(t *testing.T) {
|
||||
tr, _, proxyURL := startProxy(t)
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.WriteHeader(200)
|
||||
fl := w.(http.Flusher)
|
||||
for _, ev := range []string{
|
||||
"retry: 1500\nid: 1\nevent: greet\ndata: hello\ndata: world\n\n",
|
||||
"id: 2\ndata: second\n\n",
|
||||
"event: done\ndata: bye\n\n",
|
||||
} {
|
||||
_, _ = io.WriteString(w, ev)
|
||||
fl.Flush()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
resp, err := proxyClient(t, tr, proxyURL).Get(localhostURL(srv.URL) + "/events")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if !strings.Contains(string(body), "data: bye") {
|
||||
t.Fatalf("client body %q", body)
|
||||
}
|
||||
|
||||
var ex streamExchange
|
||||
waitFor(t, "sse end", func() bool {
|
||||
exs := streamExchanges(t, tr, "%event-stream%")
|
||||
if len(exs) != 1 || exs[0].respLen == 0 {
|
||||
return false
|
||||
}
|
||||
ex = exs[0]
|
||||
return true
|
||||
})
|
||||
if ex.status != 200 || !strings.HasPrefix(ex.url, localhostURL(srv.URL)+"/events") {
|
||||
t.Fatalf("exchange %+v", ex)
|
||||
}
|
||||
if n, _ := tr.Count(); n != 1 {
|
||||
t.Fatalf("%d exchanges recorded, want the single SSE exchange", n)
|
||||
}
|
||||
msgs := allMessages(t, tr, ex.id)
|
||||
want := []struct {
|
||||
event, id, data string
|
||||
retry int
|
||||
}{
|
||||
{"greet", "1", "hello\nworld", 1500},
|
||||
{"message", "2", "second", 0},
|
||||
{"done", "", "bye", 0},
|
||||
}
|
||||
if len(msgs) != len(want) {
|
||||
t.Fatalf("got %d events: %+v", len(msgs), msgs)
|
||||
}
|
||||
for i, w := range want {
|
||||
m := msgs[i]
|
||||
if m.Seq != int64(i+1) || m.Direction != "s2c" || m.Kind != 0 || m.Type != "event" ||
|
||||
m.Event != w.event || m.EventID != w.id || m.Payload != w.data || m.Retry != w.retry || m.Size != len(w.data) {
|
||||
t.Fatalf("event %d = %+v, want %+v", i, m, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// unitStream opens a recorder without starting the proxy and drives its hooks
|
||||
// directly with a synthetic WebSocket flow.
|
||||
func unitStream(t *testing.T, host string) (*Traffic, *sink, *mproxy.Flow) {
|
||||
t.Helper()
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = tr.Close() })
|
||||
u, _ := url.Parse("https://" + host + "/ws")
|
||||
f := &mproxy.Flow{Request: &mproxy.Request{Method: "GET", URL: u, Proto: "HTTP/1.1", Header: http.Header{}}, WebScoket: &mproxy.WebSocketData{}}
|
||||
s := tr.proxy.Addons[0].(*sink)
|
||||
s.WebSocketStart(f)
|
||||
return tr, s, f
|
||||
}
|
||||
|
||||
func addWS(f *mproxy.Flow, fromClient bool, mt int, content []byte) {
|
||||
f.WebScoket.Messages = append(f.WebScoket.Messages, &mproxy.WebSocketMessage{Type: mt, Content: content, FromClient: fromClient, Timestamp: time.Now()})
|
||||
}
|
||||
|
||||
func onlyExchange(t *testing.T, tr *Traffic) string {
|
||||
t.Helper()
|
||||
exs := streamExchanges(t, tr, "websocket")
|
||||
if len(exs) != 1 {
|
||||
t.Fatalf("%d websocket exchanges", len(exs))
|
||||
}
|
||||
return exs[0].id
|
||||
}
|
||||
|
||||
func metaCounts(t *testing.T, m StreamMessage) (msgs, bytes int64) {
|
||||
t.Helper()
|
||||
var v struct {
|
||||
DroppedMessages int64 `json:"dropped_messages"`
|
||||
DroppedBytes int64 `json:"dropped_bytes"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(m.Payload), &v); err != nil {
|
||||
t.Fatalf("meta payload %q: %v", m.Payload, err)
|
||||
}
|
||||
return v.DroppedMessages, v.DroppedBytes
|
||||
}
|
||||
|
||||
func TestStreamPayloadTruncatedToLimit(t *testing.T) {
|
||||
tr, s, f := unitStream(t, "trunc.test")
|
||||
big := bytes.Repeat([]byte("a"), maxStreamPayload+10)
|
||||
addWS(f, true, websocket.TextMessage, big)
|
||||
s.WebSocketMessage(f)
|
||||
id := onlyExchange(t, tr)
|
||||
msgs := allMessages(t, tr, id)
|
||||
if len(msgs) != 1 {
|
||||
t.Fatalf("%d messages", len(msgs))
|
||||
}
|
||||
m := msgs[0]
|
||||
if !m.Truncated || m.Size != maxStreamPayload+10 || m.Blob == "" || len(m.Payload) > streamPreview {
|
||||
t.Fatalf("message = seq %d size %d truncated %v blob %q payload %d bytes", m.Seq, m.Size, m.Truncated, m.Blob, len(m.Payload))
|
||||
}
|
||||
data, total, err := tr.BlobRange(m.Blob, 0, 16)
|
||||
if err != nil || total != maxStreamPayload || string(data) != strings.Repeat("a", 16) {
|
||||
t.Fatalf("blob total=%d data=%q err=%v, want the kept %d-byte prefix", total, data, err, maxStreamPayload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamMessageLimitStoresOneMetaRow(t *testing.T) {
|
||||
tr, s, f := unitStream(t, "count.test")
|
||||
for i := 0; i < maxStreamMessages+5; i++ {
|
||||
addWS(f, i%2 == 0, websocket.TextMessage, []byte("m"))
|
||||
}
|
||||
s.WebSocketMessage(f)
|
||||
addWS(f, true, websocket.TextMessage, []byte("late"))
|
||||
s.WebSocketMessage(f)
|
||||
s.WebSocketEnd(f)
|
||||
id := onlyExchange(t, tr)
|
||||
msgs := allMessages(t, tr, id)
|
||||
if len(msgs) != maxStreamMessages+1 {
|
||||
t.Fatalf("%d rows, want %d messages and one meta row", len(msgs), maxStreamMessages)
|
||||
}
|
||||
meta := msgs[len(msgs)-1]
|
||||
if meta.Direction != "meta" || meta.Type != "meta" || meta.Kind != 0 || meta.Seq != maxStreamMessages+1 {
|
||||
t.Fatalf("last row %+v, want the meta row", meta)
|
||||
}
|
||||
if n, b := metaCounts(t, meta); n != 6 || b != 5+4 {
|
||||
t.Fatalf("meta dropped_messages=%d dropped_bytes=%d, want 6 and 9", n, b)
|
||||
}
|
||||
for _, m := range msgs[:len(msgs)-1] {
|
||||
if m.Direction == "meta" {
|
||||
t.Fatalf("more than one meta row: %+v", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamByteLimitStoresOneMetaRow(t *testing.T) {
|
||||
tr, s, f := unitStream(t, "bytes.test")
|
||||
chunk := bytes.Repeat([]byte{7}, maxStreamPayload)
|
||||
// Each kept 1 MiB payload spills and also keeps its inline tag, so the cap
|
||||
// admits floor(maxStreamBytes / (payload + tag)) of them.
|
||||
cost := streamStoredCost(streamRow{payload: chunk, ctype: "application/octet-stream"})
|
||||
if cost <= maxStreamPayload {
|
||||
t.Fatalf("cost %d does not count the inline preview", cost)
|
||||
}
|
||||
kept := int(maxStreamBytes / cost)
|
||||
n := kept + 1
|
||||
for i := 0; i < n; i++ {
|
||||
addWS(f, false, websocket.BinaryMessage, chunk)
|
||||
}
|
||||
s.WebSocketMessage(f)
|
||||
addWS(f, false, websocket.BinaryMessage, []byte{1})
|
||||
s.WebSocketMessage(f)
|
||||
s.WebSocketEnd(f)
|
||||
id := onlyExchange(t, tr)
|
||||
msgs := allMessages(t, tr, id)
|
||||
if len(msgs) != kept+1 {
|
||||
t.Fatalf("%d rows, want %d stored and one meta row", len(msgs), kept)
|
||||
}
|
||||
meta := msgs[len(msgs)-1]
|
||||
if meta.Direction != "meta" {
|
||||
t.Fatalf("last row %+v, want the meta row", meta)
|
||||
}
|
||||
if dn, db := metaCounts(t, meta); dn != 2 || db != maxStreamPayload+1 {
|
||||
t.Fatalf("meta dropped_messages=%d dropped_bytes=%d", dn, db)
|
||||
}
|
||||
var stored, inline int64
|
||||
if err := tr.DB().QueryRow(`SELECT SUM(size), SUM(LENGTH(payload)) FROM stream_messages WHERE exchange_id=? AND direction<>'meta'`, id).Scan(&stored, &inline); err != nil ||
|
||||
stored != int64(kept*maxStreamPayload) || stored+inline > maxStreamBytes {
|
||||
t.Fatalf("stored %d payload bytes + %d inline err=%v, want %d within %d", stored, inline, err, kept*maxStreamPayload, maxStreamBytes)
|
||||
}
|
||||
var respLen int64
|
||||
if err := tr.DB().QueryRow(`SELECT resp_len FROM exchanges WHERE id=?`, id).Scan(&respLen); err != nil || respLen != int64(n*maxStreamPayload+1) {
|
||||
t.Fatalf("resp_len %d err=%v, want every observed byte", respLen, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A payload over maxStreamInline goes to the blob store; the row returns at
|
||||
// most 4 KiB and the hash, and the blob holds the whole payload.
|
||||
func TestStreamPayloadSpillsAboveInlineLimit(t *testing.T) {
|
||||
tr, s, f := unitStream(t, "spill.test")
|
||||
bigText := []byte(strings.Repeat("0123456789", 1024)) // 10 KiB
|
||||
bigBin := make([]byte, 10*1024)
|
||||
for i := range bigBin {
|
||||
bigBin[i] = byte(i)
|
||||
}
|
||||
small := []byte(strings.Repeat("s", 1024))
|
||||
addWS(f, true, websocket.TextMessage, bigText)
|
||||
addWS(f, false, websocket.BinaryMessage, bigBin)
|
||||
addWS(f, true, websocket.TextMessage, small)
|
||||
s.WebSocketMessage(f)
|
||||
msgs := allMessages(t, tr, onlyExchange(t, tr))
|
||||
if len(msgs) != 3 {
|
||||
t.Fatalf("%d messages", len(msgs))
|
||||
}
|
||||
text, bin, inline := msgs[0], msgs[1], msgs[2]
|
||||
if text.Blob == "" || len(text.Payload) > streamPreview || text.Payload != string(bigText[:streamPreview]) || text.Size != len(bigText) || text.Truncated {
|
||||
t.Fatalf("10 KiB text row: blob %q payload %d bytes size %d truncated %v", text.Blob, len(text.Payload), text.Size, text.Truncated)
|
||||
}
|
||||
if data, total, err := tr.BlobRange(text.Blob, 0, int64(len(bigText))); err != nil || total != int64(len(bigText)) || !bytes.Equal(data, bigText) {
|
||||
t.Fatalf("text blob total=%d err=%v, want the full 10 KiB", total, err)
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(bin.Payload)
|
||||
if bin.Blob == "" || err != nil || !bytes.Equal(raw, bigBin[:streamPreview]) {
|
||||
t.Fatalf("10 KiB binary row: blob %q preview %d bytes err=%v, want its first 4 KiB", bin.Blob, len(raw), err)
|
||||
}
|
||||
if data, total, err := tr.BlobRange(bin.Blob, 0, int64(len(bigBin))); err != nil || total != int64(len(bigBin)) || !bytes.Equal(data, bigBin) {
|
||||
t.Fatalf("binary blob total=%d err=%v", total, err)
|
||||
}
|
||||
if inline.Blob != "" || inline.Payload != string(small) {
|
||||
t.Fatalf("1 KiB row: blob %q payload %d bytes, want inline and complete", inline.Blob, len(inline.Payload))
|
||||
}
|
||||
}
|
||||
|
||||
// The server writes a message and closes at once; the message the client read
|
||||
// must be recorded even though WebSocketEnd fires as soon as one direction ends.
|
||||
func TestWebSocketFinalMessageOnClose(t *testing.T) {
|
||||
tr, _, proxyURL := startProxy(t)
|
||||
up := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
c, err := up.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = c.WriteMessage(websocket.TextMessage, []byte("final"))
|
||||
_ = c.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
|
||||
_ = c.Close()
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
c, _, err := wsDialer(t, tr, proxyURL).Dial("wss://"+strings.TrimPrefix(localhostURL(srv.URL), "https://")+"/ws", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, got, err := c.ReadMessage(); err != nil || string(got) != "final" {
|
||||
t.Fatalf("read %q err=%v", got, err)
|
||||
}
|
||||
_ = c.Close()
|
||||
var id string
|
||||
waitFor(t, "websocket end", func() bool {
|
||||
exs := streamExchanges(t, tr, "websocket")
|
||||
if len(exs) != 1 || exs[0].respLen == 0 {
|
||||
return false
|
||||
}
|
||||
id = exs[0].id
|
||||
return true
|
||||
})
|
||||
msgs := allMessages(t, tr, id)
|
||||
if len(msgs) != 1 || msgs[0].Direction != "s2c" || msgs[0].Payload != "final" {
|
||||
t.Fatalf("messages %+v, want the final server message", msgs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteHostRemovesStreamMessagesAndBlobs(t *testing.T) {
|
||||
tr, s, f := unitStream(t, "gone.test")
|
||||
addWS(f, false, websocket.BinaryMessage, bytes.Repeat([]byte{9}, maxInlineBody+1))
|
||||
addWS(f, true, websocket.TextMessage, []byte("small"))
|
||||
s.WebSocketMessage(f)
|
||||
s.WebSocketEnd(f)
|
||||
id := onlyExchange(t, tr)
|
||||
msgs := allMessages(t, tr, id)
|
||||
if len(msgs) != 2 || msgs[0].Blob == "" {
|
||||
t.Fatalf("messages %+v, want a spilled first payload", msgs)
|
||||
}
|
||||
blobPath := filepath.Join(tr.dir, "_blobs", "sha256", msgs[0].Blob[:2], msgs[0].Blob+".bin")
|
||||
if _, err := os.Stat(blobPath); err != nil {
|
||||
t.Fatalf("payload blob missing: %v", err)
|
||||
}
|
||||
if n, err := tr.DeleteHost("gone.test"); err != nil || n != 1 {
|
||||
t.Fatalf("DeleteHost = %d, %v", n, err)
|
||||
}
|
||||
var rows int
|
||||
if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM stream_messages`).Scan(&rows); err != nil || rows != 0 {
|
||||
t.Fatalf("%d stream rows left, err=%v", rows, err)
|
||||
}
|
||||
if _, err := os.Stat(blobPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("payload blob not collected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A recording failure must never reach the proxied connection.
|
||||
func TestWebSocketSurvivesRecordingFailure(t *testing.T) {
|
||||
tr, _, proxyURL := startProxy(t)
|
||||
srv := echoWSServer(t)
|
||||
c, _, err := wsDialer(t, tr, proxyURL).Dial("wss://"+strings.TrimPrefix(localhostURL(srv.URL), "https://")+"/ws", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer closeWS(c)
|
||||
if err := c.WriteMessage(websocket.TextMessage, []byte("before")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, got, err := c.ReadMessage(); err != nil || string(got) != "echo:before" {
|
||||
t.Fatalf("echo %q err=%v", got, err)
|
||||
}
|
||||
if err := tr.DB().Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
msg := fmt.Sprintf("after-%d", i)
|
||||
if err := c.WriteMessage(websocket.TextMessage, []byte(msg)); err != nil {
|
||||
t.Fatalf("write %s: %v", msg, err)
|
||||
}
|
||||
_ = c.SetReadDeadline(time.Now().Add(5 * time.Second))
|
||||
if _, got, err := c.ReadMessage(); err != nil || string(got) != "echo:"+msg {
|
||||
t.Fatalf("echo %q err=%v after the index closed", got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Both forwarding goroutines fire the message hook at once; every message must
|
||||
// be stored exactly once, each direction in order.
|
||||
func TestWebSocketConcurrentDirections(t *testing.T) {
|
||||
const n = 200
|
||||
tr, _, proxyURL := startProxy(t)
|
||||
up := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
c, err := up.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer c.Close()
|
||||
go func() {
|
||||
for i := 0; i < n; i++ {
|
||||
if c.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("s-%d", i))) != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
for {
|
||||
if _, _, err := c.ReadMessage(); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
c, _, err := wsDialer(t, tr, proxyURL).Dial("wss://"+strings.TrimPrefix(localhostURL(srv.URL), "https://")+"/ws", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < n; i++ {
|
||||
if err := c.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("c-%d", i))); err != nil {
|
||||
t.Errorf("write: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
for i := 0; i < n; i++ {
|
||||
if _, _, err := c.ReadMessage(); err != nil {
|
||||
t.Fatalf("read %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
var id string
|
||||
waitFor(t, "all client messages recorded", func() bool {
|
||||
exs := streamExchanges(t, tr, "websocket")
|
||||
if len(exs) != 1 {
|
||||
return false
|
||||
}
|
||||
id = exs[0].id
|
||||
count, _, _ := tr.StreamInfo(id)
|
||||
return count == 2*n
|
||||
})
|
||||
closeWS(c)
|
||||
|
||||
msgs := allMessages(t, tr, id)
|
||||
next := map[string]int{"c2s": 0, "s2c": 0}
|
||||
for i, m := range msgs {
|
||||
if m.Seq != int64(i+1) {
|
||||
t.Fatalf("row %d has seq %d", i, m.Seq)
|
||||
}
|
||||
prefix := "s-"
|
||||
if m.Direction == "c2s" {
|
||||
prefix = "c-"
|
||||
}
|
||||
if want := fmt.Sprintf("%s%d", prefix, next[m.Direction]); m.Payload != want {
|
||||
t.Fatalf("row %d %s payload %q, want %q", i, m.Direction, m.Payload, want)
|
||||
}
|
||||
next[m.Direction]++
|
||||
}
|
||||
if next["c2s"] != n || next["s2c"] != n {
|
||||
t.Fatalf("stored %v, want %d each way", next, n)
|
||||
}
|
||||
}
|
||||
|
||||
// The library's unexported lock is what makes reading its message slices safe;
|
||||
// a go-mitmproxy upgrade that renames it must fail here, not race silently.
|
||||
func TestLibMutexFound(t *testing.T) {
|
||||
if libMutex(&mproxy.WebSocketData{}) == nil || libMutex(&mproxy.SSEData{}) == nil {
|
||||
t.Fatal("go-mitmproxy WebSocketData/SSEData no longer have a sync.Mutex field named mu")
|
||||
}
|
||||
// Without the lock nothing is read: no unlocked fallback.
|
||||
if _, ok := lockLib(&struct{ Messages []int }{}); ok {
|
||||
t.Fatal("lockLib reported a lock on a type without mu")
|
||||
}
|
||||
}
|
||||
+111
-7
@@ -73,6 +73,24 @@ CREATE TABLE IF NOT EXISTS blob_refs (
|
||||
PRIMARY KEY (hash, exchange_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_blob_refs_ex ON blob_refs(exchange_id);
|
||||
|
||||
-- WebSocket messages and SSE events of a stream exchange, in arrival order.
|
||||
CREATE TABLE IF NOT EXISTS stream_messages (
|
||||
exchange_id TEXT NOT NULL,
|
||||
seq INTEGER NOT NULL,
|
||||
ts INTEGER NOT NULL, -- unix seconds, like exchanges.ts
|
||||
direction TEXT NOT NULL, -- 'c2s' | 's2c' | 'meta'
|
||||
kind INTEGER NOT NULL, -- WebSocket opcode (1 text, 2 binary), 0 for SSE and meta
|
||||
event TEXT NOT NULL DEFAULT '',
|
||||
event_id TEXT NOT NULL DEFAULT '',
|
||||
retry INTEGER NOT NULL DEFAULT 0,
|
||||
size INTEGER NOT NULL, -- original payload size
|
||||
payload BLOB, -- inline payload, or a preview when spilled
|
||||
payload_blob TEXT, -- sha256 of the spilled payload
|
||||
truncated INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (exchange_id, seq)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_stream_messages_ex ON stream_messages(exchange_id, seq);
|
||||
`
|
||||
|
||||
// ftsSchema is applied separately from indexSchema: a driver build without FTS5
|
||||
@@ -90,6 +108,14 @@ const (
|
||||
// maxInlineBody is the cutoff between a body stored inline (SQLite column) and
|
||||
// one spilled to the content-addressed blob store.
|
||||
maxInlineBody = 256 * 1024
|
||||
// maxStreamPayload is how much of one WebSocket message or SSE event is kept.
|
||||
// A longer payload keeps its prefix, with truncated=1 and its original size.
|
||||
maxStreamPayload = 1024 * 1024
|
||||
// maxStreamMessages and maxStreamBytes bound one stream exchange (bytes as
|
||||
// stored, after truncation). Past either, storing stops and a single 'meta'
|
||||
// row records how many messages and bytes were dropped.
|
||||
maxStreamMessages = 10000
|
||||
maxStreamBytes = 32 * 1024 * 1024
|
||||
// blobPreview is how much of a spilled body stays inline so readers can tell
|
||||
// what it is (JSON shape, SQL dump header, ZIP magic) without fetching it.
|
||||
blobPreview = 8 * 1024
|
||||
@@ -387,8 +413,9 @@ func (t *Traffic) DB() *sql.DB { return t.db }
|
||||
// the requests of client connections marked for pacing (see pace.go).
|
||||
type sink struct {
|
||||
mproxy.BaseAddon
|
||||
t *Traffic
|
||||
paced sync.Map // ClientConn.Id of a marked connection -> struct{}
|
||||
t *Traffic
|
||||
paced sync.Map // ClientConn.Id of a marked connection -> struct{}
|
||||
streams sync.Map // *mproxy.Flow of a live WebSocket/SSE stream -> *streamState (stream.go)
|
||||
}
|
||||
|
||||
func (s *sink) Response(f *mproxy.Flow) {
|
||||
@@ -443,9 +470,8 @@ func (t *Traffic) record(f *mproxy.Flow) {
|
||||
host := f.Request.URL.Hostname()
|
||||
method := f.Request.Method
|
||||
tmpl := db.TemplatePath(f.Request.URL.EscapedPath())
|
||||
n := t.seq.Add(1)
|
||||
now := time.Now()
|
||||
id := fmt.Sprintf("%d-%04d", now.Unix(), n%10000)
|
||||
id := t.newExchangeID(now)
|
||||
|
||||
ct := f.Response.Header.Get("Content-Type")
|
||||
url := f.Request.URL.String()
|
||||
@@ -511,6 +537,12 @@ VALUES(?,?,?,?,?,?,?)`,
|
||||
}
|
||||
}
|
||||
|
||||
// newExchangeID names a recorded exchange: unix seconds plus a rolling sequence.
|
||||
func (t *Traffic) newExchangeID(now time.Time) string {
|
||||
n := t.seq.Add(1)
|
||||
return fmt.Sprintf("%d-%04d", now.Unix(), n%10000)
|
||||
}
|
||||
|
||||
// storedBody is one body after the inline/spill decision: inline is what goes in
|
||||
// the SQLite column (the whole body, or a preview when spilled), hash names the
|
||||
// blob when it spilled, and index is the text handed to FTS (empty for binary).
|
||||
@@ -526,6 +558,12 @@ type storedBody struct {
|
||||
// are handed to the full-text index in full (up to maxIndexBody) — indexing is
|
||||
// independent of where the bytes end up.
|
||||
func (t *Traffic) spill(body []byte, contentType string) storedBody {
|
||||
return t.spillAt(body, contentType, maxInlineBody)
|
||||
}
|
||||
|
||||
// spillAt is spill with the inline threshold as a parameter: bodies up to
|
||||
// maxInline stay inline. Stream messages use a much lower one (maxStreamInline).
|
||||
func (t *Traffic) spillAt(body []byte, contentType string, maxInline int) storedBody {
|
||||
if len(body) == 0 {
|
||||
return storedBody{}
|
||||
}
|
||||
@@ -536,7 +574,7 @@ func (t *Traffic) spill(body []byte, contentType string) storedBody {
|
||||
}
|
||||
return string(clipBytes(body, maxIndexBody))
|
||||
}
|
||||
if len(body) <= maxInlineBody {
|
||||
if len(body) <= maxInline {
|
||||
return storedBody{inline: body, index: indexText()}
|
||||
}
|
||||
|
||||
@@ -1191,6 +1229,9 @@ func (t *Traffic) deleteWhere(tx *sql.Tx, where string, args ...any) (int64, err
|
||||
if _, err := tx.Exec(`DELETE FROM exchange_bodies WHERE id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil {
|
||||
return 0, fmt.Errorf("delete bodies: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM stream_messages WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil {
|
||||
return 0, fmt.Errorf("delete stream messages: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM blob_refs WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil {
|
||||
return 0, fmt.Errorf("delete blob references: %w", err)
|
||||
}
|
||||
@@ -2058,7 +2099,7 @@ func (t *Traffic) Tools() []actool.CoreTool {
|
||||
|
||||
get := actool.Build(actool.Spec{
|
||||
Name: "traffic_get",
|
||||
Description: "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling.",
|
||||
Description: "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling. A WebSocket (status 101) or SSE (text/event-stream) record shows only its handshake here; read its messages with traffic_messages.",
|
||||
Schema: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{"id": map[string]any{"type": "string", "description": "the id returned by traffic_search"}},
|
||||
@@ -2116,7 +2157,70 @@ func (t *Traffic) Tools() []actool.CoreTool {
|
||||
},
|
||||
})
|
||||
|
||||
return []actool.CoreTool{search, get, blob}
|
||||
messages := actool.Build(actool.Spec{
|
||||
Name: "traffic_messages",
|
||||
Description: "List the messages of one captured WebSocket or SSE (Server-Sent Events) record in order: direction (c2s client→server, s2c server→client), type (text/binary/event, or meta when the recorder dropped messages past its limits), size and a preview of at most 1KB (binary as hex). A payload stored as a blob carries its hash; read the rest with traffic_blob. Returns 20 messages by default, at most 50; use offset to page.",
|
||||
Schema: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"id": map[string]any{"type": "string", "description": "the record id from traffic_search (a WebSocket status-101 or SSE record)"},
|
||||
"offset": map[string]any{"type": "integer", "description": "number of messages to skip, default 0"},
|
||||
"limit": map[string]any{"type": "integer", "description": "messages to return, default 20, max 50"},
|
||||
},
|
||||
"required": []any{"id"},
|
||||
},
|
||||
ReadOnly: ro,
|
||||
Permissions: allow,
|
||||
Run: func(_ context.Context, in json.RawMessage, _ *actool.ToolContext) (actool.Result, error) {
|
||||
var a struct {
|
||||
ID string
|
||||
Offset int
|
||||
Limit int
|
||||
}
|
||||
_ = json.Unmarshal(in, &a)
|
||||
if strings.TrimSpace(a.ID) == "" {
|
||||
return actool.Errorf("id is a required parameter"), nil
|
||||
}
|
||||
if a.Limit <= 0 {
|
||||
a.Limit = 20
|
||||
}
|
||||
if a.Limit > 50 {
|
||||
a.Limit = 50
|
||||
}
|
||||
if a.Offset < 0 {
|
||||
a.Offset = 0
|
||||
}
|
||||
msgs, total, err := t.StreamMessages(a.ID, a.Offset, a.Limit)
|
||||
if err != nil {
|
||||
return actool.Errorf(err.Error()), nil
|
||||
}
|
||||
if total == 0 {
|
||||
return actool.Text("No stream messages recorded for this id (not a WebSocket/SSE record, or no message was exchanged)."), nil
|
||||
}
|
||||
type liteMsg struct {
|
||||
Seq int64 `json:"seq"`
|
||||
Direction string `json:"direction"`
|
||||
Type string `json:"type"`
|
||||
Event string `json:"event,omitempty"`
|
||||
Size int `json:"size"`
|
||||
Preview string `json:"preview"`
|
||||
Blob string `json:"blob,omitempty"`
|
||||
Truncated bool `json:"truncated,omitempty"`
|
||||
}
|
||||
lite := make([]liteMsg, 0, len(msgs))
|
||||
for _, m := range msgs {
|
||||
l := liteMsg{Seq: m.Seq, Direction: m.Direction, Type: m.Type, Event: m.Event, Size: m.Size, Preview: streamToolPreview(m), Truncated: m.Truncated}
|
||||
if m.Blob != "" {
|
||||
l.Blob = "@blob sha256:" + m.Blob
|
||||
}
|
||||
lite = append(lite, l)
|
||||
}
|
||||
b, _ := json.Marshal(lite)
|
||||
return actool.Text(fmt.Sprintf("[messages %d-%d of %d]\n", a.Offset+1, a.Offset+len(msgs), total) + string(b)), nil
|
||||
},
|
||||
})
|
||||
|
||||
return []actool.CoreTool{search, get, blob, messages}
|
||||
}
|
||||
|
||||
// SeedToolMetas returns the traffic tools built on a ZERO receiver, for seeding the
|
||||
|
||||
+28
-1
@@ -377,6 +377,22 @@ VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
seed("1-0002", "b.example.com", hb) // sole reference to blobB
|
||||
seed("1-0003", "c.example.com", hc) // shares blobC with d
|
||||
seed("1-0004", "d.example.com", hc)
|
||||
// d is also a stream exchange whose one message payload spilled to blobD,
|
||||
// referenced through blob_refs like any body.
|
||||
hd := strings.Repeat("d", 64)
|
||||
blobD := filepath.Join(dir, "_blobs", "sha256", "dd", hd+".bin")
|
||||
if err := os.MkdirAll(filepath.Dir(blobD), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(blobD, []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tr.DB().Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload,payload_blob) VALUES('1-0004',1,1,'s2c',2,1,x'78',?)`, hd); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tr.DB().Exec(`INSERT INTO blob_refs(hash,exchange_id) VALUES(?, '1-0004')`, hd); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Delete a: blobA orphaned → removed; blobB/blobC still referenced → kept.
|
||||
if n, err := tr.DeleteHost("a.example"); err != nil || n != 1 {
|
||||
@@ -399,12 +415,23 @@ VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
if _, err := os.Stat(blobC); err != nil {
|
||||
t.Fatalf("shared blobC removed after deleting one sharer: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(blobD); err != nil {
|
||||
t.Fatalf("stream payload blobD removed while d still references it: %v", err)
|
||||
}
|
||||
|
||||
// Delete d: last reference gone → blobC collected.
|
||||
// Delete d: last reference gone → blobC collected, and d's stream message
|
||||
// and its payload blob go with it.
|
||||
if n, err := tr.DeleteHost("d.example"); err != nil || n != 1 {
|
||||
t.Fatalf("DeleteHost(d.example)=%d, err=%v; want 1, nil", n, err)
|
||||
}
|
||||
if _, err := os.Stat(blobC); !os.IsNotExist(err) {
|
||||
t.Fatalf("blobC still exists after last reference removed: %v", err)
|
||||
}
|
||||
var streamRows int
|
||||
if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM stream_messages`).Scan(&streamRows); err != nil || streamRows != 0 {
|
||||
t.Fatalf("%d stream rows left after deleting d, err=%v", streamRows, err)
|
||||
}
|
||||
if _, err := os.Stat(blobD); !os.IsNotExist(err) {
|
||||
t.Fatalf("stream payload blobD still exists after its exchange was deleted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
Generated
+588
-234
File diff suppressed because it is too large
Load Diff
+4
-4
@@ -12,7 +12,8 @@
|
||||
"check": "biome check",
|
||||
"check:fix": "biome check --write",
|
||||
"prepare": "husky",
|
||||
"generate:presets": "ts-node -P tsconfig.scripts.json src/scripts/generate-theme-presets.ts"
|
||||
"generate:presets": "ts-node -P tsconfig.scripts.json src/scripts/generate-theme-presets.ts",
|
||||
"test:polling": "node --test scripts/polling.test.mjs"
|
||||
},
|
||||
"lint-staged": {
|
||||
"*.{js,ts,jsx,tsx}": [
|
||||
@@ -21,7 +22,6 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@antv/g6": "^5.1.1",
|
||||
"@antv/graphin": "^3.0.5",
|
||||
"@base-ui/react": "^1.6.0",
|
||||
"@dnd-kit/core": "^6.3.1",
|
||||
"@dnd-kit/modifiers": "^9.0.0",
|
||||
@@ -40,7 +40,7 @@
|
||||
"geist": "^1.7.2",
|
||||
"input-otp": "^1.4.2",
|
||||
"lucide-react": "^1.22.0",
|
||||
"next": "^16.2.9",
|
||||
"next": "^16.4.6",
|
||||
"next-themes": "^0.4.6",
|
||||
"radix-ui": "^1.6.0",
|
||||
"react": "^19.2.7",
|
||||
@@ -52,7 +52,6 @@
|
||||
"react-resizable-panels": "^4.12.0",
|
||||
"recharts": "^3.8.0",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"shadcn": "^4.12.0",
|
||||
"simple-icons": "^16.24.1",
|
||||
"sonner": "^2.0.7",
|
||||
"tailwind-merge": "^3.6.0",
|
||||
@@ -74,6 +73,7 @@
|
||||
"husky": "^9.1.7",
|
||||
"lint-staged": "^16.4.0",
|
||||
"postcss": "^8.5.16",
|
||||
"shadcn": "^4.12.0",
|
||||
"tailwindcss": "^4.1.5",
|
||||
"ts-node": "^10.9.2",
|
||||
"tw-animate-css": "^1.4.0",
|
||||
|
||||
Executable
+120
@@ -0,0 +1,120 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import vm from "node:vm";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import ts from "typescript";
|
||||
|
||||
const web = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
function moduleFor(name, globals = {}) {
|
||||
const output = ts.transpileModule(fs.readFileSync(path.join(web, "src/lib", name + ".ts"), "utf8"), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 },
|
||||
}).outputText;
|
||||
const exports = {};
|
||||
vm.runInNewContext(output, { exports, ...globals });
|
||||
return exports;
|
||||
}
|
||||
function setup(hidden = false) {
|
||||
let sequence = 0;
|
||||
const timers = new Map();
|
||||
const listeners = new Set();
|
||||
const document = {
|
||||
visibilityState: hidden ? "hidden" : "visible",
|
||||
addEventListener: (_, callback) => listeners.add(callback),
|
||||
removeEventListener: (_, callback) => listeners.delete(callback),
|
||||
};
|
||||
const { setVisibleInterval } = moduleFor("polling", {
|
||||
document,
|
||||
setTimeout: (callback) => {
|
||||
const id = ++sequence;
|
||||
timers.set(id, callback);
|
||||
return id;
|
||||
},
|
||||
clearTimeout: (id) => timers.delete(id),
|
||||
});
|
||||
return {
|
||||
setVisibleInterval,
|
||||
timers,
|
||||
listeners,
|
||||
visibility(hidden) {
|
||||
document.visibilityState = hidden ? "hidden" : "visible";
|
||||
for (const callback of listeners) callback();
|
||||
},
|
||||
tick() {
|
||||
const entries = [...timers];
|
||||
timers.clear();
|
||||
for (const [, callback] of entries) callback();
|
||||
},
|
||||
};
|
||||
}
|
||||
async function settle() {
|
||||
for (let i = 0; i < 8; i++) await Promise.resolve();
|
||||
}
|
||||
test("hidden tabs make no scheduled requests; returning refreshes immediately and cleanup stops work", async () => {
|
||||
const s = setup(true);
|
||||
let calls = 0;
|
||||
const stop = s.setVisibleInterval(() => {
|
||||
calls++;
|
||||
}, 5000);
|
||||
s.tick();
|
||||
await settle();
|
||||
assert.equal(calls, 0);
|
||||
assert.equal(s.timers.size, 0);
|
||||
s.visibility(false);
|
||||
await settle();
|
||||
assert.equal(calls, 1);
|
||||
assert.equal(s.timers.size, 1);
|
||||
s.visibility(true);
|
||||
assert.equal(s.timers.size, 0);
|
||||
s.visibility(false);
|
||||
await settle();
|
||||
assert.equal(calls, 2);
|
||||
stop();
|
||||
s.tick();
|
||||
s.visibility(false);
|
||||
await settle();
|
||||
assert.equal(calls, 2);
|
||||
assert.equal(s.listeners.size, 0);
|
||||
assert.equal(s.timers.size, 0);
|
||||
});
|
||||
test("a slow poll cannot overlap on timer or visibility changes; dispose while running cannot revive it", async () => {
|
||||
const s = setup();
|
||||
let calls = 0;
|
||||
let release;
|
||||
const stop = s.setVisibleInterval(() => {
|
||||
calls++;
|
||||
return new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
}, 2000);
|
||||
s.tick();
|
||||
assert.equal(calls, 1);
|
||||
s.visibility(true);
|
||||
s.visibility(false);
|
||||
s.tick();
|
||||
assert.equal(calls, 1);
|
||||
release();
|
||||
await settle();
|
||||
assert.equal(s.timers.size, 1);
|
||||
s.tick();
|
||||
assert.equal(calls, 2);
|
||||
stop();
|
||||
release();
|
||||
await settle();
|
||||
assert.equal(s.timers.size, 0);
|
||||
});
|
||||
test("a failed background refresh retries on the next interval", async () => {
|
||||
const s = setup();
|
||||
let calls = 0;
|
||||
const stop = s.setVisibleInterval(async () => {
|
||||
if (++calls === 1) throw Error("temporary");
|
||||
}, 1000);
|
||||
s.tick();
|
||||
await settle();
|
||||
assert.equal(s.timers.size, 1);
|
||||
s.tick();
|
||||
await settle();
|
||||
assert.equal(calls, 2);
|
||||
stop();
|
||||
});
|
||||
@@ -37,6 +37,7 @@ import type {
|
||||
NavMainLinkItem,
|
||||
NavMainParentItem,
|
||||
} from "@/navigation/sidebar/sidebar-items";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
|
||||
interface NavMainProps {
|
||||
readonly items: readonly NavGroup[];
|
||||
@@ -174,8 +175,8 @@ function InterceptPendingBadge() {
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
poll();
|
||||
const t = setInterval(poll, 5000);
|
||||
return () => { live = false; clearInterval(t); };
|
||||
const stopPolling = setVisibleInterval(poll, 5000);
|
||||
return () => { live = false; stopPolling(); };
|
||||
}, []);
|
||||
if (count === 0) return null;
|
||||
return (
|
||||
|
||||
@@ -40,6 +40,7 @@ import type {
|
||||
UsageStats,
|
||||
} from "@/lib/types";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
|
||||
// ── chart constants ───────────────────────────────────────────────────────────
|
||||
|
||||
@@ -172,10 +173,10 @@ export default function DashboardPage() {
|
||||
}
|
||||
};
|
||||
void load();
|
||||
const t = setInterval(load, 5000);
|
||||
const stopPolling = setVisibleInterval(load, 5000);
|
||||
return () => {
|
||||
alive = false;
|
||||
clearInterval(t);
|
||||
stopPolling();
|
||||
};
|
||||
}, []);
|
||||
|
||||
@@ -211,10 +212,10 @@ export default function DashboardPage() {
|
||||
}
|
||||
};
|
||||
void load();
|
||||
const timer = setInterval(load, 5000);
|
||||
const stopPolling = setVisibleInterval(load, 5000);
|
||||
return () => {
|
||||
alive = false;
|
||||
clearInterval(timer);
|
||||
stopPolling();
|
||||
};
|
||||
}, []);
|
||||
|
||||
@@ -247,10 +248,10 @@ export default function DashboardPage() {
|
||||
}
|
||||
};
|
||||
void load();
|
||||
const t = setInterval(load, 15000);
|
||||
const stopPolling = setVisibleInterval(load, 15000);
|
||||
return () => {
|
||||
alive = false;
|
||||
clearInterval(t);
|
||||
stopPolling();
|
||||
};
|
||||
}, []);
|
||||
|
||||
|
||||
@@ -67,6 +67,7 @@ import {
|
||||
VERIFICATION_META,
|
||||
VERIFICATION_STATUSES,
|
||||
} from "./_components/findings-table";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
|
||||
const FINDING_LIST_PREFERENCE_KEY = "artex_finding_list_preferences";
|
||||
|
||||
@@ -573,8 +574,8 @@ export default function FindingsPage() {
|
||||
};
|
||||
refresh();
|
||||
if (view === "asset") return;
|
||||
const timer = setInterval(refresh, 5000);
|
||||
return () => clearInterval(timer);
|
||||
const stopPolling = setVisibleInterval(refresh, 5000);
|
||||
return stopPolling;
|
||||
}, [activeRetestFingerprint, loadFlat, loadGroup, preferencesHydrated, refreshGroups, view]);
|
||||
|
||||
React.useEffect(() => {
|
||||
@@ -606,10 +607,10 @@ export default function FindingsPage() {
|
||||
});
|
||||
};
|
||||
load();
|
||||
const t = setInterval(load, 5000);
|
||||
const stopPolling = setVisibleInterval(load, 5000);
|
||||
return () => {
|
||||
alive = false;
|
||||
clearInterval(t);
|
||||
stopPolling();
|
||||
};
|
||||
}, []);
|
||||
|
||||
|
||||
@@ -43,6 +43,7 @@ import { parseCompanyScopeText } from "@/lib/company-scope";
|
||||
import { taskAssetSourceLabel } from "@/lib/task-assets";
|
||||
import type { Asset, NewAssetType } from "@/lib/types";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
|
||||
const PAGE_SIZES = [25, 50, 100, 200];
|
||||
|
||||
@@ -415,11 +416,11 @@ export function AssetsTab({ taskId }: { taskId: string }) {
|
||||
};
|
||||
// Debounce DSL keystrokes; plain (re)loads and polling run immediately.
|
||||
const debounce = setTimeout(() => void load(), dsl ? 400 : 0);
|
||||
const timer = setInterval(() => void load(), 10_000);
|
||||
const stopPolling = setVisibleInterval(() => void load(), 10_000);
|
||||
return () => {
|
||||
active = false;
|
||||
clearTimeout(debounce);
|
||||
clearInterval(timer);
|
||||
stopPolling();
|
||||
};
|
||||
}, [page, query, refreshKey, size, tab, taskId]);
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ import { type Tone, toneClasses, toneDot } from "@/lib/status";
|
||||
import { taskAssetTypeLabel } from "@/lib/task-assets";
|
||||
import type { Edge, ExploreKind, FindingAsset, NewAssetType, TaskNode } from "@/lib/types";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
|
||||
const PAGE_SIZES = [20, 50, 100];
|
||||
const POLL_MS = 8000;
|
||||
@@ -519,10 +520,10 @@ export function BroadcastTab({ taskId }: { taskId: string }) {
|
||||
alive = false;
|
||||
};
|
||||
}
|
||||
const timer = setInterval(load, POLL_MS);
|
||||
const stopPolling = setVisibleInterval(load, POLL_MS);
|
||||
return () => {
|
||||
alive = false;
|
||||
clearInterval(timer);
|
||||
stopPolling();
|
||||
};
|
||||
}, [taskId, page, size, kinds, query, order, live, atLive]);
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ import { Input } from "@/components/ui/input";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { api } from "@/lib/api";
|
||||
import type { TaskCredential } from "@/lib/types";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
|
||||
// Credentials tab (#3): the task's test accounts — user-declared (program alias
|
||||
// accounts) and agent-recorded (accounts the workers registered themselves).
|
||||
@@ -38,8 +39,8 @@ export function CredentialsTab({ taskId }: { taskId: string }) {
|
||||
React.useEffect(() => {
|
||||
load();
|
||||
// Agent runs can record credentials mid-task; a light poll keeps the tab current.
|
||||
const timer = setInterval(load, 15000);
|
||||
return () => clearInterval(timer);
|
||||
const stopPolling = setVisibleInterval(load, 15000);
|
||||
return stopPolling;
|
||||
}, [load]);
|
||||
|
||||
async function add() {
|
||||
|
||||
@@ -16,6 +16,7 @@ import { useStoredSortPreference } from "@/lib/sort-preference";
|
||||
import { statusMeta } from "@/lib/status";
|
||||
import type { Finding, FindingStatus } from "@/lib/types";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
|
||||
type FindingSortField = "time";
|
||||
|
||||
@@ -159,10 +160,10 @@ export function FindingsTab({ taskId }: { taskId: string }) {
|
||||
});
|
||||
};
|
||||
load();
|
||||
const t = setInterval(load, 3000);
|
||||
const stopPolling = setVisibleInterval(load, 3000);
|
||||
return () => {
|
||||
active = false;
|
||||
clearInterval(t);
|
||||
stopPolling();
|
||||
};
|
||||
}, [taskId]);
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import { Card, CardContent } from "@/components/ui/card";
|
||||
import { ExplorationGraph } from "@/components/exploration-graph";
|
||||
import { api } from "@/lib/api";
|
||||
import type { Edge, TaskNode } from "@/lib/types";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
|
||||
export function GraphTab({ taskId }: { taskId: string }) {
|
||||
const [nodes, setNodes] = React.useState<TaskNode[]>([]);
|
||||
@@ -38,10 +39,10 @@ export function GraphTab({ taskId }: { taskId: string }) {
|
||||
});
|
||||
};
|
||||
load();
|
||||
const timer = setInterval(load, 20000);
|
||||
const stopPolling = setVisibleInterval(load, 20000);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearInterval(timer);
|
||||
stopPolling();
|
||||
};
|
||||
}, [taskId]);
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ import type {
|
||||
TaskNode,
|
||||
TaskScopeRow,
|
||||
} from "@/lib/types";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
|
||||
// Compact token count formatting (12345 → 12.3k, 2000000 → 2M).
|
||||
function fmtTokens(n: number): string {
|
||||
@@ -402,7 +403,7 @@ export function OverviewTab({ taskId }: { taskId: string }) {
|
||||
void loadTokens();
|
||||
void loadGoals();
|
||||
void loadConstraints();
|
||||
const timer = setInterval(() => {
|
||||
const stopPolling = setVisibleInterval(() => {
|
||||
void load();
|
||||
void loadTokens();
|
||||
void loadGoals();
|
||||
@@ -410,7 +411,7 @@ export function OverviewTab({ taskId }: { taskId: string }) {
|
||||
}, 3000);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearInterval(timer);
|
||||
stopPolling();
|
||||
};
|
||||
}, [taskId, loadScope, loadTokens, loadGoals, loadConstraints]);
|
||||
|
||||
|
||||
@@ -69,6 +69,7 @@ import type {
|
||||
TokenTotal,
|
||||
} from "@/lib/types";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
|
||||
// resolutionLabel names the LLM a session runs on. The badge shows this alone and
|
||||
// keeps the model id in its tooltip. Env-backed configs can arrive without a name,
|
||||
@@ -794,10 +795,10 @@ export function SessionsTab({ taskId, taskPaused = false }: { taskId: string; ta
|
||||
// Polling is best-effort; the next interval retries automatically.
|
||||
});
|
||||
void load();
|
||||
const t = setInterval(load, 5000);
|
||||
const stopPolling = setVisibleInterval(load, 5000);
|
||||
return () => {
|
||||
alive = false;
|
||||
clearInterval(t);
|
||||
stopPolling();
|
||||
};
|
||||
}, [taskId]);
|
||||
|
||||
@@ -817,10 +818,10 @@ export function SessionsTab({ taskId, taskPaused = false }: { taskId: string; ta
|
||||
});
|
||||
};
|
||||
void load();
|
||||
const timer = setInterval(load, 2000);
|
||||
const stopPolling = setVisibleInterval(load, 2000);
|
||||
return () => {
|
||||
alive = false;
|
||||
clearInterval(timer);
|
||||
stopPolling();
|
||||
};
|
||||
}, [taskId]);
|
||||
|
||||
@@ -836,10 +837,10 @@ export function SessionsTab({ taskId, taskPaused = false }: { taskId: string; ta
|
||||
// Polling is best-effort; the next interval retries automatically.
|
||||
});
|
||||
void load();
|
||||
const timer = setInterval(load, 10_000);
|
||||
const stopPolling = setVisibleInterval(load, 10_000);
|
||||
return () => {
|
||||
alive = false;
|
||||
clearInterval(timer);
|
||||
stopPolling();
|
||||
};
|
||||
}, [taskId]);
|
||||
|
||||
@@ -855,10 +856,10 @@ export function SessionsTab({ taskId, taskPaused = false }: { taskId: string; ta
|
||||
// Polling is best-effort; the next interval retries automatically.
|
||||
});
|
||||
void load();
|
||||
const t = setInterval(load, 5000);
|
||||
const stopPolling = setVisibleInterval(load, 5000);
|
||||
return () => {
|
||||
alive = false;
|
||||
clearInterval(t);
|
||||
stopPolling();
|
||||
};
|
||||
}, [taskId]);
|
||||
|
||||
@@ -1059,10 +1060,10 @@ export function SessionsTab({ taskId, taskPaused = false }: { taskId: string; ta
|
||||
// The next poll retries; Worker controls and transcripts remain available.
|
||||
});
|
||||
void load();
|
||||
const timer = setInterval(load, 5000);
|
||||
const stopPolling = setVisibleInterval(load, 5000);
|
||||
return () => {
|
||||
active = false;
|
||||
clearInterval(timer);
|
||||
stopPolling();
|
||||
};
|
||||
}, [taskId]);
|
||||
|
||||
@@ -1099,10 +1100,10 @@ export function SessionsTab({ taskId, taskPaused = false }: { taskId: string; ta
|
||||
// Polling is best-effort; the next interval retries automatically.
|
||||
});
|
||||
void load();
|
||||
const t = setInterval(load, 5000);
|
||||
const stopPolling = setVisibleInterval(load, 5000);
|
||||
return () => {
|
||||
active = false;
|
||||
clearInterval(t);
|
||||
stopPolling();
|
||||
};
|
||||
}, [taskId]);
|
||||
|
||||
|
||||
@@ -50,6 +50,7 @@ import { OverviewTab } from "./_tabs/overview-tab";
|
||||
import { ReportTab } from "./_tabs/report-tab";
|
||||
import { RetestsTab } from "./_tabs/retests-tab";
|
||||
import { SessionsTab } from "./_tabs/sessions-tab";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
|
||||
const TABS = [
|
||||
{ value: "sessions", label: "Sessions" },
|
||||
@@ -235,10 +236,10 @@ function TaskDetailInner() {
|
||||
// Polling is best-effort; the next interval retries automatically.
|
||||
});
|
||||
void load();
|
||||
const t = setInterval(load, 5000);
|
||||
const stopPolling = setVisibleInterval(load, 5000);
|
||||
return () => {
|
||||
alive = false;
|
||||
clearInterval(t);
|
||||
stopPolling();
|
||||
};
|
||||
}, [id]);
|
||||
|
||||
@@ -273,8 +274,8 @@ function TaskDetailInner() {
|
||||
}, [id]);
|
||||
React.useEffect(() => {
|
||||
load();
|
||||
const timer = setInterval(load, 5000);
|
||||
return () => clearInterval(timer);
|
||||
const stopPolling = setVisibleInterval(load, 5000);
|
||||
return stopPolling;
|
||||
}, [load]);
|
||||
|
||||
async function togglePause() {
|
||||
|
||||
@@ -119,6 +119,7 @@ import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
import { api } from "@/lib/api";
|
||||
import { getLocalStorageValue, setLocalStorageValue } from "@/lib/local-storage.client";
|
||||
import { type SortDirection, useStoredSortPreference } from "@/lib/sort-preference";
|
||||
@@ -485,8 +486,8 @@ export default function TasksPage() {
|
||||
React.useEffect(() => {
|
||||
load();
|
||||
loadCategories();
|
||||
const i = setInterval(load, POLL_MS);
|
||||
return () => clearInterval(i);
|
||||
const stopPolling = setVisibleInterval(load, POLL_MS);
|
||||
return stopPolling;
|
||||
}, [load, loadCategories]);
|
||||
|
||||
const refreshCategoriesAndTasks = React.useCallback(() => {
|
||||
@@ -1694,8 +1695,8 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {
|
||||
|
||||
React.useEffect(() => {
|
||||
void load();
|
||||
const timer = setInterval(() => void load(), 2_000);
|
||||
return () => clearInterval(timer);
|
||||
const stopPolling = setVisibleInterval(() => void load(), 2_000);
|
||||
return stopPolling;
|
||||
}, [load]);
|
||||
|
||||
const selectable = React.useMemo(
|
||||
|
||||
@@ -45,6 +45,7 @@ import { api } from "@/lib/api";
|
||||
import { useStoredSortPreference } from "@/lib/sort-preference";
|
||||
import type { TrafficDetail, TrafficExchange, TrafficHost, TrafficResp } from "@/lib/types";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
|
||||
function fmtTime(ts: string) {
|
||||
return new Date(ts).toLocaleString("zh-CN", {
|
||||
@@ -227,13 +228,13 @@ export default function TrafficPage() {
|
||||
});
|
||||
};
|
||||
load();
|
||||
const t = setInterval(() => {
|
||||
const stopPolling = setVisibleInterval(() => {
|
||||
if (page !== 0) return; // only auto-refresh the newest page
|
||||
load();
|
||||
}, 5000);
|
||||
return () => {
|
||||
alive = false;
|
||||
clearInterval(t);
|
||||
stopPolling();
|
||||
};
|
||||
}, [page, size, hostQ, method, queryQ, bodyQ, pathQ, statusFilter, respMinQ, respMaxQ, sort, reloadTick]);
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ import type { LLMPoolMember, LLMPoolStatus, LLMProfile, LLMRetryOverride } from
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
import { ProfileRetryFields, RetryPolicyPanel, ZERO_OVERRIDE } from "./_components/retry";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
|
||||
// The thinking switch (thinking.type) and the thinking effort (reasoning_effort) are two [independent] fields,
|
||||
// each set on its own — some APIs have no thinking field and activate thinking via the effort parameter alone, hence the decoupling.
|
||||
@@ -125,8 +126,8 @@ function PoolSheet({
|
||||
// The cooldown countdown is the remaining seconds computed by the backend — fetched on a timer only while the drawer is open and some profile is unhealthy, to keep it ticking.
|
||||
React.useEffect(() => {
|
||||
if (!open || !pool?.enabled || !pool.chain.some((m) => m.state !== "ok")) return;
|
||||
const t = setInterval(() => void onReload(), 10_000);
|
||||
return () => clearInterval(t);
|
||||
const stopPolling = setVisibleInterval(() => void onReload(), 10_000);
|
||||
return stopPolling;
|
||||
}, [open, pool, onReload]);
|
||||
|
||||
async function toggle(patch: { llm_pool_enabled?: boolean; llm_pool_bind_fallback?: boolean }) {
|
||||
|
||||
@@ -47,6 +47,7 @@ import type {
|
||||
InterceptRisk,
|
||||
} from "@/lib/types";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
|
||||
function fmtTime(value?: string) {
|
||||
if (!value) return "—";
|
||||
@@ -863,10 +864,10 @@ export function ApprovalRecords({ taskId }: { taskId?: string }) {
|
||||
|
||||
React.useEffect(() => {
|
||||
void load();
|
||||
const timer = setInterval(() => void load(), 5000);
|
||||
const stopPolling = setVisibleInterval(() => void load(), 5000);
|
||||
return () => {
|
||||
request.current++;
|
||||
clearInterval(timer);
|
||||
stopPolling();
|
||||
};
|
||||
}, [load]);
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import { toast } from "sonner";
|
||||
|
||||
import { sseUrl } from "@/lib/api";
|
||||
import { isBtwCommand, type SideExchange, type SideHistory, sideAPI } from "@/lib/side-questions";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
|
||||
// crypto.randomUUID is only available in secure contexts (https/localhost); it degrades when accessed via IP+http.
|
||||
function newSideRequestID(): string {
|
||||
@@ -120,8 +121,8 @@ export function useSideQuestions(parent: string | null) {
|
||||
useEffect(() => {
|
||||
if (!open || !parent) return;
|
||||
void load();
|
||||
const timer = setInterval(() => void load(), 2000);
|
||||
return () => clearInterval(timer);
|
||||
const stopPolling = setVisibleInterval(() => void load(), 2000);
|
||||
return stopPolling;
|
||||
}, [open, parent, load]);
|
||||
|
||||
// biome-ignore lint/correctness/useExhaustiveDependencies: Reconnect after a clear attempt invalidates older callbacks.
|
||||
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
// Background browser tabs must not keep downloading snapshots. A poll that
|
||||
// returns a promise completes before the next interval starts; focus refreshes
|
||||
// immediately, without creating a second request while one is still running.
|
||||
export function setVisibleInterval(callback: () => unknown | Promise<unknown>, delay: number): () => void {
|
||||
let stopped = false;
|
||||
let running = false;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const visible = () => typeof document === "undefined" || document.visibilityState !== "hidden";
|
||||
const clear = () => {
|
||||
if (timer !== undefined) clearTimeout(timer);
|
||||
timer = undefined;
|
||||
};
|
||||
const schedule = () => {
|
||||
clear();
|
||||
if (!stopped && visible()) timer = setTimeout(() => void run(), delay);
|
||||
};
|
||||
const run = async () => {
|
||||
clear();
|
||||
if (stopped || running || !visible()) return;
|
||||
running = true;
|
||||
try {
|
||||
await callback();
|
||||
} catch {
|
||||
// Existing loaders own their error UI; transient polling failures retry.
|
||||
} finally {
|
||||
running = false;
|
||||
schedule();
|
||||
}
|
||||
};
|
||||
const onVisibility = () => {
|
||||
clear();
|
||||
if (visible()) void run();
|
||||
};
|
||||
if (typeof document !== "undefined") document.addEventListener("visibilitychange", onVisibility);
|
||||
schedule();
|
||||
return () => {
|
||||
stopped = true;
|
||||
clear();
|
||||
if (typeof document !== "undefined") document.removeEventListener("visibilitychange", onVisibility);
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user