mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-17 21:50:53 +08:00
The logical call now returns the usage of every physical attempt (atomic and stream), keeps the usage of failed uncommitted stream attempts without delivering their events, normalizes per-attempt echoes with MergeEcho, stops with ctx.Err() when cancelled during backoff, and treats signed/redacted thinking and tool-use signatures as committing output. The llm_usage ledger stays at one row per physical call.
287 lines
10 KiB
Go
287 lines
10 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"iter"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/Autumn-27/artex/agent"
|
|
"github.com/Autumn-27/artex/db"
|
|
"github.com/Autumn-27/artex/llmrec"
|
|
"github.com/Autumn-27/norma/llm"
|
|
)
|
|
|
|
// Usage reported by one failed attempt, and by a successful one. A stream
|
|
// reports the input side in message_start and echoes it, with the output, in
|
|
// message_delta; the atomic path returns the attempt's normalized usage.
|
|
var (
|
|
failedAttemptUsage = llm.Usage{InputTokens: 11, OutputTokens: 2, CacheReadTokens: 3, CacheWriteTokens: 2}
|
|
okStartUsage = llm.Usage{InputTokens: 13, CacheReadTokens: 3, CacheWriteTokens: 2}
|
|
okDeltaUsage = llm.Usage{InputTokens: 17, OutputTokens: 5, CacheReadTokens: 4, CacheWriteTokens: 2}
|
|
okAttemptUsage = llm.Usage{InputTokens: 17, OutputTokens: 5, CacheReadTokens: 4, CacheWriteTokens: 2}
|
|
)
|
|
|
|
// usageAttemptProvider fails its first fail attempts (always, when fail < 0)
|
|
// after reporting failedAttemptUsage, then answers "done".
|
|
type usageAttemptProvider struct {
|
|
fail int
|
|
failErr error
|
|
onFail func()
|
|
// extra events a failed stream attempt emits before its error.
|
|
failEvents []llm.StreamEvent
|
|
calls int
|
|
}
|
|
|
|
func (p *usageAttemptProvider) failing() bool {
|
|
p.calls++
|
|
failed := p.fail < 0 || p.calls <= p.fail
|
|
if failed && p.onFail != nil {
|
|
p.onFail()
|
|
}
|
|
return failed
|
|
}
|
|
|
|
func (p *usageAttemptProvider) Complete(context.Context, llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
|
|
if p.failing() {
|
|
return llm.Message{}, "", failedAttemptUsage, p.failErr
|
|
}
|
|
return llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{llm.TextBlock("done")}}, "end_turn", okAttemptUsage, nil
|
|
}
|
|
|
|
func (p *usageAttemptProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
|
return func(yield func(llm.StreamEvent, error) bool) {
|
|
if p.failing() {
|
|
events := append([]llm.StreamEvent{{Type: llm.SEMessageStart, Usage: failedAttemptUsage}}, p.failEvents...)
|
|
for _, ev := range events {
|
|
if !yield(ev, nil) {
|
|
return
|
|
}
|
|
}
|
|
yield(llm.StreamEvent{}, p.failErr)
|
|
return
|
|
}
|
|
for _, ev := range []llm.StreamEvent{
|
|
{Type: llm.SEMessageStart, Usage: okStartUsage},
|
|
{Type: llm.SETextDelta, Text: "done"},
|
|
{Type: llm.SEMessageDelta, StopReason: "end_turn", Usage: okDeltaUsage},
|
|
{Type: llm.SEMessageStop},
|
|
} {
|
|
if !yield(ev, nil) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// logicalResult is what the SDK sees for one logical call.
|
|
type logicalResult struct {
|
|
usage llm.Usage
|
|
text string
|
|
events []llm.StreamEvent
|
|
err error
|
|
}
|
|
|
|
func runLogicalCall(ctx context.Context, stream bool, taskID string, hooks taskLLMStreamHooks) logicalResult {
|
|
if !stream {
|
|
msg, _, usage, err := completeTaskLLM(ctx, taskID, llm.CompletionRequest{}, hooks)
|
|
return logicalResult{usage: usage, text: msg.Text(), err: err}
|
|
}
|
|
var r logicalResult
|
|
acc := llm.NewAccumulator()
|
|
for ev, err := range streamTaskLLM(ctx, taskID, llm.CompletionRequest{}, hooks) {
|
|
if err != nil {
|
|
r.err = err
|
|
break
|
|
}
|
|
r.events = append(r.events, ev)
|
|
acc.Add(ev)
|
|
}
|
|
r.usage, r.text = acc.Usage, acc.Message().Text()
|
|
return r
|
|
}
|
|
|
|
func singleProfileHooks(p llm.Provider, retry agent.RetryConfig) taskLLMStreamHooks {
|
|
return taskLLMStreamHooks{
|
|
current: func(context.Context) (taskLLMSelection, error) {
|
|
return taskLLMSelection{provider: p, retry: retry}, nil
|
|
},
|
|
exhaust: func(taskLLMSelection, error) (db.TaskLLMTransition, error) {
|
|
return db.TaskLLMTransition{}, nil
|
|
},
|
|
}
|
|
}
|
|
|
|
func sumUsage(us ...llm.Usage) llm.Usage {
|
|
var t llm.Usage
|
|
for _, u := range us {
|
|
t.Add(u)
|
|
}
|
|
return t
|
|
}
|
|
|
|
var usageModes = []struct {
|
|
name string
|
|
stream bool
|
|
}{{"atomic", false}, {"stream", true}}
|
|
|
|
func TestTaskLLMRetrySumsAttemptUsage(t *testing.T) {
|
|
for _, m := range usageModes {
|
|
t.Run(m.name, func(t *testing.T) {
|
|
p := &usageAttemptProvider{fail: 1, failErr: errors.New("connection reset by peer")}
|
|
r := runLogicalCall(t.Context(), m.stream, "p12-retry", singleProfileHooks(p, agent.RetryConfig{StreamAttempts: 1, StreamInterval: time.Millisecond}))
|
|
if r.err != nil {
|
|
t.Fatal(r.err)
|
|
}
|
|
want := sumUsage(failedAttemptUsage, okAttemptUsage) // {28 7 7 4}
|
|
if r.usage != want || p.calls != 2 || r.text != "done" {
|
|
t.Fatalf("usage=%+v want %+v, calls=%d, text=%q", r.usage, want, p.calls, r.text)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestTaskLLMExhaustedRetriesReturnSummedUsage(t *testing.T) {
|
|
for _, m := range usageModes {
|
|
t.Run(m.name, func(t *testing.T) {
|
|
p := &usageAttemptProvider{fail: -1, failErr: errors.New("connection reset by peer")}
|
|
r := runLogicalCall(t.Context(), m.stream, "p12-exhausted", singleProfileHooks(p, agent.RetryConfig{StreamAttempts: 1, StreamInterval: time.Millisecond}))
|
|
if r.err == nil || r.err.Error() != "connection reset by peer" {
|
|
t.Fatalf("err=%v", r.err)
|
|
}
|
|
want := sumUsage(failedAttemptUsage, failedAttemptUsage)
|
|
if r.usage != want || p.calls != 2 {
|
|
t.Fatalf("usage=%+v want %+v, calls=%d", r.usage, want, p.calls)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestTaskLLMProfileFailoverSumsUsage(t *testing.T) {
|
|
for _, m := range usageModes {
|
|
t.Run(m.name, func(t *testing.T) {
|
|
first := &usageAttemptProvider{fail: -1, failErr: errors.New("openai: status 402: insufficient_quota")}
|
|
second := &usageAttemptProvider{}
|
|
active := int64(11)
|
|
hooks := taskLLMStreamHooks{
|
|
current: func(context.Context) (taskLLMSelection, error) {
|
|
if active == 11 {
|
|
return taskLLMSelection{profileID: 11, provider: first}, nil
|
|
}
|
|
return taskLLMSelection{profileID: 22, provider: second}, nil
|
|
},
|
|
exhaust: func(taskLLMSelection, error) (db.TaskLLMTransition, error) {
|
|
active = 22
|
|
next := active
|
|
return db.TaskLLMTransition{PreviousProfileID: 11, NextProfileID: &next, Advanced: true}, nil
|
|
},
|
|
}
|
|
r := runLogicalCall(t.Context(), m.stream, "p12-failover-"+m.name, hooks)
|
|
if r.err != nil {
|
|
t.Fatal(r.err)
|
|
}
|
|
want := sumUsage(failedAttemptUsage, okAttemptUsage)
|
|
if r.usage != want || first.calls != 1 || second.calls != 1 || r.text != "done" {
|
|
t.Fatalf("usage=%+v want %+v, calls first=%d second=%d, text=%q", r.usage, want, first.calls, second.calls, r.text)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestTaskLLMCancelDuringBackoffStops(t *testing.T) {
|
|
for _, m := range usageModes {
|
|
t.Run(m.name, func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
defer cancel()
|
|
p := &usageAttemptProvider{fail: -1, failErr: errors.New("connection reset by peer")}
|
|
// Cancel once the retry is waiting out its (long) backoff.
|
|
p.onFail = func() { time.AfterFunc(20*time.Millisecond, cancel) }
|
|
r := runLogicalCall(ctx, m.stream, "p12-cancel", singleProfileHooks(p, agent.RetryConfig{StreamAttempts: 2, StreamInterval: time.Minute}))
|
|
if !errors.Is(r.err, context.Canceled) {
|
|
t.Fatalf("err=%v, want context.Canceled", r.err)
|
|
}
|
|
if p.calls != 1 || r.usage != failedAttemptUsage {
|
|
t.Fatalf("calls=%d usage=%+v", p.calls, r.usage)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestTaskLLMUncommittedStreamAttemptKeepsUsageWithoutReplay(t *testing.T) {
|
|
p := &usageAttemptProvider{
|
|
fail: 1,
|
|
failErr: errors.New("anthropic: overloaded_error"),
|
|
// Buffered, non-committing events of the failed attempt.
|
|
failEvents: []llm.StreamEvent{{Type: llm.SEThinkingDelta}, {Type: llm.SEToolInputJSON}},
|
|
}
|
|
r := runLogicalCall(t.Context(), true, "p12-uncommitted", singleProfileHooks(p, agent.RetryConfig{StreamAttempts: 1, StreamInterval: time.Millisecond}))
|
|
if r.err != nil {
|
|
t.Fatal(r.err)
|
|
}
|
|
if want := sumUsage(failedAttemptUsage, okAttemptUsage); r.usage != want {
|
|
t.Fatalf("usage=%+v want %+v", r.usage, want)
|
|
}
|
|
if r.text != "done" {
|
|
t.Fatalf("text=%q, want the second attempt's only", r.text)
|
|
}
|
|
// The failed attempt contributes one usage-only message_start, nothing else.
|
|
if len(r.events) == 0 || r.events[0].Type != llm.SEMessageStart || r.events[0].Usage != failedAttemptUsage {
|
|
t.Fatalf("first event %+v", r.events)
|
|
}
|
|
for _, ev := range r.events {
|
|
if ev.Type == llm.SEThinkingDelta || ev.Type == llm.SEToolInputJSON {
|
|
t.Fatalf("buffered event of the failed attempt was delivered: %+v", r.events)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestTaskLLMOpaqueThinkingCommitsStream(t *testing.T) {
|
|
defer withZeroRetryBackoff()()
|
|
for _, kind := range []llm.StreamEventType{llm.SEThinkingSignature, llm.SERedactedThinking, llm.SEToolUseSignature} {
|
|
t.Run(string(kind), func(t *testing.T) {
|
|
if !streamEventCommitsOutput(llm.StreamEvent{Type: kind, Text: "opaque"}) {
|
|
t.Fatalf("%s does not commit", kind)
|
|
}
|
|
// A retryable failure after the opaque block is surfaced, not replayed.
|
|
p := &scriptedLLMProvider{events: []llm.StreamEvent{{Type: kind, Text: "opaque"}}, err: errors.New("connection reset by peer")}
|
|
events, err := collectTaskLLMStream(streamTaskLLM(t.Context(), "p12-opaque", llm.CompletionRequest{}, singleProfileHooks(p, agent.RetryConfig{})))
|
|
if err == nil || p.calls != 1 || len(events) != 1 || events[0].Type != kind {
|
|
t.Fatalf("calls=%d events=%+v err=%v", p.calls, events, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The recorder below the task router writes one llm_usage row per physical
|
|
// attempt; summing the attempts for the logical call adds no row.
|
|
func TestTaskLLMRetryKeepsOneLedgerRowPerPhysicalCall(t *testing.T) {
|
|
m, err := NewManager(t.TempDir(), "")
|
|
if err != nil {
|
|
t.Skipf("postgres unavailable (%v) — skipping", err)
|
|
}
|
|
defer m.Close()
|
|
pg := m.PG()
|
|
for _, mode := range usageModes {
|
|
t.Run(mode.name, func(t *testing.T) {
|
|
taskID := fmt.Sprintf("p12-ledger-%s-%d", mode.name, time.Now().UnixNano())
|
|
t.Cleanup(func() { _, _ = pg.Exec(`DELETE FROM llm_usage WHERE task_id=$1`, taskID) })
|
|
p := &usageAttemptProvider{fail: 1, failErr: errors.New("connection reset by peer")}
|
|
rec := llmrec.Wrap(p, pg, "test-model", "test-profile", "", "", func() bool { return false })
|
|
ctx := llmrec.WithTaskID(t.Context(), taskID)
|
|
r := runLogicalCall(ctx, mode.stream, taskID, singleProfileHooks(rec, agent.RetryConfig{StreamAttempts: 1, StreamInterval: time.Millisecond}))
|
|
if r.err != nil {
|
|
t.Fatal(r.err)
|
|
}
|
|
var rows, failed int
|
|
if err := pg.QueryRow(`SELECT count(*), count(*) FILTER (WHERE status='error') FROM llm_usage WHERE task_id=$1`, taskID).Scan(&rows, &failed); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if rows != 2 || failed != 1 || p.calls != 2 {
|
|
t.Fatalf("llm_usage rows=%d (error %d), physical calls=%d; want 2 rows, 1 error", rows, failed, p.calls)
|
|
}
|
|
})
|
|
}
|
|
}
|