mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-17 21:50:53 +08:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ce096cdba1 | ||
|
|
50f6d07640 | ||
|
|
e6ab0f34a7 | ||
|
|
5ed28cde9f |
+6
-6
@@ -195,11 +195,11 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
// Always track usage (cheap); accumulate text/thinking only for bodies.
|
// Always track usage (cheap); accumulate text/thinking only for bodies.
|
||||||
// Anthropic (and the other providers) split token usage across events:
|
// message_start carries the input side (input + cache) and is added.
|
||||||
// message_start carries ONLY the input side (input + cache), message_delta
|
// Since Norma v0.4.7 message_delta also echoes that input side next to the
|
||||||
// ONLY the output. They must be FOLDED with Add — overwriting on delta
|
// output, so it is folded with llm.Usage.MergeEcho (max on echoed fields,
|
||||||
// would zero out the input already counted at start (mirrors the SDK's own
|
// sum on output), as llm.Accumulator does; Add would count the input and
|
||||||
// llm.Accumulator; see norma/llm/accumulate.go).
|
// cache twice. An output-only delta folds the same either way.
|
||||||
switch ev.Type {
|
switch ev.Type {
|
||||||
case llm.SETextDelta:
|
case llm.SETextDelta:
|
||||||
if recordBodies {
|
if recordBodies {
|
||||||
@@ -215,7 +215,7 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S
|
|||||||
if ev.StopReason != "" {
|
if ev.StopReason != "" {
|
||||||
stopReason = ev.StopReason
|
stopReason = ev.StopReason
|
||||||
}
|
}
|
||||||
usage.Add(ev.Usage)
|
usage.MergeEcho(ev.Usage)
|
||||||
}
|
}
|
||||||
if !yield(ev, err) {
|
if !yield(ev, err) {
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -0,0 +1,118 @@
|
|||||||
|
package llmrec
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"iter"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/Autumn-27/artex/db"
|
||||||
|
"github.com/Autumn-27/norma/llm"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// echoStreamProvider streams one message_start and one message_delta with the
|
||||||
|
// given (already normalized) usage, as Norma's providers emit them.
|
||||||
|
type echoStreamProvider struct {
|
||||||
|
completeProvider
|
||||||
|
start, delta llm.Usage
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p echoStreamProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
||||||
|
return func(yield func(llm.StreamEvent, error) bool) {
|
||||||
|
for _, ev := range []llm.StreamEvent{
|
||||||
|
{Type: llm.SEMessageStart, Usage: p.start},
|
||||||
|
{Type: llm.SETextDelta, Text: "answer"},
|
||||||
|
{Type: llm.SEMessageDelta, StopReason: "end_turn", Usage: p.delta},
|
||||||
|
{Type: llm.SEMessageStop},
|
||||||
|
} {
|
||||||
|
if !yield(ev, nil) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Norma normalizes Anthropic usage so InputTokens includes the cache read and
|
||||||
|
// write subsets, and since v0.4.7 message_delta echoes that input side.
|
||||||
|
var usageEchoCases = []struct {
|
||||||
|
name string
|
||||||
|
start, delta llm.Usage
|
||||||
|
want llm.Usage
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "full_echo",
|
||||||
|
start: llm.Usage{InputTokens: 23, CacheReadTokens: 5, CacheWriteTokens: 3},
|
||||||
|
delta: llm.Usage{InputTokens: 23, CacheReadTokens: 5, CacheWriteTokens: 3, OutputTokens: 5},
|
||||||
|
want: llm.Usage{InputTokens: 23, OutputTokens: 5, CacheReadTokens: 5, CacheWriteTokens: 3},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Raw start: input 500, cache read 200, cache creation 40 → normalized 740.
|
||||||
|
// Raw delta: input 500, output 77, no cache fields → normalized 500.
|
||||||
|
name: "partial_echo",
|
||||||
|
start: llm.Usage{InputTokens: 740, CacheReadTokens: 200, CacheWriteTokens: 40},
|
||||||
|
delta: llm.Usage{InputTokens: 500, OutputTokens: 77},
|
||||||
|
want: llm.Usage{InputTokens: 740, OutputTokens: 77, CacheReadTokens: 200, CacheWriteTokens: 40},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no_echo",
|
||||||
|
start: llm.Usage{InputTokens: 23, CacheReadTokens: 5, CacheWriteTokens: 3},
|
||||||
|
delta: llm.Usage{OutputTokens: 5},
|
||||||
|
want: llm.Usage{InputTokens: 23, OutputTokens: 5, CacheReadTokens: 5, CacheWriteTokens: 3},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func drainRecorderStream(t *testing.T, ctx context.Context, r *Recorder) {
|
||||||
|
t.Helper()
|
||||||
|
for _, err := range r.Stream(ctx, llm.CompletionRequest{}) {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The metered usage of a streamed call counts an echoed input side once.
|
||||||
|
func TestStreamUsageEchoCountedOnce(t *testing.T) {
|
||||||
|
for _, tc := range usageEchoCases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var got []llm.Usage
|
||||||
|
ctx := WithUsageSink(t.Context(), func(u llm.Usage) { got = append(got, u) })
|
||||||
|
drainRecorderStream(t, ctx, Wrap(echoStreamProvider{start: tc.start, delta: tc.delta}, nil, "fixture", "profile", "", "", func() bool { return false }))
|
||||||
|
if len(got) != 1 || got[0] != tc.want {
|
||||||
|
t.Fatalf("metered %+v, want one call with %+v", got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The llm_usage ledger gets one row per streamed call, with the echo counted once.
|
||||||
|
func TestStreamUsageEchoLedgerRow(t *testing.T) {
|
||||||
|
dsn, _, err := db.DSN()
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("no database config (%v)", err)
|
||||||
|
}
|
||||||
|
pg, err := db.Open(dsn)
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("postgres unavailable (%v)", err)
|
||||||
|
}
|
||||||
|
defer pg.Close()
|
||||||
|
if err := pg.EnsureLLMUsageTable(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, tc := range usageEchoCases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
profile := "usage-echo-" + uuid.NewString()
|
||||||
|
t.Cleanup(func() { _, _ = pg.Exec(`DELETE FROM llm_usage WHERE profile_name=$1`, profile) })
|
||||||
|
drainRecorderStream(t, t.Context(), Wrap(echoStreamProvider{start: tc.start, delta: tc.delta}, pg, "fixture", profile, "", "", func() bool { return false }))
|
||||||
|
var rows int
|
||||||
|
var got llm.Usage
|
||||||
|
err := pg.QueryRow(`SELECT count(*), COALESCE(sum(input_tokens),0), COALESCE(sum(output_tokens),0), COALESCE(sum(cache_read),0), COALESCE(sum(cache_write),0)
|
||||||
|
FROM llm_usage WHERE profile_name=$1`, profile).Scan(&rows, &got.InputTokens, &got.OutputTokens, &got.CacheReadTokens, &got.CacheWriteTokens)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if rows != 1 || got != tc.want {
|
||||||
|
t.Fatalf("ledger rows=%d usage=%+v, want 1 row with %+v", rows, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
+75
-19
@@ -317,10 +317,14 @@ func (r *taskLLMRuntime) Complete(ctx context.Context, req llm.CompletionRequest
|
|||||||
}
|
}
|
||||||
|
|
||||||
func completeTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest, hooks taskLLMStreamHooks) (llm.Message, string, llm.Usage, error) {
|
func completeTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest, hooks taskLLMStreamHooks) (llm.Message, string, llm.Usage, error) {
|
||||||
|
// total is the usage of the logical call: the sum of every physical attempt
|
||||||
|
// (same-provider retries and profile failover). Each attempt is already billed
|
||||||
|
// once by the recorder below; this sum only feeds the caller's accounting.
|
||||||
|
var total llm.Usage
|
||||||
for {
|
for {
|
||||||
selection, err := hooks.current(ctx)
|
selection, err := hooks.current(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return llm.Message{}, "", llm.Usage{}, err
|
return llm.Message{}, "", total, err
|
||||||
}
|
}
|
||||||
var (
|
var (
|
||||||
msg llm.Message
|
msg llm.Message
|
||||||
@@ -333,13 +337,14 @@ func completeTaskLLM(ctx context.Context, taskID string, req llm.CompletionReque
|
|||||||
retries, backoffOf := sameProviderRetryPolicy(selection.retry)
|
retries, backoffOf := sameProviderRetryPolicy(selection.retry)
|
||||||
for attempt := 0; ; attempt++ {
|
for attempt := 0; ; attempt++ {
|
||||||
msg, sr, usage, callErr = selection.provider.Complete(ctx, req)
|
msg, sr, usage, callErr = selection.provider.Complete(ctx, req)
|
||||||
|
total.Add(usage)
|
||||||
if callErr != nil && ctx.Err() == nil &&
|
if callErr != nil && ctx.Err() == nil &&
|
||||||
attempt < retries && isRetryableStreamError(callErr) {
|
attempt < retries && isRetryableStreamError(callErr) {
|
||||||
backoff := backoffOf(attempt)
|
backoff := backoffOf(attempt)
|
||||||
log.Printf("[task-llm] task %s non-streaming call failed, retrying on the same provider after %v (%d/%d): %v",
|
log.Printf("[task-llm] task %s non-streaming call failed, retrying on the same provider after %v (%d/%d): %v",
|
||||||
taskID, backoff, attempt+1, retries, callErr)
|
taskID, backoff, attempt+1, retries, callErr)
|
||||||
if sleepCtx(ctx, backoff) {
|
if sleepCtx(ctx, backoff) {
|
||||||
break // ctx cancelled during backoff → stop retrying
|
return llm.Message{}, "", total, ctx.Err() // cancelled during backoff → stop
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -347,7 +352,7 @@ func completeTaskLLM(ctx context.Context, taskID string, req llm.CompletionReque
|
|||||||
}
|
}
|
||||||
if callErr == nil {
|
if callErr == nil {
|
||||||
resetLLMFailStreak(taskID, selection.profileID)
|
resetLLMFailStreak(taskID, selection.profileID)
|
||||||
return msg, sr, usage, nil
|
return msg, sr, total, nil
|
||||||
}
|
}
|
||||||
// Non-quota failures on a chain profile escalate after a consecutive
|
// Non-quota failures on a chain profile escalate after a consecutive
|
||||||
// streak (#99): same advance path as quota exhaust, truthful reason.
|
// streak (#99): same advance path as quota exhaust, truthful reason.
|
||||||
@@ -356,31 +361,31 @@ func completeTaskLLM(ctx context.Context, taskID string, req llm.CompletionReque
|
|||||||
cause := fmt.Errorf("profile #%d failed %d consecutive calls (last: %w)", selection.profileID, llmFailStreakThreshold, callErr)
|
cause := fmt.Errorf("profile #%d failed %d consecutive calls (last: %w)", selection.profileID, llmFailStreakThreshold, callErr)
|
||||||
transition, markErr := hooks.exhaust(selection, cause)
|
transition, markErr := hooks.exhaust(selection, cause)
|
||||||
if markErr != nil {
|
if markErr != nil {
|
||||||
return llm.Message{}, "", llm.Usage{}, fmt.Errorf("mark profile exhausted after %v: %w", callErr, markErr)
|
return llm.Message{}, "", total, fmt.Errorf("mark profile exhausted after %v: %w", callErr, markErr)
|
||||||
}
|
}
|
||||||
if transition.Advanced && !transition.Stale && hooks.transition != nil {
|
if transition.Advanced && !transition.Stale && hooks.transition != nil {
|
||||||
hooks.transition(selection, transition, callErr)
|
hooks.transition(selection, transition, callErr)
|
||||||
}
|
}
|
||||||
if !transition.Stale && transition.NextProfileID == nil {
|
if !transition.Stale && transition.NextProfileID == nil {
|
||||||
return llm.Message{}, "", llm.Usage{}, &taskLLMError{taskID: taskID, chainExhausted: transition.ChainExhausted, cause: callErr}
|
return llm.Message{}, "", total, &taskLLMError{taskID: taskID, chainExhausted: transition.ChainExhausted, cause: callErr}
|
||||||
}
|
}
|
||||||
continue // replay the same logical request on the next profile
|
continue // replay the same logical request on the next profile
|
||||||
}
|
}
|
||||||
return llm.Message{}, "", llm.Usage{}, callErr
|
return llm.Message{}, "", total, callErr
|
||||||
}
|
}
|
||||||
// profileID=0: the explicit chain was cleared; non-quota error: pass through. Neither changes the task failover state.
|
// profileID=0: the explicit chain was cleared; non-quota error: pass through. Neither changes the task failover state.
|
||||||
if selection.profileID == 0 || !isQuotaExhaustedError(callErr) {
|
if selection.profileID == 0 || !isQuotaExhaustedError(callErr) {
|
||||||
return llm.Message{}, "", llm.Usage{}, callErr
|
return llm.Message{}, "", total, callErr
|
||||||
}
|
}
|
||||||
transition, markErr := hooks.exhaust(selection, callErr)
|
transition, markErr := hooks.exhaust(selection, callErr)
|
||||||
if markErr != nil {
|
if markErr != nil {
|
||||||
return llm.Message{}, "", llm.Usage{}, fmt.Errorf("mark profile quota exhausted after %v: %w", callErr, markErr)
|
return llm.Message{}, "", total, fmt.Errorf("mark profile quota exhausted after %v: %w", callErr, markErr)
|
||||||
}
|
}
|
||||||
if transition.Advanced && !transition.Stale && hooks.transition != nil {
|
if transition.Advanced && !transition.Stale && hooks.transition != nil {
|
||||||
hooks.transition(selection, transition, callErr)
|
hooks.transition(selection, transition, callErr)
|
||||||
}
|
}
|
||||||
if !transition.Stale && transition.NextProfileID == nil {
|
if !transition.Stale && transition.NextProfileID == nil {
|
||||||
return llm.Message{}, "", llm.Usage{}, &taskLLMError{taskID: taskID, chainExhausted: transition.ChainExhausted, cause: callErr}
|
return llm.Message{}, "", total, &taskLLMError{taskID: taskID, chainExhausted: transition.ChainExhausted, cause: callErr}
|
||||||
}
|
}
|
||||||
// No output has been delivered to the caller, so replaying the same logical request on the next profile is safe.
|
// No output has been delivered to the caller, so replaying the same logical request on the next profile is safe.
|
||||||
}
|
}
|
||||||
@@ -397,6 +402,35 @@ func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest
|
|||||||
committed := false
|
committed := false
|
||||||
var pending []llm.StreamEvent
|
var pending []llm.StreamEvent
|
||||||
var streamErr error
|
var streamErr error
|
||||||
|
// The caller's llm.Accumulator sums usage across the whole logical call
|
||||||
|
// (every attempt and profile): message_start is added, message_delta is
|
||||||
|
// folded with MergeEcho. Folding one attempt's echo against the sum of
|
||||||
|
// earlier attempts would lose input/cache tokens, so each attempt's echoes
|
||||||
|
// are normalized here against that attempt alone and only the increment
|
||||||
|
// is passed on, as a message_start. The recorder below already billed the
|
||||||
|
// physical attempt; this only shapes what the caller adds up.
|
||||||
|
var attemptUsage llm.Usage
|
||||||
|
forward := func(event llm.StreamEvent) bool {
|
||||||
|
switch event.Type {
|
||||||
|
case llm.SEMessageStart:
|
||||||
|
attemptUsage.Add(event.Usage)
|
||||||
|
case llm.SEMessageDelta:
|
||||||
|
next := attemptUsage
|
||||||
|
next.MergeEcho(event.Usage)
|
||||||
|
inc := llm.Usage{
|
||||||
|
InputTokens: next.InputTokens - attemptUsage.InputTokens,
|
||||||
|
OutputTokens: next.OutputTokens - attemptUsage.OutputTokens,
|
||||||
|
CacheReadTokens: next.CacheReadTokens - attemptUsage.CacheReadTokens,
|
||||||
|
CacheWriteTokens: next.CacheWriteTokens - attemptUsage.CacheWriteTokens,
|
||||||
|
}
|
||||||
|
attemptUsage = next
|
||||||
|
if inc != (llm.Usage{}) && !yield(llm.StreamEvent{Type: llm.SEMessageStart, Usage: inc}, nil) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
event.Usage = llm.Usage{}
|
||||||
|
}
|
||||||
|
return yield(event, nil)
|
||||||
|
}
|
||||||
retries, backoffOf := sameProviderRetryPolicy(selection.retry)
|
retries, backoffOf := sameProviderRetryPolicy(selection.retry)
|
||||||
// Same-provider safe-window retry: a transient failure before committed
|
// Same-provider safe-window retry: a transient failure before committed
|
||||||
// (nothing delivered to the caller yet) can be replayed verbatim without
|
// (nothing delivered to the caller yet) can be replayed verbatim without
|
||||||
@@ -405,6 +439,7 @@ func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest
|
|||||||
committed = false
|
committed = false
|
||||||
pending = nil
|
pending = nil
|
||||||
streamErr = nil
|
streamErr = nil
|
||||||
|
attemptUsage = llm.Usage{}
|
||||||
for event, err := range selection.provider.Stream(ctx, req) {
|
for event, err := range selection.provider.Stream(ctx, req) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
streamErr = err
|
streamErr = err
|
||||||
@@ -416,24 +451,34 @@ func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest
|
|||||||
}
|
}
|
||||||
if !committed {
|
if !committed {
|
||||||
for _, buffered := range pending {
|
for _, buffered := range pending {
|
||||||
if !yield(buffered, nil) {
|
if !forward(buffered) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pending = nil
|
pending = nil
|
||||||
committed = true
|
committed = true
|
||||||
}
|
}
|
||||||
if !yield(event, nil) {
|
if !forward(event) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if streamErr != nil && !committed {
|
||||||
|
// A failed uncommitted attempt delivers none of its events (they are
|
||||||
|
// never replayed), but the usage it reported survives as one
|
||||||
|
// message_start so the logical call still counts it.
|
||||||
|
if u := pendingUsage(pending); u != (llm.Usage{}) && !yield(llm.StreamEvent{Type: llm.SEMessageStart, Usage: u}, nil) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pending = nil
|
||||||
|
}
|
||||||
if streamErr != nil && !committed && ctx.Err() == nil &&
|
if streamErr != nil && !committed && ctx.Err() == nil &&
|
||||||
attempt < retries && isRetryableStreamError(streamErr) {
|
attempt < retries && isRetryableStreamError(streamErr) {
|
||||||
backoff := backoffOf(attempt)
|
backoff := backoffOf(attempt)
|
||||||
log.Printf("[task-llm] task %s pre-commit stream failure, retrying on the same provider after %v (%d/%d): %v",
|
log.Printf("[task-llm] task %s pre-commit stream failure, retrying on the same provider after %v (%d/%d): %v",
|
||||||
taskID, backoff, attempt+1, retries, streamErr)
|
taskID, backoff, attempt+1, retries, streamErr)
|
||||||
if sleepCtx(ctx, backoff) {
|
if sleepCtx(ctx, backoff) {
|
||||||
break // ctx cancelled during backoff → stop retrying
|
yield(llm.StreamEvent{}, ctx.Err()) // cancelled during backoff → stop
|
||||||
|
return
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -442,7 +487,7 @@ func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest
|
|||||||
if streamErr == nil {
|
if streamErr == nil {
|
||||||
resetLLMFailStreak(taskID, selection.profileID)
|
resetLLMFailStreak(taskID, selection.profileID)
|
||||||
for _, buffered := range pending {
|
for _, buffered := range pending {
|
||||||
if !yield(buffered, nil) {
|
if !forward(buffered) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -474,11 +519,6 @@ func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest
|
|||||||
// bundle was still in use. Agent/global fallback errors follow the legacy
|
// bundle was still in use. Agent/global fallback errors follow the legacy
|
||||||
// behavior and never mutate task failover state.
|
// behavior and never mutate task failover state.
|
||||||
if selection.profileID == 0 || !isQuotaExhaustedError(streamErr) {
|
if selection.profileID == 0 || !isQuotaExhaustedError(streamErr) {
|
||||||
for _, buffered := range pending {
|
|
||||||
if !yield(buffered, nil) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
yield(llm.StreamEvent{}, streamErr)
|
yield(llm.StreamEvent{}, streamErr)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -508,9 +548,25 @@ func streamTaskLLM(ctx context.Context, taskID string, req llm.CompletionRequest
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pendingUsage is the usage the buffered (uncommitted) events of one attempt
|
||||||
|
// reported. Only message_start can be buffered: message_delta commits.
|
||||||
|
func pendingUsage(pending []llm.StreamEvent) llm.Usage {
|
||||||
|
var u llm.Usage
|
||||||
|
for _, ev := range pending {
|
||||||
|
if ev.Type == llm.SEMessageStart {
|
||||||
|
u.Add(ev.Usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return u
|
||||||
|
}
|
||||||
|
|
||||||
|
// streamEventCommitsOutput reports whether delivering event makes the attempt
|
||||||
|
// unreplayable. Signed and redacted thinking and tool-use signatures are opaque
|
||||||
|
// output the model bound to this exact response, so they commit like text.
|
||||||
func streamEventCommitsOutput(event llm.StreamEvent) bool {
|
func streamEventCommitsOutput(event llm.StreamEvent) bool {
|
||||||
switch event.Type {
|
switch event.Type {
|
||||||
case llm.SETextDelta, llm.SEThinkingDelta, llm.SEToolInputJSON:
|
case llm.SETextDelta, llm.SEThinkingDelta, llm.SEToolInputJSON,
|
||||||
|
llm.SEThinkingSignature, llm.SERedactedThinking, llm.SEToolUseSignature:
|
||||||
return event.Text != ""
|
return event.Text != ""
|
||||||
case llm.SEToolUseStart, llm.SEMessageDelta, llm.SEMessageStop:
|
case llm.SEToolUseStart, llm.SEMessageDelta, llm.SEMessageStop:
|
||||||
return true
|
return true
|
||||||
|
|||||||
@@ -0,0 +1,286 @@
|
|||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"iter"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Autumn-27/artex/agent"
|
||||||
|
"github.com/Autumn-27/artex/db"
|
||||||
|
"github.com/Autumn-27/artex/llmrec"
|
||||||
|
"github.com/Autumn-27/norma/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Usage reported by one failed attempt, and by a successful one. A stream
|
||||||
|
// reports the input side in message_start and echoes it, with the output, in
|
||||||
|
// message_delta; the atomic path returns the attempt's normalized usage.
|
||||||
|
var (
|
||||||
|
failedAttemptUsage = llm.Usage{InputTokens: 11, OutputTokens: 2, CacheReadTokens: 3, CacheWriteTokens: 2}
|
||||||
|
okStartUsage = llm.Usage{InputTokens: 13, CacheReadTokens: 3, CacheWriteTokens: 2}
|
||||||
|
okDeltaUsage = llm.Usage{InputTokens: 17, OutputTokens: 5, CacheReadTokens: 4, CacheWriteTokens: 2}
|
||||||
|
okAttemptUsage = llm.Usage{InputTokens: 17, OutputTokens: 5, CacheReadTokens: 4, CacheWriteTokens: 2}
|
||||||
|
)
|
||||||
|
|
||||||
|
// usageAttemptProvider fails its first fail attempts (always, when fail < 0)
|
||||||
|
// after reporting failedAttemptUsage, then answers "done".
|
||||||
|
type usageAttemptProvider struct {
|
||||||
|
fail int
|
||||||
|
failErr error
|
||||||
|
onFail func()
|
||||||
|
// extra events a failed stream attempt emits before its error.
|
||||||
|
failEvents []llm.StreamEvent
|
||||||
|
calls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *usageAttemptProvider) failing() bool {
|
||||||
|
p.calls++
|
||||||
|
failed := p.fail < 0 || p.calls <= p.fail
|
||||||
|
if failed && p.onFail != nil {
|
||||||
|
p.onFail()
|
||||||
|
}
|
||||||
|
return failed
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *usageAttemptProvider) Complete(context.Context, llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
|
||||||
|
if p.failing() {
|
||||||
|
return llm.Message{}, "", failedAttemptUsage, p.failErr
|
||||||
|
}
|
||||||
|
return llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{llm.TextBlock("done")}}, "end_turn", okAttemptUsage, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *usageAttemptProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
||||||
|
return func(yield func(llm.StreamEvent, error) bool) {
|
||||||
|
if p.failing() {
|
||||||
|
events := append([]llm.StreamEvent{{Type: llm.SEMessageStart, Usage: failedAttemptUsage}}, p.failEvents...)
|
||||||
|
for _, ev := range events {
|
||||||
|
if !yield(ev, nil) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
yield(llm.StreamEvent{}, p.failErr)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, ev := range []llm.StreamEvent{
|
||||||
|
{Type: llm.SEMessageStart, Usage: okStartUsage},
|
||||||
|
{Type: llm.SETextDelta, Text: "done"},
|
||||||
|
{Type: llm.SEMessageDelta, StopReason: "end_turn", Usage: okDeltaUsage},
|
||||||
|
{Type: llm.SEMessageStop},
|
||||||
|
} {
|
||||||
|
if !yield(ev, nil) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// logicalResult is what the SDK sees for one logical call.
|
||||||
|
type logicalResult struct {
|
||||||
|
usage llm.Usage
|
||||||
|
text string
|
||||||
|
events []llm.StreamEvent
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func runLogicalCall(ctx context.Context, stream bool, taskID string, hooks taskLLMStreamHooks) logicalResult {
|
||||||
|
if !stream {
|
||||||
|
msg, _, usage, err := completeTaskLLM(ctx, taskID, llm.CompletionRequest{}, hooks)
|
||||||
|
return logicalResult{usage: usage, text: msg.Text(), err: err}
|
||||||
|
}
|
||||||
|
var r logicalResult
|
||||||
|
acc := llm.NewAccumulator()
|
||||||
|
for ev, err := range streamTaskLLM(ctx, taskID, llm.CompletionRequest{}, hooks) {
|
||||||
|
if err != nil {
|
||||||
|
r.err = err
|
||||||
|
break
|
||||||
|
}
|
||||||
|
r.events = append(r.events, ev)
|
||||||
|
acc.Add(ev)
|
||||||
|
}
|
||||||
|
r.usage, r.text = acc.Usage, acc.Message().Text()
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
func singleProfileHooks(p llm.Provider, retry agent.RetryConfig) taskLLMStreamHooks {
|
||||||
|
return taskLLMStreamHooks{
|
||||||
|
current: func(context.Context) (taskLLMSelection, error) {
|
||||||
|
return taskLLMSelection{provider: p, retry: retry}, nil
|
||||||
|
},
|
||||||
|
exhaust: func(taskLLMSelection, error) (db.TaskLLMTransition, error) {
|
||||||
|
return db.TaskLLMTransition{}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func sumUsage(us ...llm.Usage) llm.Usage {
|
||||||
|
var t llm.Usage
|
||||||
|
for _, u := range us {
|
||||||
|
t.Add(u)
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
|
var usageModes = []struct {
|
||||||
|
name string
|
||||||
|
stream bool
|
||||||
|
}{{"atomic", false}, {"stream", true}}
|
||||||
|
|
||||||
|
func TestTaskLLMRetrySumsAttemptUsage(t *testing.T) {
|
||||||
|
for _, m := range usageModes {
|
||||||
|
t.Run(m.name, func(t *testing.T) {
|
||||||
|
p := &usageAttemptProvider{fail: 1, failErr: errors.New("connection reset by peer")}
|
||||||
|
r := runLogicalCall(t.Context(), m.stream, "p12-retry", singleProfileHooks(p, agent.RetryConfig{StreamAttempts: 1, StreamInterval: time.Millisecond}))
|
||||||
|
if r.err != nil {
|
||||||
|
t.Fatal(r.err)
|
||||||
|
}
|
||||||
|
want := sumUsage(failedAttemptUsage, okAttemptUsage) // {28 7 7 4}
|
||||||
|
if r.usage != want || p.calls != 2 || r.text != "done" {
|
||||||
|
t.Fatalf("usage=%+v want %+v, calls=%d, text=%q", r.usage, want, p.calls, r.text)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTaskLLMExhaustedRetriesReturnSummedUsage(t *testing.T) {
|
||||||
|
for _, m := range usageModes {
|
||||||
|
t.Run(m.name, func(t *testing.T) {
|
||||||
|
p := &usageAttemptProvider{fail: -1, failErr: errors.New("connection reset by peer")}
|
||||||
|
r := runLogicalCall(t.Context(), m.stream, "p12-exhausted", singleProfileHooks(p, agent.RetryConfig{StreamAttempts: 1, StreamInterval: time.Millisecond}))
|
||||||
|
if r.err == nil || r.err.Error() != "connection reset by peer" {
|
||||||
|
t.Fatalf("err=%v", r.err)
|
||||||
|
}
|
||||||
|
want := sumUsage(failedAttemptUsage, failedAttemptUsage)
|
||||||
|
if r.usage != want || p.calls != 2 {
|
||||||
|
t.Fatalf("usage=%+v want %+v, calls=%d", r.usage, want, p.calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTaskLLMProfileFailoverSumsUsage(t *testing.T) {
|
||||||
|
for _, m := range usageModes {
|
||||||
|
t.Run(m.name, func(t *testing.T) {
|
||||||
|
first := &usageAttemptProvider{fail: -1, failErr: errors.New("openai: status 402: insufficient_quota")}
|
||||||
|
second := &usageAttemptProvider{}
|
||||||
|
active := int64(11)
|
||||||
|
hooks := taskLLMStreamHooks{
|
||||||
|
current: func(context.Context) (taskLLMSelection, error) {
|
||||||
|
if active == 11 {
|
||||||
|
return taskLLMSelection{profileID: 11, provider: first}, nil
|
||||||
|
}
|
||||||
|
return taskLLMSelection{profileID: 22, provider: second}, nil
|
||||||
|
},
|
||||||
|
exhaust: func(taskLLMSelection, error) (db.TaskLLMTransition, error) {
|
||||||
|
active = 22
|
||||||
|
next := active
|
||||||
|
return db.TaskLLMTransition{PreviousProfileID: 11, NextProfileID: &next, Advanced: true}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
r := runLogicalCall(t.Context(), m.stream, "p12-failover-"+m.name, hooks)
|
||||||
|
if r.err != nil {
|
||||||
|
t.Fatal(r.err)
|
||||||
|
}
|
||||||
|
want := sumUsage(failedAttemptUsage, okAttemptUsage)
|
||||||
|
if r.usage != want || first.calls != 1 || second.calls != 1 || r.text != "done" {
|
||||||
|
t.Fatalf("usage=%+v want %+v, calls first=%d second=%d, text=%q", r.usage, want, first.calls, second.calls, r.text)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTaskLLMCancelDuringBackoffStops(t *testing.T) {
|
||||||
|
for _, m := range usageModes {
|
||||||
|
t.Run(m.name, func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
defer cancel()
|
||||||
|
p := &usageAttemptProvider{fail: -1, failErr: errors.New("connection reset by peer")}
|
||||||
|
// Cancel once the retry is waiting out its (long) backoff.
|
||||||
|
p.onFail = func() { time.AfterFunc(20*time.Millisecond, cancel) }
|
||||||
|
r := runLogicalCall(ctx, m.stream, "p12-cancel", singleProfileHooks(p, agent.RetryConfig{StreamAttempts: 2, StreamInterval: time.Minute}))
|
||||||
|
if !errors.Is(r.err, context.Canceled) {
|
||||||
|
t.Fatalf("err=%v, want context.Canceled", r.err)
|
||||||
|
}
|
||||||
|
if p.calls != 1 || r.usage != failedAttemptUsage {
|
||||||
|
t.Fatalf("calls=%d usage=%+v", p.calls, r.usage)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTaskLLMUncommittedStreamAttemptKeepsUsageWithoutReplay(t *testing.T) {
|
||||||
|
p := &usageAttemptProvider{
|
||||||
|
fail: 1,
|
||||||
|
failErr: errors.New("anthropic: overloaded_error"),
|
||||||
|
// Buffered, non-committing events of the failed attempt.
|
||||||
|
failEvents: []llm.StreamEvent{{Type: llm.SEThinkingDelta}, {Type: llm.SEToolInputJSON}},
|
||||||
|
}
|
||||||
|
r := runLogicalCall(t.Context(), true, "p12-uncommitted", singleProfileHooks(p, agent.RetryConfig{StreamAttempts: 1, StreamInterval: time.Millisecond}))
|
||||||
|
if r.err != nil {
|
||||||
|
t.Fatal(r.err)
|
||||||
|
}
|
||||||
|
if want := sumUsage(failedAttemptUsage, okAttemptUsage); r.usage != want {
|
||||||
|
t.Fatalf("usage=%+v want %+v", r.usage, want)
|
||||||
|
}
|
||||||
|
if r.text != "done" {
|
||||||
|
t.Fatalf("text=%q, want the second attempt's only", r.text)
|
||||||
|
}
|
||||||
|
// The failed attempt contributes one usage-only message_start, nothing else.
|
||||||
|
if len(r.events) == 0 || r.events[0].Type != llm.SEMessageStart || r.events[0].Usage != failedAttemptUsage {
|
||||||
|
t.Fatalf("first event %+v", r.events)
|
||||||
|
}
|
||||||
|
for _, ev := range r.events {
|
||||||
|
if ev.Type == llm.SEThinkingDelta || ev.Type == llm.SEToolInputJSON {
|
||||||
|
t.Fatalf("buffered event of the failed attempt was delivered: %+v", r.events)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTaskLLMOpaqueThinkingCommitsStream(t *testing.T) {
|
||||||
|
defer withZeroRetryBackoff()()
|
||||||
|
for _, kind := range []llm.StreamEventType{llm.SEThinkingSignature, llm.SERedactedThinking, llm.SEToolUseSignature} {
|
||||||
|
t.Run(string(kind), func(t *testing.T) {
|
||||||
|
if !streamEventCommitsOutput(llm.StreamEvent{Type: kind, Text: "opaque"}) {
|
||||||
|
t.Fatalf("%s does not commit", kind)
|
||||||
|
}
|
||||||
|
// A retryable failure after the opaque block is surfaced, not replayed.
|
||||||
|
p := &scriptedLLMProvider{events: []llm.StreamEvent{{Type: kind, Text: "opaque"}}, err: errors.New("connection reset by peer")}
|
||||||
|
events, err := collectTaskLLMStream(streamTaskLLM(t.Context(), "p12-opaque", llm.CompletionRequest{}, singleProfileHooks(p, agent.RetryConfig{})))
|
||||||
|
if err == nil || p.calls != 1 || len(events) != 1 || events[0].Type != kind {
|
||||||
|
t.Fatalf("calls=%d events=%+v err=%v", p.calls, events, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The recorder below the task router writes one llm_usage row per physical
|
||||||
|
// attempt; summing the attempts for the logical call adds no row.
|
||||||
|
func TestTaskLLMRetryKeepsOneLedgerRowPerPhysicalCall(t *testing.T) {
|
||||||
|
m, err := NewManager(t.TempDir(), "")
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||||
|
}
|
||||||
|
defer m.Close()
|
||||||
|
pg := m.PG()
|
||||||
|
for _, mode := range usageModes {
|
||||||
|
t.Run(mode.name, func(t *testing.T) {
|
||||||
|
taskID := fmt.Sprintf("p12-ledger-%s-%d", mode.name, time.Now().UnixNano())
|
||||||
|
t.Cleanup(func() { _, _ = pg.Exec(`DELETE FROM llm_usage WHERE task_id=$1`, taskID) })
|
||||||
|
p := &usageAttemptProvider{fail: 1, failErr: errors.New("connection reset by peer")}
|
||||||
|
rec := llmrec.Wrap(p, pg, "test-model", "test-profile", "", "", func() bool { return false })
|
||||||
|
ctx := llmrec.WithTaskID(t.Context(), taskID)
|
||||||
|
r := runLogicalCall(ctx, mode.stream, taskID, singleProfileHooks(rec, agent.RetryConfig{StreamAttempts: 1, StreamInterval: time.Millisecond}))
|
||||||
|
if r.err != nil {
|
||||||
|
t.Fatal(r.err)
|
||||||
|
}
|
||||||
|
var rows, failed int
|
||||||
|
if err := pg.QueryRow(`SELECT count(*), count(*) FILTER (WHERE status='error') FROM llm_usage WHERE task_id=$1`, taskID).Scan(&rows, &failed); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if rows != 2 || failed != 1 || p.calls != 2 {
|
||||||
|
t.Fatalf("llm_usage rows=%d (error %d), physical calls=%d; want 2 rows, 1 error", rows, failed, p.calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -32,8 +32,11 @@ func (s SideQuestionService) Answer(ctx context.Context, req llm.CompletionReque
|
|||||||
out.Text += ev.Text
|
out.Text += ev.Text
|
||||||
case llm.SEToolUseStart:
|
case llm.SEToolUseStart:
|
||||||
out.ToolUse = true
|
out.ToolUse = true
|
||||||
case llm.SEMessageStart, llm.SEMessageDelta:
|
case llm.SEMessageStart:
|
||||||
out.Usage.Add(ev.Usage)
|
out.Usage.Add(ev.Usage)
|
||||||
|
case llm.SEMessageDelta:
|
||||||
|
// message_delta may echo message_start's input side: MergeEcho, not Add.
|
||||||
|
out.Usage.MergeEcho(ev.Usage)
|
||||||
case llm.SEMessageStop:
|
case llm.SEMessageStop:
|
||||||
complete = true
|
complete = true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
package sidequestion
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"iter"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/Autumn-27/norma/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A streamed side answer counts message_delta's echo of the input side once.
|
||||||
|
func TestAnswerStreamUsageEchoCountedOnce(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
start, delta llm.Usage
|
||||||
|
want llm.Usage
|
||||||
|
}{
|
||||||
|
{"full_echo",
|
||||||
|
llm.Usage{InputTokens: 23, CacheReadTokens: 5, CacheWriteTokens: 3},
|
||||||
|
llm.Usage{InputTokens: 23, CacheReadTokens: 5, CacheWriteTokens: 3, OutputTokens: 5},
|
||||||
|
llm.Usage{InputTokens: 23, OutputTokens: 5, CacheReadTokens: 5, CacheWriteTokens: 3}},
|
||||||
|
{"partial_echo",
|
||||||
|
llm.Usage{InputTokens: 740, CacheReadTokens: 200, CacheWriteTokens: 40},
|
||||||
|
llm.Usage{InputTokens: 500, OutputTokens: 77},
|
||||||
|
llm.Usage{InputTokens: 740, OutputTokens: 77, CacheReadTokens: 200, CacheWriteTokens: 40}},
|
||||||
|
{"no_echo",
|
||||||
|
llm.Usage{InputTokens: 23, CacheReadTokens: 5, CacheWriteTokens: 3},
|
||||||
|
llm.Usage{OutputTokens: 5},
|
||||||
|
llm.Usage{InputTokens: 23, OutputTokens: 5, CacheReadTokens: 5, CacheWriteTokens: 3}},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
p := fakeProvider{stream: func(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
||||||
|
return func(yield func(llm.StreamEvent, error) bool) {
|
||||||
|
for _, ev := range []llm.StreamEvent{
|
||||||
|
{Type: llm.SEMessageStart, Usage: tc.start},
|
||||||
|
{Type: llm.SETextDelta, Text: "answer"},
|
||||||
|
{Type: llm.SEMessageDelta, StopReason: "end_turn", Usage: tc.delta},
|
||||||
|
{Type: llm.SEMessageStop},
|
||||||
|
} {
|
||||||
|
if !yield(ev, nil) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
out, err := (SideQuestionService{Provider: p}).Answer(t.Context(), llm.CompletionRequest{}, true, nil)
|
||||||
|
if err != nil || out.Text != "answer" || out.Usage != tc.want {
|
||||||
|
t.Fatalf("answer=%+v err=%v, want usage %+v", out, err, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user