Files
ARTEX/llmrec/usage_echo_test.go
T
BaaRai ce096cdba1 fix(llmrec,sidequestion): fold echoed stream usage with MergeEcho
Since Norma v0.4.7 message_delta echoes the input side reported by message_start. The recorder and side answers added both events, counting input and cache twice in llm_usage and side-answer usage. message_start is still added; message_delta is now folded with llm.Usage.MergeEcho, as llm.Accumulator does.
2026-11-15 17:51:53 +01:00

119 lines
4.0 KiB
Go

package llmrec
import (
"context"
"iter"
"testing"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/norma/llm"
"github.com/google/uuid"
)
// echoStreamProvider streams one message_start and one message_delta with the
// given (already normalized) usage, as Norma's providers emit them.
type echoStreamProvider struct {
completeProvider
start, delta llm.Usage
}
func (p echoStreamProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
return func(yield func(llm.StreamEvent, error) bool) {
for _, ev := range []llm.StreamEvent{
{Type: llm.SEMessageStart, Usage: p.start},
{Type: llm.SETextDelta, Text: "answer"},
{Type: llm.SEMessageDelta, StopReason: "end_turn", Usage: p.delta},
{Type: llm.SEMessageStop},
} {
if !yield(ev, nil) {
return
}
}
}
}
// Norma normalizes Anthropic usage so InputTokens includes the cache read and
// write subsets, and since v0.4.7 message_delta echoes that input side.
var usageEchoCases = []struct {
name string
start, delta llm.Usage
want llm.Usage
}{
{
name: "full_echo",
start: llm.Usage{InputTokens: 23, CacheReadTokens: 5, CacheWriteTokens: 3},
delta: llm.Usage{InputTokens: 23, CacheReadTokens: 5, CacheWriteTokens: 3, OutputTokens: 5},
want: llm.Usage{InputTokens: 23, OutputTokens: 5, CacheReadTokens: 5, CacheWriteTokens: 3},
},
{
// Raw start: input 500, cache read 200, cache creation 40 → normalized 740.
// Raw delta: input 500, output 77, no cache fields → normalized 500.
name: "partial_echo",
start: llm.Usage{InputTokens: 740, CacheReadTokens: 200, CacheWriteTokens: 40},
delta: llm.Usage{InputTokens: 500, OutputTokens: 77},
want: llm.Usage{InputTokens: 740, OutputTokens: 77, CacheReadTokens: 200, CacheWriteTokens: 40},
},
{
name: "no_echo",
start: llm.Usage{InputTokens: 23, CacheReadTokens: 5, CacheWriteTokens: 3},
delta: llm.Usage{OutputTokens: 5},
want: llm.Usage{InputTokens: 23, OutputTokens: 5, CacheReadTokens: 5, CacheWriteTokens: 3},
},
}
func drainRecorderStream(t *testing.T, ctx context.Context, r *Recorder) {
t.Helper()
for _, err := range r.Stream(ctx, llm.CompletionRequest{}) {
if err != nil {
t.Fatal(err)
}
}
}
// The metered usage of a streamed call counts an echoed input side once.
func TestStreamUsageEchoCountedOnce(t *testing.T) {
for _, tc := range usageEchoCases {
t.Run(tc.name, func(t *testing.T) {
var got []llm.Usage
ctx := WithUsageSink(t.Context(), func(u llm.Usage) { got = append(got, u) })
drainRecorderStream(t, ctx, Wrap(echoStreamProvider{start: tc.start, delta: tc.delta}, nil, "fixture", "profile", "", "", func() bool { return false }))
if len(got) != 1 || got[0] != tc.want {
t.Fatalf("metered %+v, want one call with %+v", got, tc.want)
}
})
}
}
// The llm_usage ledger gets one row per streamed call, with the echo counted once.
func TestStreamUsageEchoLedgerRow(t *testing.T) {
dsn, _, err := db.DSN()
if err != nil {
t.Skipf("no database config (%v)", err)
}
pg, err := db.Open(dsn)
if err != nil {
t.Skipf("postgres unavailable (%v)", err)
}
defer pg.Close()
if err := pg.EnsureLLMUsageTable(); err != nil {
t.Fatal(err)
}
for _, tc := range usageEchoCases {
t.Run(tc.name, func(t *testing.T) {
profile := "usage-echo-" + uuid.NewString()
t.Cleanup(func() { _, _ = pg.Exec(`DELETE FROM llm_usage WHERE profile_name=$1`, profile) })
drainRecorderStream(t, t.Context(), Wrap(echoStreamProvider{start: tc.start, delta: tc.delta}, pg, "fixture", profile, "", "", func() bool { return false }))
var rows int
var got llm.Usage
err := pg.QueryRow(`SELECT count(*), COALESCE(sum(input_tokens),0), COALESCE(sum(output_tokens),0), COALESCE(sum(cache_read),0), COALESCE(sum(cache_write),0)
FROM llm_usage WHERE profile_name=$1`, profile).Scan(&rows, &got.InputTokens, &got.OutputTokens, &got.CacheReadTokens, &got.CacheWriteTokens)
if err != nil {
t.Fatal(err)
}
if rows != 1 || got != tc.want {
t.Fatalf("ledger rows=%d usage=%+v, want 1 row with %+v", rows, got, tc.want)
}
})
}
}