Files
ARTEX/server/engine_emptyturn_test.go
T
Mike ReadandClaude Opus 5.5 03e216d032 fix: report provider refusals, do not rerun them as idle turns
A turn the provider's safety system declines (Anthropic stop_reason
"refusal", OpenAI-compatible finish_reason "content_filter") has no text
and no tool calls. Norma passes the stop reason through and the harness
ends any tool-less turn with ReasonCompleted, so the run summary said the
model "ended this turn normally", and steerHooks.Stop took the empty turn
for an idle one and resent the conversation up to the empty-response retry
count.

The LLM recorder is the one layer that sees the stop reason. It now hands
each completed call's stop reason to an optional context sink
(llmrec.WithStopSink). captureRunSession installs a per-run tracker
(agent.WithStopTracker) on that sink. With it:

- steerHooks.Stop does not inject the idle-turn rerun after a refusal;
- terminalText reports "provider refusal (stop_reason ...)" and says the
  intent was not completed, instead of the completed wording.

Other terminal states and ordinary empty turns are unchanged.

Fixes #327

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 00:41:18 -06:00

208 lines
8.2 KiB
Go

package server
import (
"context"
"iter"
"sync/atomic"
"testing"
"github.com/Autumn-27/artex/agent"
"github.com/Autumn-27/artex/llmrec"
"github.com/Autumn-27/norma/llm"
)
// stopReasonProvider ends a streamed call with the given stop reason and no
// content, the shape of a provider refusal before any output.
type stopReasonProvider struct{ stop string }
func (p stopReasonProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
return func(yield func(llm.StreamEvent, error) bool) {
if !yield(llm.StreamEvent{Type: llm.SEMessageStart, Usage: llm.Usage{InputTokens: 41}}, nil) {
return
}
if !yield(llm.StreamEvent{Type: llm.SEMessageDelta, StopReason: p.stop}, nil) {
return
}
yield(llm.StreamEvent{Type: llm.SEMessageStop}, nil)
}
}
func (p stopReasonProvider) Complete(context.Context, llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
return llm.Message{Role: llm.RoleAssistant}, p.stop, llm.Usage{}, nil
}
// runCtxAfter returns a run context whose latest recorded model call ended with stop.
func runCtxAfter(t *testing.T, stop string) context.Context {
t.Helper()
ctx := agent.WithStopTracker(context.Background())
rec := llmrec.Wrap(stopReasonProvider{stop: stop}, nil, "model", "profile", "", "", func() bool { return false })
for _, err := range rec.Stream(ctx, llm.CompletionRequest{}) {
if err != nil {
t.Fatal(err)
}
}
return ctx
}
// A provider refusal leaves an assistant turn with no text and no tools, the same
// shape as an idle turn, but rerunning it only sends the same conversation to be
// refused again.
func TestSteerHooksStopDoesNotRerunProviderRefusal(t *testing.T) {
refused := []llm.Message{llm.UserText("start"), {Role: llm.RoleAssistant}}
for _, stop := range []string{"refusal", "content_filter"} {
h := steerHooks{nudges: &atomic.Int64{}, limit: defaultEmptyTurnNudges, label: "worker-1 · #1"}
prevent, blocking, _ := h.Stop(runCtxAfter(t, stop), refused)
if prevent || blocking != nil {
t.Fatalf("%s: refusal was rerun: prevent=%v blocking=%v", stop, prevent, blocking)
}
if n := h.nudges.Load(); n != 0 {
t.Fatalf("%s: refusal consumed the idle-turn quota: %d", stop, n)
}
}
// Control: the same empty turn after an ordinary end of turn is still nudged.
h := steerHooks{nudges: &atomic.Int64{}, limit: defaultEmptyTurnNudges, label: "worker-1 · #1"}
if _, blocking, _ := h.Stop(runCtxAfter(t, "end_turn"), refused); len(blocking) != 1 || blocking[0] != emptyTurnNudge {
t.Fatalf("idle turn after end_turn not nudged: %v", blocking)
}
}
// Identifying and nudging empty turns (thinking only, no text, no tools); see steerHooks.Stop.
func assistantThinking(text string) llm.Message {
return llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{
{Type: llm.BlockThinking, Thinking: text, Signature: "sig"},
}}
}
func TestIsThinkingOnlyTurn(t *testing.T) {
toolUse := llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{
{Type: llm.BlockThinking, Thinking: "scan ports first"},
{Type: llm.BlockToolUse, ID: "t1", Name: "run_nuclei"},
}}
cases := []struct {
name string
msgs []llm.Message
want bool
}{
{"thinking only", []llm.Message{llm.UserText("start"), assistantThinking("thinking")}, true},
{"thinking+tool", []llm.Message{llm.UserText("start"), toolUse}, false},
{"thinking+text", []llm.Message{assistantThinking("thinking"), {
Role: llm.RoleAssistant,
Content: []llm.ContentBlock{{Type: llm.BlockThinking, Thinking: "x"}, llm.TextBlock("conclusion")},
}}, false},
{"text with only whitespace", []llm.Message{{
Role: llm.RoleAssistant,
Content: []llm.ContentBlock{{Type: llm.BlockThinking, Thinking: "x"}, llm.TextBlock(" \n ")},
}}, true},
{"completely empty assistant turn", []llm.Message{{Role: llm.RoleAssistant}}, true},
// Tool results have the user role; the check must walk back to the assistant message
// before it instead of misjudging the nearest message.
{"last message is a tool result", []llm.Message{toolUse, {
Role: llm.RoleUser,
Content: []llm.ContentBlock{{Type: llm.BlockToolResult, ToolUseID: "t1"}},
}}, false},
{"no assistant message", []llm.Message{llm.UserText("start")}, false},
{"empty history", nil, false},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if got := isThinkingOnlyTurn(c.msgs); got != c.want {
t.Fatalf("isThinkingOnlyTurn = %v, want %v", got, c.want)
}
})
}
}
// fakeHooks is a programmable inner HookRunner used to verify that steerHooks respects
// the inner decision.
type fakeHooks struct {
prevent bool
blocking []string
msg string
}
func (f fakeHooks) PreToolUse(context.Context, string, []byte) (bool, string, []byte) {
return false, "", nil
}
func (f fakeHooks) PostToolUse(context.Context, string, []byte, []byte, bool) {}
func (f fakeHooks) Stop(context.Context, []llm.Message) (bool, []string, string) {
return f.prevent, f.blocking, f.msg
}
func TestSteerHooksStopNudgesEmptyTurn(t *testing.T) {
empty := []llm.Message{assistantThinking("I should enumerate subdomains first")}
t.Run("empty turn injects a continue directive", func(t *testing.T) {
h := steerHooks{nudges: &atomic.Int64{}, limit: defaultEmptyTurnNudges, label: "worker-1 · #1"}
prevent, blocking, _ := h.Stop(context.Background(), empty)
if prevent {
t.Fatal("empty turn should not be hard-stopped")
}
if len(blocking) != 1 || blocking[0] != emptyTurnNudge {
t.Fatalf("blocking = %v, want [emptyTurnNudge]", blocking)
}
})
t.Run("does not intervene when text or tools are present", func(t *testing.T) {
h := steerHooks{nudges: &atomic.Int64{}, limit: defaultEmptyTurnNudges}
normal := []llm.Message{{
Role: llm.RoleAssistant,
Content: []llm.ContentBlock{llm.TextBlock("scan complete, no open ports found")},
}}
if _, blocking, _ := h.Stop(context.Background(), normal); blocking != nil {
t.Fatalf("normal wrap-up misjudged as an empty turn: %v", blocking)
}
if n := h.nudges.Load(); n != 0 {
t.Fatalf("should not count when not intervening, got %d", n)
}
})
t.Run("allows wrap-up after the limit is reached", func(t *testing.T) {
const limit = 5 // the user configured "empty response retries" to 5
h := steerHooks{nudges: &atomic.Int64{}, limit: limit}
for i := 1; i <= limit; i++ {
if _, blocking, _ := h.Stop(context.Background(), empty); len(blocking) != 1 {
t.Fatalf("attempt %d should still be within quota, blocking = %v", i, blocking)
}
}
if _, blocking, _ := h.Stop(context.Background(), empty); blocking != nil {
t.Fatalf("still injecting beyond the limit: %v", blocking)
}
})
// "empty response retries" set to -1 disables this layer; emptyTurnNudgeLimit parses to 0.
t.Run("does not intervene when disabled by config", func(t *testing.T) {
h := steerHooks{nudges: &atomic.Int64{}, limit: 0}
if _, blocking, _ := h.Stop(context.Background(), empty); blocking != nil {
t.Fatalf("still injecting when disabled: %v", blocking)
}
})
t.Run("does not stack when inner decides a hard stop", func(t *testing.T) {
h := steerHooks{inner: fakeHooks{prevent: true, msg: "guard refused to finish"}, nudges: &atomic.Int64{}, limit: defaultEmptyTurnNudges}
prevent, blocking, msg := h.Stop(context.Background(), empty)
if !prevent || msg != "guard refused to finish" || blocking != nil {
t.Fatalf("inner hard stop was rewritten: prevent=%v blocking=%v msg=%q", prevent, blocking, msg)
}
if n := h.nudges.Load(); n != 0 {
t.Fatalf("should not consume quota when yielding to inner, got %d", n)
}
})
t.Run("does not stack when inner already wants to continue", func(t *testing.T) {
h := steerHooks{inner: fakeHooks{blocking: []string{"guard's reason to continue"}}, nudges: &atomic.Int64{}, limit: defaultEmptyTurnNudges}
_, blocking, _ := h.Stop(context.Background(), empty)
if len(blocking) != 1 || blocking[0] != "guard's reason to continue" {
t.Fatalf("inner continue message was rewritten: %v", blocking)
}
})
t.Run("behavior unchanged without a counter", func(t *testing.T) {
h := steerHooks{limit: defaultEmptyTurnNudges} // e.g. a future call site forgets to pass nudges
if _, blocking, _ := h.Stop(context.Background(), empty); blocking != nil {
t.Fatalf("should not inject without a counter: %v", blocking)
}
})
}