mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-18 22:20:56 +08:00
A turn the provider's safety system declines (Anthropic stop_reason "refusal", OpenAI-compatible finish_reason "content_filter") has no text and no tool calls. Norma passes the stop reason through and the harness ends any tool-less turn with ReasonCompleted, so the run summary said the model "ended this turn normally", and steerHooks.Stop took the empty turn for an idle one and resent the conversation up to the empty-response retry count. The LLM recorder is the one layer that sees the stop reason. It now hands each completed call's stop reason to an optional context sink (llmrec.WithStopSink). captureRunSession installs a per-run tracker (agent.WithStopTracker) on that sink. With it: - steerHooks.Stop does not inject the idle-turn rerun after a refusal; - terminalText reports "provider refusal (stop_reason ...)" and says the intent was not completed, instead of the completed wording. Other terminal states and ordinary empty turns are unchanged. Fixes #327 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
208 lines
8.2 KiB
Go
208 lines
8.2 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"iter"
|
|
"sync/atomic"
|
|
"testing"
|
|
|
|
"github.com/Autumn-27/artex/agent"
|
|
"github.com/Autumn-27/artex/llmrec"
|
|
"github.com/Autumn-27/norma/llm"
|
|
)
|
|
|
|
// stopReasonProvider ends a streamed call with the given stop reason and no
|
|
// content, the shape of a provider refusal before any output.
|
|
type stopReasonProvider struct{ stop string }
|
|
|
|
func (p stopReasonProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
|
return func(yield func(llm.StreamEvent, error) bool) {
|
|
if !yield(llm.StreamEvent{Type: llm.SEMessageStart, Usage: llm.Usage{InputTokens: 41}}, nil) {
|
|
return
|
|
}
|
|
if !yield(llm.StreamEvent{Type: llm.SEMessageDelta, StopReason: p.stop}, nil) {
|
|
return
|
|
}
|
|
yield(llm.StreamEvent{Type: llm.SEMessageStop}, nil)
|
|
}
|
|
}
|
|
|
|
func (p stopReasonProvider) Complete(context.Context, llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
|
|
return llm.Message{Role: llm.RoleAssistant}, p.stop, llm.Usage{}, nil
|
|
}
|
|
|
|
// runCtxAfter returns a run context whose latest recorded model call ended with stop.
|
|
func runCtxAfter(t *testing.T, stop string) context.Context {
|
|
t.Helper()
|
|
ctx := agent.WithStopTracker(context.Background())
|
|
rec := llmrec.Wrap(stopReasonProvider{stop: stop}, nil, "model", "profile", "", "", func() bool { return false })
|
|
for _, err := range rec.Stream(ctx, llm.CompletionRequest{}) {
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return ctx
|
|
}
|
|
|
|
// A provider refusal leaves an assistant turn with no text and no tools, the same
|
|
// shape as an idle turn, but rerunning it only sends the same conversation to be
|
|
// refused again.
|
|
func TestSteerHooksStopDoesNotRerunProviderRefusal(t *testing.T) {
|
|
refused := []llm.Message{llm.UserText("start"), {Role: llm.RoleAssistant}}
|
|
for _, stop := range []string{"refusal", "content_filter"} {
|
|
h := steerHooks{nudges: &atomic.Int64{}, limit: defaultEmptyTurnNudges, label: "worker-1 · #1"}
|
|
prevent, blocking, _ := h.Stop(runCtxAfter(t, stop), refused)
|
|
if prevent || blocking != nil {
|
|
t.Fatalf("%s: refusal was rerun: prevent=%v blocking=%v", stop, prevent, blocking)
|
|
}
|
|
if n := h.nudges.Load(); n != 0 {
|
|
t.Fatalf("%s: refusal consumed the idle-turn quota: %d", stop, n)
|
|
}
|
|
}
|
|
|
|
// Control: the same empty turn after an ordinary end of turn is still nudged.
|
|
h := steerHooks{nudges: &atomic.Int64{}, limit: defaultEmptyTurnNudges, label: "worker-1 · #1"}
|
|
if _, blocking, _ := h.Stop(runCtxAfter(t, "end_turn"), refused); len(blocking) != 1 || blocking[0] != emptyTurnNudge {
|
|
t.Fatalf("idle turn after end_turn not nudged: %v", blocking)
|
|
}
|
|
}
|
|
|
|
// Identifying and nudging empty turns (thinking only, no text, no tools); see steerHooks.Stop.
|
|
|
|
func assistantThinking(text string) llm.Message {
|
|
return llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{
|
|
{Type: llm.BlockThinking, Thinking: text, Signature: "sig"},
|
|
}}
|
|
}
|
|
|
|
func TestIsThinkingOnlyTurn(t *testing.T) {
|
|
toolUse := llm.Message{Role: llm.RoleAssistant, Content: []llm.ContentBlock{
|
|
{Type: llm.BlockThinking, Thinking: "scan ports first"},
|
|
{Type: llm.BlockToolUse, ID: "t1", Name: "run_nuclei"},
|
|
}}
|
|
cases := []struct {
|
|
name string
|
|
msgs []llm.Message
|
|
want bool
|
|
}{
|
|
{"thinking only", []llm.Message{llm.UserText("start"), assistantThinking("thinking")}, true},
|
|
{"thinking+tool", []llm.Message{llm.UserText("start"), toolUse}, false},
|
|
{"thinking+text", []llm.Message{assistantThinking("thinking"), {
|
|
Role: llm.RoleAssistant,
|
|
Content: []llm.ContentBlock{{Type: llm.BlockThinking, Thinking: "x"}, llm.TextBlock("conclusion")},
|
|
}}, false},
|
|
{"text with only whitespace", []llm.Message{{
|
|
Role: llm.RoleAssistant,
|
|
Content: []llm.ContentBlock{{Type: llm.BlockThinking, Thinking: "x"}, llm.TextBlock(" \n ")},
|
|
}}, true},
|
|
{"completely empty assistant turn", []llm.Message{{Role: llm.RoleAssistant}}, true},
|
|
// Tool results have the user role; the check must walk back to the assistant message
|
|
// before it instead of misjudging the nearest message.
|
|
{"last message is a tool result", []llm.Message{toolUse, {
|
|
Role: llm.RoleUser,
|
|
Content: []llm.ContentBlock{{Type: llm.BlockToolResult, ToolUseID: "t1"}},
|
|
}}, false},
|
|
{"no assistant message", []llm.Message{llm.UserText("start")}, false},
|
|
{"empty history", nil, false},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
if got := isThinkingOnlyTurn(c.msgs); got != c.want {
|
|
t.Fatalf("isThinkingOnlyTurn = %v, want %v", got, c.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// fakeHooks is a programmable inner HookRunner used to verify that steerHooks respects
|
|
// the inner decision.
|
|
type fakeHooks struct {
|
|
prevent bool
|
|
blocking []string
|
|
msg string
|
|
}
|
|
|
|
func (f fakeHooks) PreToolUse(context.Context, string, []byte) (bool, string, []byte) {
|
|
return false, "", nil
|
|
}
|
|
func (f fakeHooks) PostToolUse(context.Context, string, []byte, []byte, bool) {}
|
|
func (f fakeHooks) Stop(context.Context, []llm.Message) (bool, []string, string) {
|
|
return f.prevent, f.blocking, f.msg
|
|
}
|
|
|
|
func TestSteerHooksStopNudgesEmptyTurn(t *testing.T) {
|
|
empty := []llm.Message{assistantThinking("I should enumerate subdomains first")}
|
|
|
|
t.Run("empty turn injects a continue directive", func(t *testing.T) {
|
|
h := steerHooks{nudges: &atomic.Int64{}, limit: defaultEmptyTurnNudges, label: "worker-1 · #1"}
|
|
prevent, blocking, _ := h.Stop(context.Background(), empty)
|
|
if prevent {
|
|
t.Fatal("empty turn should not be hard-stopped")
|
|
}
|
|
if len(blocking) != 1 || blocking[0] != emptyTurnNudge {
|
|
t.Fatalf("blocking = %v, want [emptyTurnNudge]", blocking)
|
|
}
|
|
})
|
|
|
|
t.Run("does not intervene when text or tools are present", func(t *testing.T) {
|
|
h := steerHooks{nudges: &atomic.Int64{}, limit: defaultEmptyTurnNudges}
|
|
normal := []llm.Message{{
|
|
Role: llm.RoleAssistant,
|
|
Content: []llm.ContentBlock{llm.TextBlock("scan complete, no open ports found")},
|
|
}}
|
|
if _, blocking, _ := h.Stop(context.Background(), normal); blocking != nil {
|
|
t.Fatalf("normal wrap-up misjudged as an empty turn: %v", blocking)
|
|
}
|
|
if n := h.nudges.Load(); n != 0 {
|
|
t.Fatalf("should not count when not intervening, got %d", n)
|
|
}
|
|
})
|
|
|
|
t.Run("allows wrap-up after the limit is reached", func(t *testing.T) {
|
|
const limit = 5 // the user configured "empty response retries" to 5
|
|
h := steerHooks{nudges: &atomic.Int64{}, limit: limit}
|
|
for i := 1; i <= limit; i++ {
|
|
if _, blocking, _ := h.Stop(context.Background(), empty); len(blocking) != 1 {
|
|
t.Fatalf("attempt %d should still be within quota, blocking = %v", i, blocking)
|
|
}
|
|
}
|
|
if _, blocking, _ := h.Stop(context.Background(), empty); blocking != nil {
|
|
t.Fatalf("still injecting beyond the limit: %v", blocking)
|
|
}
|
|
})
|
|
|
|
// "empty response retries" set to -1 disables this layer; emptyTurnNudgeLimit parses to 0.
|
|
t.Run("does not intervene when disabled by config", func(t *testing.T) {
|
|
h := steerHooks{nudges: &atomic.Int64{}, limit: 0}
|
|
if _, blocking, _ := h.Stop(context.Background(), empty); blocking != nil {
|
|
t.Fatalf("still injecting when disabled: %v", blocking)
|
|
}
|
|
})
|
|
|
|
t.Run("does not stack when inner decides a hard stop", func(t *testing.T) {
|
|
h := steerHooks{inner: fakeHooks{prevent: true, msg: "guard refused to finish"}, nudges: &atomic.Int64{}, limit: defaultEmptyTurnNudges}
|
|
prevent, blocking, msg := h.Stop(context.Background(), empty)
|
|
if !prevent || msg != "guard refused to finish" || blocking != nil {
|
|
t.Fatalf("inner hard stop was rewritten: prevent=%v blocking=%v msg=%q", prevent, blocking, msg)
|
|
}
|
|
if n := h.nudges.Load(); n != 0 {
|
|
t.Fatalf("should not consume quota when yielding to inner, got %d", n)
|
|
}
|
|
})
|
|
|
|
t.Run("does not stack when inner already wants to continue", func(t *testing.T) {
|
|
h := steerHooks{inner: fakeHooks{blocking: []string{"guard's reason to continue"}}, nudges: &atomic.Int64{}, limit: defaultEmptyTurnNudges}
|
|
_, blocking, _ := h.Stop(context.Background(), empty)
|
|
if len(blocking) != 1 || blocking[0] != "guard's reason to continue" {
|
|
t.Fatalf("inner continue message was rewritten: %v", blocking)
|
|
}
|
|
})
|
|
|
|
t.Run("behavior unchanged without a counter", func(t *testing.T) {
|
|
h := steerHooks{limit: defaultEmptyTurnNudges} // e.g. a future call site forgets to pass nudges
|
|
if _, blocking, _ := h.Stop(context.Background(), empty); blocking != nil {
|
|
t.Fatalf("should not inject without a counter: %v", blocking)
|
|
}
|
|
})
|
|
}
|