Files
ARTEX/intercept/closed_judge_pg_test.go
BaaRai 9a927846ef Intercept judge: closed-question mode, fail-closed to ask
The LLM fallback judge gets a closed mode (default when the judge is on):
the model answers eight code-owned questions with yes|no|unknown and one
evidence line each, in one call; the decision is taken in code. Every
failure (invalid output, missing key, value outside the vocabulary, a
failed call, arguments above the 256 KiB cap) is ask. The custom judge
prompt becomes bounded operator-policy data (8000 runes). The answers are
stored in the intercept audit and shown in the approval detail.

freeform keeps the previous behaviour; defaultJudgeFailAction changes
from allow to ask.
2026-11-12 22:35:11 +01:00

34 lines
1.4 KiB
Go

package intercept
import (
"slices"
"testing"
)
// The closed-question answers of a judge ask are stored in
// intercept_pending.audit (JSONB), with the reason prefix that makes the row a
// model decision.
func TestClosedJudgeAnswersStored(t *testing.T) {
d := targetTestDB(t)
const task = "closed-judge-audit"
t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM intercept_pending WHERE task_id=$1`, task) })
f := &fakeCompleter{reply: closedReply(map[string]string{QPentestAction: "yes", QLiveConfig: "unknown"})}
dec, _ := judgeWith(t.Context(), closedCfg(), nil, f.complete, "Bash", benignArgs)
const reason = "[model] ask live_config=unknown: ev live_config"
if dec.Action != "ask" || dec.Message != reason {
t.Fatalf("decision %s %q, want ask %q", dec.Action, dec.Message, reason)
}
id, err := d.CreateInterceptPending(dec.RuleID, 0, task, "test", "Bash", benignArgs, dec.Message, auditFor(t.Context(), dec, benignArgs, "pending"))
if err != nil {
t.Fatal(err)
}
detail, err := d.GetInterceptDetail(id)
if err != nil || detail == nil || detail.Audit == nil {
t.Fatalf("detail %+v: %v", detail, err)
}
if detail.Reason != reason || detail.DecisionSource != "model" || len(dec.JudgeAnswers) != len(closedQuestions) ||
!slices.Equal(detail.Audit.JudgeAnswers, dec.JudgeAnswers) {
t.Fatalf("stored reason=%q source=%q answers=%+v", detail.Reason, detail.DecisionSource, detail.Audit.JudgeAnswers)
}
}