mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-19 14:40:53 +08:00
The LLM fallback judge gets a closed mode (default when the judge is on): the model answers eight code-owned questions with yes|no|unknown and one evidence line each, in one call; the decision is taken in code. Every failure (invalid output, missing key, value outside the vocabulary, a failed call, arguments above the 256 KiB cap) is ask. The custom judge prompt becomes bounded operator-policy data (8000 runes). The answers are stored in the intercept audit and shown in the approval detail. freeform keeps the previous behaviour; defaultJudgeFailAction changes from allow to ask.
34 lines
1.4 KiB
Go
34 lines
1.4 KiB
Go
package intercept
|
|
|
|
import (
|
|
"slices"
|
|
"testing"
|
|
)
|
|
|
|
// The closed-question answers of a judge ask are stored in
|
|
// intercept_pending.audit (JSONB), with the reason prefix that makes the row a
|
|
// model decision.
|
|
func TestClosedJudgeAnswersStored(t *testing.T) {
|
|
d := targetTestDB(t)
|
|
const task = "closed-judge-audit"
|
|
t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM intercept_pending WHERE task_id=$1`, task) })
|
|
f := &fakeCompleter{reply: closedReply(map[string]string{QPentestAction: "yes", QLiveConfig: "unknown"})}
|
|
dec, _ := judgeWith(t.Context(), closedCfg(), nil, f.complete, "Bash", benignArgs)
|
|
const reason = "[model] ask live_config=unknown: ev live_config"
|
|
if dec.Action != "ask" || dec.Message != reason {
|
|
t.Fatalf("decision %s %q, want ask %q", dec.Action, dec.Message, reason)
|
|
}
|
|
id, err := d.CreateInterceptPending(dec.RuleID, 0, task, "test", "Bash", benignArgs, dec.Message, auditFor(t.Context(), dec, benignArgs, "pending"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
detail, err := d.GetInterceptDetail(id)
|
|
if err != nil || detail == nil || detail.Audit == nil {
|
|
t.Fatalf("detail %+v: %v", detail, err)
|
|
}
|
|
if detail.Reason != reason || detail.DecisionSource != "model" || len(dec.JudgeAnswers) != len(closedQuestions) ||
|
|
!slices.Equal(detail.Audit.JudgeAnswers, dec.JudgeAnswers) {
|
|
t.Fatalf("stored reason=%q source=%q answers=%+v", detail.Reason, detail.DecisionSource, detail.Audit.JudgeAnswers)
|
|
}
|
|
}
|