Compare commits

..
Author SHA1 Message Date
BaaRai 8c9c1fc0ab feat(findings): optional counterevidence and confidence_rationale
Add two optional free-text fields to findings so a reporter can record
what was tested and did not work (counterevidence) and why the
conclusion holds (confidence_rationale).

- db: idempotent ADD COLUMN IF NOT EXISTS, TEXT NOT NULL DEFAULT '',
  no CHECK; RecordFindingInput, the INSERT, the shared select list and
  all three scan sites carry them.
- agent: report_finding accepts both as optional strings; the worker
  and pentest prompts say to fill them when confidence is not high.
- server: FindingDTO (list and detail) carries both, omitted when empty;
  existing databases get the two schema properties added to the stored
  report_finding row once, without overwriting user edits.
- web: the finding detail page shows each section only when non-empty.

Both fields stay optional: a missing field never refuses a finding.
2026-11-11 23:43:12 +01:00
16 changed files with 318 additions and 92 deletions
+70
View File
@@ -0,0 +1,70 @@
package agent
import (
"encoding/json"
"slices"
"strings"
"testing"
"github.com/Autumn-27/artex/db"
)
// Pure: report_finding advertises counterevidence / confidence_rationale as
// optional string properties.
func TestReportFindingSchemaReasoningFieldsOptional(t *testing.T) {
schema := NewToolSet(nil, "").addFinding().InputSchema()
props, _ := schema["properties"].(map[string]any)
required, _ := schema["required"].([]string)
if !slices.Contains(required, "vulnclass") {
t.Fatalf("unexpected required list shape: %#v", schema["required"])
}
for _, name := range []string{"counterevidence", "confidence_rationale"} {
p, ok := props[name].(map[string]any)
if !ok {
t.Fatalf("report_finding schema missing %s", name)
}
if p["type"] != "string" {
t.Fatalf("%s type = %v, want string", name, p["type"])
}
if desc, _ := p["description"].(string); !strings.HasPrefix(desc, "optional") {
t.Fatalf("%s description should say optional: %q", name, desc)
}
if slices.Contains(required, name) {
t.Fatalf("%s must not be required: %v", name, required)
}
}
}
// PG: the tool passes both fields into storage, and omitting them never blocks
// a finding.
func TestReportFindingStoresReasoningFields(t *testing.T) {
d := testDB(t)
defer d.Close()
task, err := d.CreateTask("reasoning fields", "fixture", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
defer d.DeleteTask(task.ID)
ts := NewToolSet(nil, "")
ts.ts, ts.taskID = d.Exploration(task.ExplorationID), task.ID
for _, tc := range []struct{ extra, counter, rationale string }{
{`,"counterevidence":"WAF blocked 4 of 5 encodings","confidence_rationale":"the 5th returned the other tenant's row"`, "WAF blocked 4 of 5 encodings", "the 5th returned the other tenant's row"},
{"", "", ""},
} {
res, err := ts.addFinding().Call(t.Context(), json.RawMessage(`{"vulnclass":"SQLI","severity":"high","summary":"reasoning fixture","evidence":"output"`+tc.extra+`}`), nil)
if err != nil || !strings.HasPrefix(res.Flatten(), "finding recorded: ") {
t.Fatalf("report_finding rejected: %v %s", err, res.Flatten())
}
var out db.RecordedFinding
if err := json.Unmarshal([]byte(strings.SplitN(res.Flatten(), "\n", 2)[1]), &out); err != nil {
t.Fatal(err)
}
f, err := d.GetFinding(out.FindingID)
if err != nil || f == nil {
t.Fatalf("GetFinding: %v %v", f, err)
}
if f.Counterevidence != tc.counter || f.ConfidenceRationale != tc.rationale {
t.Fatalf("stored (%q, %q), want (%q, %q)", f.Counterevidence, f.ConfidenceRationale, tc.counter, tc.rationale)
}
}
}
+1 -1
View File
@@ -56,7 +56,7 @@ const pentestDefaultTmpl = `You are the "solo pentest agent" of an authorized pe
- Land every result **immediately**; don't save it for the end (if the session's step budget runs out, everything unrecorded is lost; only what's recorded counts — what lives in your head doesn't). These records are also your long-term memory against compaction.
- **Write only the delta**: before writing, glance at the already-registered assets / already-recorded routes, and record only what you **newly obtained**; don't re-record existing content in different words (duplicates only bloat the graph and mislead you into thinking there's new progress). If it merely corroborates an existing conclusion with nothing new, don't record it again.
- **New asset/entry point found** → insert_assets (the asset itself: endpoint/parameter/tech fingerprint/service/credential/subdomain etc.; structured attributes go on the asset's props). Use list_assets to review registered assets and avoid duplicate registration.
- **Confirmed vulnerability** → report_finding (with a reproducible PoC). **Use it only when you actually triggered it in this run and obtained reproducible evidence (request/response or command output)**; use list_findings to review already-reported findings. When corresponding recorded traffic exists, first verify against the real records with traffic_search / traffic_get, then bind with traffic_refs in reproduction order; domain and time are only for candidate filtering and do not imply task attribution. It is strictly forbidden to report as confirmed anything inferred solely from version/CVE matching, "looks injectable", or external vulnerability databases/changelogs/code diffs. **Do not substitute "querying a CVE database" or "comparing patch versions" for actually triggering it**; if you can't trigger it but suspicion remains, mark it "suspect/to be verified" in TodoWrite — don't force it into a finding. Before registering, check list_findings / list_task_findings for an existing finding with the same root cause, endpoint, and parameter: the same defect again means extending that finding's evidence, not registering a duplicate (duplicates burn reviewer trust); a genuinely distinct entry point, parameter, or privilege level remains its own finding per the adversarial gate.
- **Confirmed vulnerability** → report_finding (with a reproducible PoC). **Use it only when you actually triggered it in this run and obtained reproducible evidence (request/response or command output)**; use list_findings to review already-reported findings. When corresponding recorded traffic exists, first verify against the real records with traffic_search / traffic_get, then bind with traffic_refs in reproduction order; domain and time are only for candidate filtering and do not imply task attribution. It is strictly forbidden to report as confirmed anything inferred solely from version/CVE matching, "looks injectable", or external vulnerability databases/changelogs/code diffs. **Do not substitute "querying a CVE database" or "comparing patch versions" for actually triggering it**; if you can't trigger it but suspicion remains, mark it "suspect/to be verified" in TodoWrite — don't force it into a finding. Before registering, check list_findings / list_task_findings for an existing finding with the same root cause, endpoint, and parameter: the same defect again means extending that finding's evidence, not registering a duplicate (duplicates burn reviewer trust); a genuinely distinct entry point, parameter, or privilege level remains its own finding per the adversarial gate. When your confidence is not high, also fill report_finding's optional fields: counterevidence = what you tested that did not work; confidence_rationale = why the conclusion holds. Both are optional and never block reporting.
Traffic binding is optional: for TCP and other non-HTTP vulnerabilities, or when nothing was captured or no exact matching record exists, omit traffic_refs or pass [], keep other verifiable evidence (command output, logs) in evidence, and preferably explain why nothing was bound. Don't guess IDs, and don't re-probe just to pad the packet count.
+5 -1
View File
@@ -1388,9 +1388,13 @@ func (t *ToolSet) addFinding() actool.CoreTool {
"evidence_hint_id": idp("optional: the hint node ID in this task corresponding to this finding; automatically carries its structured traffic_refs; cannot reference an inherited hint or another finding's hint"),
"traffic_refs": map[string]any{"type": "array", "description": "optional; for HTTP/HTTPS findings, first search and verify entry by entry that the requests/responses actually support the finding's conclusion, then fill in real IDs in reproduction order. For TCP and other non-HTTP findings, or when nothing was captured or no exact record is found, omit or pass [] — it does not block reporting; explain why in evidence and provide other verifiable evidence. Don't guess IDs, infer association by domain/time, or re-probe just to pad the packet count. Roles: baseline = normal control / proof = vulnerability proof / verification = supplementary verification / supporting = auxiliary evidence.",
"items": obj(map[string]any{"traffic_id": str("a real traffic ID returned by traffic_search"), "role": map[string]any{"type": "string", "enum": []string{"baseline", "proof", "verification", "supporting"}}, "note": str("how this traffic supports the finding's conclusion")}, "traffic_id")},
"counterevidence": str("optional: what you tested that did not work (variants, paths, or bypasses that were blocked or came back negative). Fill it when your confidence is not high; omitting it never blocks reporting."),
"confidence_rationale": str("optional: why the conclusion holds (which observation rules out the benign explanation). Fill it when your confidence is not high; omitting it never blocks reporting."),
}, "vulnclass", "severity", "summary"), func(ctx context.Context, in json.RawMessage) (actool.Result, error) {
var a struct {
VulnClass, Name, Severity, Summary, Evidence string
Counterevidence string `json:"counterevidence"`
ConfidenceRationale string `json:"confidence_rationale"`
IntentID json.RawMessage `json:"intent_id"`
AssetIDs []json.RawMessage `json:"asset_ids"`
TrafficRefs []db.TrafficRef `json:"traffic_refs"`
@@ -1418,7 +1422,7 @@ func (t *ToolSet) addFinding() actool.CoreTool {
if err != nil {
return actool.Errorf(err.Error()), nil
}
input := db.RecordFindingInput{TaskID: t.taskID, ExplorationID: t.ts.ID(), IntentID: pid(a.IntentID), VulnClass: a.VulnClass, Name: a.Name, Severity: a.Severity, Summary: a.Summary, Evidence: a.Evidence, Worker: t.worker, AssetIDs: pidList(a.AssetIDs)}
input := db.RecordFindingInput{TaskID: t.taskID, ExplorationID: t.ts.ID(), IntentID: pid(a.IntentID), VulnClass: a.VulnClass, Name: a.Name, Severity: a.Severity, Summary: a.Summary, Evidence: a.Evidence, Worker: t.worker, AssetIDs: pidList(a.AssetIDs), Counterevidence: a.Counterevidence, ConfidenceRationale: a.ConfidenceRationale}
var recorded *db.RecordedFinding
if t.findingRecorder != nil {
recorded, err = t.findingRecorder.Record(ctx, input, refs)
+1 -1
View File
@@ -249,7 +249,7 @@ const workerDefaultTmpl = `You are an "executor" (work agent) of a network-secur
**Write back as you discover** (only what's written into the graph counts; what's in your head/text doesn't; write every result immediately — don't save it for the end and lose it when the step budget runs out). Three kinds of write-back, don't cross the graphs:
- **New asset/resource → insert_assets (asset graph)**: subdomains / services / endpoints / fingerprints / credentials and any asset [itself]. **Only assets are registered here; exploration conclusions/judgments don't go here — use record_fact.**
- **Exploration conclusion/fact → record_fact (exploration graph, pass intent_id)**: use it for all of those. **Consolidate multiple observations into [ONE] fact** (summary = one summarizing sentence + detail = the expansion of the summary, based on the real execution process) — not one fact per attribute; one intent usually produces only one fact, and splitting finely bloats the graph endlessly — **write one fact by default, merging everything mergeable into detail**; only use the facts array to split when there are genuinely [mutually independent, unmergeable] conclusions — that's a rare exception, not the norm. **Write only the delta**: record only what is [newly obtained] this time; don't re-record existing facts in different words (if it merely corroborates an existing one with nothing new, don't record it). **Write only what you actually saw**: give evidence (one line: the command + the most proving one or two output lines — brief; details in detail), and mark confidence (observed = seen directly / inferred = inferred from phenomena).
- **Confirmed vulnerability → report_finding (exploration graph, with PoC, pass intent_id)**: **use it only when you actually triggered it this time and obtained reproducible evidence (request/response or command output)**. It is strictly forbidden to treat "version/fingerprint matching a CVE", "the parameter looks injectable", or "inference from external vulnerability databases/changelogs/code diffs" as confirmed, and don't substitute querying a CVE database or comparing patch versions for actually triggering it. Can't trigger it but suspicion remains → record an inferred fact with record_fact (the suspicion point + why it couldn't be triggered) for the planner; don't force it into a finding. Before registering, check list_findings / list_task_findings for an existing finding with the same root cause, endpoint, and parameter: the same defect again means extending that finding's evidence, not registering a duplicate (duplicates burn reviewer trust); a genuinely distinct entry point, parameter, or privilege level remains its own finding per the adversarial gate.
- **Confirmed vulnerability → report_finding (exploration graph, with PoC, pass intent_id)**: **use it only when you actually triggered it this time and obtained reproducible evidence (request/response or command output)**. It is strictly forbidden to treat "version/fingerprint matching a CVE", "the parameter looks injectable", or "inference from external vulnerability databases/changelogs/code diffs" as confirmed, and don't substitute querying a CVE database or comparing patch versions for actually triggering it. Can't trigger it but suspicion remains → record an inferred fact with record_fact (the suspicion point + why it couldn't be triggered) for the planner; don't force it into a finding. Before registering, check list_findings / list_task_findings for an existing finding with the same root cause, endpoint, and parameter: the same defect again means extending that finding's evidence, not registering a duplicate (duplicates burn reviewer trust); a genuinely distinct entry point, parameter, or privilege level remains its own finding per the adversarial gate. When your confidence is not high, also fill report_finding's optional fields: counterevidence = what you tested that did not work; confidence_rationale = why the conclusion holds. Both are optional and never block reporting.
**Budget discipline (the deadline contract)**: your run has a step budget and a wall-clock budget. Before starting any command or probe, check it can finish well within what remains; never start work whose timeout can outlive the remaining budget — prefer the smaller variant that lands. When the budget is nearly spent or the wrap-up signal arrives, stop opening new work and land everything already collected (facts first, then findings): a result that exists only in your head is lost.
+67
View File
@@ -0,0 +1,67 @@
package db
import (
"strconv"
"testing"
)
// counterevidence / confidence_rationale are optional: they round-trip through
// RecordFinding and every read path, and default to "" when not given.
func TestFindingReasoningFieldsRoundTrip(t *testing.T) {
d, err := Open(testDSN(t))
if err != nil {
t.Skipf("postgres unavailable (%v) — skipping", err)
}
defer d.Close()
tk, err := d.CreateTask("finding reasoning fields", "goal", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
defer d.DeleteTask(tk.ID)
es := d.Exploration(tk.ExplorationID)
with, err := es.RecordFinding(t.Context(), RecordFindingInput{TaskID: tk.ID, VulnClass: "IDOR", Severity: "medium", Summary: "with reasoning", Worker: "test",
Counterevidence: "UUID guessing returned 404", ConfidenceRationale: "two accounts, same response body"})
if err != nil {
t.Fatal(err)
}
without, err := es.RecordFinding(t.Context(), RecordFindingInput{TaskID: tk.ID, VulnClass: "IDOR", Severity: "low", Summary: "without reasoning", Worker: "test"})
if err != nil {
t.Fatal(err)
}
want := map[int64][2]string{
with.FindingID: {"UUID guessing returned 404", "two accounts, same response body"},
without.FindingID: {"", ""},
}
check := func(src string, f *DBFinding) {
t.Helper()
w, ok := want[f.ID]
if !ok {
return
}
if f.Counterevidence != w[0] || f.ConfidenceRationale != w[1] {
t.Fatalf("%s finding %d: got (%q, %q), want (%q, %q)", src, f.ID, f.Counterevidence, f.ConfidenceRationale, w[0], w[1])
}
}
for id := range want {
f, err := d.GetFinding(id)
if err != nil || f == nil {
t.Fatalf("GetFinding(%d): %v %v", id, f, err)
}
check("GetFinding", f)
}
filter := FindingFilter{TaskID: strconv.FormatInt(tk.ID, 10)}
page, total, err := d.ListFindingsPage(filter, 1, 50)
if err != nil || total != 2 {
t.Fatalf("ListFindingsPage: total=%d err=%v", total, err)
}
for _, f := range page {
check("ListFindingsPage", f)
}
exported, err := d.ListFindingsForExport(filter, nil)
if err != nil || len(exported) != 2 {
t.Fatalf("ListFindingsForExport: n=%d err=%v", len(exported), err)
}
for _, f := range exported {
check("ListFindingsForExport", f)
}
}
+3 -2
View File
@@ -328,6 +328,7 @@ func (d *DB) EditFindingTraffic(ctx context.Context, findingID, bindingID, versi
type RecordFindingInput struct {
TaskID, ExplorationID, IntentID int64
VulnClass, Name, Severity, Summary, Evidence, Worker string
Counterevidence, ConfidenceRationale string // optional; "" = not given
AssetIDs []int64
}
@@ -384,8 +385,8 @@ VALUES($1,'finding',$2,9,'confirmed',$3) RETURNING id`, in.ExplorationID, string
assets = []int64{}
}
raw, _ := json.Marshal(assets)
if err := tx.QueryRow(`INSERT INTO findings(task_id,node_id,vulnclass,name,severity,summary,evidence,worker,asset_ids)
VALUES(NULLIF($1,0),$2,$3,$4,$5,$6,$7,$8,$9) RETURNING id`, in.TaskID, out.NodeID, in.VulnClass, in.Name, in.Severity, in.Summary, in.Evidence, in.Worker, string(raw)).Scan(&out.FindingID); err != nil {
if err := tx.QueryRow(`INSERT INTO findings(task_id,node_id,vulnclass,name,severity,summary,evidence,worker,asset_ids,counterevidence,confidence_rationale)
VALUES(NULLIF($1,0),$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) RETURNING id`, in.TaskID, out.NodeID, in.VulnClass, in.Name, in.Severity, in.Summary, in.Evidence, in.Worker, string(raw), in.Counterevidence, in.ConfidenceRationale).Scan(&out.FindingID); err != nil {
return nil, err
}
// Register a push event in the **same transaction**: committing guarantees
+7 -3
View File
@@ -37,6 +37,9 @@ type DBFinding struct {
Report string // detailed report (Markdown); only populated by GetFinding, not by list queries
CreatedAt time.Time
TaskDescription string // populated via LEFT JOIN on tasks
Counterevidence string // what was tested and did not work; optional, "" = not given
ConfidenceRationale string // why the conclusion holds; optional, "" = not given
}
// Finding triage states (findings.status).
@@ -109,6 +112,7 @@ const findingSelectCols = `f.id, f.task_id, f.node_id, f.vulnclass, COALESCE(f.n
f.evidence, f.worker, f.asset_ids, COALESCE(f.status, 'pending'), f.created_at,
COALESCE(t.description, '') AS task_description, f.evidence_version, f.report_evidence_version,
f.poe_level, COALESCE(f.verification_status, 'unverified'), COALESCE(f.verification_note, ''),
f.counterevidence, f.confidence_rationale,
(SELECT count(*) FROM finding_traffic_bindings b WHERE b.finding_id=f.id)`
// scanFindings materializes rows selected via findingSelectCols.
@@ -122,7 +126,7 @@ func scanFindings(rows interface {
f := &DBFinding{}
var aidsJSON string
if err := rows.Scan(&f.ID, &f.TaskID, &f.NodeID, &f.VulnClass, &f.Name, &f.Severity,
&f.Summary, &f.Evidence, &f.Worker, &aidsJSON, &f.Status, &f.CreatedAt, &f.TaskDescription, &f.EvidenceVersion, &f.ReportEvidenceVersion, &f.PoELevel, &f.VerificationStatus, &f.VerificationNote, &f.TrafficCount); err != nil {
&f.Summary, &f.Evidence, &f.Worker, &aidsJSON, &f.Status, &f.CreatedAt, &f.TaskDescription, &f.EvidenceVersion, &f.ReportEvidenceVersion, &f.PoELevel, &f.VerificationStatus, &f.VerificationNote, &f.Counterevidence, &f.ConfidenceRationale, &f.TrafficCount); err != nil {
return nil, err
}
_ = json.Unmarshal([]byte(aidsJSON), &f.AssetIDs)
@@ -525,7 +529,7 @@ func (d *DB) ListFindingsForExport(f FindingFilter, ids []int64) ([]*DBFinding,
var aidsJSON string
if err := rows.Scan(&f.ID, &f.TaskID, &f.NodeID, &f.VulnClass, &f.Name, &f.Severity,
&f.Summary, &f.Evidence, &f.Worker, &aidsJSON, &f.Status, &f.CreatedAt,
&f.TaskDescription, &f.EvidenceVersion, &f.ReportEvidenceVersion, &f.PoELevel, &f.VerificationStatus, &f.VerificationNote, &f.TrafficCount, &f.Report); err != nil {
&f.TaskDescription, &f.EvidenceVersion, &f.ReportEvidenceVersion, &f.PoELevel, &f.VerificationStatus, &f.VerificationNote, &f.Counterevidence, &f.ConfidenceRationale, &f.TrafficCount, &f.Report); err != nil {
return nil, err
}
_ = json.Unmarshal([]byte(aidsJSON), &f.AssetIDs)
@@ -655,7 +659,7 @@ func (d *DB) GetFinding(id int64) (*DBFinding, error) {
WHERE f.id = $1`, id).Scan(
&f.ID, &f.TaskID, &f.NodeID, &f.VulnClass, &f.Name, &f.Severity,
&f.Summary, &f.Evidence, &f.Worker, &aidsJSON, &f.Status, &f.CreatedAt,
&f.TaskDescription, &f.EvidenceVersion, &f.ReportEvidenceVersion, &f.PoELevel, &f.VerificationStatus, &f.VerificationNote, &f.TrafficCount, &f.Report)
&f.TaskDescription, &f.EvidenceVersion, &f.ReportEvidenceVersion, &f.PoELevel, &f.VerificationStatus, &f.VerificationNote, &f.Counterevidence, &f.ConfidenceRationale, &f.TrafficCount, &f.Report)
if err == sql.ErrNoRows {
return nil, nil
}
+4
View File
@@ -1247,6 +1247,10 @@ ALTER TABLE findings ADD COLUMN IF NOT EXISTS report_evidence_version BIGINT NOT
ALTER TABLE findings ADD COLUMN IF NOT EXISTS poe_level SMALLINT NOT NULL DEFAULT 0;
ALTER TABLE findings ADD COLUMN IF NOT EXISTS verification_status TEXT NOT NULL DEFAULT 'unverified';
ALTER TABLE findings ADD COLUMN IF NOT EXISTS verification_note TEXT NOT NULL DEFAULT '';
-- Reporter-supplied reasoning, both optional (empty = not given): counterevidence is what was
-- tested and did not work; confidence_rationale is why the conclusion holds.
ALTER TABLE findings ADD COLUMN IF NOT EXISTS counterevidence TEXT NOT NULL DEFAULT '';
ALTER TABLE findings ADD COLUMN IF NOT EXISTS confidence_rationale TEXT NOT NULL DEFAULT '';
CREATE TABLE IF NOT EXISTS traffic_evidence_snapshots (
id TEXT PRIMARY KEY,
+2 -22
View File
@@ -252,7 +252,7 @@ var targetGuidance = map[string]string{
// always final (an approval can still arrive, a lookup can succeed later), so
// it does not end with "This operation is forbidden."
func TargetBlockMessage(res intercept.TargetResult) string {
reason := capBlockReason(res.Reason)
reason := res.Reason
if res.Host != "" && !strings.Contains(reason, res.Host) {
target := res.Host
if res.Port != 0 {
@@ -277,27 +277,7 @@ func TargetBlockMessage(res intercept.TargetResult) string {
// model-facing tool_result carries this framing.
func systemBlockMessage(reason string) string {
return "[ARTEX platform control, not target defense] This call was intercepted by the platform. " +
"Reason: " + capBlockReason(reason) + ". This operation is forbidden."
}
// maxBlockReasonRunes bounds the reason a block message carries back to the
// model. Rule messages are operator text and judge or target reasons can embed
// command input, so a reason has no natural bound; 4000 runes keeps every
// realistic reason whole while one oversized reason cannot flood the context.
// Only the model-facing text is capped: Interceptor.Log stores the raw reason.
const maxBlockReasonRunes = 4000
// capBlockReason returns s unchanged when it has at most maxBlockReasonRunes
// runes, and otherwise its first maxBlockReasonRunes runes and a marker.
func capBlockReason(s string) string {
n := 0
for i := range s {
if n == maxBlockReasonRunes {
return s[:i] + " …(truncated)"
}
n++
}
return s
"Reason: " + reason + ". This operation is forbidden."
}
var reBlocked = regexp.MustCompile(`(?i)\b(403|forbidden|waf|blocked|rate.?limit|429|captcha|denied)\b`)
-62
View File
@@ -6,7 +6,6 @@ import (
"strings"
"testing"
"time"
"unicode/utf8"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/intercept"
@@ -151,65 +150,4 @@ func TestTargetOnlyGuardAppliesBuiltinDenyRules(t *testing.T) {
}
}
// A deny rule with an oversized message reaches the model capped, with the
// platform framing and the guidance intact.
func TestDenyRuleReasonIsCapped(t *testing.T) {
ic := intercept.New(nil)
ic.SetRulesForTest([]db.InterceptRule{
{ID: 1, Name: "Huge message", Enabled: true, Priority: 50,
MatchTarget: "tool_input", MatchType: "string", Action: "deny",
Pattern: "whoami", Message: strings.Repeat("日", 20000)},
})
block, msg := pre(NewWithInterceptor(ic), context.Background(), "Bash", `{"command":"whoami"}`)
if !block {
t.Fatal("the deny rule did not block")
}
limit := maxBlockReasonRunes + utf8.RuneCountInString(systemBlockMessage("")+" …(truncated)")
if n := utf8.RuneCountInString(msg); n > limit {
t.Fatalf("message has %d runes, want at most %d", n, limit)
}
if !utf8.ValidString(msg) ||
!strings.HasPrefix(msg, "[ARTEX platform control, not target defense] This call was intercepted by the platform. Reason: ") ||
!strings.HasSuffix(msg, strings.Repeat("日", maxBlockReasonRunes)+" …(truncated). This operation is forbidden.") {
t.Errorf("capped message lost its framing or guidance: %q...%q", msg[:120], msg[len(msg)-120:])
}
}
// A target block with an oversized reason is capped the same way, and keeps the
// target and the guidance of its code.
func TestTargetBlockReasonIsCapped(t *testing.T) {
res := intercept.TargetResult{Code: intercept.TargetDenied, Host: "app.example.com", Port: 443,
Reason: strings.Repeat("é", 20000)}
msg := TargetBlockMessage(res)
empty := res
empty.Reason = ""
limit := maxBlockReasonRunes + utf8.RuneCountInString(TargetBlockMessage(empty)+" …(truncated)")
if n := utf8.RuneCountInString(msg); n > limit {
t.Fatalf("message has %d runes, want at most %d", n, limit)
}
if !utf8.ValidString(msg) ||
!strings.HasPrefix(msg, "[ARTEX platform control, not target defense] TARGET_DENIED: target app.example.com:443: ") ||
!strings.HasSuffix(msg, strings.Repeat("é", maxBlockReasonRunes)+" …(truncated). "+targetGuidance[intercept.TargetDenied]) {
t.Errorf("capped message lost its target or guidance: %q...%q", msg[:120], msg[len(msg)-120:])
}
}
// A reason within the cap passes through byte for byte.
func TestShortBlockReasonUnchanged(t *testing.T) {
for _, s := range []string{"", "Destructive command rejected", strings.Repeat("日", maxBlockReasonRunes)} {
if got := capBlockReason(s); got != s {
t.Errorf("capBlockReason changed a reason of %d runes", utf8.RuneCountInString(s))
}
}
if got, want := systemBlockMessage("Destructive command rejected"),
"[ARTEX platform control, not target defense] This call was intercepted by the platform. Reason: Destructive command rejected. This operation is forbidden."; got != want {
t.Errorf("systemBlockMessage = %q, want %q", got, want)
}
res := intercept.TargetResult{Code: intercept.TargetDenied, Host: "app.example.com", Port: 443, Reason: "the request was denied"}
if got, want := TargetBlockMessage(res),
"[ARTEX platform control, not target defense] TARGET_DENIED: target app.example.com:443: the request was denied. "+targetGuidance[intercept.TargetDenied]; got != want {
t.Errorf("TargetBlockMessage = %q, want %q", got, want)
}
}
var _ = hook.PreToolUse
+3
View File
@@ -317,6 +317,8 @@ type FindingDTO struct {
PoELevel int `json:"poe_level"`
VerificationStatus string `json:"verification_status"`
VerificationNote string `json:"verification_note"`
Counterevidence string `json:"counterevidence,omitempty"` // what was tested and did not work; optional
ConfidenceRationale string `json:"confidence_rationale,omitempty"` // why the conclusion holds; optional
ID string `json:"id"`
FindingID string `json:"finding_id,omitempty"` // standalone findings-table id — the handle for status updates
@@ -458,6 +460,7 @@ func findingFromDB(f *db.DBFinding, assets map[int64]*db.Asset) FindingDTO {
TrafficCount: f.TrafficCount, EvidenceVersion: f.EvidenceVersion, ReportEvidenceVersion: f.ReportEvidenceVersion,
ReportStale: f.Report != "" && f.EvidenceVersion != f.ReportEvidenceVersion, TrafficBindings: f.TrafficBindings,
PoELevel: f.PoELevel, VerificationStatus: f.VerificationStatus, VerificationNote: f.VerificationNote,
Counterevidence: f.Counterevidence, ConfidenceRationale: f.ConfidenceRationale,
VulnClass: f.VulnClass,
Name: f.Name,
Severity: f.Severity,
+94
View File
@@ -0,0 +1,94 @@
package server
import (
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"github.com/Autumn-27/artex/db"
)
// Pure: FindingDTO carries both reasoning fields and omits them when empty.
func TestFindingDTOReasoningFields(t *testing.T) {
d := findingFromDB(&db.DBFinding{ID: 1, Counterevidence: "blocked by WAF", ConfidenceRationale: "row of tenant B returned"}, nil)
if d.Counterevidence != "blocked by WAF" || d.ConfidenceRationale != "row of tenant B returned" {
t.Fatalf("DTO lost reasoning fields: %+v", d)
}
raw, _ := json.Marshal(d)
if !strings.Contains(string(raw), `"counterevidence":"blocked by WAF"`) || !strings.Contains(string(raw), `"confidence_rationale":"row of tenant B returned"`) {
t.Fatalf("JSON missing reasoning fields: %s", raw)
}
raw, _ = json.Marshal(findingFromDB(&db.DBFinding{ID: 2}, nil))
if strings.Contains(string(raw), "counterevidence") || strings.Contains(string(raw), "confidence_rationale") {
t.Fatalf("empty reasoning fields should be omitted: %s", raw)
}
}
// PG: the detail endpoint returns the stored fields, and the report_finding
// schema upgrade adds both properties to an existing (customized) row without
// overwriting it.
func TestFindingReasoningDetailAndSchemaUpgrade(t *testing.T) {
s, f, req := trafficEvidenceServer(t)
pg := s.m.pg
var taskID int64
if err := pg.QueryRow(`SELECT task_id FROM findings WHERE id=$1`, f.FindingID).Scan(&taskID); err != nil {
t.Fatal(err)
}
var expID int64
if err := pg.QueryRow(`SELECT exploration_id FROM tasks WHERE id=$1`, taskID).Scan(&expID); err != nil {
t.Fatal(err)
}
rec, err := pg.Exploration(expID).RecordFinding(context.Background(), db.RecordFindingInput{TaskID: taskID, VulnClass: "TEST", Severity: "low", Summary: "reasoning detail", Worker: "test",
Counterevidence: "admin path returned 403", ConfidenceRationale: "user path returned the admin record"})
if err != nil {
t.Fatal(err)
}
w := req("GET", fmt.Sprintf("/api/exploration/findings/%d", rec.FindingID), "")
if w.Code != 200 {
t.Fatalf("detail status %d: %s", w.Code, w.Body)
}
var got FindingDTO
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {
t.Fatal(err)
}
if got.Counterevidence != "admin path returned 403" || got.ConfidenceRationale != "user path returned the admin record" {
t.Fatalf("detail DTO lost reasoning fields: %+v", got)
}
old, err := pg.GetTool("report_finding")
if err != nil || old == nil {
t.Fatal("missing report_finding", err)
}
t.Cleanup(func() {
bindings, _ := json.Marshal(old.Agents)
pg.UpdateTool(old.Key, old.Description, old.Schema, bindings, old.Enabled)
})
custom := json.RawMessage(`{"type":"object","properties":{"evidence":{"type":"string","description":"user evidence instructions"}},"required":["evidence"]}`)
if err := pg.UpdateTool("report_finding", "user description", custom, json.RawMessage(`["worker"]`), false); err != nil {
t.Fatal(err)
}
for range 2 { // idempotent
s.seedFindingEvidenceSchema()
}
tool, err := pg.GetTool("report_finding")
if err != nil || tool == nil {
t.Fatal(err)
}
var schema struct {
Properties map[string]map[string]any `json:"properties"`
Required []string `json:"required"`
}
if err := json.Unmarshal(tool.Schema, &schema); err != nil {
t.Fatal(err)
}
for _, name := range []string{"counterevidence", "confidence_rationale"} {
if schema.Properties[name]["type"] != "string" {
t.Fatalf("upgrade did not add %s: %s", name, tool.Schema)
}
}
if tool.Enabled || tool.Description != "user description" || len(schema.Required) != 1 || schema.Required[0] != "evidence" || schema.Properties["evidence"]["description"] != "user evidence instructions" {
t.Fatalf("custom configuration overwritten: %+v %s", tool, tool.Schema)
}
}
+38
View File
@@ -13,6 +13,7 @@ import (
"strings"
"unicode/utf8"
"github.com/Autumn-27/artex/agent"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/evidence"
)
@@ -242,3 +243,40 @@ WHERE key=$1 AND system AND NOT(COALESCE(schema->'properties','{}'::jsonb) ? $2)
}
}
}
// seedFindingEvidenceSchema adds the optional counterevidence /
// confidence_rationale properties to the report_finding schema of existing
// databases (SeedTool is first-insert only, so without this the model would never
// see them). Idempotent: a property is added only when the stored schema lacks it,
// and user edits are preserved.
func (s *Server) seedFindingEvidenceSchema() {
for _, seed := range agent.BuiltinToolSeeds() {
if seed.Key != "report_finding" {
continue
}
raw, err := json.Marshal(seed.Schema)
if err != nil {
log.Printf("[evidence] report_finding schema: %v", err)
return
}
var obj map[string]json.RawMessage
if err := json.Unmarshal(raw, &obj); err != nil {
return
}
var properties map[string]json.RawMessage
if err := json.Unmarshal(obj["properties"], &properties); err != nil {
return
}
for _, name := range []string{"counterevidence", "confidence_rationale"} {
if len(properties[name]) == 0 {
continue
}
_, err := s.m.pg.Exec(`UPDATE tools SET schema=jsonb_set(schema,ARRAY['properties',$2::text],$3::jsonb,true),updated_at=now()
WHERE key=$1 AND system AND NOT(COALESCE(schema->'properties','{}'::jsonb) ? $2)`, "report_finding", name, string(properties[name]))
if err != nil {
log.Printf("[evidence] upgrade report_finding.%s: %v", name, err)
}
}
return
}
}
+1
View File
@@ -570,6 +570,7 @@ func (s *Server) seedOrchestrationTools() {
s.seedFindingTrafficTools() // adds optional evidence params and read-only evidence tools, preserving user config
s.seedFindingWorkflowTools()
s.seedFindingVerificationSchema() // S1: poe_level / evidence_quotes on update_finding_report, added once per DB
s.seedFindingEvidenceSchema() // optional counterevidence / confidence_rationale on report_finding, added once per DB
// Note: pentest's default tool bindings need no migration — BuiltinToolSeeds already seeds
// list_assets/insert_assets/report_finding/list_findings/list_companies together with
// pentest at fresh initialization (no old DBs exist in this project, so no migration).
@@ -189,6 +189,26 @@ function FindingDetailInner() {
)}
</CardContent>
</Card>
{finding.counterevidence?.trim() && (
<Card>
<CardHeader>
<CardTitle className="text-sm">Counterevidence</CardTitle>
</CardHeader>
<CardContent>
<p className="text-sm leading-relaxed whitespace-pre-wrap">{finding.counterevidence}</p>
</CardContent>
</Card>
)}
{finding.confidence_rationale?.trim() && (
<Card>
<CardHeader>
<CardTitle className="text-sm">Confidence rationale</CardTitle>
</CardHeader>
<CardContent>
<p className="text-sm leading-relaxed whitespace-pre-wrap">{finding.confidence_rationale}</p>
</CardContent>
</Card>
)}
<FindingTrafficPanel
key={id}
findingId={id}
+2
View File
@@ -500,6 +500,8 @@ export interface Finding {
poe_level?: number; // proof-of-exploitation level (3 or 4 when verified); 0 = none declared
verification_status?: VerificationStatus; // automated verification outcome (S1)
verification_note?: string; // why the status was reached; empty when verified
counterevidence?: string; // what was tested and did not work; optional, omitted when empty
confidence_rationale?: string; // why the conclusion holds; optional, omitted when empty
id: string;
finding_id?: string; // row id in the standalone findings table, the handle for status updates (may be missing for old in-task nodes)
vulnclass: string;