From a0adc209ad903fbb9f00607868709fc599707f9a Mon Sep 17 00:00:00 2001 From: Xiaobo <269351206+liuxiaobo0920@users.noreply.github.com> Date: Tue, 17 Nov 2026 02:09:49 +0800 Subject: [PATCH] feat(report): surface finding proof strength in the report and the export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The task report and the findings export asserted every finding with no qualification: nothing said how far exploitation actually reached (PoE level), whether the claim was machine-verified (verification status and note), whether counterevidence was recorded, or whether any traffic was bound. The web UI badges all of it (VERIFICATION_META, the detail page), so the report — the artifact a client actually reads — was the last surface still asserting bare claims. Render it from one place: - report/quality.go (new): two constructors and two pure renderers, no I/O, no DB handle, so the block is unit-tested without a database. - Both surfaces call the same helpers, which is the point: the task report and the per-finding export cannot drift apart the way duplicated formatting does. - Missing is explicit: a finding node with no standalone row prints "not assessed (no finding record)" instead of an invented zero — the honest gap the existing coverage work (#207) also insists on. - Labels match the shipped UI wording (Verification, Confidence rationale, Counterevidence) and states are humanised ("Not assessed", "False positive"), with an unrecognised value printed verbatim so a new state cannot silently vanish from a report. - Bound-traffic count appears once per document: as a Proof bullet in the task report (it has no traffic section), and in the export only when nothing is bound — otherwise the traffic section already states it. - The detailed report is capped at 8 KiB in the task report only, rune-safe, with the cap derived into the truncation message; the export is the canonical per-finding document and is never silently shortened. - The export keeps its document structure: the single-finding document gets "## Detailed Report" (a section like its siblings), the multi-finding summary gets a bold label. Read paths only: no change to the write side of poe_level or verification_status, no schema change (all five columns already exist), no API/DTO change (the widened FindingMeta stays internal to the report path, and the findings list keeps selecting only what it renders). One extra task-scoped query on GET /api/report, because findings bodies stay off the list path. docs/vulnerability-traffic-evidence.md §Export describes the new Markdown. --- db/finding_quality_test.go | 108 ++++++++++ db/findings.go | 77 ++++++- docs/vulnerability-traffic-evidence.md | 2 +- report/findings.go | 28 ++- report/quality.go | 249 +++++++++++++++++++++++ report/report.go | 26 ++- report/report_test.go | 267 +++++++++++++++++++++++++ server/server.go | 23 ++- 8 files changed, 759 insertions(+), 21 deletions(-) create mode 100644 db/finding_quality_test.go create mode 100644 report/quality.go diff --git a/db/finding_quality_test.go b/db/finding_quality_test.go new file mode 100644 index 0000000..a40cfa6 --- /dev/null +++ b/db/finding_quality_test.go @@ -0,0 +1,108 @@ +package db + +import "testing" + +// The report path reads proof strength, the two attestations, both evidence +// versions and the reporter-written body in one query; the task findings list keeps +// its own lean meta query. Both are exercised here because the 13-column Scan in +// FindingQualityByNodeID has no other coverage: a column added, dropped or reordered +// silently shifts fields into each other, and nothing else would notice. +func TestFindingQualityByNodeID(t *testing.T) { + d, err := Open(testDSN(t)) + if err != nil { + t.Skipf("postgres unavailable (%v) — skipping", err) + } + defer d.Close() + tk, err := d.CreateTask("finding quality lookup", "goal", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + defer d.DeleteTask(tk.ID) + + es := d.Exploration(tk.ExplorationID) + rec, err := es.RecordFinding(t.Context(), RecordFindingInput{ + TaskID: tk.ID, VulnClass: "IDOR", Severity: "high", Summary: "with proof", Worker: "test", + Counterevidence: "guessing a UUID returned 404", + ConfidenceRationale: "two accounts returned each other's order", + }) + if err != nil { + t.Fatal(err) + } + if err := d.SetFindingVerification(rec.FindingID, VerificationVerified, 3, "reproduced twice on staging"); err != nil { + t.Fatal(err) + } + const body = "## Overview\n\nThe 200 carries the victim's order body.\n\n## Fix\n\nbind the owner server-side." + if _, err := d.SetFindingReportVersionByNodeID(t.Context(), rec.NodeID, body, nil); err != nil { + t.Fatal(err) + } + // Binding or re-binding evidence is what bumps evidence_version; moving it after + // the report was written is exactly what makes that report stale. + if _, err := d.Exec(`UPDATE findings SET evidence_version=evidence_version+1 WHERE id=$1`, rec.FindingID); err != nil { + t.Fatal(err) + } + + // 1) The report-only query carries the whole proof-strength footprint. + got, err := d.FindingQualityByNodeID(tk.ID) + if err != nil { + t.Fatal(err) + } + q, ok := got[rec.NodeID] + if !ok { + t.Fatalf("node %d missing from the quality lookup (%d rows)", rec.NodeID, len(got)) + } + if q.ID != rec.FindingID { + t.Errorf("record id = %d, want %d", q.ID, rec.FindingID) + } + if q.Status != FindingPending { + t.Errorf("status = %q, want %q", q.Status, FindingPending) + } + if q.PoeLevel != 3 || q.VerificationStatus != VerificationVerified { + t.Errorf("poe/verification = %d/%q, want 3/%q", q.PoeLevel, q.VerificationStatus, VerificationVerified) + } + if q.VerificationNote != "reproduced twice on staging" { + t.Errorf("verification note = %q", q.VerificationNote) + } + if q.Counterevidence != "guessing a UUID returned 404" { + t.Errorf("counterevidence = %q", q.Counterevidence) + } + if q.ConfidenceRationale != "two accounts returned each other's order" { + t.Errorf("confidence rationale = %q", q.ConfidenceRationale) + } + if q.Report != body { + t.Errorf("report body = %q, want %q", q.Report, body) + } + if q.EvidenceVersion != 1 || q.ReportEvidenceVersion != 0 { + t.Errorf("versions = %d/%d, want 1/0", q.EvidenceVersion, q.ReportEvidenceVersion) + } + if !q.ReportStale() { + t.Error("a report written before the evidence moved must be stale") + } + if q.TrafficCount != 0 { + t.Errorf("traffic count = %d, want 0", q.TrafficCount) + } + + // 2) The findings list keeps its lean query, and it must still resolve the row. + meta, err := d.FindingMetaByNodeID(tk.ID) + if err != nil { + t.Fatal(err) + } + m, ok := meta[rec.NodeID] + if !ok { + t.Fatalf("node %d missing from the meta lookup", rec.NodeID) + } + if m.ID != rec.FindingID || m.Status != FindingPending || m.TrafficCount != 0 { + t.Errorf("meta = {id:%d status:%q traffic:%d}, want {id:%d status:%q traffic:0}", m.ID, m.Status, m.TrafficCount, rec.FindingID, FindingPending) + } + + // 3) Rows are keyed by task: another task sees its own (empty) set, not this one. + other, err := d.CreateTask("finding quality lookup (other)", "goal", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + defer d.DeleteTask(other.ID) + if rows, err := d.FindingQualityByNodeID(other.ID); err != nil { + t.Fatal(err) + } else if len(rows) != 0 { + t.Errorf("other task sees %d rows, want 0", len(rows)) + } +} diff --git a/db/findings.go b/db/findings.go index ae80c24..56e8e0b 100644 --- a/db/findings.go +++ b/db/findings.go @@ -775,6 +775,11 @@ func (d *DB) SetFindingVulnClass(id int64, vulnclass string) (int64, error) { // FindingMeta is the standalone-row data (id, triage state, anchored assets) the // per-task view grafts onto its exploration-node findings. +// +// It stays deliberately lean: the task findings list calls FindingMetaByNodeID for +// up to a few hundred nodes per page and renders none of the proof-strength fields, +// so the multi-KB report body and the two prose attestations must not ride along on +// this query. The report path uses FindingQualityByNodeID instead. type FindingMeta struct { TrafficCount int @@ -783,6 +788,33 @@ type FindingMeta struct { AssetIDs []int64 } +// FindingQuality is the report-only view of one standalone row: the triage meta +// plus the proof-strength fields, the reporter-written body and the two evidence +// versions that decide whether that body still describes the current evidence. +// +// VerificationNote / Counterevidence / ConfidenceRationale are optional; "" means +// "not given" and stays distinguishable from an empty assertion. Report "" means +// the reporter never wrote one. +type FindingQuality struct { + FindingMeta + + PoeLevel int + VerificationStatus string + VerificationNote string + Counterevidence string + ConfidenceRationale string + Report string + EvidenceVersion int64 + ReportEvidenceVersion int64 +} + +// ReportStale reports whether the stored report predates the evidence it +// describes: it was written against an older evidence version, so it must be read +// as outdated rather than as the current claim. +func (q FindingQuality) ReportStale() bool { + return q.Report != "" && q.EvidenceVersion != q.ReportEvidenceVersion +} + // FindingMetaByNodeID maps a task's finding node ids to their standalone-row // metadata (status + anchored asset ids) via the asset store, so callers holding // only an AssetStore (e.g. the agent ToolSet) can reach it without a raw *DB. @@ -799,7 +831,8 @@ func (d *DB) FindingMetaByNodeID(taskID int64) (map[int64]FindingMeta, error) { if taskID <= 0 { return out, nil } - rows, err := d.Query(`SELECT node_id, id, COALESCE(status,'pending'), asset_ids, (SELECT count(*) FROM finding_traffic_bindings b WHERE b.finding_id=findings.id) FROM findings + rows, err := d.Query(`SELECT node_id, id, COALESCE(status,'pending'), asset_ids, `+findingTrafficCountCol+` + FROM findings WHERE task_id=$1 AND node_id IS NOT NULL`, taskID) if err != nil { return out, err @@ -817,3 +850,45 @@ func (d *DB) FindingMetaByNodeID(taskID int64) (map[int64]FindingMeta, error) { } return out, rows.Err() } + +// findingTrafficCountCol is the per-row bound-evidence count, spelled once so both +// node-id lookups count the same way. +const findingTrafficCountCol = `(SELECT count(*) FROM finding_traffic_bindings b WHERE b.finding_id=findings.id)` + +// FindingQualityByNodeID maps a task's finding node ids to everything the report +// renders about them: triage meta, poe level, verification state and note, the two +// attestations, the reporter-written body and both evidence versions. One query, +// because the report shows all of it together and a report render is a single +// on-demand document. +// +// The column order below is load-bearing: it is the FindingMetaByNodeID order +// followed by the report-only columns, so the Scan target order can be checked +// column by column against the SELECT instead of being guessed. +func (d *DB) FindingQualityByNodeID(taskID int64) (map[int64]FindingQuality, error) { + out := map[int64]FindingQuality{} + if taskID <= 0 { + return out, nil + } + rows, err := d.Query(`SELECT node_id, id, COALESCE(status,'pending'), asset_ids, `+findingTrafficCountCol+`, + COALESCE(poe_level,0), COALESCE(verification_status,'unverified'), COALESCE(verification_note,''), COALESCE(counterevidence,''), COALESCE(confidence_rationale,''), + evidence_version, report_evidence_version, COALESCE(report,'') + FROM findings + WHERE task_id=$1 AND node_id IS NOT NULL`, taskID) + if err != nil { + return out, err + } + defer rows.Close() + for rows.Next() { + var nid int64 + var q FindingQuality + var aidsJSON string + if err := rows.Scan(&nid, &q.ID, &q.Status, &aidsJSON, &q.TrafficCount, + &q.PoeLevel, &q.VerificationStatus, &q.VerificationNote, &q.Counterevidence, &q.ConfidenceRationale, + &q.EvidenceVersion, &q.ReportEvidenceVersion, &q.Report); err != nil { + return out, err + } + _ = json.Unmarshal([]byte(aidsJSON), &q.AssetIDs) + out[nid] = q + } + return out, rows.Err() +} diff --git a/docs/vulnerability-traffic-evidence.md b/docs/vulnerability-traffic-evidence.md index 240168d..dd4ddc1 100644 --- a/docs/vulnerability-traffic-evidence.md +++ b/docs/vulnerability-traffic-evidence.md @@ -67,7 +67,7 @@ The cleaner runs hourly and only reclaims unreferenced content with no in-flight ## Export and Archiving -Markdown includes the ordered evidence list and version; JSON includes metadata; CSV adds counts and binding IDs. `md-zip` keeps the vulnerability Markdown and additionally provides: +Markdown includes the ordered evidence list and version; JSON includes metadata; CSV adds counts and binding IDs. The Markdown also carries the finding's proof strength — PoE level, verification state and note, bound-evidence count, confidence rationale, counterevidence and the detailed report body — rendered by the same helpers the task report uses, so the two surfaces cannot drift apart. The multi-finding summary labels that body (**Detailed report:**) while the single-finding document keeps it a section (`## Detailed Report`); the binding count is stated once, as a Proof bullet when nothing is bound and otherwise by the traffic section itself. `md-zip` keeps the vulnerability Markdown and additionally provides: ```text evidence/// diff --git a/report/findings.go b/report/findings.go index 3c3b003..a059e69 100644 --- a/report/findings.go +++ b/report/findings.go @@ -73,18 +73,17 @@ func FindingsMarkdown(fs []*db.DBFinding, generatedAt time.Time) string { if desc := strings.TrimSpace(f.TaskDescription); desc != "" { fmt.Fprintf(&b, "- **Task**: %s\n", desc) } - fmt.Fprintf(&b, "- **Found at**: %s\n\n", f.CreatedAt.Format("2006-01-02 15:04:05")) + fmt.Fprintf(&b, "- **Found at**: %s\n", f.CreatedAt.Format("2006-01-02 15:04:05")) + q := qualityFromDBFinding(f) + b.WriteString(qualityBullets(q, false)) if s := strings.TrimSpace(f.Summary); s != "" { fmt.Fprintf(&b, "%s\n\n", s) } if e := strings.TrimSpace(f.Evidence); e != "" { fmt.Fprintf(&b, "**Evidence:**\n\n```\n%s\n```\n\n", e) } - if rep := strings.TrimSpace(f.Report); rep != "" { - b.WriteString("**Detailed report:**\n\n") - b.WriteString(rep) - b.WriteString("\n\n") - } + b.WriteString(qualityNarrative(q)) + b.WriteString(detailedReportFull(f.Report, "**Detailed report:**")) b.WriteString(findingTrafficMarkdown(f, false)) b.WriteString("---\n\n") } @@ -104,18 +103,17 @@ func SingleFindingMarkdown(f *db.DBFinding, generatedAt time.Time) string { fmt.Fprintf(&b, "- **Task**: %s\n", desc) } fmt.Fprintf(&b, "- **Found at**: %s\n", f.CreatedAt.Format("2006-01-02 15:04:05")) - fmt.Fprintf(&b, "- **Generated at**: %s\n\n", generatedAt.Format("2006-01-02 15:04:05")) + fmt.Fprintf(&b, "- **Generated at**: %s\n", generatedAt.Format("2006-01-02 15:04:05")) + q := qualityFromDBFinding(f) + b.WriteString(qualityBullets(q, false)) if s := strings.TrimSpace(f.Summary); s != "" { fmt.Fprintf(&b, "## Summary\n\n%s\n\n", s) } if e := strings.TrimSpace(f.Evidence); e != "" { fmt.Fprintf(&b, "## Evidence\n\n```\n%s\n```\n\n", e) } - if rep := strings.TrimSpace(f.Report); rep != "" { - b.WriteString("## Detailed Report\n\n") - b.WriteString(rep) - b.WriteString("\n") - } + b.WriteString(qualityNarrative(q)) + b.WriteString(detailedReportFull(f.Report, "## Detailed Report")) b.WriteString(findingTrafficMarkdown(f, true)) return b.String() } @@ -177,16 +175,14 @@ func findingTrafficIDs(f *db.DBFinding) string { } func findingTrafficMarkdown(f *db.DBFinding, attachments bool) string { - stale := f.Report != "" && f.EvidenceVersion != f.ReportEvidenceVersion + stale := reportStale(f.Report, f.EvidenceVersion, f.ReportEvidenceVersion) if len(f.TrafficBindings) == 0 && !stale { return "" } var out strings.Builder out.WriteString("\n## Related Traffic Evidence\n\n") fmt.Fprintf(&out, "Evidence version: %d; bindings: %d.\n\n", f.EvidenceVersion, len(f.TrafficBindings)) - if stale { - out.WriteString("The evidence has changed; the detailed report needs updating.\n\n") - } + out.WriteString(staleReportNotice(stale)) for i, b := range f.TrafficBindings { fmt.Fprintf(&out, "%d. **Evidence #%d · %s** — `%s %s`, status %d\n", i+1, b.ID, b.Role, b.Snapshot.Method, strings.ReplaceAll(b.Snapshot.URL, "`", "%60"), b.Snapshot.Status) if b.Note != "" { diff --git a/report/quality.go b/report/quality.go new file mode 100644 index 0000000..38ba6ff --- /dev/null +++ b/report/quality.go @@ -0,0 +1,249 @@ +package report + +import ( + "fmt" + "strings" + "unicode/utf8" + + "github.com/Autumn-27/artex/db" +) + +// Quality is the proof-strength descriptor of one finding: how far exploitation +// actually reached (PoE), whether the claim was machine-verified, what was tried +// and did not work, why the conclusion still holds, and how many traffic +// exchanges are bound to it as evidence. +// +// The task report and the findings exports render it through the same helpers so +// one finding never says two different things in two deliverables. Deliberate +// design rule: an honest gap (unverified, zero bound traffic, unreported) is +// printed, never silently dropped — a report that hides its own weakness is +// worth less than one that states it. +type Quality struct { + PoE int + Verification string + VerificationNote string + ConfidenceRationale string + Counterevidence string + TrafficCount int + RecordID int64 // standalone findings-table id; 0 = no row + Report string // reporter-written detailed report body (Markdown) + EvidenceVersion int64 // version of the evidence bound to the row + ReportEvidenceVersion int64 // evidence version the stored report was written against + + // Missing marks a finding node with no standalone findings row at all: no + // PoE level, no verification state and no traffic binding was ever recorded. + // It is distinct from a row that exists and says unverified. + Missing bool + + // Unavailable marks a lookup that could not be read at all (the row map is + // nil, i.e. the query failed). It is deliberately a third state: a failed + // lookup is not evidence about the finding, so it must never be rendered as + // "no finding record" and never as "unverified". + Unavailable bool +} + +// ReportStale reports whether the stored report predates the evidence it +// describes. Both surfaces ask this one function, so the task report and the +// findings export cannot disagree about the same report. +func (q Quality) ReportStale() bool { + return reportStale(q.Report, q.EvidenceVersion, q.ReportEvidenceVersion) +} + +// reportStale is the predicate behind ReportStale, taking the raw row fields so +// callers holding a db.DBFinding (the export path) ask the same question. +func reportStale(report string, evidenceVersion, reportEvidenceVersion int64) bool { + return report != "" && evidenceVersion != reportEvidenceVersion +} + +// qualityFromRows builds a Quality from the report's single quality lookup, +// keyed by finding node id. A nil map means the lookup itself failed +// (unavailable); an empty map means the task genuinely has no standalone rows. +func qualityFromRows(nodeID int64, rows map[int64]db.FindingQuality) Quality { + if rows == nil { + return Quality{Unavailable: true} + } + q, ok := rows[nodeID] + if !ok { + return Quality{Missing: true} + } + return Quality{ + PoE: q.PoeLevel, + Verification: q.VerificationStatus, + VerificationNote: q.VerificationNote, + ConfidenceRationale: q.ConfidenceRationale, + Counterevidence: q.Counterevidence, + TrafficCount: q.TrafficCount, + RecordID: q.ID, + Report: q.Report, + EvidenceVersion: q.EvidenceVersion, + ReportEvidenceVersion: q.ReportEvidenceVersion, + } +} + +// qualityFromDBFinding builds a Quality from a standalone findings row. +func qualityFromDBFinding(f *db.DBFinding) Quality { + return Quality{ + PoE: f.PoELevel, + Verification: f.VerificationStatus, + VerificationNote: f.VerificationNote, + ConfidenceRationale: f.ConfidenceRationale, + Counterevidence: f.Counterevidence, + TrafficCount: f.TrafficCount, + RecordID: f.ID, + Report: f.Report, + EvidenceVersion: f.EvidenceVersion, + ReportEvidenceVersion: f.ReportEvidenceVersion, + } +} + +// qualityBullets renders the proof-strength bullets (record id, PoE level, +// verification state, bound evidence count, verification note). It always emits +// a verification line and always accounts for bound traffic, including the +// honest "nothing bound" case, and ends with a blank line when non-empty. +func qualityBullets(q Quality, alwaysProof bool) string { + var b strings.Builder + // A failed lookup is not an absence: say the fields could not be read, and say + // nothing else. A DB error must never turn into a claim about the finding. + if q.Unavailable { + b.WriteString("- **Verification**: unknown (quality lookup failed — proof strength unavailable)\n\n") + return b.String() + } + if q.Missing { + b.WriteString("- **Verification**: not assessed (no finding record — quality fields unavailable)\n\n") + return b.String() + } + if q.RecordID > 0 { + fmt.Fprintf(&b, "- **Finding record**: #%d\n", q.RecordID) + } + fmt.Fprintf(&b, "- **PoE level**: %d\n", q.PoE) + fmt.Fprintf(&b, "- **Verification**: %s\n", verificationLabel(q.Verification)) + // alwaysProof is true for the task report, which has no traffic section of its own. + // The findings export passes false: when anything is bound its traffic section + // already states the count ("bindings: N"), so a bullet would be the same number + // twice in one document. The empty case is kept either way — that section renders + // nothing when nothing is bound, and "no evidence was bound" is a finding about + // the finding, not a duplicate of a count. + if alwaysProof || q.TrafficCount == 0 { + if q.TrafficCount > 0 { + fmt.Fprintf(&b, "- **Proof**: %d bound traffic exchange(s)\n", q.TrafficCount) + } else { + b.WriteString("- **Proof**: no bound traffic evidence\n") + } + } + if note := oneLine(q.VerificationNote); note != "" { + fmt.Fprintf(&b, "- **Verification note**: %s\n", note) + } + b.WriteString("\n") + return b.String() +} + +// qualityNarrative renders the two prose attestations, each only when it was +// actually written — an absent one produces no heading at all, so a reader can +// never mistake an empty field for an empty claim. +func qualityNarrative(q Quality) string { + if q.Missing || q.Unavailable { + return "" + } + var b strings.Builder + if s := strings.TrimSpace(q.ConfidenceRationale); s != "" { + fmt.Fprintf(&b, "**Confidence rationale**:\n\n%s\n\n", s) + } + if s := strings.TrimSpace(q.Counterevidence); s != "" { + fmt.Fprintf(&b, "**Counterevidence**:\n\n%s\n\n", s) + } + return b.String() +} + +// staleReportNotice is the one warning both surfaces print when the evidence moved +// after the report was written. The task report and the findings export both call +// it, so the two deliverables can never drift on the same claim. It sits next to +// Quality.ReportStale so the predicate and its wording stay together. +func staleReportNotice(stale bool) string { + if !stale { + return "" + } + return "The evidence has changed; the detailed report needs updating.\n\n" +} + +// reportMaxReportBytes caps the detailed report inlined into the task report, so +// one finding cannot dominate a document that also has to list every other one. +const reportMaxReportBytes = 8 << 10 + +// detailedReport renders the reporter-written body (overview / impact / +// reproduction / evidence / fix), capped, for the task report. heading lets the +// caller keep the document's own structure: the task report uses a bold label +// (it is one block among many findings), the single-finding export uses an H2 so +// the body stays a section like its siblings. Nothing is rendered when the body +// was never written. +func detailedReport(body, heading string) string { + return detailedReportLimit(body, heading, reportMaxReportBytes) +} + +// detailedReportFull renders the same body uncapped. The findings export is the +// canonical per-finding document, so it must never silently shorten the report. +func detailedReportFull(body, heading string) string { + return detailedReportLimit(body, heading, 0) +} + +func detailedReportLimit(body, heading string, limit int) string { + s := strings.TrimSpace(body) + if s == "" { + return "" + } + if limit > 0 && len(s) > limit { + cut := s[:limit] + for len(cut) > 0 && !utf8.ValidString(cut) { + cut = cut[:len(cut)-1] // never split a rune + } + cut = strings.TrimSpace(cut) + // Rune safety is not enough: a cut inside an open ``` fence would swallow the + // truncation note — and every section after it, other findings included — into + // the stale code block. Close the fence before the note so the rest of the + // document still parses as the document. + if unbalancedFence(cut) { + cut += "\n```" + } + return heading + "\n\n" + cut + + fmt.Sprintf("\n\n_Detailed report truncated at %d KiB; read the full text from the finding record._\n\n", limit>>10) + } + return heading + "\n\n" + s + "\n\n" +} + +// unbalancedFence reports whether s ends inside an open fenced code block, i.e. +// whether it holds an odd number of fence delimiters. A delimiter is a line whose +// first non-space content is ``` — an opening fence with an info string and a bare +// closing fence both count, which is exactly what a Markdown renderer pairs up. +func unbalancedFence(s string) bool { + fences := 0 + for _, line := range strings.Split(s, "\n") { + if strings.HasPrefix(strings.TrimLeft(line, " \t"), "```") { + fences++ + } + } + return fences%2 == 1 +} + +// verificationLabel renders a verification state for a human-readable document. +// The five labels match the ones the web UI already shows for the same values; an +// unrecognised value is printed verbatim instead of hidden, so a new state can +// never silently disappear from a report. +func verificationLabel(s string) string { + switch s { + case "", db.VerificationUnverified: + return "Unverified" + case db.VerificationVerified: + return "Verified" + case db.VerificationBlocked: + return "Blocked" + case db.VerificationNotAssessed: + return "Not assessed" + case db.VerificationFalsePositive: + return "False positive" + } + return s +} + +// oneLine collapses a note onto a single line so it can sit inside a bullet. +func oneLine(s string) string { + return strings.Join(strings.Fields(s), " ") +} diff --git a/report/report.go b/report/report.go index a1ab7e8..2cfd599 100644 --- a/report/report.go +++ b/report/report.go @@ -20,6 +20,19 @@ type Input struct { AssetCounts map[string]int Findings []*db.Node Coverage *db.CoverageLedger // nil when the task's coverage feature is off + + // Quality carries the standalone findings-row data (poe level, verification + // state, the two attestations, both evidence versions and the reporter-written + // body) keyed by finding node id, from the report's single quality lookup. The + // exploration node payload has none of it; without this map every finding would + // render as an unqualified claim. + // + // Three states, deliberately distinct: an empty but non-nil map means the task + // has no standalone rows (nothing was assessed); a nil map means the lookup + // failed, which says nothing about the findings themselves; an entry means the + // row exists. Collapsing failure into absence would turn a database error into a + // statement about the finding. + Quality map[int64]db.FindingQuality } // reportMaxGaps caps the coverage gap list of the report; the rest is counted. @@ -31,6 +44,7 @@ type findingView struct { Severity string Summary string PoC string + Q Quality // proof-strength fields grafted from the standalone row } func parseFinding(n *db.Node) findingView { @@ -44,7 +58,7 @@ func parseFinding(n *db.Node) findingView { } `json:"evidence"` } _ = json.Unmarshal(n.Payload, &p) - return findingView{p.VulnClass, p.Name, p.Severity, p.Summary, p.Evidence.PoC} + return findingView{VulnClass: p.VulnClass, Name: p.Name, Severity: p.Severity, Summary: p.Summary, PoC: p.Evidence.PoC} } var sevRank = map[string]int{"critical": 0, "high": 1, "medium": 2, "low": 3, "": 4} @@ -80,15 +94,23 @@ func Markdown(in Input) string { } else { fs := make([]findingView, 0, len(in.Findings)) for _, n := range in.Findings { - fs = append(fs, parseFinding(n)) + v := parseFinding(n) + v.Q = qualityFromRows(n.ID, in.Quality) + fs = append(fs, v) } sort.SliceStable(fs, func(i, j int) bool { return sevRank[fs[i].Severity] < sevRank[fs[j].Severity] }) for i, f := range fs { fmt.Fprintf(&b, "### %d. [%s] %s\n\n", i+1, strings.ToUpper(nz(f.Severity, "info")), nz(f.Name, nz(f.VulnClass, "Uncategorized"))) fmt.Fprintf(&b, "%s\n\n", nz(f.Summary, "")) + b.WriteString(qualityBullets(f.Q, true)) // honest proof strength, including its gaps if f.PoC != "" { fmt.Fprintf(&b, "**PoC / Evidence:**\n\n```\n%s\n```\n\n", f.PoC) } + b.WriteString(qualityNarrative(f.Q)) // why it holds / what failed, only when written + // The same warning, from the same helper, that the findings export prints: a + // report written against older evidence is stale on both surfaces or on neither. + b.WriteString(staleReportNotice(f.Q.ReportStale())) + b.WriteString(detailedReport(f.Q.Report, "**Detailed report:**")) } } diff --git a/report/report_test.go b/report/report_test.go index 18ed9b0..6b5750e 100644 --- a/report/report_test.go +++ b/report/report_test.go @@ -1,8 +1,11 @@ package report import ( + "fmt" "strings" "testing" + "time" + "unicode/utf8" "github.com/Autumn-27/artex/db" ) @@ -47,3 +50,267 @@ func TestMarkdownCoverageGapsCap(t *testing.T) { t.Fatalf("missing count line in\n%s", md) } } + +// reportNode builds a finding node with only the exploration payload the graph +// actually stores (name/summary/evidence/severity/vulnclass). +func reportNode(id int64, severity, name string) *db.Node { + payload := fmt.Sprintf(`{"vulnclass":"cwe-602","name":%q,"severity":%q,"summary":"s%d","evidence":{"poc":"proof-%d"}}`, name, severity, id, id) + return &db.Node{ID: id, Kind: db.KindFinding, Payload: []byte(payload)} +} + +// A finding with a populated standalone row must show its whole defensibility +// footprint: record id, poe level, verification state, bound evidence count, the +// two attestations, and the reporter-written body. +func TestMarkdownFindingQuality(t *testing.T) { + rows := map[int64]db.FindingQuality{ + 8: { + FindingMeta: db.FindingMeta{ID: 1, Status: db.FindingConfirmed, TrafficCount: 2}, + PoeLevel: 3, + VerificationStatus: db.VerificationVerified, + ConfidenceRationale: "the 200 carries the victim's order body", + Counterevidence: "PATCH on the same id returned 403", + Report: "## Overview\n\nclient-supplied header\n\n## Fix\n\nbind role server-side", + EvidenceVersion: 4, + ReportEvidenceVersion: 4, + }, + // Row present but nothing proven: the gaps must be stated, not omitted. + 9: {FindingMeta: db.FindingMeta{ID: 2, Status: db.FindingPending}, PoeLevel: 0, VerificationStatus: db.VerificationUnverified}, + } + + md := Markdown(Input{ + Title: "t", + GeneratedAt: time.Unix(0, 0), + Findings: []*db.Node{reportNode(8, "high", "BOLA"), reportNode(9, "critical", "role header")}, + Quality: rows, + }) + + for _, want := range []string{ + "- **Finding record**: #1", + "- **PoE level**: 3", + "- **Verification**: Verified", + "- **Proof**: 2 bound traffic exchange(s)", + "**Confidence rationale**:\n\nthe 200 carries the victim's order body", + "**Counterevidence**:\n\nPATCH on the same id returned 403", + "**Detailed report:**\n\n## Overview\n\nclient-supplied header", + "- **Finding record**: #2", + "- **PoE level**: 0", + "- **Verification**: Unverified", + "- **Proof**: no bound traffic evidence", + } { + if !strings.Contains(md, want) { + t.Errorf("missing %q in\n%s", want, md) + } + } + // Absent optional fields produce no heading at all, and the verified finding's + // narrative must not leak onto the unverified one. + if n := strings.Count(md, "**Counterevidence**"); n != 1 { + t.Errorf("counterevidence rendered %d times, want 1 (empty field must stay empty)\n%s", n, md) + } + if n := strings.Count(md, "**Detailed report:**"); n != 1 { + t.Errorf("detailed report rendered %d times, want 1\n%s", n, md) + } + // A report written against the current evidence version is not stale. + if strings.Contains(md, "The evidence has changed") { + t.Errorf("up-to-date report flagged stale\n%s", md) + } +} + +// A finding node without a standalone row was never assessed: say so, and do not +// print a poe level or verification verdict that nobody produced. +func TestMarkdownFindingQualityNoRow(t *testing.T) { + md := Markdown(Input{Title: "t", Findings: []*db.Node{reportNode(3, "low", "x")}, Quality: map[int64]db.FindingQuality{}}) + if !strings.Contains(md, "- **Verification**: not assessed (no finding record") { + t.Errorf("missing not-assessed line in\n%s", md) + } + if strings.Contains(md, "- **PoE level**") { + t.Errorf("claimed a poe level with no row\n%s", md) + } +} + +// A lookup that failed is not an absence: a nil map must render "could not be +// read", never "no finding record", or a database error becomes a claim about the +// finding (and silently drops a report body that does exist). +func TestMarkdownFindingQualityUnavailable(t *testing.T) { + md := Markdown(Input{Title: "t", Findings: []*db.Node{reportNode(3, "low", "x")}}) + if !strings.Contains(md, "- **Verification**: unknown (quality lookup failed") { + t.Errorf("missing lookup-failed line in\n%s", md) + } + if strings.Contains(md, "no finding record") { + t.Errorf("a failed lookup is rendered as an absent row\n%s", md) + } + if strings.Contains(md, "- **PoE level**") || strings.Contains(md, "- **Proof**") { + t.Errorf("claimed proof strength that was never read\n%s", md) + } +} + +// The stale-evidence warning is one sentence printed from one helper, so the task +// report and the findings export cannot drift on the same finding. +func TestStaleReportWarningOnBothSurfaces(t *testing.T) { + const warning = "The evidence has changed; the detailed report needs updating." + row := db.FindingQuality{ + FindingMeta: db.FindingMeta{ID: 5}, + PoeLevel: 2, + VerificationStatus: db.VerificationVerified, + Report: "## Overview\n\nstale body", + EvidenceVersion: 7, // evidence moved after the report was written + ReportEvidenceVersion: 6, + } + md := Markdown(Input{Title: "t", Findings: []*db.Node{reportNode(4, "high", "stale one")}, Quality: map[int64]db.FindingQuality{4: row}}) + f := &db.DBFinding{ID: 5, VulnClass: "cwe-602", Severity: "high", Status: db.FindingConfirmed, + PoELevel: 2, VerificationStatus: db.VerificationVerified, Report: row.Report, + EvidenceVersion: 7, ReportEvidenceVersion: 6} + fresh := *f + fresh.EvidenceVersion = 6 + + if n := strings.Count(md, warning); n != 1 { + t.Errorf("task report prints the stale warning %d times, want 1\n%s", n, md) + } + // It has to sit before the body it is about, not after it. + if strings.Index(md, warning) > strings.Index(md, "**Detailed report:**") { + t.Errorf("stale warning printed after the body it flags\n%s", md) + } + for name, out := range map[string]string{ + "summary export": FindingsMarkdown([]*db.DBFinding{f}, time.Unix(0, 0)), + "single-finding export": SingleFindingMarkdown(f, time.Unix(0, 0)), + } { + if n := strings.Count(out, warning); n != 1 { + t.Errorf("%s prints the stale warning %d times, want 1\n%s", name, n, out) + } + } + // An up-to-date report gets the warning on neither surface. + if strings.Contains(SingleFindingMarkdown(&fresh, time.Unix(0, 0)), warning) { + t.Errorf("export flags an up-to-date report stale") + } + if strings.Contains(FindingsMarkdown([]*db.DBFinding{&fresh}, time.Unix(0, 0)), warning) { + t.Errorf("summary export flags an up-to-date report stale") + } +} + +// A cut inside an open fence would swallow the truncation marker and every later +// section (other findings included) into the stale code block. The cap has to land +// on a balanced fence. Third party report on the same defect: +// TestMarkdownDetailedReportFenceBalance in ChaosLabsInc/ARTEX. +func TestMarkdownDetailedReportFenceBalance(t *testing.T) { + // An unclosed ```http fence that opens well before the 8 KiB cap and is still + // open where the body gets cut. + unclosed := "## Evidence\n\n```http\n" + strings.Repeat("GET /orders/7 HTTP/1.1\nHost: victim\n\n", 400) + if !unbalancedFence(strings.TrimSpace(unclosed[:reportMaxReportBytes])) { + t.Fatalf("fixture does not cut inside an open fence") + } + rows := map[int64]db.FindingQuality{ + 1: {FindingMeta: db.FindingMeta{ID: 1}, PoeLevel: 3, VerificationStatus: db.VerificationVerified, Report: unclosed}, + 2: {FindingMeta: db.FindingMeta{ID: 2}, PoeLevel: 1, VerificationStatus: db.VerificationUnverified}, + } + md := Markdown(Input{Title: "t", Findings: []*db.Node{reportNode(1, "high", "first"), reportNode(2, "low", "second")}, Quality: rows}) + + if n := strings.Count(md, "```"); n%2 != 0 { + t.Fatalf("report has %d fences (odd), so a code block is left open\n%s", n, md) + } + // The truncated body must be closed before the note, and the next finding must + // still render as a heading rather than as text inside a code block. + block := md[strings.Index(md, "**Detailed report:**"):strings.Index(md, "### 2.")] + if !strings.Contains(block, "_Detailed report truncated at 8 KiB") { + t.Fatalf("truncation note missing from the body block\n%s", block) + } + if strings.Index(block, "\n```\n") > strings.Index(block, "_Detailed report truncated") { + t.Errorf("the fence is closed after the note, so the note stays inside the block\n%s", block) + } + if !strings.Contains(md, "### 2. [LOW] second") || !strings.Contains(md[strings.Index(md, "### 2."):], "- **PoE level**: 1") { + t.Errorf("the second finding did not survive the cut\n%s", md) + } + + // A body with no fence at all must not gain one. + plain := strings.Repeat("word ", 3000) + if out := detailedReport(plain, "**Detailed report:**"); strings.Contains(out, "```") { + t.Errorf("a fence-free body gained a fence\n%s", out) + } + // A body that ends with a closed fence stays as it is. + closed := "## Overview\n\n```\nstill readable\n```\n" + strings.Repeat("trailer line\n", 700) + if out := detailedReport(closed, "**Detailed report:**"); strings.Count(out, "```")%2 != 0 { + t.Errorf("balanced body ended up unbalanced\n%s", out) + } +} + +// The inlined detail body is capped so one finding cannot dominate the report, +// and truncation never splits a multi-byte rune. +func TestDetailedReportTruncation(t *testing.T) { + // A multi-byte rune straddling the cap must be dropped whole, never split. + big := strings.Repeat("a", reportMaxReportBytes-2) + "€" + strings.Repeat("b", 100) + got := detailedReport(big, "**Detailed report:**") + if !strings.Contains(got, "_Detailed report truncated at 8 KiB") { + t.Fatalf("missing truncation marker\n%s", got) + } + if !utf8.ValidString(got) { + t.Errorf("truncation produced invalid UTF-8") + } + if strings.Contains(got, "€") || strings.Contains(got, "bbbb") { + t.Errorf("truncation kept text past the cap") + } + // A rune fully inside the cap stays intact. + kept := detailedReport("€"+strings.Repeat("a", 10), "**Detailed report:**") + if !strings.Contains(kept, "€") { + t.Errorf("truncation dropped a rune inside the cap") + } + if detailedReport(" ", "**Detailed report:**") != "" { + t.Errorf("empty body must render nothing") + } + // The findings export is the canonical per-finding document: it must never + // shorten the body the way the multi-finding task report does. + if out := detailedReportFull(big, "## Detailed Report"); !strings.Contains(out, strings.Repeat("b", 100)) { + t.Errorf("export must not truncate the detailed report") + } +} + +// One finding must read the same in the task report and in the findings exports: +// the proof-strength block comes from the same helpers. +func TestFindingsExportsCarryQuality(t *testing.T) { + f := &db.DBFinding{ + ID: 7, VulnClass: "cwe-602", Name: "role header", Severity: "critical", + Status: db.FindingConfirmed, Summary: "sum", Evidence: "ev", + PoELevel: 2, VerificationStatus: db.VerificationUnverified, + VerificationNote: "no\nverbatim\nquote", ConfidenceRationale: "reason", + Counterevidence: "tried X", TrafficCount: 0, + Report: "## Overview\n\nbody", + } + for name, md := range map[string]string{ + "summary": FindingsMarkdown([]*db.DBFinding{f}, time.Unix(0, 0)), + "single finding": SingleFindingMarkdown(f, time.Unix(0, 0)), + } { + for _, want := range []string{ + "- **Finding record**: #7", + "- **PoE level**: 2", + "- **Verification**: Unverified", + "- **Verification note**: no verbatim quote", + "**Confidence rationale**:\n\nreason", + "**Counterevidence**:\n\ntried X", + } { + if !strings.Contains(md, want) { + t.Errorf("%s: missing %q in\n%s", name, want, md) + } + } + // With nothing bound the traffic section renders nothing at all, so the export + // must still say so; with bindings present the traffic section states the count + // and a bullet would be the same number twice (covered below). + if n := strings.Count(md, "- **Proof**"); n != 1 { + t.Errorf("%s: want exactly one Proof bullet when nothing is bound, got %d\n%s", name, n, md) + } + } + // The detailed report keeps the document's own structure: a per-finding export is + // a section of its own, while the summary lists many findings under one heading. + if md := SingleFindingMarkdown(f, time.Unix(0, 0)); !strings.Contains(md, "## Detailed Report\n\n## Overview\n\nbody") { + t.Errorf("single-finding export lost its Detailed Report section heading\n%s", md) + } + if md := FindingsMarkdown([]*db.DBFinding{f}, time.Unix(0, 0)); !strings.Contains(md, "**Detailed report:**\n\n## Overview\n\nbody") { + t.Errorf("multi-finding export should label the body, not open a new section\n%s", md) + } + + // With bindings present the export states the count once, in its traffic section. + bound := *f + bound.TrafficCount = 2 + bound.TrafficBindings = []db.FindingTrafficBinding{{ID: 1, Role: "proof"}} + if md := SingleFindingMarkdown(&bound, time.Unix(0, 0)); strings.Contains(md, "- **Proof**") { + t.Errorf("export repeats the binding count next to its traffic section\n%s", md) + } else if !strings.Contains(md, "bindings: 1") { + t.Errorf("export lost the binding count entirely\n%s", md) + } +} diff --git a/server/server.go b/server/server.go index 245790a..a41275b 100644 --- a/server/server.go +++ b/server/server.go @@ -4219,14 +4219,35 @@ func (s *Server) getReport(w http.ResponseWriter, r *http.Request) { counts[ty] = len(ns) } } + taskID, _ := strconv.ParseInt(t.ID, 10, 64) var ledger *db.CoverageLedger if t.CoverageEnabled { - taskID, _ := strconv.ParseInt(t.ID, 10, 64) ledger, _ = s.m.Assets().CoverageLedger(taskID) } + // The exploration node payload carries only name/summary/evidence/severity/ + // vulnclass. Everything the report says about proof strength (poe level, + // verification state and note, counterevidence, confidence rationale, both + // evidence versions, the reporter-written body) lives on the standalone findings + // row, joined by node_id, and one report-only query fetches all of it. The + // findings list keeps its own lean meta query: folding report bodies into it + // would fetch multi-KB text for rows that never render it. + // + // A lookup failure is not an absence. The report is generated on demand and can + // be retried, so a failed read fails loudly instead of rendering every finding as + // "not assessed" — a database error must never become a claim about the finding. + quality := map[int64]db.FindingQuality{} // pg-less deployments have no standalone rows at all + if s.m.pg != nil { + var err error + if quality, err = s.m.pg.FindingQualityByNodeID(taskID); err != nil { + log.Printf("[report] task=%s finding quality: %v", t.ID, err) + writeErr(w, 500, "report unavailable: the finding quality lookup failed, retry") + return + } + } md := report.Markdown(report.Input{ Title: t.Description, Goal: t.Goal, GeneratedAt: time.Now(), AssetCounts: counts, Findings: findings, Coverage: ledger, + Quality: quality, }) w.Header().Set("Content-Type", "text/markdown; charset=utf-8") w.WriteHeader(200)