Files
ARTEX/report/report_test.go
T
Xiaobo a0adc209ad feat(report): surface finding proof strength in the report and the export
The task report and the findings export asserted every finding with no
qualification: nothing said how far exploitation actually reached (PoE level),
whether the claim was machine-verified (verification status and note), whether
counterevidence was recorded, or whether any traffic was bound. The web UI
badges all of it (VERIFICATION_META, the detail page), so the report — the
artifact a client actually reads — was the last surface still asserting bare
claims.

Render it from one place:

- report/quality.go (new): two constructors and two pure renderers, no I/O, no
  DB handle, so the block is unit-tested without a database.
- Both surfaces call the same helpers, which is the point: the task report and
  the per-finding export cannot drift apart the way duplicated formatting does.
- Missing is explicit: a finding node with no standalone row prints
  "not assessed (no finding record)" instead of an invented zero — the honest
  gap the existing coverage work (#207) also insists on.
- Labels match the shipped UI wording (Verification, Confidence rationale,
  Counterevidence) and states are humanised ("Not assessed", "False positive"),
  with an unrecognised value printed verbatim so a new state cannot silently
  vanish from a report.
- Bound-traffic count appears once per document: as a Proof bullet in the task
  report (it has no traffic section), and in the export only when nothing is
  bound — otherwise the traffic section already states it.
- The detailed report is capped at 8 KiB in the task report only, rune-safe, with
  the cap derived into the truncation message; the export is the canonical
  per-finding document and is never silently shortened.
- The export keeps its document structure: the single-finding document gets
  "## Detailed Report" (a section like its siblings), the multi-finding summary
  gets a bold label.

Read paths only: no change to the write side of poe_level or verification_status,
no schema change (all five columns already exist), no API/DTO change (the widened
FindingMeta stays internal to the report path, and the findings list keeps
selecting only what it renders). One extra task-scoped query on GET /api/report,
because findings bodies stay off the list path.

docs/vulnerability-traffic-evidence.md §Export describes the new Markdown.
2026-11-17 08:40:37 +08:00

317 lines
14 KiB
Go

package report
import (
"fmt"
"strings"
"testing"
"time"
"unicode/utf8"
"github.com/Autumn-27/artex/db"
)
// The exported report carries the coverage gaps and the completeness caveat;
// nothing is added when the task's coverage feature is off (nil ledger).
func TestMarkdownCoverageGaps(t *testing.T) {
l := db.BuildCoverageLedger([]db.CoverageEntry{{Surface: "/a", RiskArea: "xss", Outcome: db.OutcomeNeedsFollowUp}},
[]string{"sqli"}, db.CoverageCompletenessFor("timeout", false, false, true))
cases := []struct {
name string
l *db.CoverageLedger
want []string
}{
{"ledger", l, []string{"## Coverage gaps", "timeout", "Incomplete scan", "open gaps: 2", "- unrecorded_risk_class sqli", "- needs_follow_up /a xss"}},
{"coverage off", nil, nil},
}
for _, c := range cases {
md := Markdown(Input{Title: "t", Coverage: c.l})
if c.want == nil && strings.Contains(md, "Coverage gaps") {
t.Fatalf("%s: unexpected coverage section", c.name)
}
for _, w := range c.want {
if !strings.Contains(md, w) {
t.Errorf("%s: missing %q in\n%s", c.name, w, md)
}
}
}
}
// The report lists at most reportMaxGaps gaps, then a count line.
func TestMarkdownCoverageGapsCap(t *testing.T) {
var entries []db.CoverageEntry
for i := 0; i < reportMaxGaps+30; i++ {
entries = append(entries, db.CoverageEntry{Surface: "/p", RiskArea: "xss", Outcome: db.OutcomeNeedsFollowUp})
}
md := Markdown(Input{Title: "t", Coverage: db.BuildCoverageLedger(entries, nil, db.CoverageCompleteness{})})
if n := strings.Count(md, "\n- needs_follow_up /p xss"); n != reportMaxGaps {
t.Fatalf("listed %d gaps, want %d", n, reportMaxGaps)
}
if !strings.Contains(md, "- and 30 more (see the coverage API)") {
t.Fatalf("missing count line in\n%s", md)
}
}
// reportNode builds a finding node with only the exploration payload the graph
// actually stores (name/summary/evidence/severity/vulnclass).
func reportNode(id int64, severity, name string) *db.Node {
payload := fmt.Sprintf(`{"vulnclass":"cwe-602","name":%q,"severity":%q,"summary":"s%d","evidence":{"poc":"proof-%d"}}`, name, severity, id, id)
return &db.Node{ID: id, Kind: db.KindFinding, Payload: []byte(payload)}
}
// A finding with a populated standalone row must show its whole defensibility
// footprint: record id, poe level, verification state, bound evidence count, the
// two attestations, and the reporter-written body.
func TestMarkdownFindingQuality(t *testing.T) {
rows := map[int64]db.FindingQuality{
8: {
FindingMeta: db.FindingMeta{ID: 1, Status: db.FindingConfirmed, TrafficCount: 2},
PoeLevel: 3,
VerificationStatus: db.VerificationVerified,
ConfidenceRationale: "the 200 carries the victim's order body",
Counterevidence: "PATCH on the same id returned 403",
Report: "## Overview\n\nclient-supplied header\n\n## Fix\n\nbind role server-side",
EvidenceVersion: 4,
ReportEvidenceVersion: 4,
},
// Row present but nothing proven: the gaps must be stated, not omitted.
9: {FindingMeta: db.FindingMeta{ID: 2, Status: db.FindingPending}, PoeLevel: 0, VerificationStatus: db.VerificationUnverified},
}
md := Markdown(Input{
Title: "t",
GeneratedAt: time.Unix(0, 0),
Findings: []*db.Node{reportNode(8, "high", "BOLA"), reportNode(9, "critical", "role header")},
Quality: rows,
})
for _, want := range []string{
"- **Finding record**: #1",
"- **PoE level**: 3",
"- **Verification**: Verified",
"- **Proof**: 2 bound traffic exchange(s)",
"**Confidence rationale**:\n\nthe 200 carries the victim's order body",
"**Counterevidence**:\n\nPATCH on the same id returned 403",
"**Detailed report:**\n\n## Overview\n\nclient-supplied header",
"- **Finding record**: #2",
"- **PoE level**: 0",
"- **Verification**: Unverified",
"- **Proof**: no bound traffic evidence",
} {
if !strings.Contains(md, want) {
t.Errorf("missing %q in\n%s", want, md)
}
}
// Absent optional fields produce no heading at all, and the verified finding's
// narrative must not leak onto the unverified one.
if n := strings.Count(md, "**Counterevidence**"); n != 1 {
t.Errorf("counterevidence rendered %d times, want 1 (empty field must stay empty)\n%s", n, md)
}
if n := strings.Count(md, "**Detailed report:**"); n != 1 {
t.Errorf("detailed report rendered %d times, want 1\n%s", n, md)
}
// A report written against the current evidence version is not stale.
if strings.Contains(md, "The evidence has changed") {
t.Errorf("up-to-date report flagged stale\n%s", md)
}
}
// A finding node without a standalone row was never assessed: say so, and do not
// print a poe level or verification verdict that nobody produced.
func TestMarkdownFindingQualityNoRow(t *testing.T) {
md := Markdown(Input{Title: "t", Findings: []*db.Node{reportNode(3, "low", "x")}, Quality: map[int64]db.FindingQuality{}})
if !strings.Contains(md, "- **Verification**: not assessed (no finding record") {
t.Errorf("missing not-assessed line in\n%s", md)
}
if strings.Contains(md, "- **PoE level**") {
t.Errorf("claimed a poe level with no row\n%s", md)
}
}
// A lookup that failed is not an absence: a nil map must render "could not be
// read", never "no finding record", or a database error becomes a claim about the
// finding (and silently drops a report body that does exist).
func TestMarkdownFindingQualityUnavailable(t *testing.T) {
md := Markdown(Input{Title: "t", Findings: []*db.Node{reportNode(3, "low", "x")}})
if !strings.Contains(md, "- **Verification**: unknown (quality lookup failed") {
t.Errorf("missing lookup-failed line in\n%s", md)
}
if strings.Contains(md, "no finding record") {
t.Errorf("a failed lookup is rendered as an absent row\n%s", md)
}
if strings.Contains(md, "- **PoE level**") || strings.Contains(md, "- **Proof**") {
t.Errorf("claimed proof strength that was never read\n%s", md)
}
}
// The stale-evidence warning is one sentence printed from one helper, so the task
// report and the findings export cannot drift on the same finding.
func TestStaleReportWarningOnBothSurfaces(t *testing.T) {
const warning = "The evidence has changed; the detailed report needs updating."
row := db.FindingQuality{
FindingMeta: db.FindingMeta{ID: 5},
PoeLevel: 2,
VerificationStatus: db.VerificationVerified,
Report: "## Overview\n\nstale body",
EvidenceVersion: 7, // evidence moved after the report was written
ReportEvidenceVersion: 6,
}
md := Markdown(Input{Title: "t", Findings: []*db.Node{reportNode(4, "high", "stale one")}, Quality: map[int64]db.FindingQuality{4: row}})
f := &db.DBFinding{ID: 5, VulnClass: "cwe-602", Severity: "high", Status: db.FindingConfirmed,
PoELevel: 2, VerificationStatus: db.VerificationVerified, Report: row.Report,
EvidenceVersion: 7, ReportEvidenceVersion: 6}
fresh := *f
fresh.EvidenceVersion = 6
if n := strings.Count(md, warning); n != 1 {
t.Errorf("task report prints the stale warning %d times, want 1\n%s", n, md)
}
// It has to sit before the body it is about, not after it.
if strings.Index(md, warning) > strings.Index(md, "**Detailed report:**") {
t.Errorf("stale warning printed after the body it flags\n%s", md)
}
for name, out := range map[string]string{
"summary export": FindingsMarkdown([]*db.DBFinding{f}, time.Unix(0, 0)),
"single-finding export": SingleFindingMarkdown(f, time.Unix(0, 0)),
} {
if n := strings.Count(out, warning); n != 1 {
t.Errorf("%s prints the stale warning %d times, want 1\n%s", name, n, out)
}
}
// An up-to-date report gets the warning on neither surface.
if strings.Contains(SingleFindingMarkdown(&fresh, time.Unix(0, 0)), warning) {
t.Errorf("export flags an up-to-date report stale")
}
if strings.Contains(FindingsMarkdown([]*db.DBFinding{&fresh}, time.Unix(0, 0)), warning) {
t.Errorf("summary export flags an up-to-date report stale")
}
}
// A cut inside an open fence would swallow the truncation marker and every later
// section (other findings included) into the stale code block. The cap has to land
// on a balanced fence. Third party report on the same defect:
// TestMarkdownDetailedReportFenceBalance in ChaosLabsInc/ARTEX.
func TestMarkdownDetailedReportFenceBalance(t *testing.T) {
// An unclosed ```http fence that opens well before the 8 KiB cap and is still
// open where the body gets cut.
unclosed := "## Evidence\n\n```http\n" + strings.Repeat("GET /orders/7 HTTP/1.1\nHost: victim\n\n", 400)
if !unbalancedFence(strings.TrimSpace(unclosed[:reportMaxReportBytes])) {
t.Fatalf("fixture does not cut inside an open fence")
}
rows := map[int64]db.FindingQuality{
1: {FindingMeta: db.FindingMeta{ID: 1}, PoeLevel: 3, VerificationStatus: db.VerificationVerified, Report: unclosed},
2: {FindingMeta: db.FindingMeta{ID: 2}, PoeLevel: 1, VerificationStatus: db.VerificationUnverified},
}
md := Markdown(Input{Title: "t", Findings: []*db.Node{reportNode(1, "high", "first"), reportNode(2, "low", "second")}, Quality: rows})
if n := strings.Count(md, "```"); n%2 != 0 {
t.Fatalf("report has %d fences (odd), so a code block is left open\n%s", n, md)
}
// The truncated body must be closed before the note, and the next finding must
// still render as a heading rather than as text inside a code block.
block := md[strings.Index(md, "**Detailed report:**"):strings.Index(md, "### 2.")]
if !strings.Contains(block, "_Detailed report truncated at 8 KiB") {
t.Fatalf("truncation note missing from the body block\n%s", block)
}
if strings.Index(block, "\n```\n") > strings.Index(block, "_Detailed report truncated") {
t.Errorf("the fence is closed after the note, so the note stays inside the block\n%s", block)
}
if !strings.Contains(md, "### 2. [LOW] second") || !strings.Contains(md[strings.Index(md, "### 2."):], "- **PoE level**: 1") {
t.Errorf("the second finding did not survive the cut\n%s", md)
}
// A body with no fence at all must not gain one.
plain := strings.Repeat("word ", 3000)
if out := detailedReport(plain, "**Detailed report:**"); strings.Contains(out, "```") {
t.Errorf("a fence-free body gained a fence\n%s", out)
}
// A body that ends with a closed fence stays as it is.
closed := "## Overview\n\n```\nstill readable\n```\n" + strings.Repeat("trailer line\n", 700)
if out := detailedReport(closed, "**Detailed report:**"); strings.Count(out, "```")%2 != 0 {
t.Errorf("balanced body ended up unbalanced\n%s", out)
}
}
// The inlined detail body is capped so one finding cannot dominate the report,
// and truncation never splits a multi-byte rune.
func TestDetailedReportTruncation(t *testing.T) {
// A multi-byte rune straddling the cap must be dropped whole, never split.
big := strings.Repeat("a", reportMaxReportBytes-2) + "€" + strings.Repeat("b", 100)
got := detailedReport(big, "**Detailed report:**")
if !strings.Contains(got, "_Detailed report truncated at 8 KiB") {
t.Fatalf("missing truncation marker\n%s", got)
}
if !utf8.ValidString(got) {
t.Errorf("truncation produced invalid UTF-8")
}
if strings.Contains(got, "€") || strings.Contains(got, "bbbb") {
t.Errorf("truncation kept text past the cap")
}
// A rune fully inside the cap stays intact.
kept := detailedReport("€"+strings.Repeat("a", 10), "**Detailed report:**")
if !strings.Contains(kept, "€") {
t.Errorf("truncation dropped a rune inside the cap")
}
if detailedReport(" ", "**Detailed report:**") != "" {
t.Errorf("empty body must render nothing")
}
// The findings export is the canonical per-finding document: it must never
// shorten the body the way the multi-finding task report does.
if out := detailedReportFull(big, "## Detailed Report"); !strings.Contains(out, strings.Repeat("b", 100)) {
t.Errorf("export must not truncate the detailed report")
}
}
// One finding must read the same in the task report and in the findings exports:
// the proof-strength block comes from the same helpers.
func TestFindingsExportsCarryQuality(t *testing.T) {
f := &db.DBFinding{
ID: 7, VulnClass: "cwe-602", Name: "role header", Severity: "critical",
Status: db.FindingConfirmed, Summary: "sum", Evidence: "ev",
PoELevel: 2, VerificationStatus: db.VerificationUnverified,
VerificationNote: "no\nverbatim\nquote", ConfidenceRationale: "reason",
Counterevidence: "tried X", TrafficCount: 0,
Report: "## Overview\n\nbody",
}
for name, md := range map[string]string{
"summary": FindingsMarkdown([]*db.DBFinding{f}, time.Unix(0, 0)),
"single finding": SingleFindingMarkdown(f, time.Unix(0, 0)),
} {
for _, want := range []string{
"- **Finding record**: #7",
"- **PoE level**: 2",
"- **Verification**: Unverified",
"- **Verification note**: no verbatim quote",
"**Confidence rationale**:\n\nreason",
"**Counterevidence**:\n\ntried X",
} {
if !strings.Contains(md, want) {
t.Errorf("%s: missing %q in\n%s", name, want, md)
}
}
// With nothing bound the traffic section renders nothing at all, so the export
// must still say so; with bindings present the traffic section states the count
// and a bullet would be the same number twice (covered below).
if n := strings.Count(md, "- **Proof**"); n != 1 {
t.Errorf("%s: want exactly one Proof bullet when nothing is bound, got %d\n%s", name, n, md)
}
}
// The detailed report keeps the document's own structure: a per-finding export is
// a section of its own, while the summary lists many findings under one heading.
if md := SingleFindingMarkdown(f, time.Unix(0, 0)); !strings.Contains(md, "## Detailed Report\n\n## Overview\n\nbody") {
t.Errorf("single-finding export lost its Detailed Report section heading\n%s", md)
}
if md := FindingsMarkdown([]*db.DBFinding{f}, time.Unix(0, 0)); !strings.Contains(md, "**Detailed report:**\n\n## Overview\n\nbody") {
t.Errorf("multi-finding export should label the body, not open a new section\n%s", md)
}
// With bindings present the export states the count once, in its traffic section.
bound := *f
bound.TrafficCount = 2
bound.TrafficBindings = []db.FindingTrafficBinding{{ID: 1, Role: "proof"}}
if md := SingleFindingMarkdown(&bound, time.Unix(0, 0)); strings.Contains(md, "- **Proof**") {
t.Errorf("export repeats the binding count next to its traffic section\n%s", md)
} else if !strings.Contains(md, "bindings: 1") {
t.Errorf("export lost the binding count entirely\n%s", md)
}
}