Coverage ledger: record examined, ruled-out and untested risk classes

Workers record per (surface, risk_area) outcomes with the new record_coverage
tool (reported, no_issue_found, ruled_out, not_applicable, needs_follow_up).
Gaps are needs_follow_up entries, entries missing their required evidence or
reason, and goal vulnclass values with no entry (unrecorded_risk_class), so an
absent finding never reads as a clean target. Completeness marks timeout,
budget wrap-up, pause, abort and running tasks as incomplete with a caveat.

Surfaced in the planner context (one line per round, full gap list on the
timeout/budget wrap-up round), GET /api/tasks/{id}/coverage, the coverage card
and the Markdown report. Follows coverage_enabled; no new env var.
This commit is contained in:
BaaRai
2026-11-12 20:16:23 +01:00
parent afa6f4faab
commit 6f3a59e205
18 changed files with 788 additions and 7 deletions
+1 -1
View File
@@ -212,7 +212,7 @@ func (m *IntentMeter) Finish() {
// BudgetWrapupBlocks reports whether a worker tool is refused once its budget is
// exhausted: all but the write-back tools, so the run records facts and stops.
func BudgetWrapupBlocks(tool string) bool {
return tool != "record_fact" && tool != "insert_assets" && tool != "report_finding" && tool != "update_finding_report"
return tool != "record_fact" && tool != "insert_assets" && tool != "report_finding" && tool != "update_finding_report" && tool != "record_coverage"
}
// WorkerBudgetWrapup is returned in place of a refused tool call.
+106
View File
@@ -0,0 +1,106 @@
package agent
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/Autumn-27/artex/db"
actool "github.com/Autumn-27/norma/tool"
)
// recordCoverage writes one coverage-ledger entry (db/coverage_ledger.go). It is a
// coverage-only tool: dropped from the tool list when the task's coverage_enabled
// is false, and its handler refuses to write in that case. Missing evidence or
// reason is accepted and surfaced as a gap (warn, do not block).
func (t *ToolSet) recordCoverage() actool.CoreTool {
return writeTool(
"record_coverage",
"Record what you examined on a surface for one risk class (use the goal's vulnclass as risk_area when it applies), including negative results, so an absent finding is never read as a clean target.",
obj(map[string]any{
"surface": str("what was examined: a URL, endpoint, parameter, host or feature"),
"risk_area": str("risk class, e.g. sqli, xss, idor"),
"outcome": map[string]any{"type": "string", "enum": []string{db.OutcomeReported, db.OutcomeNoIssueFound, db.OutcomeRuledOut, db.OutcomeNotApplicable, db.OutcomeNeedsFollowUp}, "description": "result for this surface and risk class"},
"evidence": str("required for no_issue_found: a fact id, intent id or traffic id that shows the test"),
"reason": str("required for ruled_out and not_applicable: why"),
}, "surface", "risk_area", "outcome"),
func(_ context.Context, in json.RawMessage) (actool.Result, error) {
if t.coverageDisabled {
return actool.Errorf("record_coverage is disabled: the coverage feature is off for this task"), nil
}
if t.as == nil || t.taskID <= 0 {
return actool.Errorf("record_coverage requires a task context"), nil
}
var e db.CoverageEntry
if err := json.Unmarshal(in, &e); err != nil {
return actool.Errorf(err.Error()), nil
}
e.Outcome = strings.ToLower(strings.TrimSpace(e.Outcome))
e.Worker, e.IntentID = t.worker, t.ownerNode
if err := t.as.RecordCoverage(t.taskID, e); err != nil {
return actool.Errorf(err.Error()), nil
}
gaps := db.BuildCoverageLedger([]db.CoverageEntry{e}, nil, db.CoverageCompleteness{}).Gaps
if len(gaps) > 0 {
return actool.Text("recorded, but counted as a gap: " + gaps[0].Kind), nil
}
return actool.Text("recorded"), nil
})
}
// coverageLedgerBlock is the planner's compact ledger line for one round: counters
// plus the top gaps, absent when the feature is off or nothing was recorded yet. On
// a wrap-up round (task timeout or budget) it lists every remaining gap instead,
// since an unexamined class must not read as clean in the final judgment.
func (t *ToolSet) coverageLedgerBlock(wrapup bool) string {
if t.coverageDisabled || t.as == nil || t.taskID <= 0 {
return ""
}
l, err := t.as.CoverageLedger(t.taskID)
if err != nil {
return ""
}
return renderCoverageLedger(l, wrapup)
}
const plannerLedgerTopGaps = 3
func renderCoverageLedger(l *db.CoverageLedger, wrapup bool) string {
if l == nil {
return ""
}
if wrapup {
if len(l.Gaps) == 0 {
return ""
}
var b strings.Builder
fmt.Fprintf(&b, "\n\n[Coverage ledger: remaining gaps at wrap-up (%d); prove_goal stays allowed, but these were not covered and are not clean]", len(l.Gaps))
for _, g := range l.Gaps {
b.WriteString("\n- " + g.String())
}
return b.String()
}
if len(l.Entries) == 0 {
return ""
}
counts := map[string]int{}
for _, e := range l.Entries {
counts[e.Outcome]++
}
var parts []string
for _, o := range []string{db.OutcomeReported, db.OutcomeNoIssueFound, db.OutcomeRuledOut, db.OutcomeNotApplicable, db.OutcomeNeedsFollowUp} {
if counts[o] > 0 {
parts = append(parts, fmt.Sprintf("%s %d", o, counts[o]))
}
}
s := fmt.Sprintf("\n\n[Coverage ledger] entries %d (%s); gaps %d", len(l.Entries), strings.Join(parts, ", "), len(l.Gaps))
for i, g := range l.Gaps {
if i == plannerLedgerTopGaps {
s += "; …"
break
}
s += "; " + g.String()
}
return s
}
+142
View File
@@ -0,0 +1,142 @@
package agent
import (
"encoding/json"
"strings"
"testing"
"github.com/Autumn-27/artex/db"
)
func hasTool(ts *ToolSet, name string) bool {
for _, tool := range ts.DropCoverageTools(ts.WorkerTools()) {
if tool.Name() == name {
return true
}
}
return false
}
// coverage_enabled=false: no tool, no write, no planner line.
func TestRecordCoverageDisabledPassthrough(t *testing.T) {
ts := NewToolSet(nil, "worker-1")
ts.SetTaskID(1)
if !hasTool(ts, "record_coverage") {
t.Fatal("record_coverage missing with coverage on")
}
ts.SetCoverageEnabled(false)
if hasTool(ts, "record_coverage") {
t.Fatal("record_coverage exposed with coverage off")
}
res := callGuardTool(t, ts.recordCoverage(), `{"surface":"/q","risk_area":"sqli","outcome":"no_issue_found"}`)
if !res.IsError {
t.Fatal("record_coverage wrote with coverage off")
}
if s := ts.coverageLedgerBlock(true); s != "" {
t.Fatalf("planner line with coverage off: %q", s)
}
}
func TestRenderCoverageLedger(t *testing.T) {
gapsOnly := db.BuildCoverageLedger(nil, []string{"sqli"}, db.CoverageCompleteness{})
mixed := db.BuildCoverageLedger([]db.CoverageEntry{
{Surface: "/a", RiskArea: "xss", Outcome: db.OutcomeNeedsFollowUp},
{Surface: "/b", RiskArea: "idor", Outcome: db.OutcomeRuledOut},
{Surface: "/c", RiskArea: "rce", Outcome: db.OutcomeNoIssueFound},
{Surface: "/d", RiskArea: "ssrf", Outcome: db.OutcomeNoIssueFound, Evidence: "fact 3"},
}, []string{"sqli"}, db.CoverageCompleteness{})
cases := []struct {
name string
l *db.CoverageLedger
wrapup bool
want []string
notWant []string
}{
{"no entries: no planner line", gapsOnly, false, nil, []string{"Coverage ledger"}},
{"no ledger", nil, true, nil, []string{"Coverage ledger"}},
{"counters and top gaps", mixed, false,
[]string{"entries 4", "no_issue_found 2", "gaps 4", "unrecorded_risk_class sqli", "…"}, []string{"rce"}},
{"wrap-up lists every gap", mixed, true,
[]string{"remaining gaps at wrap-up (4)", "- unrecorded_risk_class sqli", "- missing_evidence /c rce"}, nil},
{"wrap-up lists unrecorded classes without entries", gapsOnly, true, []string{"- unrecorded_risk_class sqli"}, nil},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got := renderCoverageLedger(c.l, c.wrapup)
if c.want == nil && got != "" {
t.Fatalf("want empty, got %q", got)
}
for _, w := range c.want {
if !strings.Contains(got, w) {
t.Errorf("missing %q in %q", w, got)
}
}
for _, w := range c.notWant {
if strings.Contains(got, w) {
t.Errorf("unexpected %q in %q", w, got)
}
}
})
}
}
// PG: record_coverage is a new seeded key; on an existing database the seed
// inserts it and leaves already-seeded (possibly edited) rows untouched.
func TestRecordCoverageSeededOnExistingDB(t *testing.T) {
d := testDB(t)
t.Cleanup(func() { _ = d.Close() })
seed := func() {
for _, s := range BuiltinToolSeeds() {
schema, _ := json.Marshal(s.Schema)
agents, _ := json.Marshal(s.Agents)
if err := d.SeedTool(s.Key, s.Desc, schema, agents); err != nil {
t.Fatal(err)
}
}
}
seed()
var desc string
if err := d.QueryRow(`SELECT description FROM tools WHERE key='record_fact'`).Scan(&desc); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _, _ = d.Exec(`UPDATE tools SET description=$1 WHERE key='record_fact'`, desc) })
if _, err := d.Exec(`UPDATE tools SET description='operator edit' WHERE key='record_fact'`); err != nil {
t.Fatal(err)
}
if _, err := d.Exec(`DELETE FROM tools WHERE key='record_coverage'`); err != nil { // a database from before this key
t.Fatal(err)
}
seed()
tool, err := d.GetTool("record_coverage")
if err != nil || tool == nil || !tool.System || strings.Join(tool.Agents, ",") != "worker" {
t.Fatalf("record_coverage row=%+v err=%v", tool, err)
}
if fact, _ := d.GetTool("record_fact"); fact == nil || fact.Description != "operator edit" {
t.Fatalf("existing tool overwritten: %+v", fact)
}
}
// PG: the tool writes the entry, flags a missing evidence as a gap, and the planner
// line appears once an entry exists.
func TestRecordCoverageToolWritesEntry(t *testing.T) {
d := testDB(t)
t.Cleanup(func() { _ = d.Close() })
task, err := d.CreateTaskWithOptions("record_coverage", "g", db.TaskCreateOptions{})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = d.DeleteTask(task.ID) })
ts := NewToolSet(d.Exploration(task.ExplorationID), "worker-1")
ts.SetTaskID(task.ID)
ts.SetAssetStore(d.Assets(), d.Companies())
if s := ts.coverageLedgerBlock(false); s != "" {
t.Fatalf("planner line before any entry: %q", s)
}
res := callGuardTool(t, ts.recordCoverage(), `{"surface":"/q","risk_area":"SQLi","outcome":"no_issue_found"}`)
if res.IsError || !strings.Contains(res.Flatten(), "missing_evidence") {
t.Fatalf("result=%+v", res)
}
if s := ts.coverageLedgerBlock(false); !strings.Contains(s, "missing_evidence /q sqli") {
t.Fatalf("planner line=%q", s)
}
}
+3
View File
@@ -503,6 +503,9 @@ func (p *Planner) Plan(ctx context.Context, taskID int64, as *db.AssetStore, ts
// Task budget (off by default, see budget.go): the remaining-budget line, and on
// the forced wrap-up round the budget wrap-up instruction. Empty without a budget.
situational += plannerBudgetBlock(ctx)
// Coverage ledger (coverage-only): compact line each round; full gap list on a wrap-up round.
budgetRound, _ := ctx.Value(budgetRoundKey{}).(BudgetRound)
situational += tsx.coverageLedgerBlock(tc.Final || budgetRound.WrapUp)
// This task's work dir <workDir>/tasks/<taskID>; create it first.
taskDir := ensureRunDir(p.workDir, taskID, 0)
ctx = intercept.WithReviewContext(ctx, taskDir, intercept.ReviewBackground{})
+1 -1
View File
@@ -274,7 +274,7 @@ func (t *ToolSet) CoverageDisabled() bool { return t.coverageDisabled }
// (the filter basis for asset queries), not merely a coverage denominator, so the
// agents that own scope definition keep it either way — in lockstep with
// insertAssets' auto-scope hook, which also runs regardless of the switch.
var coverageOnlyTools = map[string]bool{"list_untested_assets": true}
var coverageOnlyTools = map[string]bool{"list_untested_assets": true, "record_coverage": true}
// DropCoverageTools returns tools with the coverage-only ones removed when this
// task has the feature disabled; otherwise it returns tools unchanged.
+2
View File
@@ -668,6 +668,8 @@ func (t *ToolSet) WorkerTools() []actool.CoreTool {
// avoid reporting the same finding twice.
t.listFindings(),
t.addFinding(), t.recordFact(),
// coverage ledger (coverage-only: dropped when coverage_enabled=false).
t.recordCoverage(),
// asset management (handlers guard nil store internally).
// add_company_scope is not given to workers: defining a company's asset scope is
// the duty of planning/main-control/Auto; a worker only executes exploration.
+1 -1
View File
@@ -408,7 +408,7 @@ func (w *Worker) execute(ctx context.Context, name string, taskID int64, as *db.
// base = built-in worker tools ∪ host tools (traffic) ∪ default tools (incl. Bash);
// then augment with the agent's visible skills/MCP. During the SDK settlement
// phase, Bash is hidden via Settlement.DisabledTools (no local gating needed).
base := append(tsx.WorkerTools(), w.extraTools...)
base := append(tsx.DropCoverageTools(tsx.WorkerTools()), w.extraTools...)
// The worker deliberately gets no MultiEdit/Glob/Grep: precise file edits use Edit,
// searching goes through Bash(grep/find) — narrowing the tool surface and reducing
// low-value calls. The rest of the SDK default tools (Read/Write/Edit/LS/Bash/Sleep)
+260
View File
@@ -0,0 +1,260 @@
package db
import (
"fmt"
"sort"
"strings"
"time"
)
// Coverage ledger (modeled on Strix strix/report/coverage.py, skill_coverage_gaps).
// Two layers: agent_reported = the entries workers write with record_coverage;
// machine_observed = execution facts (asset coverage from TaskCoverageWithSources,
// findings). The ledger adds the risk-class dimension and the negative results; it
// never recomputes tested assets. A risk class with no entry is unexamined, never
// clean, and a no_issue_found is never derived from the absence of a finding.
// Ledger outcomes.
const (
OutcomeReported = "reported"
OutcomeNoIssueFound = "no_issue_found"
OutcomeRuledOut = "ruled_out"
OutcomeNotApplicable = "not_applicable"
OutcomeNeedsFollowUp = "needs_follow_up"
)
// Gap kinds.
const (
GapNeedsFollowUp = "needs_follow_up"
GapUnrecordedRiskClass = "unrecorded_risk_class"
GapMissingEvidence = "missing_evidence"
GapMissingReason = "missing_reason"
)
// EndReasonBudget is the activity metadata end_reason of a task stopped by the
// run-budget wrap-up (server/engine_timeout.go); completeness reads it back.
const EndReasonBudget = "budget"
// ValidCoverageOutcome reports whether o is one of the five ledger outcomes.
func ValidCoverageOutcome(o string) bool {
switch o {
case OutcomeReported, OutcomeNoIssueFound, OutcomeRuledOut, OutcomeNotApplicable, OutcomeNeedsFollowUp:
return true
}
return false
}
// CoverageEntry is one agent_reported row: surface + risk_area + outcome + evidence + reason.
type CoverageEntry struct {
Surface string `json:"surface"`
RiskArea string `json:"risk_area"`
Outcome string `json:"outcome"`
Evidence string `json:"evidence"`
Reason string `json:"reason"`
Worker string `json:"worker,omitempty"`
IntentID int64 `json:"intent_id,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
}
// CoverageGap is one open item: a follow-up, an unrecorded mandatory class, or an
// entry missing its required evidence/reason.
type CoverageGap struct {
Kind string `json:"kind"`
Surface string `json:"surface,omitempty"`
RiskArea string `json:"risk_area"`
}
func (g CoverageGap) String() string {
if g.Surface == "" {
return g.Kind + " " + g.RiskArea
}
return g.Kind + " " + g.Surface + " " + g.RiskArea
}
// CoverageCompleteness says whether the run ended as a complete scan. Caveat is
// always set: an incomplete run is never reported silently.
type CoverageCompleteness struct {
Complete bool `json:"complete"`
State string `json:"state"` // complete | in_progress | paused | timeout | budget | aborted
Caveat string `json:"caveat"`
}
// CoverageLedger is the ledger document of one task.
type CoverageLedger struct {
Entries []CoverageEntry `json:"entries"`
Gaps []CoverageGap `json:"gaps"`
Unexamined []string `json:"unexamined"` // mandatory classes with no entry
FindingsByClass map[string]int `json:"findings_by_class"` // machine_observed
Completeness *CoverageCompleteness `json:"completeness"`
}
func normRiskArea(s string) string { return strings.ToLower(strings.TrimSpace(s)) }
// RecordCoverage upserts the current row of (task, surface, risk_area) and appends
// the write to the history, in one transaction.
func (s *AssetStore) RecordCoverage(taskID int64, e CoverageEntry) error {
e.Surface, e.RiskArea = strings.TrimSpace(e.Surface), normRiskArea(e.RiskArea)
if taskID <= 0 || e.Surface == "" || e.RiskArea == "" {
return fmt.Errorf("task, surface and risk_area are required")
}
if !ValidCoverageOutcome(e.Outcome) {
return fmt.Errorf("outcome must be one of reported, no_issue_found, ruled_out, not_applicable, needs_follow_up")
}
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
args := []any{taskID, e.Surface, e.RiskArea, e.Outcome, strings.TrimSpace(e.Evidence), strings.TrimSpace(e.Reason), e.Worker, nullID(e.IntentID)}
if _, err := tx.Exec(`INSERT INTO coverage_ledger_history(task_id, surface, risk_area, outcome, evidence, reason, worker, intent_id)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`, args...); err != nil {
return err
}
if _, err := tx.Exec(`INSERT INTO coverage_ledger(task_id, surface, risk_area, outcome, evidence, reason, worker, intent_id)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
ON CONFLICT (task_id, surface, risk_area) DO UPDATE SET outcome=EXCLUDED.outcome, evidence=EXCLUDED.evidence,
reason=EXCLUDED.reason, worker=EXCLUDED.worker, intent_id=EXCLUDED.intent_id, updated_at=now()`, args...); err != nil {
return err
}
return tx.Commit()
}
func nullID(id int64) any {
if id <= 0 {
return nil
}
return id
}
// CoverageLedger loads the ledger document of a task: its entries, the mandatory
// classes (goal vulnclass), the findings per class and the task's end state.
func (s *AssetStore) CoverageLedger(taskID int64) (*CoverageLedger, error) {
var entries []CoverageEntry
rows, err := s.db.Query(`SELECT surface, risk_area, outcome, evidence, reason, worker, COALESCE(intent_id,0), updated_at
FROM coverage_ledger WHERE task_id=$1 ORDER BY risk_area, surface`, taskID)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var e CoverageEntry
if err := rows.Scan(&e.Surface, &e.RiskArea, &e.Outcome, &e.Evidence, &e.Reason, &e.Worker, &e.IntentID, &e.UpdatedAt); err != nil {
return nil, err
}
entries = append(entries, e)
}
if err := rows.Err(); err != nil {
return nil, err
}
var classes []string
if err := s.scanStrings(&classes, `SELECT DISTINCT lower(btrim(n.payload->>'vulnclass')) FROM exploration_nodes n
JOIN tasks t ON t.exploration_id=n.exploration_id
WHERE t.id=$1 AND n.kind='goal' AND COALESCE(btrim(n.payload->>'vulnclass'),'')<>'' ORDER BY 1`, taskID); err != nil {
return nil, err
}
byClass := map[string]int{}
frows, err := s.db.Query(`SELECT lower(btrim(vulnclass)), COUNT(*) FROM findings WHERE task_id=$1 AND btrim(vulnclass)<>'' GROUP BY 1`, taskID)
if err != nil {
return nil, err
}
defer frows.Close()
for frows.Next() {
var c string
var n int
if err := frows.Scan(&c, &n); err != nil {
return nil, err
}
byClass[c] = n
}
if err := frows.Err(); err != nil {
return nil, err
}
var status string
var paused, budget, pastDeadline bool
if err := s.db.QueryRow(`SELECT t.status, COALESCE(t.paused,false),
EXISTS (SELECT 1 FROM activity a WHERE a.exploration_id=t.exploration_id AND a.worker='system' AND a.metadata->>'end_reason'=$2),
COALESCE(t.deadline_at IS NOT NULL AND t.completed_at IS NOT NULL AND t.completed_at >= t.deadline_at, false)
FROM tasks t WHERE t.id=$1`, taskID, EndReasonBudget).Scan(&status, &paused, &budget, &pastDeadline); err != nil {
return nil, err
}
l := BuildCoverageLedger(entries, classes, CoverageCompletenessFor(status, paused, budget, pastDeadline))
l.FindingsByClass = byClass
return l, nil
}
func (s *AssetStore) scanStrings(out *[]string, q string, args ...any) error {
rows, err := s.db.Query(q, args...)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var v string
if err := rows.Scan(&v); err != nil {
return err
}
*out = append(*out, v)
}
return rows.Err()
}
// BuildCoverageLedger derives the gaps from the entries and the mandatory classes.
// Gap sources: (a) needs_follow_up entries; (b) a mandatory class with no entry at
// all (unrecorded_risk_class, also listed as unexamined); (c) an entry missing its
// required evidence (no_issue_found) or reason (ruled_out, not_applicable). Nothing
// here reads findings: an absent finding never closes a class.
func BuildCoverageLedger(entries []CoverageEntry, mandatory []string, c CoverageCompleteness) *CoverageLedger {
l := &CoverageLedger{Entries: entries, Gaps: []CoverageGap{}, Unexamined: []string{}, FindingsByClass: map[string]int{}, Completeness: &c}
if l.Entries == nil {
l.Entries = []CoverageEntry{}
}
seen := map[string]bool{}
for _, e := range entries {
seen[normRiskArea(e.RiskArea)] = true
switch {
case e.Outcome == OutcomeNeedsFollowUp:
l.Gaps = append(l.Gaps, CoverageGap{GapNeedsFollowUp, e.Surface, e.RiskArea})
case e.Outcome == OutcomeNoIssueFound && strings.TrimSpace(e.Evidence) == "":
l.Gaps = append(l.Gaps, CoverageGap{GapMissingEvidence, e.Surface, e.RiskArea})
case (e.Outcome == OutcomeRuledOut || e.Outcome == OutcomeNotApplicable) && strings.TrimSpace(e.Reason) == "":
l.Gaps = append(l.Gaps, CoverageGap{GapMissingReason, e.Surface, e.RiskArea})
}
}
classes := map[string]bool{}
for _, m := range mandatory {
if m = normRiskArea(m); m != "" && !seen[m] && !classes[m] {
classes[m] = true
l.Unexamined = append(l.Unexamined, m)
}
}
sort.Strings(l.Unexamined)
unrecorded := make([]CoverageGap, 0, len(l.Unexamined))
for _, m := range l.Unexamined {
unrecorded = append(unrecorded, CoverageGap{Kind: GapUnrecordedRiskClass, RiskArea: m})
}
l.Gaps = append(unrecorded, l.Gaps...) // the central invariant first
return l
}
// CoverageCompletenessFor maps a task's end state to completeness. Only a task that
// reached done on its own is complete; timeout, budget wrap-up (even when the final
// round met the goals), pause, abort (failed) and a still-running task are not.
func CoverageCompletenessFor(status string, paused, budget, pastDeadline bool) CoverageCompleteness {
state := "complete"
switch {
case budget:
state = "budget"
case status == "timeout" || (status == "done" && pastDeadline):
state = "timeout"
case status == "failed":
state = "aborted"
case paused || status == "paused":
state = "paused"
case status != "done":
state = "in_progress"
}
if state == "complete" {
return CoverageCompleteness{true, state, "Run ended normally. Classes listed as unexamined were not tested and are not clean."}
}
return CoverageCompleteness{false, state, "Incomplete scan (" + state + "): the absence of a finding does not mean the target is clean; unexamined classes and open gaps were not covered."}
}
+153
View File
@@ -0,0 +1,153 @@
package db
import (
"encoding/json"
"reflect"
"strings"
"testing"
)
func TestBuildCoverageLedgerGaps(t *testing.T) {
e := func(surface, risk, outcome, evidence, reason string) CoverageEntry {
return CoverageEntry{Surface: surface, RiskArea: risk, Outcome: outcome, Evidence: evidence, Reason: reason}
}
cases := []struct {
name string
entries []CoverageEntry
mandatory []string
want []string // CoverageGap.String()
unexamined []string
}{
{"mandatory class with no entry is unexamined, never clean", nil, []string{"SQLi"},
[]string{"unrecorded_risk_class sqli"}, []string{"sqli"}},
{"needs_follow_up is a gap", []CoverageEntry{e("/login", "idor", OutcomeNeedsFollowUp, "", "")}, nil,
[]string{"needs_follow_up /login idor"}, nil},
{"no_issue_found without evidence is a gap", []CoverageEntry{e("/q", "sqli", OutcomeNoIssueFound, " ", "")}, []string{"sqli"},
[]string{"missing_evidence /q sqli"}, nil},
{"no_issue_found with evidence closes the class", []CoverageEntry{e("/q", "sqli", OutcomeNoIssueFound, "fact 12", "")}, []string{"SQLi "},
nil, nil},
{"ruled_out without reason is a gap", []CoverageEntry{e("/a", "xss", OutcomeRuledOut, "", "")}, nil,
[]string{"missing_reason /a xss"}, nil},
{"not_applicable without reason is a gap", []CoverageEntry{e("/a", "ssrf", OutcomeNotApplicable, "", "")}, nil,
[]string{"missing_reason /a ssrf"}, nil},
{"all three sources, unrecorded first", []CoverageEntry{e("/b", "xss", OutcomeNeedsFollowUp, "", ""), e("/c", "rce", OutcomeNoIssueFound, "", "")}, []string{"xss", "idor"},
[]string{"unrecorded_risk_class idor", "needs_follow_up /b xss", "missing_evidence /c rce"}, []string{"idor"}},
{"no goals and no entries: no obligation, no gap", nil, nil, nil, nil},
{"reported closes the class", []CoverageEntry{e("/r", "sqli", OutcomeReported, "", "")}, []string{"sqli"}, nil, nil},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
l := BuildCoverageLedger(c.entries, c.mandatory, CoverageCompleteness{})
got := []string{}
for _, g := range l.Gaps {
got = append(got, g.String())
}
if c.want == nil {
c.want = []string{}
}
if c.unexamined == nil {
c.unexamined = []string{}
}
if !reflect.DeepEqual(got, c.want) || !reflect.DeepEqual(l.Unexamined, c.unexamined) {
t.Fatalf("gaps=%v unexamined=%v, want %v %v", got, l.Unexamined, c.want, c.unexamined)
}
})
}
}
func TestCoverageCompletenessFor(t *testing.T) {
cases := []struct {
status string
paused, budget, past bool
state string
}{
{"done", false, false, false, "complete"},
{"timeout", false, false, true, "timeout"},
{"done", false, false, true, "timeout"}, // goals met on the final timeout round
{"timeout", false, true, false, "budget"},
{"done", false, true, false, "budget"}, // budget wrap-up that met the goals
{"failed", false, false, false, "aborted"},
{"running", true, false, false, "paused"},
{"running", false, false, false, "in_progress"},
}
for _, c := range cases {
got := CoverageCompletenessFor(c.status, c.paused, c.budget, c.past)
if got.State != c.state || got.Complete != (c.state == "complete") || got.Caveat == "" {
t.Errorf("%+v: got %+v", c, got)
}
}
}
// The GET coverage document carries entries, gaps and completeness at top level.
func TestCoverageJSONCarriesLedger(t *testing.T) {
l := BuildCoverageLedger(nil, []string{"sqli"}, CoverageCompletenessFor("timeout", false, false, true))
b, _ := json.Marshal(&Coverage{Enabled: true, ByType: []CoverageByType{}, CoverageLedger: l})
var m map[string]json.RawMessage
_ = json.Unmarshal(b, &m)
for _, k := range []string{"entries", "gaps", "unexamined", "completeness", "by_type"} {
if _, ok := m[k]; !ok {
t.Fatalf("missing %q in %s", k, b)
}
}
if !strings.Contains(string(m["gaps"]), "unrecorded_risk_class") {
t.Fatalf("gaps=%s", m["gaps"])
}
}
// PG: upsert keeps one current row per (task, surface, risk_area) and appends history;
// the loaded ledger reads goal vulnclass and the budget end marker.
func TestRecordCoverageUpsertHistory(t *testing.T) {
d, err := Open(testDSN(t))
if err != nil {
t.Skipf("postgres unavailable (%v) — skipping", err)
}
defer d.Close()
task, err := d.CreateTaskWithOptions("coverage ledger", "g", TaskCreateOptions{})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = d.DeleteTask(task.ID) })
as := d.Assets()
store := d.Exploration(task.ExplorationID)
for _, vc := range []string{"SQLi", "XSS"} {
if _, err := store.AddGoal(map[string]any{"text": "g " + vc, "vulnclass": vc}, "test"); err != nil {
t.Fatal(err)
}
}
for _, e := range []CoverageEntry{
{Surface: "/q", RiskArea: "SQLi", Outcome: OutcomeNeedsFollowUp},
{Surface: "/q", RiskArea: "sqli", Outcome: OutcomeNoIssueFound, Evidence: "fact 7"},
} {
if err := as.RecordCoverage(task.ID, e); err != nil {
t.Fatal(err)
}
}
if err := as.RecordCoverage(task.ID, CoverageEntry{Surface: "/q", RiskArea: "x", Outcome: "clean"}); err == nil {
t.Fatal("invalid outcome accepted")
}
var current, history int
_ = d.QueryRow(`SELECT count(*) FROM coverage_ledger WHERE task_id=$1`, task.ID).Scan(&current)
_ = d.QueryRow(`SELECT count(*) FROM coverage_ledger_history WHERE task_id=$1`, task.ID).Scan(&history)
if current != 1 || history != 2 {
t.Fatalf("current=%d history=%d, want 1 and 2", current, history)
}
l, err := as.CoverageLedger(task.ID)
if err != nil {
t.Fatal(err)
}
if len(l.Entries) != 1 || l.Entries[0].Outcome != OutcomeNoIssueFound || len(l.Gaps) != 1 || l.Gaps[0].String() != "unrecorded_risk_class xss" {
t.Fatalf("ledger=%+v", l)
}
if l.Completeness.State != "in_progress" {
t.Fatalf("completeness=%+v", l.Completeness)
}
if _, err := store.AppendActivity(Activity{Worker: "system", Kind: "text", Metadata: json.RawMessage(`{"end_reason":"budget"}`)}); err != nil {
t.Fatal(err)
}
if err := d.SetStatus(task.ID, "done"); err != nil {
t.Fatal(err)
}
if l, _ = as.CoverageLedger(task.ID); l.Completeness.Complete || l.Completeness.State != "budget" {
t.Fatalf("completeness after budget=%+v", l.Completeness)
}
}
+32
View File
@@ -1568,3 +1568,35 @@ CREATE TABLE IF NOT EXISTS rsi_lesson_usage (
shown_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (lesson_id, task_id, mode)
);
-- Coverage ledger (agent_reported layer, see db/coverage_ledger.go): what a worker
-- examined, ruled out or left open, per (task, surface, risk_area). One current row
-- per key; every write is also appended to coverage_ledger_history. A risk class with
-- no row is unexamined, never clean. Rows exist only for tasks with coverage_enabled.
CREATE TABLE IF NOT EXISTS coverage_ledger (
id BIGSERIAL PRIMARY KEY,
task_id BIGINT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE,
surface TEXT NOT NULL,
risk_area TEXT NOT NULL, -- stored lower-case, matched against goal vulnclass
outcome TEXT NOT NULL
CHECK (outcome IN ('reported','no_issue_found','ruled_out','not_applicable','needs_follow_up')),
evidence TEXT NOT NULL DEFAULT '',
reason TEXT NOT NULL DEFAULT '',
worker TEXT NOT NULL DEFAULT '',
intent_id BIGINT,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
UNIQUE (task_id, surface, risk_area)
);
CREATE TABLE IF NOT EXISTS coverage_ledger_history (
id BIGSERIAL PRIMARY KEY,
task_id BIGINT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE,
surface TEXT NOT NULL,
risk_area TEXT NOT NULL,
outcome TEXT NOT NULL,
evidence TEXT NOT NULL DEFAULT '',
reason TEXT NOT NULL DEFAULT '',
worker TEXT NOT NULL DEFAULT '',
intent_id BIGINT,
recorded_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_coverage_ledger_history_task ON coverage_ledger_history(task_id, id);
+3
View File
@@ -290,6 +290,9 @@ type Coverage struct {
Tested int `json:"tested"` // tested (approximate)
Pct *float64 `json:"pct"` // coverage; null when the denominator is 0
ByType []CoverageByType `json:"by_type"` // total/tested per asset type
// Coverage ledger (entries, gaps, unexamined, findings_by_class, completeness),
// flattened into the same document; nil when the feature is off.
*CoverageLedger
}
// CoverageEnabled reports whether a task has the asset-coverage feature turned on
+1 -1
View File
@@ -28,7 +28,7 @@ var noTargetTools = toolSet(
"set_goals", "set_constraints", "list_goals", "prove_goal", "goal_met",
"insert_assets", "add_company_scope", "add_task_scope", "list_assets",
"list_untested_assets", "list_companies", "report_finding", "record_fact",
"save_task_credential", "list_task_credentials", "ingest_bbot",
"save_task_credential", "list_task_credentials", "ingest_bbot", "record_coverage",
// cross-task orchestration, findings and retests (server)
"list_tasks", "list_llm_profiles", "spawn_task", "pause_task", "get_task_graph",
"list_task_findings", "add_task_hint", "get_task_worker_trace",
+13
View File
@@ -19,6 +19,7 @@ type Input struct {
GeneratedAt time.Time
AssetCounts map[string]int
Findings []*db.Node
Coverage *db.CoverageLedger // nil when the task's coverage feature is off
}
type findingView struct {
@@ -88,6 +89,18 @@ func Markdown(in Input) string {
}
}
if c := in.Coverage; c != nil {
fmt.Fprintf(&b, "## Coverage gaps\n\n")
if c.Completeness != nil {
fmt.Fprintf(&b, "- **Completeness**: %s. %s\n", c.Completeness.State, c.Completeness.Caveat)
}
fmt.Fprintf(&b, "- Ledger entries: %d, open gaps: %d\n", len(c.Entries), len(c.Gaps))
for _, g := range c.Gaps {
fmt.Fprintf(&b, "- %s\n", g)
}
b.WriteString("\n")
}
return b.String()
}
+34
View File
@@ -0,0 +1,34 @@
package report
import (
"strings"
"testing"
"github.com/Autumn-27/artex/db"
)
// The exported report carries the coverage gaps and the completeness caveat;
// nothing is added when the task's coverage feature is off (nil ledger).
func TestMarkdownCoverageGaps(t *testing.T) {
l := db.BuildCoverageLedger([]db.CoverageEntry{{Surface: "/a", RiskArea: "xss", Outcome: db.OutcomeNeedsFollowUp}},
[]string{"sqli"}, db.CoverageCompletenessFor("timeout", false, false, true))
cases := []struct {
name string
l *db.CoverageLedger
want []string
}{
{"ledger", l, []string{"## Coverage gaps", "timeout", "Incomplete scan", "open gaps: 2", "- unrecorded_risk_class sqli", "- needs_follow_up /a xss"}},
{"coverage off", nil, nil},
}
for _, c := range cases {
md := Markdown(Input{Title: "t", Coverage: c.l})
if c.want == nil && strings.Contains(md, "Coverage gaps") {
t.Fatalf("%s: unexpected coverage section", c.name)
}
for _, w := range c.want {
if !strings.Contains(md, w) {
t.Errorf("%s: missing %q in\n%s", c.name, w, md)
}
}
}
}
+4 -1
View File
@@ -2,6 +2,7 @@ package server
import (
"context"
"encoding/json"
"fmt"
"log"
"strconv"
@@ -235,7 +236,9 @@ func (e *Engine) settleTask(ctx context.Context, t *Task, budget bool) {
}
if budget {
e.emitActivity(t, db.Activity{Worker: "system", Kind: "text",
Summary: "Task run budget exhausted: forced wrap-up done, task stopped (final state " + status + ")"})
Summary: "Task run budget exhausted: forced wrap-up done, task stopped (final state " + status + ")",
// read back by the coverage ledger's completeness (db.CoverageLedger)
Metadata: json.RawMessage(`{"end_reason":"` + db.EndReasonBudget + `"}`)})
}
won, err := e.m.SetTaskStatusGuarded(t.ID, status)
switch {
+10 -1
View File
@@ -1943,6 +1943,10 @@ func (s *Server) taskCoverage(w http.ResponseWriter, r *http.Request) {
return
}
cov.Enabled = true
if cov.CoverageLedger, err = as.CoverageLedger(taskID); err != nil {
writeErr(w, 500, err.Error())
return
}
writeJSON(w, 200, cov)
}
@@ -4067,9 +4071,14 @@ func (s *Server) getReport(w http.ResponseWriter, r *http.Request) {
counts[ty] = len(ns)
}
}
var ledger *db.CoverageLedger
if t.CoverageEnabled {
taskID, _ := strconv.ParseInt(t.ID, 10, 64)
ledger, _ = s.m.Assets().CoverageLedger(taskID)
}
md := report.Markdown(report.Input{
Title: t.Description, Goal: t.Goal, GeneratedAt: time.Now(),
AssetCounts: counts, Findings: findings,
AssetCounts: counts, Findings: findings, Coverage: ledger,
})
w.Header().Set("Content-Type", "text/markdown; charset=utf-8")
w.WriteHeader(200)
@@ -115,6 +115,8 @@ export function OverviewTab({ taskId }: { taskId: string }) {
tested: number;
pct: number | null;
by_type: { type: string; total: number; tested: number }[];
gaps?: { kind: string; surface?: string; risk_area: string }[];
completeness?: { complete: boolean; state: string; caveat: string };
} | null>(null);
// Intent ids currently re-running ("__all__" means the batch); used to disable buttons + spinners.
const [rerunning, setRerunning] = React.useState<Set<string>>(new Set());
@@ -701,7 +703,7 @@ export function OverviewTab({ taskId }: { taskId: string }) {
</CardContent>
</Card>
<TaskInterceptRulesCard taskId={taskId} />
{coverage && coverage.enabled && coverage.scope_rows > 0 && (
{coverage && coverage.enabled && (coverage.scope_rows > 0 || (coverage.gaps?.length ?? 0) > 0) && (
<Card>
<CardHeader>
<CardTitle className="flex items-center gap-2 text-base">
@@ -731,6 +733,22 @@ export function OverviewTab({ taskId }: { taskId: string }) {
))}
</div>
)}
{coverage.completeness && !coverage.completeness.complete && (
<p className="text-muted-foreground text-xs">{coverage.completeness.caveat}</p>
)}
{coverage.gaps && (
<div className="text-xs">
<span className="font-medium">Coverage gaps: {coverage.gaps.length}</span>
<ul className="text-muted-foreground mt-1 list-disc pl-4">
{coverage.gaps.map((g, i) => (
<li key={i}>
{g.kind} {g.surface ? g.surface + " " : ""}
{g.risk_area}
</li>
))}
</ul>
</div>
)}
</CardContent>
</Card>
)}
+3
View File
@@ -385,6 +385,9 @@ export const api = {
tested: number;
pct: number | null;
by_type: { type: string; total: number; tested: number }[];
// coverage ledger (absent when the feature is off): gaps = open items; an unexamined class is never clean
gaps?: { kind: string; surface?: string; risk_area: string }[];
completeness?: { complete: boolean; state: string; caveat: string };
}>(`/tasks/${id}/coverage`),
// Asset coverage graph: every in-scope asset + root-domain/company nodes used for connection, with tested/in_scope.
taskCoverageGraph: (id: string) => get<CoverageGraphData>(`/tasks/${id}/coverage-graph`),