Files
ARTEX/agent/scan_ingest_tool_pg_test.go
BaaRai 07fe075c12 agent: ingest nuclei, httpx and ffuf output in ingest_scan_output
Extends ingest_scan_output with three formats:

- nuclei-jsonl: one tool_output fact per result and an asset for the
  scanned target when it is valid; no automatic finding
- httpx-jsonl: one HTTP service asset and one fact per URL
- ffuf-json: one URL asset and one fact per result, read token by token
  with encoding/json; a file cut short keeps the complete results

JSONL lines go through readJSONLines, taken from the bbot ingester
(64 KiB per line; longer lines and invalid JSON are skipped and
counted). bbot now uses the same reader, with no behaviour change.
One file yields at most 50000 assets and facts; records past the cap
are counted in over_cap and the result is marked truncated.
2026-11-13 19:15:59 +01:00

202 lines
7.0 KiB
Go

package agent
import (
"context"
"encoding/json"
"os"
"path/filepath"
"testing"
"github.com/Autumn-27/artex/db"
actool "github.com/Autumn-27/norma/tool"
)
// ingest_scan_output (PG): an nmap file writes its assets through insert_assets
// and its facts through record_fact; a second ingestion of the same file keeps
// the counters, adds no asset row and no fact; a path leaving the working
// directory through a symlink is refused.
func TestIngestScanOutputPG(t *testing.T) {
d := testDB(t)
t.Cleanup(func() { _ = d.Close() })
task, err := d.CreateTaskWithOptions("ingest_scan_output", "scan-ingest.example.com", db.TaskCreateOptions{})
if err != nil {
t.Fatal(err)
}
const assetsWhere = `ip IN ('198.51.100.77','2001:db8::77') OR domain LIKE '%scan-ingest.example.com' OR url LIKE '%198.51.100.77%' OR url LIKE '%2001:db8::77%'`
t.Cleanup(func() {
_ = d.DeleteTask(task.ID)
_, _ = d.Exec(`DELETE FROM assets WHERE ` + assetsWhere)
})
store := d.Exploration(task.ExplorationID)
tsx := NewToolSet(store, "worker-1")
tsx.SetTaskID(task.ID)
tsx.SetAssetStore(d.Assets(), d.Companies())
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "scan.xml"), []byte(nmapFixture), 0o600); err != nil {
t.Fatal(err)
}
tool := tsx.ingestScanOutput()
call := func(path string) (map[string]any, actool.Result) {
t.Helper()
in, _ := json.Marshal(map[string]string{"format": "nmap-xml", "path": path})
res, err := tool.Call(context.Background(), in, &actool.ToolContext{WorkingDir: dir})
if err != nil {
t.Fatal(err)
}
var out map[string]any
_ = json.Unmarshal([]byte(res.Flatten()), &out)
return out, res
}
count := func(q string) int {
t.Helper()
var n int
if err := d.QueryRow(q).Scan(&n); err != nil {
t.Fatal(err)
}
return n
}
facts := func() int {
all, err := store.ListByKind(db.KindFact, 500)
if err != nil {
t.Fatal(err)
}
n := 0
for _, f := range all {
if f.Origin == "worker-1" {
n++
}
}
return n
}
first, res := call("scan.xml")
if res.IsError || first["facts_written"] != 2.0 || first["assets_written"] != 6.0 || first["fact_errors"] != nil || first["asset_errors"] != nil {
t.Fatalf("first ingestion: %s", res.Flatten())
}
if n := count(`SELECT count(*) FROM assets WHERE type='ip' AND ip IN ('198.51.100.77','2001:db8::77')`); n != 2 {
t.Fatalf("%d ip assets, want 2", n)
}
if n := count(`SELECT count(*) FROM assets WHERE type='service' AND url IN ('http://198.51.100.77:80','https://198.51.100.77:443','https://[2001:db8::77]:8443')`); n != 3 {
t.Fatalf("%d http service assets, want 3", n)
}
if n := count(`SELECT count(*) FROM assets WHERE url LIKE '%evil%' OR domain LIKE '%evil%' OR domain = 'ptr.example.net'`); n != 0 {
t.Fatalf("%d assets from target text or a PTR name", n)
}
assets, nFacts := count(`SELECT count(*) FROM assets WHERE `+assetsWhere), facts()
if nFacts != 2 {
t.Fatalf("%d facts, want 2", nFacts)
}
// Re-ingestion: same counters, no new asset row, no new fact.
again, res := call("scan.xml")
if res.IsError || again["facts_written"] != 0.0 || again["facts_existing"] != 2.0 {
t.Fatalf("re-ingestion: %s", res.Flatten())
}
for _, k := range []string{"hosts", "ports", "urls", "events", "skipped", "assets_written"} {
if again[k] != first[k] {
t.Fatalf("re-ingestion: %s = %v, first run %v", k, again[k], first[k])
}
}
if a, f := count(`SELECT count(*) FROM assets WHERE `+assetsWhere), facts(); a != assets || f != nFacts {
t.Fatalf("re-ingestion: %d assets and %d facts, first run %d and %d", a, f, assets, nFacts)
}
// A symlink in the working directory leading outside it is refused.
outside := filepath.Join(t.TempDir(), "secret.xml")
if err := os.WriteFile(outside, []byte(nmapFixture), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(dir, "escape.xml")); err != nil {
t.Fatal(err)
}
for _, p := range []string{"escape.xml", outside, "../secret.xml"} {
if _, res := call(p); !res.IsError {
t.Fatalf("%s accepted: %s", p, res.Flatten())
}
}
}
// ingest_scan_output (PG), JSON formats: nuclei, httpx and ffuf write their
// assets and facts; a second ingestion keeps the counters and adds no asset
// row and no fact.
func TestIngestScanOutputJSONPG(t *testing.T) {
d := testDB(t)
t.Cleanup(func() { _ = d.Close() })
task, err := d.CreateTaskWithOptions("ingest_scan_output_json", "scan-ingest.example.com", db.TaskCreateOptions{})
if err != nil {
t.Fatal(err)
}
const assetsWhere = `domain LIKE '%scan-ingest.example.com' OR url LIKE '%scan-ingest.example.com%'`
t.Cleanup(func() {
_ = d.DeleteTask(task.ID)
_, _ = d.Exec(`DELETE FROM assets WHERE ` + assetsWhere)
})
store := d.Exploration(task.ExplorationID)
tsx := NewToolSet(store, "worker-1")
tsx.SetTaskID(task.ID)
tsx.SetAssetStore(d.Assets(), d.Companies())
dir := t.TempDir()
tool := tsx.ingestScanOutput()
formats := map[string]string{"nuclei-jsonl": nucleiFixture, "httpx-jsonl": httpxFixture, "ffuf-json": ffufFixture}
ingest := func(format string) map[string]any {
t.Helper()
if err := os.WriteFile(filepath.Join(dir, format+".out"), []byte(formats[format]), 0o600); err != nil {
t.Fatal(err)
}
in, _ := json.Marshal(map[string]string{"format": format, "path": format + ".out"})
res, err := tool.Call(context.Background(), in, &actool.ToolContext{WorkingDir: dir})
var out map[string]any
if err != nil || res.IsError || json.Unmarshal([]byte(res.Flatten()), &out) != nil ||
out["asset_errors"] != nil || out["fact_errors"] != nil {
t.Fatalf("%s: %v %s", format, err, res.Flatten())
}
return out
}
state := func() (assets, facts int) {
t.Helper()
if err := d.QueryRow(`SELECT count(*) FROM assets WHERE ` + assetsWhere).Scan(&assets); err != nil {
t.Fatal(err)
}
all, err := store.ListByKind(db.KindFact, 500)
if err != nil {
t.Fatal(err)
}
for _, f := range all {
if f.Origin == "worker-1" {
facts++
}
}
return assets, facts
}
first := map[string]map[string]any{}
for format := range formats {
if first[format] = ingest(format); first[format]["facts_written"] != 2.0 || first[format]["assets_written"] != 2.0 {
t.Fatalf("%s first ingestion: %v", format, first[format])
}
}
assets, facts := state()
if facts != 6 || assets == 0 {
t.Fatalf("%d assets and %d facts after the first ingestion, want some assets and 6 facts", assets, facts)
}
var n int
if err := d.QueryRow(`SELECT count(*) FROM assets WHERE url LIKE '%evil%' OR domain LIKE '%evil%'`).Scan(&n); err != nil || n != 0 {
t.Fatalf("%d assets from target text (%v)", n, err)
}
for format := range formats {
out := ingest(format)
if out["facts_written"] != 0.0 || out["facts_existing"] != 2.0 {
t.Fatalf("%s re-ingestion: %v", format, out)
}
for _, k := range []string{"hosts", "urls", "events", "skipped", "oversized_lines", "assets_written"} {
if out[k] != first[format][k] {
t.Fatalf("%s re-ingestion: %s = %v, first run %v", format, k, out[k], first[format][k])
}
}
}
if a, f := state(); a != assets || f != facts {
t.Fatalf("re-ingestion: %d assets and %d facts, first run %d and %d", a, f, assets, facts)
}
}