diff --git a/agent/bbot_ingest.go b/agent/bbot_ingest.go index 6ef52d1..3b9a51b 100644 --- a/agent/bbot_ingest.go +++ b/agent/bbot_ingest.go @@ -114,6 +114,7 @@ func bbotTyped(raw bbotRawEvent) bbotEvent { } if u, err := url.Parse(strings.TrimSpace(raw.URL)); err == nil && ev.Host != "" && (u.Scheme == "http" || u.Scheme == "https") && strings.EqualFold(u.Hostname(), ev.Host) { + u.User = nil // credentials never reach an asset ev.URL = u.String() } return ev diff --git a/agent/bbot_ingest_test.go b/agent/bbot_ingest_test.go index b78253d..60c4379 100644 --- a/agent/bbot_ingest_test.go +++ b/agent/bbot_ingest_test.go @@ -71,3 +71,15 @@ func TestParseBbotJSONLTypedOnly(t *testing.T) { } } } + +// The typed url of an in-scope URL event loses its userinfo, keeps its query. +func TestParseBbotURLUserinfo(t *testing.T) { + res, err := parseBbotJSONL(strings.NewReader(`{"type":"URL","host":"www.example.com","url":"https://scanuser:s3cr3t@www.example.com/app?id=7","scope_distance":0}` + "\n")) + if err != nil || len(res.InScope) != 1 { + t.Fatalf("parse: %v, %+v", err, res) + } + want := []assetInputItem{{Type: "service", URL: "https://www.example.com/app?id=7"}} + if got := res.assetItems(); !reflect.DeepEqual(got, want) { + t.Errorf("assetItems = %+v, want %+v", got, want) + } +} diff --git a/agent/scan_ingest.go b/agent/scan_ingest.go index 6634387..d3a435d 100644 --- a/agent/scan_ingest.go +++ b/agent/scan_ingest.go @@ -83,8 +83,8 @@ func scanHost(h string) string { return "" } -// scanURL returns raw re-serialized and its host when it is an http(s) URL -// with a valid host, "" otherwise. +// scanURL returns raw re-serialized without its userinfo, and its host, when +// it is an http(s) URL with a valid host, "" otherwise. func scanURL(raw string) (string, string) { u, err := url.Parse(strings.TrimSpace(raw)) if err != nil || (u.Scheme != "http" && u.Scheme != "https") { @@ -94,9 +94,31 @@ func scanURL(raw string) (string, string) { if h == "" { return "", "" } + u.User = nil // credentials never reach an asset, a note or a fact return u.String(), h } +// stripUserinfo removes the userinfo (user:pass@) from raw, a URL or a bare +// host[:port] as a scanner prints it, and keeps every other byte. It is for +// the raw values kept in a fact detail: unlike scanURL it never rejects, and a +// value with no userinfo, or a path with no authority, is returned unchanged. +func stripUserinfo(raw string) string { + start := 0 // the authority follows "scheme://" or "//"; a bare value is all authority + if i := strings.Index(raw, "//"); i == 0 || i > 0 && raw[i-1] == ':' && !strings.ContainsAny(raw[:i-1], "/?#@") { + start = i + 2 + } else if strings.ContainsAny(raw, "/?#") { + return raw + } + auth := raw[start:] + if j := strings.IndexAny(auth, "/?#"); j >= 0 { + auth = auth[:j] + } + if at := strings.LastIndex(auth, "@"); at >= 0 { + return raw[:start] + raw[start+at+1:] + } + return raw +} + // scanHostOf reads a host from a URL, a host:port or a bare host. func scanHostOf(s string) string { if _, h := scanURL(s); h != "" { @@ -342,7 +364,7 @@ func (s *scanIngest) addNuclei(line []byte) { } s.note("results", tid+" ("+sev+") on "+host) s.fact(fmt.Sprintf("nuclei: template %s (severity %s) matched on %s; unverified scanner result", tid, sev, host), - map[string]any{"name": r.Info.Name, "host": r.Host, "matched_at": r.MatchedAt, + map[string]any{"name": r.Info.Name, "host": stripUserinfo(r.Host), "matched_at": stripUserinfo(r.MatchedAt), "matcher_name": r.MatcherName, "extracted_results": r.Extracted}) } @@ -394,7 +416,7 @@ func (s *scanIngest) addHttpx(line []byte) { s.note("urls", fmt.Sprintf("%s %d", u, r.StatusCode)) s.fact(fmt.Sprintf("httpx: %s responded with status %d", u, r.StatusCode), map[string]any{"title": r.Title, "tech": r.Tech, "webserver": r.Webserver, "content_length": r.ContentLength, - "host": r.Host, "ip": r.IP, "tls": r.TLS}) + "host": stripUserinfo(r.Host), "ip": r.IP, "tls": r.TLS}) } // ---- ffuf -of json ---- @@ -480,5 +502,5 @@ func (s *scanIngest) addFfuf(raw json.RawMessage) { s.Assets = append(s.Assets, item) s.note("urls", fmt.Sprintf("%s %d", u, r.Status)) s.fact(fmt.Sprintf("ffuf: %s returned status %d, %d bytes, %d words, %d lines", u, r.Status, r.Length, r.Words, r.Lines), - map[string]any{"input": r.Input, "redirectlocation": r.Redirect}) + map[string]any{"input": r.Input, "redirectlocation": stripUserinfo(r.Redirect)}) } diff --git a/agent/scan_ingest_test.go b/agent/scan_ingest_test.go index d273781..f571c84 100644 --- a/agent/scan_ingest_test.go +++ b/agent/scan_ingest_test.go @@ -248,6 +248,137 @@ func TestScanIngestJSONLLineLimits(t *testing.T) { } } +// scanURL drops the userinfo and keeps scheme, host, port, path and query; an +// @ in the path is not userinfo; a URL it refused before is still refused. +func TestScanURLUserinfo(t *testing.T) { + cases := []struct{ raw, url, host string }{ + {"http://scanuser:s3cr3t@scan-ingest.example.com:8080/app?id=7&x=y", "http://scan-ingest.example.com:8080/app?id=7&x=y", "scan-ingest.example.com"}, + {"https://scanuser@scan-ingest.example.com/app", "https://scan-ingest.example.com/app", "scan-ingest.example.com"}, + {"https://scan-ingest.example.com/a@b/c?x=@y", "https://scan-ingest.example.com/a@b/c?x=@y", "scan-ingest.example.com"}, + {"http://scanuser:s3cr3t@scan-ingest.example.com/%zz", "", ""}, + {"javascript:alert(1)", "", ""}, + {"http://scanuser:s3cr3t@bad host/", "", ""}, + {"not a url", "", ""}, + } + for _, c := range cases { + if u, h := scanURL(c.raw); u != c.url || h != c.host { + t.Errorf("scanURL(%q) = %q, %q; want %q, %q", c.raw, u, h, c.url, c.host) + } + } +} + +func TestStripUserinfo(t *testing.T) { + cases := map[string]string{ + "https://scanuser:s3cr3t@scan-ingest.example.com:443/login?next=1": "https://scan-ingest.example.com:443/login?next=1", + "http://scanuser:s3cr3t@scan-ingest.example.com/%zz": "http://scan-ingest.example.com/%zz", + "//scanuser:s3cr3t@scan-ingest.example.com/": "//scan-ingest.example.com/", + "scanuser:s3cr3t@scan-ingest.example.com:443": "scan-ingest.example.com:443", + "https://scan-ingest.example.com/a@b?x=@y": "https://scan-ingest.example.com/a@b?x=@y", + "https://scan-ingest.example.com/?r=http://scanuser:s3cr3t@x/": "https://scan-ingest.example.com/?r=http://scanuser:s3cr3t@x/", + "/admin@x/": "/admin@x/", + "scan-ingest.example.com:443": "scan-ingest.example.com:443", + "198.51.100.77": "198.51.100.77", + "": "", + } + for raw, want := range cases { + if got := stripUserinfo(raw); got != want { + t.Errorf("stripUserinfo(%q) = %q, want %q", raw, got, want) + } + } +} + +// Scanner lines carrying user:pass: no asset, url, notable item or fact keeps +// the credentials, and the query survives. +const ( + nucleiUserinfoFixture = `{"template-id":"tech-detect","info":{"severity":"info"},"host":"https://scanuser:s3cr3t@scan-ingest.example.com","matched-at":"https://scanuser:s3cr3t@scan-ingest.example.com/login?next=1"} +{"template-id":"tech-detect","info":{"severity":"info"},"host":"scanuser:s3cr3t@scan-ingest.example.com:443","matched-at":"https://scanuser:s3cr3t@scan-ingest.example.com:443/x"} +` + httpxUserinfoFixture = `{"url":"https://scanuser:s3cr3t@scan-ingest.example.com/app?id=7","status_code":200,"host":"198.51.100.77"} +` + ffufUserinfoFixture = `{"results":[{"input":{"FUZZ":"app"},"status":301,"url":"https://scanuser:s3cr3t@scan-ingest.example.com/app?id=7","redirectlocation":"https://scanuser:s3cr3t@scan-ingest.example.com/app/"}]} +` +) + +// hasScanCreds reports whether v holds a user or password of the fixtures. +func hasScanCreds(v string) bool { + for _, c := range []string{"s3cr3t", "scanuser", "alice", "bob"} { + if strings.Contains(v, c) { + return true + } + } + return false +} + +func checkNoUserinfo(t *testing.T, format string, s *scanIngest) { + t.Helper() + var all []string + for _, a := range s.Assets { + all = append(all, a.URL) + } + for u := range s.urls { + all = append(all, u) + } + for _, items := range s.Notable { + all = append(all, items...) + } + for _, f := range s.Facts { + d, _ := strconv.Unquote(f.Detail) + all = append(all, f.Summary, d) + } + for _, v := range all { + if hasScanCreds(v) { + t.Fatalf("%s: credentials kept in %q", format, v) + } + } +} + +func TestScanIngestStripsUserinfo(t *testing.T) { + cases := []struct { + format, content string + want string // events urls facts assets + url string // first asset URL + }{ + {"nuclei-jsonl", nucleiUserinfoFixture, "[2 2 2 2]", "https://scan-ingest.example.com"}, + {"httpx-jsonl", httpxUserinfoFixture, "[1 1 1 1]", "https://scan-ingest.example.com/app?id=7"}, + {"ffuf-json", ffufUserinfoFixture, "[1 1 1 1]", "https://scan-ingest.example.com/app?id=7"}, + } + for _, c := range cases { + s := parseScan(t, c.format, c.content) + if got := fmt.Sprint([]int{s.Events, len(s.urls), len(s.Facts), len(s.Assets)}); got != c.want || s.Skipped != 0 { + t.Fatalf("%s: events,urls,facts,assets = %s, want %s (skipped %d)", c.format, got, c.want, s.Skipped) + } + if s.Assets[0].URL != c.url { + t.Fatalf("%s: asset URL %q, want %q", c.format, s.Assets[0].URL, c.url) + } + checkNoUserinfo(t, c.format, s) + } + // The raw values kept in the fact detail lose only their userinfo. + s := parseScan(t, "nuclei-jsonl", nucleiUserinfoFixture) + if d, _ := strconv.Unquote(s.Facts[1].Detail); !strings.Contains(d, `"host":"scan-ingest.example.com:443"`) || + !strings.Contains(d, `"matched_at":"https://scan-ingest.example.com:443/x"`) { + t.Fatalf("nuclei detail %s", d) + } + s = parseScan(t, "ffuf-json", ffufUserinfoFixture) + if d, _ := strconv.Unquote(s.Facts[0].Detail); !strings.Contains(d, `"redirectlocation":"https://scan-ingest.example.com/app/"`) { + t.Fatalf("ffuf detail %s", d) + } +} + +// The same URL with different credentials is one URL after normalisation: +// both events are counted, both assets carry the same URL. +func TestScanIngestUserinfoDedup(t *testing.T) { + s := parseScan(t, "httpx-jsonl", httpxUserinfoDedupFixture) + if s.Events != 2 || len(s.urls) != 1 || len(s.Assets) != 2 || s.Assets[0].URL != s.Assets[1].URL || + s.Assets[0].URL != "https://scan-ingest.example.com/app?id=7" { + t.Fatalf("events=%d urls=%d assets=%+v", s.Events, len(s.urls), s.Assets) + } + checkNoUserinfo(t, "httpx-jsonl", s) +} + +const httpxUserinfoDedupFixture = `{"url":"https://alice:s3cr3t-one@scan-ingest.example.com/app?id=7","status_code":200} +{"url":"https://bob:s3cr3t-two@scan-ingest.example.com/app?id=7","status_code":200} +` + // Past scanMaxItems assets and facts, records are counted, not ingested. func TestScanIngestItemCap(t *testing.T) { var b strings.Builder diff --git a/agent/scan_ingest_tool_pg_test.go b/agent/scan_ingest_tool_pg_test.go index b9e5cf6..d952914 100644 --- a/agent/scan_ingest_tool_pg_test.go +++ b/agent/scan_ingest_tool_pg_test.go @@ -199,3 +199,78 @@ func TestIngestScanOutputJSONPG(t *testing.T) { t.Fatalf("re-ingestion: %d assets and %d facts, first run %d and %d", a, f, assets, facts) } } + +// ingest_scan_output (PG), credentials in URLs: no asset row and no fact keeps +// them, and the same URL with different credentials is one asset row and one +// fact. +func TestIngestScanOutputUserinfoPG(t *testing.T) { + d := testDB(t) + t.Cleanup(func() { _ = d.Close() }) + task, err := d.CreateTaskWithOptions("ingest_scan_output_userinfo", "scan-ingest.example.com", db.TaskCreateOptions{}) + if err != nil { + t.Fatal(err) + } + const assetsWhere = `domain LIKE '%scan-ingest.example.com' OR url LIKE '%scan-ingest.example.com%'` + t.Cleanup(func() { + _ = d.DeleteTask(task.ID) + _, _ = d.Exec(`DELETE FROM assets WHERE ` + assetsWhere) + }) + store := d.Exploration(task.ExplorationID) + tsx := NewToolSet(store, "worker-1") + tsx.SetTaskID(task.ID) + tsx.SetAssetStore(d.Assets(), d.Companies()) + dir := t.TempDir() + tool := tsx.ingestScanOutput() + ingest := func(format, content string) map[string]any { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, "scan.out"), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + in, _ := json.Marshal(map[string]string{"format": format, "path": "scan.out"}) + res, err := tool.Call(context.Background(), in, &actool.ToolContext{WorkingDir: dir}) + var out map[string]any + if err != nil || res.IsError || json.Unmarshal([]byte(res.Flatten()), &out) != nil || + out["asset_errors"] != nil || out["fact_errors"] != nil { + t.Fatalf("%s: %v %s", format, err, res.Flatten()) + } + return out + } + count := func(q string) int { + t.Helper() + var n int + if err := d.QueryRow(q).Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + + out := ingest("httpx-jsonl", httpxUserinfoDedupFixture) + if out["events"] != 2.0 || out["urls"] != 1.0 || out["facts_written"] != 1.0 || out["facts_existing"] != 1.0 { + t.Fatalf("same URL, different credentials: %v", out) + } + const appURL = `SELECT count(*) FROM assets WHERE type='service' AND url='https://scan-ingest.example.com/app?id=7'` + if n := count(appURL); n != 1 { + t.Fatalf("%d asset rows for the normalised URL, want 1", n) + } + ingest("httpx-jsonl", httpxUserinfoFixture) + ingest("ffuf-json", ffufUserinfoFixture) + ingest("nuclei-jsonl", nucleiUserinfoFixture) + if n := count(appURL); n != 1 { + t.Fatalf("%d asset rows for the normalised URL after httpx and ffuf, want 1", n) + } + if n := count(`SELECT count(*) FROM assets a WHERE (` + assetsWhere + `) AND row_to_json(a)::text ~ 's3cr3t|scanuser|alice|bob'`); n != 0 { + t.Fatalf("%d asset rows keep credentials", n) + } + facts, err := store.ListByKind(db.KindFact, 500) + if err != nil { + t.Fatal(err) + } + if len(facts) == 0 { + t.Fatal("no facts written") + } + for _, f := range facts { + if hasScanCreds(string(f.Payload)) { + t.Fatalf("fact keeps credentials: %s", f.Payload) + } + } +}