mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-14 04:00:54 +08:00
Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0cc22cb794 | ||
|
|
3b6eb2fd2c | ||
|
|
2ab1046746 | ||
|
|
809436e111 | ||
|
|
c84f124cb4 | ||
|
|
cf0b0c1002 | ||
|
|
b17a0ea796 | ||
|
|
59c3702a5e | ||
|
|
05df40eecb | ||
|
|
f4f0df128a | ||
|
|
00f5d8c103 | ||
|
|
a44af3ca2b | ||
|
|
81eab87d16 | ||
|
|
06453409ea | ||
|
|
c73e2872a3 | ||
|
|
a83f494d03 | ||
|
|
a66c716c0a |
@@ -17,3 +17,8 @@ ARTEX_LLM_PROVIDER=
|
||||
ARTEX_LLM_MODEL=
|
||||
ARTEX_LLM_BASE_URL=
|
||||
ARTEX_LLM_PROXY=
|
||||
|
||||
# Untrusted-output boundary (optional). ARTEX_UNTRUSTED_BOUNDARY=1 frames every tool
|
||||
# output between nonce-tagged markers so the model treats it as data, not instructions.
|
||||
# Off by default; test it against a local target before enabling it in production.
|
||||
# ARTEX_UNTRUSTED_BOUNDARY=1
|
||||
|
||||
@@ -160,6 +160,9 @@ func (c *ChatAgent) Chat(ctx context.Context, agentKey, sessionID, message strin
|
||||
Settlement: wrapupSettlement(agentKey, nil),
|
||||
NonStreaming: c.nonStreaming(), // when this profile selects non-streaming, use Provider.Complete
|
||||
MaxTokens: c.maxTokens(), // 0 = send no cap, the server's default decides
|
||||
|
||||
// Opt-in via ARTEX_UNTRUSTED_BOUNDARY (default off): nonce-frame tool output as untrusted data.
|
||||
UntrustedOutputBoundary: untrustedBoundaryEnabled(),
|
||||
}
|
||||
if c.guard != nil {
|
||||
opts.Hooks = c.guard.Hooks()
|
||||
|
||||
@@ -169,6 +169,9 @@ func (m *MainAgent) Chat(ctx context.Context, taskID int64, mainSeg int, as *db.
|
||||
Settlement: wrapupSettlement("mainagent", nil),
|
||||
NonStreaming: m.nonStreaming(), // when this profile selects non-streaming, use Provider.Complete
|
||||
MaxTokens: m.maxTokens(), // 0 = send no cap, the server's default decides
|
||||
|
||||
// Opt-in via ARTEX_UNTRUSTED_BOUNDARY (default off): nonce-frame tool output as untrusted data.
|
||||
UntrustedOutputBoundary: untrustedBoundaryEnabled(),
|
||||
}
|
||||
if m.tx != nil { // persist raw human↔AI conversation; one accumulating file per segment
|
||||
opts.Transcript = m.tx
|
||||
|
||||
@@ -457,6 +457,9 @@ func (p *Planner) Plan(ctx context.Context, taskID int64, as *db.AssetStore, ts
|
||||
Settlement: settle,
|
||||
NonStreaming: p.nonStreaming(), // when this profile selects non-streaming, use Provider.Complete
|
||||
MaxTokens: p.maxTokens(), // 0 = send no cap; the server default decides
|
||||
|
||||
// Opt-in via ARTEX_UNTRUSTED_BOUNDARY (default off): nonce-frame tool output as untrusted data.
|
||||
UntrustedOutputBoundary: untrustedBoundaryEnabled(),
|
||||
}
|
||||
if p.tx != nil { // persist raw LLM conversation; one accumulating file per task's planner
|
||||
opts.Transcript = p.tx
|
||||
|
||||
@@ -3,6 +3,7 @@ package agent
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
actool "github.com/Autumn-27/norma/tool"
|
||||
@@ -46,6 +47,46 @@ func TestBuiltinToolSeeds(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestObjDropsBlankRequired: obj never emits a blank required name, and omits the
|
||||
// "required" key entirely when no real name is left.
|
||||
func TestObjDropsBlankRequired(t *testing.T) {
|
||||
if got, ok := obj(map[string]any{}, "")["required"]; ok {
|
||||
t.Errorf(`obj(props, "") required = %v, want no required key`, got)
|
||||
}
|
||||
got, _ := obj(map[string]any{}, " ", "x")["required"].([]string)
|
||||
if len(got) != 1 || got[0] != "x" {
|
||||
t.Errorf(`obj(props, " ", "x") required = %v, want [x]`, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestListTaskCredentialsSchemaHasNoBlankRequired guards the seeded schema of the
|
||||
// parameterless list_task_credentials tool: "required": [""] is rejected by strict
|
||||
// providers, which fails every worker run.
|
||||
func TestListTaskCredentialsSchemaHasNoBlankRequired(t *testing.T) {
|
||||
for _, s := range BuiltinToolSeeds() {
|
||||
if s.Key != "list_task_credentials" {
|
||||
continue
|
||||
}
|
||||
raw, err := json.Marshal(s.Schema)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var schema struct {
|
||||
Required []string `json:"required"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &schema); err != nil {
|
||||
t.Fatalf("schema %s: %v", raw, err)
|
||||
}
|
||||
for _, r := range schema.Required {
|
||||
if strings.TrimSpace(r) == "" {
|
||||
t.Errorf("blank required entry in schema %s", raw)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatal("list_task_credentials not seeded")
|
||||
}
|
||||
|
||||
// TestDecorateToolInjectsDefaults verifies a schema "default" fills a missing param
|
||||
// before the underlying handler runs, and an explicitly-provided value is kept.
|
||||
func TestDecorateToolInjectsDefaults(t *testing.T) {
|
||||
|
||||
+10
-2
@@ -351,8 +351,16 @@ func pidList(raw []json.RawMessage) []int64 {
|
||||
|
||||
func obj(props map[string]any, required ...string) map[string]any {
|
||||
m := map[string]any{"type": "object", "properties": props}
|
||||
if len(required) > 0 {
|
||||
m["required"] = required
|
||||
// Blank names are dropped, and "required" is omitted when nothing is left:
|
||||
// strict providers reject "required": [""] and fail every run carrying the tool.
|
||||
req := make([]string, 0, len(required))
|
||||
for _, r := range required {
|
||||
if strings.TrimSpace(r) != "" {
|
||||
req = append(req, r)
|
||||
}
|
||||
}
|
||||
if len(req) > 0 {
|
||||
m["required"] = req
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
@@ -64,7 +64,7 @@ func (t *ToolSet) listTaskCredentials() actool.CoreTool {
|
||||
return writeTool(
|
||||
"list_task_credentials",
|
||||
"List this task's test accounts: user-declared accounts (e.g. program-provided aliases; a user-declared entry with an empty password means: register an account on that domain with a strong password, then record it via save_task_credential) plus accounts recorded by earlier runs. Check this BEFORE registering a new account, and use matching entries for authenticated testing.",
|
||||
obj(map[string]any{}, ""),
|
||||
obj(map[string]any{}),
|
||||
func(_ context.Context, _ json.RawMessage) (actool.Result, error) {
|
||||
if t.ts == nil {
|
||||
return actool.Errorf("list_task_credentials is not enabled: task store is not initialized"), nil
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// untrustedBoundaryEnabled reports whether ARTEX_UNTRUSTED_BOUNDARY opts in to
|
||||
// Norma's untrusted-output boundary (agentcore.Options.UntrustedOutputBoundary:
|
||||
// tool results framed between nonce-tagged markers). Only "1", "true" and "on"
|
||||
// (case- and space-insensitive) enable it; unset, empty or anything else keeps
|
||||
// it off.
|
||||
func untrustedBoundaryEnabled() bool {
|
||||
switch strings.ToLower(strings.TrimSpace(os.Getenv("ARTEX_UNTRUSTED_BOUNDARY"))) {
|
||||
case "1", "true", "on":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUntrustedBoundaryEnabled(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
env string
|
||||
want bool
|
||||
}{
|
||||
{"", false},
|
||||
{"0", false},
|
||||
{"false", false},
|
||||
{"yes", false},
|
||||
{"1", true},
|
||||
{"true", true},
|
||||
{"on", true},
|
||||
{" TRUE ", true},
|
||||
{"On", true},
|
||||
} {
|
||||
t.Setenv("ARTEX_UNTRUSTED_BOUNDARY", tc.env)
|
||||
if got := untrustedBoundaryEnabled(); got != tc.want {
|
||||
t.Errorf("ARTEX_UNTRUSTED_BOUNDARY=%q: got %v, want %v", tc.env, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUntrustedBoundaryDisabledWhenUnset(t *testing.T) {
|
||||
t.Setenv("ARTEX_UNTRUSTED_BOUNDARY", "1") // registers restoration of the original value
|
||||
os.Unsetenv("ARTEX_UNTRUSTED_BOUNDARY")
|
||||
if untrustedBoundaryEnabled() {
|
||||
t.Fatal("ARTEX_UNTRUSTED_BOUNDARY unset: got true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
// Structural, not behavioural: the four agentcore.Options literals live inside
|
||||
// run methods that need a database and an LLM provider, so this only checks
|
||||
// that each source file passes the helper to the option exactly once.
|
||||
func TestUntrustedBoundaryWiredInOptionsLiterals(t *testing.T) {
|
||||
const want = "UntrustedOutputBoundary: untrustedBoundaryEnabled(),"
|
||||
for _, f := range []string{"worker.go", "planner.go", "chat.go", "mainagent.go"} {
|
||||
src, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n := 0
|
||||
for _, line := range strings.Split(string(src), "\n") {
|
||||
if strings.TrimSpace(line) == want {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("%s: %d lines equal to %q, want 1", f, n, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -526,6 +526,9 @@ func (w *Worker) execute(ctx context.Context, name string, taskID int64, as *db.
|
||||
Todos: actool.NewTodoStore(), // session-level scratch todos (TodoWrite), pure planning, discarded on exit
|
||||
NonStreaming: w.nonStreaming(), // when this profile selects non-streaming, use Provider.Complete
|
||||
MaxTokens: w.maxTokens(), // 0 = send no cap; the server default decides
|
||||
|
||||
// Opt-in via ARTEX_UNTRUSTED_BOUNDARY (default off): nonce-frame tool output as untrusted data.
|
||||
UntrustedOutputBoundary: untrustedBoundaryEnabled(),
|
||||
}
|
||||
if hooks != nil { // typed-nil guard: only set when concrete (avoids harness panic)
|
||||
opts.Hooks = hooks
|
||||
|
||||
+8
-2
@@ -633,14 +633,20 @@ LIMIT $1`, malformedIPAssetsSampled)
|
||||
if total == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return malformedIPAssetWarningText(total, samples), nil
|
||||
}
|
||||
|
||||
// malformedIPAssetWarningText words the warning for total offending assets, of
|
||||
// which only samples are named; the ones left out are counted in the suffix.
|
||||
func malformedIPAssetWarningText(total int, samples []string) string {
|
||||
warning := fmt.Sprintf(
|
||||
"%d assets have an ip field that is not a valid IP; skipped for IP/CIDR scope matching (these assets will not be attributed to a company by subnet rules): %s",
|
||||
total, strings.Join(samples, ", "),
|
||||
)
|
||||
if total > len(samples) {
|
||||
warning += fmt.Sprintf(" and %d more", total)
|
||||
warning += fmt.Sprintf(" and %d more", total-len(samples))
|
||||
}
|
||||
return warning, nil
|
||||
return warning
|
||||
}
|
||||
|
||||
// UpdateScope replaces all scope rules for a company and reattributes.
|
||||
|
||||
@@ -458,3 +458,21 @@ func TestListCompanies(t *testing.T) {
|
||||
t.Error("ListCompanies: created company not found")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMalformedIPAssetWarningTextCountsOmitted(t *testing.T) {
|
||||
samples := []string{"#1 a", "#2 b", "#3 c", "#4 d", "#5 e"}
|
||||
// The suffix must count the assets left out, not repeat the total, otherwise
|
||||
// the operator is told about more bad rows than exist.
|
||||
got := malformedIPAssetWarningText(8, samples)
|
||||
if !strings.HasPrefix(got, "8 assets have an ip field") {
|
||||
t.Fatalf("the warning should lead with the total of 8, got %q", got)
|
||||
}
|
||||
if !strings.HasSuffix(got, ": #1 a, #2 b, #3 c, #4 d, #5 e and 3 more") {
|
||||
t.Fatalf("8 assets with 5 named should note the 3 omitted, got %q", got)
|
||||
}
|
||||
// When every offending asset is named there is nothing to add.
|
||||
got = malformedIPAssetWarningText(5, samples)
|
||||
if !strings.HasSuffix(got, ": #1 a, #2 b, #3 c, #4 d, #5 e") {
|
||||
t.Fatalf("5 assets with 5 named should have no suffix, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ services:
|
||||
ARTEX_LLM_PROXY: ${ARTEX_LLM_PROXY:-}
|
||||
ARTEX_JWT_KEY_DIR: /app/keys
|
||||
ARTEX_TRUSTED_PROXIES: ${ARTEX_TRUSTED_PROXIES:-}
|
||||
ARTEX_UNTRUSTED_BOUNDARY: ${ARTEX_UNTRUSTED_BOUNDARY:-}
|
||||
ports:
|
||||
- "${ARTEX_BIND_ADDR:-127.0.0.1}:8787:8787"
|
||||
- "127.0.0.1:8788:8788"
|
||||
|
||||
@@ -79,4 +79,28 @@ type Message struct {
|
||||
// HomeURL is the platform dashboard address (global public_base_url); empty
|
||||
// means no dashboard entry is included.
|
||||
HomeURL string
|
||||
// Limits are the truncation limits to render with; nil means the defaults.
|
||||
// A pointer because the zero value is meaningful (no truncation).
|
||||
Limits *Limits
|
||||
}
|
||||
|
||||
// Limits are the render-time truncation limits of a message, configured
|
||||
// globally (see the server's notify_summary_max_runes / notify_assets_max_shown
|
||||
// settings). In both fields 0 means no truncation; each channel's own length
|
||||
// cap remains the final guard.
|
||||
type Limits struct {
|
||||
// SummaryRunes is how many characters the summary of a single push is
|
||||
// compressed to.
|
||||
SummaryRunes int
|
||||
// AssetsShown is the maximum number of assets listed per vulnerability.
|
||||
AssetsShown int
|
||||
}
|
||||
|
||||
// limits returns the limits to render with, falling back to the defaults when
|
||||
// none were passed in.
|
||||
func (m Message) limits() Limits {
|
||||
if m.Limits == nil {
|
||||
return Limits{SummaryRunes: DefaultMaxSummaryRunes, AssetsShown: DefaultMaxAssetsShown}
|
||||
}
|
||||
return *m.Limits
|
||||
}
|
||||
|
||||
+17
-12
@@ -121,15 +121,20 @@ func feishuSeverityTemplate(severity string) string {
|
||||
}
|
||||
}
|
||||
|
||||
// feishuMaxCardBytes is a conservative cap on card content. Feishu enforces a
|
||||
// size limit on cards and rejects the whole message when exceeded; pick a value
|
||||
// clearly below the official limit to also cover JSON wrapping overhead.
|
||||
const feishuMaxCardBytes = 24000
|
||||
// feishuMaxCardBytes is a conservative cap on card content. Feishu custom bots
|
||||
// reject a request body over 20 KB as a whole (not truncated); pick a value
|
||||
// clearly below that limit to also cover JSON wrapping overhead.
|
||||
const feishuMaxCardBytes = 16000
|
||||
|
||||
// feishuCard builds the interactive card and returns it along with the **number
|
||||
// of items actually written**. kept serves the same purpose as in markdownBody:
|
||||
// only items that actually made it into the card should be marked as sent.
|
||||
func feishuCard(m Message) (map[string]any, int) {
|
||||
lim := m.limits()
|
||||
// budget is the final truncation backstop for one item's text, the same role
|
||||
// TruncateBytes plays in markdownBody. With the summary or asset limit set to
|
||||
// 0 an item has no other bound, and an oversized card loses the whole message.
|
||||
budget := feishuMaxCardBytes - markdownReservedBytes
|
||||
elements := []any{}
|
||||
kept := 0
|
||||
if m.Batch {
|
||||
@@ -137,12 +142,12 @@ func feishuCard(m Message) (map[string]any, int) {
|
||||
// "the remaining N items will continue in the next message", and N must
|
||||
// come from the number of items that actually fit.
|
||||
kept = packItemCount(m.Items, feishuMaxCardBytes, markdownReservedBytes, "", byteSize, func(it Item, idx int) string {
|
||||
return feishuBatchLine(it, idx+1)
|
||||
return feishuBatchLine(it, idx+1, lim)
|
||||
})
|
||||
items := m.Items[:kept]
|
||||
elements = append(elements, feishuMarkdownDiv(markdownBatchIntro(m, items, len(m.Items))))
|
||||
for i, it := range items {
|
||||
elements = append(elements, feishuMarkdownDiv(feishuBatchLine(it, i+1)))
|
||||
elements = append(elements, feishuMarkdownDiv(TruncateBytes(feishuBatchLine(it, i+1, lim), budget)))
|
||||
}
|
||||
if m.HomeURL != "" {
|
||||
elements = append(elements, feishuButton("View all on the platform", m.HomeURL))
|
||||
@@ -150,7 +155,7 @@ func feishuCard(m Message) (map[string]any, int) {
|
||||
} else if len(m.Items) > 0 {
|
||||
kept = 1
|
||||
it := m.Items[0]
|
||||
elements = append(elements, feishuMarkdownDiv(feishuItemLines(it)))
|
||||
elements = append(elements, feishuMarkdownDiv(TruncateBytes(feishuItemLines(it, lim), budget)))
|
||||
if it.DetailURL != "" {
|
||||
elements = append(elements, feishuButton("View details", it.DetailURL))
|
||||
}
|
||||
@@ -189,7 +194,7 @@ func feishuButton(label, url string) map[string]any {
|
||||
// emphasis, so every field coming from outside goes through markdownText
|
||||
// (single-lining + escaping) — otherwise a single vulnerability title could turn
|
||||
// into a clickable external link inside Feishu.
|
||||
func feishuItemLines(it Item) string {
|
||||
func feishuItemLines(it Item, lim Limits) string {
|
||||
out := fmt.Sprintf("**%s · %s**", SeverityLabel(it.Severity), markdownText(it.Title(), 0))
|
||||
if it.IsStatusChange() {
|
||||
out += fmt.Sprintf("\n**Status change**: %s → %s",
|
||||
@@ -198,11 +203,11 @@ func feishuItemLines(it Item) string {
|
||||
if it.VulnClass != "" && it.VulnClass != it.Title() {
|
||||
out += fmt.Sprintf("\n**Type**: %s", markdownText(it.VulnClass, 0))
|
||||
}
|
||||
if a := assetLine(it.Assets, maxAssetsShown); a != "" {
|
||||
if a := assetLine(it.Assets, lim.AssetsShown); a != "" {
|
||||
out += fmt.Sprintf("\n**Assets**: %s", markdownText(a, 0))
|
||||
}
|
||||
if it.Summary != "" {
|
||||
if s := markdownText(it.Summary, maxSummaryRunes); s != "" {
|
||||
if s := markdownText(it.Summary, lim.SummaryRunes); s != "" {
|
||||
out += fmt.Sprintf("\n**Summary**: %s", s)
|
||||
}
|
||||
}
|
||||
@@ -210,9 +215,9 @@ func feishuItemLines(it Item) string {
|
||||
}
|
||||
|
||||
// feishuBatchLine renders one entry of a digest card.
|
||||
func feishuBatchLine(it Item, index int) string {
|
||||
func feishuBatchLine(it Item, index int, lim Limits) string {
|
||||
line := fmt.Sprintf("**%d. %s · %s**", index, SeverityLabel(it.Severity), markdownText(it.Title(), 0))
|
||||
if a := assetLine(it.Assets, maxAssetsShown); a != "" {
|
||||
if a := assetLine(it.Assets, lim.AssetsShown); a != "" {
|
||||
line += " — " + markdownText(a, 0)
|
||||
}
|
||||
return line
|
||||
|
||||
+6
-6
@@ -39,10 +39,10 @@ func htmlBody(m Message, maxRunes int) string {
|
||||
if m.Batch {
|
||||
b.WriteString(htmlBatchIntro(m))
|
||||
for _, it := range m.Items {
|
||||
b.WriteString(htmlItem(it, false))
|
||||
b.WriteString(htmlItem(it, false, m.limits()))
|
||||
}
|
||||
} else if len(m.Items) > 0 {
|
||||
b.WriteString(htmlItem(m.Items[0], true))
|
||||
b.WriteString(htmlItem(m.Items[0], true, m.limits()))
|
||||
}
|
||||
if m.HomeURL != "" {
|
||||
fmt.Fprintf(&b, `<p style="margin:16px 0 0;"><a href="%s" style="color:#1677ff;">View all on the platform</a></p>`, htmlEscapeAttr(m.HomeURL))
|
||||
@@ -79,7 +79,7 @@ func htmlBatchIntro(m Message) string {
|
||||
|
||||
// htmlItem renders a single vulnerability. full=true includes the summary and
|
||||
// backlink (single push); false compresses it into one line (digest list).
|
||||
func htmlItem(it Item, full bool) string {
|
||||
func htmlItem(it Item, full bool, lim Limits) string {
|
||||
color := htmlSeverityColor(it.Severity)
|
||||
var b strings.Builder
|
||||
if full {
|
||||
@@ -92,7 +92,7 @@ func htmlItem(it Item, full bool) string {
|
||||
|
||||
if !full {
|
||||
var extras []string
|
||||
if a := assetLine(it.Assets, maxAssetsShown); a != "" {
|
||||
if a := assetLine(it.Assets, lim.AssetsShown); a != "" {
|
||||
extras = append(extras, htmlEscape(a))
|
||||
}
|
||||
if it.Summary != "" {
|
||||
@@ -112,10 +112,10 @@ func htmlItem(it Item, full bool) string {
|
||||
if it.VulnClass != "" && it.VulnClass != it.Title() {
|
||||
fmt.Fprintf(&b, `<div><b>Type</b>: %s</div>`, htmlEscape(it.VulnClass))
|
||||
}
|
||||
if a := assetLine(it.Assets, maxAssetsShown); a != "" {
|
||||
if a := assetLine(it.Assets, lim.AssetsShown); a != "" {
|
||||
fmt.Fprintf(&b, `<div><b>Assets</b>: %s</div>`, htmlEscape(a))
|
||||
}
|
||||
if s := OneLine(it.Summary, maxSummaryRunes); s != "" {
|
||||
if s := OneLine(it.Summary, lim.SummaryRunes); s != "" {
|
||||
fmt.Fprintf(&b, `<div><b>Summary</b>: %s</div>`, htmlEscape(s))
|
||||
}
|
||||
if it.DetailURL != "" {
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// This file covers the configurable render limits (Message.Limits). The summary
|
||||
// length and the number of assets listed used to be the fixed constants 120 / 3;
|
||||
// they now come from the caller, with 0 meaning "no truncation" and nil meaning
|
||||
// "the old defaults".
|
||||
|
||||
// limitsItem returns an item whose summary (200 characters) and asset list (8)
|
||||
// both exceed the defaults. It contains no markdown/HTML metacharacters, so
|
||||
// every renderer emits the text verbatim and one set of assertions fits all.
|
||||
func limitsItem() Item {
|
||||
assets := make([]string, 8)
|
||||
for i := range assets {
|
||||
assets[i] = "host" + itoa(i+1)
|
||||
}
|
||||
return Item{Severity: "high", Name: "title", Summary: strings.Repeat("Z", 200), Assets: assets}
|
||||
}
|
||||
|
||||
// feishuContent returns the lark_md text of the card's element idx.
|
||||
func feishuContent(t *testing.T, m Message, idx int) string {
|
||||
t.Helper()
|
||||
card, _ := feishuCard(m)
|
||||
elements, _ := card["elements"].([]any)
|
||||
if idx >= len(elements) {
|
||||
t.Fatalf("card has %d elements, cannot get element %d", len(elements), idx)
|
||||
}
|
||||
div, _ := elements[idx].(map[string]any)
|
||||
text, _ := div["text"].(map[string]any)
|
||||
content, _ := text["content"].(string)
|
||||
return content
|
||||
}
|
||||
|
||||
// limitsRenderings returns the body of every channel family that applies the
|
||||
// limits. For a digest, the Feishu entry is the first item line (element 0 is
|
||||
// the intro).
|
||||
func limitsRenderings(t *testing.T, m Message) map[string]string {
|
||||
t.Helper()
|
||||
md, _ := markdownBody(m, 0)
|
||||
tg, _ := telegramHTML(m)
|
||||
feishuIdx := 0
|
||||
if m.Batch {
|
||||
feishuIdx = 1
|
||||
}
|
||||
return map[string]string{
|
||||
"markdown": md,
|
||||
"feishu": feishuContent(t, m, feishuIdx),
|
||||
"telegram": tg,
|
||||
"email": htmlBody(m, 0),
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageLimitsApplyToSinglePush(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
lim *Limits
|
||||
must []string
|
||||
wrong []string
|
||||
}{
|
||||
{
|
||||
name: "unset keeps the 120 / 3 defaults",
|
||||
lim: nil,
|
||||
must: []string{strings.Repeat("Z", 119) + ellipsis, "host1, host2, host3 and 5 more"},
|
||||
wrong: []string{strings.Repeat("Z", 120), "host4"},
|
||||
},
|
||||
{
|
||||
name: "zero means no truncation",
|
||||
lim: &Limits{},
|
||||
must: []string{strings.Repeat("Z", 200), "host7, host8"},
|
||||
wrong: []string{ellipsis, " more"},
|
||||
},
|
||||
{
|
||||
name: "custom values",
|
||||
lim: &Limits{SummaryRunes: 10, AssetsShown: 1},
|
||||
must: []string{strings.Repeat("Z", 9) + ellipsis, "host1 and 7 more"},
|
||||
wrong: []string{strings.Repeat("Z", 10), "host2"},
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
m := Message{Items: []Item{limitsItem()}, Limits: tc.lim}
|
||||
for channel, got := range limitsRenderings(t, m) {
|
||||
for _, want := range tc.must {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("%s: missing %q:\n%s", channel, want, got)
|
||||
}
|
||||
}
|
||||
for _, bad := range tc.wrong {
|
||||
if strings.Contains(got, bad) {
|
||||
t.Errorf("%s: %q should not appear:\n%s", channel, bad, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMessageLimitsApplyToDigestAssets covers the digest lines, which list the
|
||||
// assets too. The digest summary is not governed by Limits.SummaryRunes: it
|
||||
// stays at its own fixed 60 characters (markdown and email), a digest being a
|
||||
// list of one-liners.
|
||||
func TestMessageLimitsApplyToDigestAssets(t *testing.T) {
|
||||
m := Message{Batch: true, Items: []Item{limitsItem()}}
|
||||
for channel, got := range limitsRenderings(t, m) {
|
||||
if !strings.Contains(got, "host1, host2, host3 and 5 more") {
|
||||
t.Errorf("%s: unset should keep 3 assets:\n%s", channel, got)
|
||||
}
|
||||
}
|
||||
m.Limits = &Limits{}
|
||||
for channel, got := range limitsRenderings(t, m) {
|
||||
if !strings.Contains(got, "host7, host8") || strings.Contains(got, " more") {
|
||||
t.Errorf("%s: 0 should list every asset:\n%s", channel, got)
|
||||
}
|
||||
if strings.Contains(got, strings.Repeat("Z", 60)) {
|
||||
t.Errorf("%s: the digest summary should stay within 60 characters:\n%s", channel, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFeishuUntruncatedItemStaysWithinBodyLimit is the guard that goes with
|
||||
// "0 = no truncation". Feishu custom bots reject a request body over 20 KB as a
|
||||
// whole, so an unbounded summary or asset list would not arrive shortened — it
|
||||
// would not arrive at all. The body measured here is the real one sent on the
|
||||
// wire, signature included.
|
||||
func TestFeishuUntruncatedItemStaysWithinBodyLimit(t *testing.T) {
|
||||
const feishuBodyLimit = 20000
|
||||
|
||||
huge := limitsItem()
|
||||
huge.DetailURL = "https://artex.local/function/findings/detail?id=42"
|
||||
// 36000 bytes of CJK: also checks that the cut lands on a character boundary.
|
||||
huge.Summary = strings.Repeat("漏洞摘要", 3000)
|
||||
huge.Assets = nil
|
||||
for i := 0; i < 1500; i++ {
|
||||
huge.Assets = append(huge.Assets, "host"+itoa(i+1)+".example.com")
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
m Message
|
||||
idx int
|
||||
}{
|
||||
{"single push", Message{Items: []Item{huge}, Limits: &Limits{}}, 0},
|
||||
{"digest whose first item is oversized", Message{
|
||||
Batch: true, WindowMinutes: 30, HomeURL: "https://artex.local/function/findings",
|
||||
Items: []Item{huge, huge}, Limits: &Limits{},
|
||||
}, 1},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
size := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
raw, _ := io.ReadAll(r.Body)
|
||||
size = len(raw)
|
||||
_, _ = io.WriteString(w, `{"code":0,"msg":"success"}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
kept, err := (feishuChannel{}).Send(context.Background(), map[string]any{"webhook": srv.URL, "secret": "SECtest123"}, tc.m)
|
||||
if err != nil {
|
||||
t.Fatalf("delivery failed: %v", err)
|
||||
}
|
||||
if kept != 1 {
|
||||
t.Fatalf("exactly the first item should be reported as delivered, got %d", kept)
|
||||
}
|
||||
if size == 0 || size > feishuBodyLimit {
|
||||
t.Fatalf("request body of %d bytes, want 1..%d", size, feishuBodyLimit)
|
||||
}
|
||||
content := feishuContent(t, tc.m, tc.idx)
|
||||
if !utf8.ValidString(content) || !strings.HasSuffix(content, ellipsis) {
|
||||
t.Fatalf("the item should be cut on a character boundary and end with an ellipsis, got tail %q", content[len(content)-12:])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+16
-16
@@ -9,16 +9,16 @@ import (
|
||||
// (DingTalk, WeCom). Feishu uses card JSON, Telegram uses HTML, and email uses
|
||||
// HTML; each renders in its own adapter.
|
||||
|
||||
// maxAssetsShown is the maximum number of assets listed in a message. A
|
||||
// vulnerability can be anchored to dozens of assets; listing them all would blow
|
||||
// up the message with no informational value — nobody reads the 4th domain
|
||||
// onwards in an IM.
|
||||
const maxAssetsShown = 3
|
||||
// DefaultMaxAssetsShown is the default maximum number of assets listed in a
|
||||
// message (Limits.AssetsShown). A vulnerability can be anchored to dozens of
|
||||
// assets; listing them all would blow up the message with no informational
|
||||
// value — nobody reads the 4th domain onwards in an IM.
|
||||
const DefaultMaxAssetsShown = 3
|
||||
|
||||
// maxSummaryRunes is how many characters the summary is compressed to. An IM
|
||||
// message is a "go look at the details" hint, not the report itself; the full
|
||||
// content lives on the platform.
|
||||
const maxSummaryRunes = 120
|
||||
// DefaultMaxSummaryRunes is the default number of characters the summary is
|
||||
// compressed to (Limits.SummaryRunes). An IM message is a "go look at the
|
||||
// details" hint, not the report itself; the full content lives on the platform.
|
||||
const DefaultMaxSummaryRunes = 120
|
||||
|
||||
// markdownReservedBytes is reserved for the message header (digest line +
|
||||
// severity distribution + possible truncation notice) and footer (platform
|
||||
@@ -114,7 +114,7 @@ func markdownBody(m Message, maxBytes int) (string, int) {
|
||||
return "", 0
|
||||
}
|
||||
var b strings.Builder
|
||||
writeItem(&b, m.Items[0], "", true)
|
||||
writeItem(&b, m.Items[0], "", true, m.limits())
|
||||
// A single message is sent even if oversized (with final truncation as
|
||||
// the backstop): partial information about one vulnerability is better
|
||||
// than sending nothing at all.
|
||||
@@ -127,7 +127,7 @@ func markdownBody(m Message, maxBytes int) (string, int) {
|
||||
}
|
||||
kept := packItemCount(m.Items, maxBytes, markdownReservedBytes, footer, byteSize, func(it Item, idx int) string {
|
||||
var b strings.Builder
|
||||
writeItem(&b, it, fmt.Sprintf("%d. ", idx+1), false)
|
||||
writeItem(&b, it, fmt.Sprintf("%d. ", idx+1), false, m.limits())
|
||||
return b.String()
|
||||
})
|
||||
|
||||
@@ -135,7 +135,7 @@ func markdownBody(m Message, maxBytes int) (string, int) {
|
||||
var b strings.Builder
|
||||
b.WriteString(markdownBatchIntro(m, items, len(m.Items)))
|
||||
for i, it := range items {
|
||||
writeItem(&b, it, fmt.Sprintf("%d. ", i+1), false)
|
||||
writeItem(&b, it, fmt.Sprintf("%d. ", i+1), false, m.limits())
|
||||
}
|
||||
b.WriteString(footer)
|
||||
return TruncateBytes(b.String(), maxBytes), kept
|
||||
@@ -191,13 +191,13 @@ func markdownBatchIntro(m Message, items []Item, total int) string {
|
||||
// markdownText: single-lining + escaping. The backlink is built from the
|
||||
// admin-configured public_base_url, is not untrusted content, and must be a
|
||||
// clickable link, so it is emitted as-is.
|
||||
func writeItem(b *strings.Builder, it Item, prefix string, single bool) {
|
||||
func writeItem(b *strings.Builder, it Item, prefix string, single bool, lim Limits) {
|
||||
line := fmt.Sprintf("%s**%s · %s**", prefix, SeverityLabel(it.Severity), markdownText(it.Title(), 0))
|
||||
if !single {
|
||||
// Digest mode: single-line presentation, with assets and summary
|
||||
// compressed and appended.
|
||||
var extras []string
|
||||
if a := assetLine(it.Assets, maxAssetsShown); a != "" {
|
||||
if a := assetLine(it.Assets, lim.AssetsShown); a != "" {
|
||||
extras = append(extras, markdownText(a, 0))
|
||||
}
|
||||
if it.Summary != "" {
|
||||
@@ -217,11 +217,11 @@ func writeItem(b *strings.Builder, it Item, prefix string, single bool) {
|
||||
if it.VulnClass != "" && it.VulnClass != it.Title() {
|
||||
fmt.Fprintf(b, "**Type**: %s\n", markdownText(it.VulnClass, 0))
|
||||
}
|
||||
if a := assetLine(it.Assets, maxAssetsShown); a != "" {
|
||||
if a := assetLine(it.Assets, lim.AssetsShown); a != "" {
|
||||
fmt.Fprintf(b, "**Assets**: %s\n", markdownText(a, 0))
|
||||
}
|
||||
if it.Summary != "" {
|
||||
if s := markdownText(it.Summary, maxSummaryRunes); s != "" {
|
||||
if s := markdownText(it.Summary, lim.SummaryRunes); s != "" {
|
||||
fmt.Fprintf(b, "**Summary**: %s\n", s)
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -178,7 +178,7 @@ func TestMarkdownEscapesUntrustedContent(t *testing.T) {
|
||||
// writeItem in single mode is the rendering path shared by all three
|
||||
// markdown channels.
|
||||
var b strings.Builder
|
||||
writeItem(&b, tc.item, "", true)
|
||||
writeItem(&b, tc.item, "", true, m.limits())
|
||||
got := b.String()
|
||||
for _, want := range tc.must {
|
||||
if !strings.Contains(got, want) {
|
||||
|
||||
+2
-2
@@ -149,7 +149,7 @@ func byteSize(s string) int { return len(s) }
|
||||
func runeSize(s string) int { return utf8.RuneCountInString(s) }
|
||||
|
||||
// assetLine renders an asset list into one line of display text; when there are
|
||||
// more than limit assets, the rest are omitted and the total is noted. A
|
||||
// more than limit assets, the rest are omitted and their count is noted. A
|
||||
// vulnerability can be anchored to dozens of assets, and listing them all would
|
||||
// blow up the message.
|
||||
func assetLine(assets []string, limit int) string {
|
||||
@@ -159,7 +159,7 @@ func assetLine(assets []string, limit int) string {
|
||||
if limit <= 0 || len(assets) <= limit {
|
||||
return strings.Join(assets, ", ")
|
||||
}
|
||||
return strings.Join(assets[:limit], ", ") + " and " + itoa(len(assets)) + " more"
|
||||
return strings.Join(assets[:limit], ", ") + " and " + itoa(len(assets)-limit) + " more"
|
||||
}
|
||||
|
||||
// itoa is a short alias for strconv.Itoa, used only for assembling display text,
|
||||
|
||||
@@ -113,11 +113,13 @@ func TestAssetLineOmitsExcess(t *testing.T) {
|
||||
if got := assetLine([]string{"a", "b"}, 3); got != "a, b" {
|
||||
t.Fatalf("within the limit everything should be listed, got %q", got)
|
||||
}
|
||||
// Over the limit the total must be noted, otherwise readers cannot tell how
|
||||
// many assets were left out.
|
||||
got := assetLine([]string{"a", "b", "c", "d", "e"}, 2)
|
||||
if !strings.Contains(got, "and 5 more") {
|
||||
t.Fatalf("the total of 5 should be noted, got %q", got)
|
||||
// Over the limit the number of omitted assets must be noted, otherwise
|
||||
// readers cannot tell how many assets were left out.
|
||||
if got := assetLine([]string{"a", "b", "c", "d", "e"}, 2); got != "a, b and 3 more" {
|
||||
t.Fatalf("5 assets with limit 2 should note the 3 omitted, got %q", got)
|
||||
}
|
||||
if got := assetLine([]string{"a", "b", "c", "d", "e", "f", "g", "h"}, 3); got != "a, b, c and 5 more" {
|
||||
t.Fatalf("8 assets with limit 3 should note the 5 omitted, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+7
-6
@@ -115,6 +115,7 @@ func telegramEndpoint(cfg map[string]any) (string, error) {
|
||||
// telegramHTML renders the HTML body and returns it together with the number of
|
||||
// items actually written (see Channel.Send).
|
||||
func telegramHTML(m Message) (string, int) {
|
||||
lim := m.limits()
|
||||
var b strings.Builder
|
||||
b.WriteString("<b>" + telegramEscape(markdownTitle(m)) + "</b>\n")
|
||||
if m.Batch {
|
||||
@@ -125,13 +126,13 @@ func telegramHTML(m Message) (string, int) {
|
||||
footer = fmt.Sprintf("\n\n<a href=\"%s\">View all on the platform</a>", telegramEscapeAttr(m.HomeURL))
|
||||
}
|
||||
kept := packItemCount(m.Items, telegramTextLimit, telegramReservedRunes, footer, runeSize, func(it Item, idx int) string {
|
||||
return telegramBatchLine(it, idx+1)
|
||||
return telegramBatchLine(it, idx+1, lim)
|
||||
})
|
||||
items := m.Items[:kept]
|
||||
b.Reset()
|
||||
b.WriteString("<b>" + telegramEscape(telegramBatchTitle(m, items, len(m.Items))) + "</b>")
|
||||
for i, it := range items {
|
||||
b.WriteString("\n" + telegramEscape(telegramBatchLine(it, i+1)))
|
||||
b.WriteString("\n" + telegramEscape(telegramBatchLine(it, i+1, lim)))
|
||||
}
|
||||
b.WriteString(footer)
|
||||
return TruncateHTML(b.String(), telegramTextLimit), kept
|
||||
@@ -147,10 +148,10 @@ func telegramHTML(m Message) (string, int) {
|
||||
if it.VulnClass != "" && it.VulnClass != it.Title() {
|
||||
b.WriteString("\n<b>Type</b>: " + telegramEscape(it.VulnClass))
|
||||
}
|
||||
if a := assetLine(it.Assets, maxAssetsShown); a != "" {
|
||||
if a := assetLine(it.Assets, lim.AssetsShown); a != "" {
|
||||
b.WriteString("\n<b>Assets</b>: " + telegramEscape(a))
|
||||
}
|
||||
if s := OneLine(it.Summary, maxSummaryRunes); s != "" {
|
||||
if s := OneLine(it.Summary, lim.SummaryRunes); s != "" {
|
||||
b.WriteString("\n<b>Summary</b>: " + telegramEscape(s))
|
||||
}
|
||||
if it.DetailURL != "" {
|
||||
@@ -165,8 +166,8 @@ const telegramReservedRunes = 160
|
||||
|
||||
// telegramBatchLine renders one entry of a digest (unescaped; the caller escapes
|
||||
// it uniformly).
|
||||
func telegramBatchLine(it Item, idx int) string {
|
||||
if a := assetLine(it.Assets, maxAssetsShown); a != "" {
|
||||
func telegramBatchLine(it Item, idx int, lim Limits) string {
|
||||
if a := assetLine(it.Assets, lim.AssetsShown); a != "" {
|
||||
return fmt.Sprintf("%d. %s · %s — %s", idx, SeverityLabel(it.Severity), it.Title(), a)
|
||||
}
|
||||
return fmt.Sprintf("%d. %s · %s", idx, SeverityLabel(it.Severity), it.Title())
|
||||
|
||||
@@ -2,6 +2,9 @@ package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/flate"
|
||||
"compress/gzip"
|
||||
"compress/zlib"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -40,8 +43,60 @@ func normalizeEvidenceText(s string) string {
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
|
||||
// decodeEvidenceBody undoes the HTTP Content-Encoding declared in respHead on a
|
||||
// captured response body, returning at most limit decoded bytes. Only identity
|
||||
// and one layer of gzip or deflate are handled. Any other coding, a stacked
|
||||
// coding or a stream that does not decode is an error: the raw bytes are never
|
||||
// returned in place of a failed decode.
|
||||
func decodeEvidenceBody(respHead string, raw []byte, limit int) (out []byte, truncated bool, err error) {
|
||||
coding := ""
|
||||
for _, line := range strings.Split(respHead, "\n") {
|
||||
name, value, ok := strings.Cut(line, ":")
|
||||
if !ok || !strings.EqualFold(strings.TrimSpace(name), "Content-Encoding") {
|
||||
continue
|
||||
}
|
||||
for _, v := range strings.Split(value, ",") {
|
||||
switch v = strings.ToLower(strings.TrimSpace(v)); v {
|
||||
case "", "identity":
|
||||
case "gzip", "deflate":
|
||||
if coding != "" {
|
||||
return nil, false, fmt.Errorf("unsupported content-encoding: more than one compressed layer")
|
||||
}
|
||||
coding = v
|
||||
default:
|
||||
return nil, false, fmt.Errorf("unsupported content-encoding: %q", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
if coding == "" || len(raw) == 0 {
|
||||
return raw, false, nil
|
||||
}
|
||||
var zr io.Reader
|
||||
switch {
|
||||
case coding == "gzip":
|
||||
zr, err = gzip.NewReader(bytes.NewReader(raw))
|
||||
case len(raw) >= 2 && raw[0]&0x0f == 8 && raw[0]>>4 <= 7 && (uint(raw[0])<<8|uint(raw[1]))%31 == 0:
|
||||
// HTTP "deflate" is the zlib format (RFC 9110); some servers send a bare
|
||||
// deflate stream instead, told apart by the zlib header.
|
||||
zr, err = zlib.NewReader(bytes.NewReader(raw))
|
||||
default:
|
||||
zr = flate.NewReader(bytes.NewReader(raw))
|
||||
}
|
||||
if err == nil {
|
||||
out, err = io.ReadAll(io.LimitReader(zr, int64(limit)+1))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%s body: %w", coding, err)
|
||||
}
|
||||
if len(out) > limit {
|
||||
return out[:limit], true, nil
|
||||
}
|
||||
return out, false, nil
|
||||
}
|
||||
|
||||
// readBoundResponse reads the response side of a bound snapshot through the
|
||||
// evidence store (same bytes the UI shows), capped at verificationReadCap.
|
||||
// evidence store (same bytes the UI shows), capped at verificationReadCap, and
|
||||
// decodes its Content-Encoding under the same cap.
|
||||
func readBoundResponse(store *evidence.Store, snap db.TrafficEvidenceSnapshot) (boundBody, error) {
|
||||
f, _, err := store.OpenBody(snap, "response")
|
||||
if err != nil {
|
||||
@@ -56,6 +111,15 @@ func readBoundResponse(store *evidence.Store, snap db.TrafficEvidenceSnapshot) (
|
||||
if len(raw) > verificationReadCap {
|
||||
bb.truncated = true
|
||||
raw = raw[:verificationReadCap]
|
||||
}
|
||||
raw, cut, err := decodeEvidenceBody(snap.RespHead, raw, verificationReadCap)
|
||||
if err != nil {
|
||||
return boundBody{}, err
|
||||
}
|
||||
if cut {
|
||||
bb.truncated = true
|
||||
}
|
||||
if bb.truncated {
|
||||
// The cap may cut a multi-byte rune: drop at most UTFMax-1 trailing bytes.
|
||||
for i := 0; i < utf8.UTFMax-1 && !utf8.Valid(raw) && len(raw) > 0; i++ {
|
||||
raw = raw[:len(raw)-1]
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/flate"
|
||||
"compress/gzip"
|
||||
"compress/zlib"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/evidence"
|
||||
)
|
||||
|
||||
const (
|
||||
decodeGzipHead = "HTTP 200\nContent-Type: text/html\nContent-Encoding: gzip\n"
|
||||
decodeHTML = "<html>\n <body>user admin:\tx7\n end of dump</body>\n</html>"
|
||||
decodeQuote = "user admin: x7 end of dump"
|
||||
)
|
||||
|
||||
// compressBody runs data through the compressor built by newWriter.
|
||||
func compressBody(t *testing.T, newWriter func(io.Writer) io.WriteCloser, data []byte) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
w := newWriter(&buf)
|
||||
if _, err := w.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func gzipBody(t *testing.T, data []byte) []byte {
|
||||
return compressBody(t, func(w io.Writer) io.WriteCloser { return gzip.NewWriter(w) }, data)
|
||||
}
|
||||
|
||||
// decodeSnapshot stores raw as a response body in a database-less evidence
|
||||
// store (same layout as evidence.hashPath) and returns the snapshot bound to it.
|
||||
func decodeSnapshot(t *testing.T, respHead string, raw []byte) (*evidence.Store, db.TrafficEvidenceSnapshot) {
|
||||
t.Helper()
|
||||
sum := sha256.Sum256(raw)
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "blobs", hash[:2], hash+".bin")
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, raw, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &evidence.Store{Dir: dir}, db.TrafficEvidenceSnapshot{RespHead: respHead, RespHash: hash, RespLen: int64(len(raw))}
|
||||
}
|
||||
|
||||
func TestDecodeEvidenceBodyGzipQuoteFound(t *testing.T) {
|
||||
raw := gzipBody(t, []byte(decodeHTML))
|
||||
if strings.Contains(string(raw), "admin") {
|
||||
t.Fatal("fixture is not compressed")
|
||||
}
|
||||
out, truncated, err := decodeEvidenceBody(decodeGzipHead, raw, verificationReadCap)
|
||||
if err != nil || truncated {
|
||||
t.Fatalf("decode: truncated=%v err=%v", truncated, err)
|
||||
}
|
||||
if string(out) != decodeHTML {
|
||||
t.Fatalf("decoded %q, want %q", out, decodeHTML)
|
||||
}
|
||||
if !strings.Contains(normalizeEvidenceText(string(out)), normalizeEvidenceText(decodeQuote)) {
|
||||
t.Fatalf("quote %q not found in decoded body", decodeQuote)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeEvidenceBodyTruncatedGzipFails(t *testing.T) {
|
||||
raw := gzipBody(t, []byte(strings.Repeat(decodeHTML, 50)))
|
||||
for _, cut := range []int{5, len(raw) / 2, len(raw) - 1} {
|
||||
out, truncated, err := decodeEvidenceBody(decodeGzipHead, raw[:cut], verificationReadCap)
|
||||
if err == nil || out != nil || truncated {
|
||||
t.Fatalf("cut at %d: out=%d bytes truncated=%v err=%v, want an error and no bytes", cut, len(out), truncated, err)
|
||||
}
|
||||
}
|
||||
// Not gzip at all: the raw bytes must not come back either.
|
||||
out, _, err := decodeEvidenceBody(decodeGzipHead, []byte(decodeHTML), verificationReadCap)
|
||||
if err == nil || out != nil {
|
||||
t.Fatalf("plain text declared as gzip: out=%q err=%v, want an error and no bytes", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeEvidenceBodyIdentityAndNoHeaderUnchanged(t *testing.T) {
|
||||
raw := []byte(decodeHTML)
|
||||
for _, head := range []string{
|
||||
"",
|
||||
"HTTP 200\nContent-Type: text/html\n",
|
||||
"HTTP 200\nContent-Encoding: identity\n",
|
||||
"HTTP 200\r\ncontent-encoding: Identity \r\n",
|
||||
} {
|
||||
out, truncated, err := decodeEvidenceBody(head, raw, verificationReadCap)
|
||||
if err != nil || truncated || !bytes.Equal(out, raw) {
|
||||
t.Fatalf("head %q: out=%q truncated=%v err=%v, want the raw bytes", head, out, truncated, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeEvidenceBodyUnknownEncodingFails(t *testing.T) {
|
||||
raw := gzipBody(t, []byte(decodeHTML))
|
||||
for _, head := range []string{
|
||||
"HTTP 200\nContent-Encoding: br\n",
|
||||
"HTTP 200\nContent-Encoding: gzip, br\n",
|
||||
"HTTP 200\nContent-Encoding: gzip\nContent-Encoding: zstd\n",
|
||||
} {
|
||||
out, _, err := decodeEvidenceBody(head, raw, verificationReadCap)
|
||||
if err == nil || !strings.Contains(err.Error(), "unsupported content-encoding") || out != nil {
|
||||
t.Fatalf("head %q: out=%d bytes err=%v, want unsupported content-encoding", head, len(out), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeEvidenceBodyTwoLayersFails(t *testing.T) {
|
||||
raw := gzipBody(t, gzipBody(t, []byte(decodeHTML)))
|
||||
for _, head := range []string{
|
||||
"HTTP 200\nContent-Encoding: gzip, gzip\n",
|
||||
"HTTP 200\nContent-Encoding: gzip\nContent-Encoding: deflate\n",
|
||||
} {
|
||||
out, _, err := decodeEvidenceBody(head, raw, verificationReadCap)
|
||||
if err == nil || !strings.Contains(err.Error(), "unsupported content-encoding") || out != nil {
|
||||
t.Fatalf("head %q: out=%d bytes err=%v, want unsupported content-encoding", head, len(out), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeEvidenceBodyOverLimitTruncated(t *testing.T) {
|
||||
const limit = 1000
|
||||
plain := []byte(strings.Repeat("0123456789", 500))
|
||||
out, truncated, err := decodeEvidenceBody(decodeGzipHead, gzipBody(t, plain), limit)
|
||||
if err != nil || !truncated || len(out) != limit || !bytes.Equal(out, plain[:limit]) {
|
||||
t.Fatalf("over limit: len=%d truncated=%v err=%v, want %d bytes truncated", len(out), truncated, err, limit)
|
||||
}
|
||||
// Exactly at the limit is complete, not truncated.
|
||||
out, truncated, err = decodeEvidenceBody(decodeGzipHead, gzipBody(t, plain[:limit]), limit)
|
||||
if err != nil || truncated || len(out) != limit {
|
||||
t.Fatalf("at limit: len=%d truncated=%v err=%v, want %d bytes not truncated", len(out), truncated, err, limit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeEvidenceBodyUppercaseHeader(t *testing.T) {
|
||||
raw := gzipBody(t, []byte(decodeHTML))
|
||||
for _, head := range []string{
|
||||
"HTTP 200\nContent-Encoding: GZIP\n",
|
||||
"HTTP 200\r\nCONTENT-ENCODING: GZIP\r\n",
|
||||
} {
|
||||
out, truncated, err := decodeEvidenceBody(head, raw, verificationReadCap)
|
||||
if err != nil || truncated || string(out) != decodeHTML {
|
||||
t.Fatalf("head %q: out=%q truncated=%v err=%v", head, out, truncated, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeEvidenceBodyDeflate(t *testing.T) {
|
||||
const head = "HTTP 200\nContent-Encoding: deflate\n"
|
||||
bodies := map[string][]byte{
|
||||
"zlib": compressBody(t, func(w io.Writer) io.WriteCloser { return zlib.NewWriter(w) }, []byte(decodeHTML)),
|
||||
"raw": compressBody(t, func(w io.Writer) io.WriteCloser {
|
||||
fw, err := flate.NewWriter(w, flate.DefaultCompression)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return fw
|
||||
}, []byte(decodeHTML)),
|
||||
}
|
||||
for name, raw := range bodies {
|
||||
out, truncated, err := decodeEvidenceBody(head, raw, verificationReadCap)
|
||||
if err != nil || truncated || string(out) != decodeHTML {
|
||||
t.Fatalf("%s deflate: out=%q truncated=%v err=%v", name, out, truncated, err)
|
||||
}
|
||||
if out, _, err := decodeEvidenceBody(head, raw[:len(raw)-1], verificationReadCap); err == nil || out != nil {
|
||||
t.Fatalf("truncated %s deflate: out=%d bytes err=%v, want an error and no bytes", name, len(out), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeBoundResponseGzipTextIsSearchable(t *testing.T) {
|
||||
store, snap := decodeSnapshot(t, decodeGzipHead, gzipBody(t, []byte(decodeHTML)))
|
||||
bb, err := readBoundResponse(store, snap)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bb.nonText || bb.truncated {
|
||||
t.Fatalf("gzip html: nonText=%v truncated=%v", bb.nonText, bb.truncated)
|
||||
}
|
||||
if !quoteInBodies(normalizeEvidenceText(decodeQuote), []boundBody{bb}) {
|
||||
t.Fatalf("quote %q not found in %q", decodeQuote, bb.text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeBoundResponseGzipBinaryStaysNonText(t *testing.T) {
|
||||
binary := append([]byte("PNG\x00\x01\x02 user admin: x7 end of dump "), 0xff, 0xfe)
|
||||
store, snap := decodeSnapshot(t, decodeGzipHead, gzipBody(t, binary))
|
||||
bb, err := readBoundResponse(store, snap)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bb.nonText || bb.text != "" {
|
||||
t.Fatalf("gzip binary: nonText=%v text=%q, want non-text", bb.nonText, bb.text)
|
||||
}
|
||||
if quoteInBodies(normalizeEvidenceText(decodeQuote), []boundBody{bb}) {
|
||||
t.Fatal("quote matched inside a non-text body")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeBoundResponseBadGzipIsAnError(t *testing.T) {
|
||||
raw := gzipBody(t, []byte(decodeHTML))
|
||||
store, snap := decodeSnapshot(t, decodeGzipHead, raw[:len(raw)-4])
|
||||
if bb, err := readBoundResponse(store, snap); err == nil {
|
||||
t.Fatalf("truncated gzip: got %+v, want an error", bb)
|
||||
}
|
||||
store, snap = decodeSnapshot(t, "HTTP 200\nContent-Encoding: br\n", raw)
|
||||
if bb, err := readBoundResponse(store, snap); err == nil || !strings.Contains(err.Error(), "unsupported content-encoding") {
|
||||
t.Fatalf("br: got %+v err=%v, want unsupported content-encoding", bb, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A response captured with Content-Encoding: gzip and no body at all (HEAD,
|
||||
// 304) is not a decode failure: it stays an empty body, without error.
|
||||
func TestDecodeEvidenceBodyEmptyBodyWithEncodingStaysEmpty(t *testing.T) {
|
||||
for _, head := range []string{
|
||||
decodeGzipHead,
|
||||
"HTTP 304\r\nCONTENT-ENCODING: GZIP\r\n",
|
||||
"HTTP 200\nContent-Encoding: deflate\n",
|
||||
} {
|
||||
for _, raw := range [][]byte{nil, {}} {
|
||||
out, truncated, err := decodeEvidenceBody(head, raw, verificationReadCap)
|
||||
if err != nil || truncated || len(out) != 0 {
|
||||
t.Fatalf("head %q: out=%q truncated=%v err=%v, want an empty body and no error", head, out, truncated, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An empty bound body, compressed or not, is read without error but can never
|
||||
// satisfy a quote. In assessFindingVerification a quote that matches no body
|
||||
// only leads to unverified or not_assessed, never verified, so this is what
|
||||
// keeps an empty response (HEAD, 304) from verifying a finding.
|
||||
func TestDecodeBoundResponseEmptyBodyNeverMatchesQuote(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
head string
|
||||
raw []byte
|
||||
}{
|
||||
{"gzip header, no bytes", decodeGzipHead, nil},
|
||||
{"deflate header, no bytes", "HTTP 200\nContent-Encoding: deflate\n", nil},
|
||||
{"identity header, no bytes", "HTTP 200\nContent-Encoding: identity\n", nil},
|
||||
{"no encoding header, no bytes", "HTTP 304\nContent-Type: text/html\n", nil},
|
||||
{"gzip stream of an empty payload", decodeGzipHead, gzipBody(t, nil)},
|
||||
}
|
||||
quotes := []string{
|
||||
decodeQuote,
|
||||
"x",
|
||||
// Present in the response head: the head is not searchable text.
|
||||
"Content-Encoding: gzip",
|
||||
"HTTP 200",
|
||||
}
|
||||
for _, c := range cases {
|
||||
store, snap := decodeSnapshot(t, c.head, c.raw)
|
||||
bb, err := readBoundResponse(store, snap)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", c.name, err)
|
||||
}
|
||||
if bb.text != "" || bb.nonText || bb.truncated {
|
||||
t.Fatalf("%s: got %+v, want an empty text body", c.name, bb)
|
||||
}
|
||||
for _, q := range quotes {
|
||||
nq := normalizeEvidenceText(q)
|
||||
if nq == "" {
|
||||
t.Fatalf("quote %q normalizes to nothing", q)
|
||||
}
|
||||
if quoteInBodies(nq, []boundBody{bb}) {
|
||||
t.Fatalf("%s: quote %q matched an empty body", c.name, q)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+43
-5
@@ -5,6 +5,8 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -25,6 +27,12 @@ const (
|
||||
settingNotifyPublicBaseURL = "notify_public_base_url"
|
||||
// settingNotifyDigestMinutes is the digest-mode period (minutes).
|
||||
settingNotifyDigestMinutes = "notify_digest_interval_min"
|
||||
// settingNotifySummaryMaxRunes is how many characters the summary of a single
|
||||
// push is cut to, and settingNotifyAssetsMaxShown how many asset names a
|
||||
// message lists. 0 means no truncation; unset means the notify package
|
||||
// defaults (120 / 3).
|
||||
settingNotifySummaryMaxRunes = "notify_summary_max_runes"
|
||||
settingNotifyAssetsMaxShown = "notify_assets_max_shown"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -43,6 +51,10 @@ const (
|
||||
notifyFanOutPerTick = 200
|
||||
// notifyDefaultDigestMinutes is the default digest period.
|
||||
notifyDefaultDigestMinutes = 30
|
||||
// notifySummaryMaxRunesCap / notifyAssetsMaxShownCap are the upper bounds the
|
||||
// settings API accepts for the two render limits.
|
||||
notifySummaryMaxRunesCap = 4000
|
||||
notifyAssetsMaxShownCap = 50
|
||||
// notifyUnlimitedBurstPerTick is the per-tick delivery cap when a channel has
|
||||
// no rate limit configured. It exists to prevent "one channel configured as
|
||||
// unlimited + a single scan finding thousands of vulnerabilities" from dragging
|
||||
@@ -223,8 +235,9 @@ func (n *Notifier) stepRealtime(ctx context.Context, ch *db.NotificationChannel,
|
||||
_ = n.pg.FailDeliveries(ctx, deliveryIDs(deliveries), fmt.Sprintf("channel type %q is not registered", ch.Kind))
|
||||
return
|
||||
}
|
||||
limits := notifyLimits(n.pg)
|
||||
for _, dl := range deliveries {
|
||||
msg, err := n.renderSingle(ctx, dl, baseURL)
|
||||
msg, err := n.renderSingle(ctx, dl, baseURL, limits)
|
||||
if err != nil {
|
||||
// A render failure is a local data problem; retrying will not help.
|
||||
_ = n.pg.FailDeliveries(ctx, []int64{dl.ID}, err.Error())
|
||||
@@ -259,7 +272,7 @@ func (n *Notifier) stepDigest(ctx context.Context, ch *db.NotificationChannel, a
|
||||
_ = n.pg.FailDeliveries(ctx, deliveryIDs(deliveries), fmt.Sprintf("channel type %q is not registered", ch.Kind))
|
||||
return
|
||||
}
|
||||
msg, included, err := n.renderBatch(ctx, deliveries, baseURL, int(window.Minutes()))
|
||||
msg, included, err := n.renderBatch(ctx, deliveries, baseURL, int(window.Minutes()), notifyLimits(n.pg))
|
||||
if err != nil {
|
||||
_ = n.pg.FailDeliveries(ctx, deliveryIDs(deliveries), err.Error())
|
||||
return
|
||||
@@ -423,7 +436,7 @@ func (n *Notifier) adapt(ch *db.NotificationChannel) (notify.Channel, map[string
|
||||
}
|
||||
|
||||
// renderSingle renders a single-vulnerability message.
|
||||
func (n *Notifier) renderSingle(ctx context.Context, dl *db.NotificationDelivery, baseURL string) (notify.Message, error) {
|
||||
func (n *Notifier) renderSingle(ctx context.Context, dl *db.NotificationDelivery, baseURL string, limits notify.Limits) (notify.Message, error) {
|
||||
snap, err := parseSnapshot(dl)
|
||||
if err != nil {
|
||||
return notify.Message{}, err
|
||||
@@ -432,7 +445,7 @@ func (n *Notifier) renderSingle(ctx context.Context, dl *db.NotificationDelivery
|
||||
if err != nil {
|
||||
return notify.Message{}, err
|
||||
}
|
||||
return notify.Message{Items: []notify.Item{item}, HomeURL: baseURL}, nil
|
||||
return notify.Message{Items: []notify.Item{item}, HomeURL: baseURL, Limits: &limits}, nil
|
||||
}
|
||||
|
||||
// renderBatch renders a digest message. Snapshots are parsed one by one — a
|
||||
@@ -446,7 +459,7 @@ func (n *Notifier) renderSingle(ctx context.Context, dl *db.NotificationDelivery
|
||||
// entry that should have failed would be marked sent, and a good entry would be
|
||||
// misjudged as undelivered. The broken ones are marked failed explicitly by the
|
||||
// caller, see stepDigest.
|
||||
func (n *Notifier) renderBatch(ctx context.Context, deliveries []*db.NotificationDelivery, baseURL string, windowMinutes int) (notify.Message, []*db.NotificationDelivery, error) {
|
||||
func (n *Notifier) renderBatch(ctx context.Context, deliveries []*db.NotificationDelivery, baseURL string, windowMinutes int, limits notify.Limits) (notify.Message, []*db.NotificationDelivery, error) {
|
||||
items := make([]notify.Item, 0, len(deliveries))
|
||||
included := make([]*db.NotificationDelivery, 0, len(deliveries))
|
||||
for _, dl := range deliveries {
|
||||
@@ -473,6 +486,7 @@ func (n *Notifier) renderBatch(ctx context.Context, deliveries []*db.Notificatio
|
||||
Batch: true,
|
||||
WindowMinutes: windowMinutes,
|
||||
HomeURL: baseURL,
|
||||
Limits: &limits,
|
||||
}, included, nil
|
||||
}
|
||||
|
||||
@@ -582,6 +596,30 @@ func (n *Notifier) digestInterval() time.Duration {
|
||||
return time.Duration(m) * time.Minute
|
||||
}
|
||||
|
||||
// notifyLimits reads the summary/asset render limits from the settings.
|
||||
func notifyLimits(pg *db.DB) notify.Limits {
|
||||
summary, sOK, _ := pg.GetSetting(settingNotifySummaryMaxRunes)
|
||||
assets, aOK, _ := pg.GetSetting(settingNotifyAssetsMaxShown)
|
||||
return notify.Limits{
|
||||
SummaryRunes: notifyLimitValue(summary, sOK, notify.DefaultMaxSummaryRunes, notifySummaryMaxRunesCap),
|
||||
AssetsShown: notifyLimitValue(assets, aOK, notify.DefaultMaxAssetsShown, notifyAssetsMaxShownCap),
|
||||
}
|
||||
}
|
||||
|
||||
// notifyLimitValue resolves one stored render limit: def when the key is unset,
|
||||
// unparseable, or outside [0, max]. 0 is a valid value (no truncation), which is
|
||||
// why "unset" is carried by ok and not by the number.
|
||||
func notifyLimitValue(v string, ok bool, def, max int) int {
|
||||
if !ok {
|
||||
return def
|
||||
}
|
||||
n, err := strconv.Atoi(strings.TrimSpace(v))
|
||||
if err != nil || n < 0 || n > max {
|
||||
return def
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// parseSnapshot parses the snapshot of the event a delivery corresponds to.
|
||||
func parseSnapshot(dl *db.NotificationDelivery) (notify.Snapshot, error) {
|
||||
var snap notify.Snapshot
|
||||
|
||||
@@ -447,6 +447,8 @@ func (s *Server) notifyTestChannel(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
msg := notifyTestMessage(s.notifierBaseURL(pg))
|
||||
limits := notifyLimits(pg)
|
||||
msg.Limits = &limits
|
||||
start := time.Now()
|
||||
// The test message is a single entry, so the delivered count is not needed
|
||||
// here (for a single message the channel length limit falls back to
|
||||
|
||||
@@ -700,6 +700,108 @@ func TestNotifyMetaAndSettingsRoundTrip(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestNotifyLimitSettingsRoundTrip covers the two render-limit keys through the
|
||||
// settings API and the settings table: unset reads as the 120 / 3 defaults, a
|
||||
// stored 0 reads back as 0 (no truncation) and not as the default, and
|
||||
// out-of-range values are rejected without touching the stored value.
|
||||
func TestNotifyLimitSettingsRoundTrip(t *testing.T) {
|
||||
f := newNotifyFixture(t)
|
||||
clear := func() {
|
||||
f.pg.Exec(`DELETE FROM settings WHERE key IN ($1,$2)`, settingNotifySummaryMaxRunes, settingNotifyAssetsMaxShown)
|
||||
}
|
||||
clear()
|
||||
t.Cleanup(clear)
|
||||
expect := func(step string, summary, assets int) {
|
||||
t.Helper()
|
||||
payload := f.s.settingsPayload()
|
||||
if payload["notify_summary_max_runes"] != summary || payload["notify_assets_max_shown"] != assets {
|
||||
t.Fatalf("%s: settings payload should report %d / %d, got %v / %v", step, summary, assets,
|
||||
payload["notify_summary_max_runes"], payload["notify_assets_max_shown"])
|
||||
}
|
||||
if got := notifyLimits(f.pg); got != (notify.Limits{SummaryRunes: summary, AssetsShown: assets}) {
|
||||
t.Fatalf("%s: the delivery engine should read %d / %d, got %+v", step, summary, assets, got)
|
||||
}
|
||||
}
|
||||
expect("unset", notify.DefaultMaxSummaryRunes, notify.DefaultMaxAssetsShown)
|
||||
|
||||
for _, tc := range []struct{ summary, assets int }{{500, 10}, {0, 0}, {4000, 50}} {
|
||||
body := fmt.Sprintf(`{"notify_summary_max_runes":%d,"notify_assets_max_shown":%d}`, tc.summary, tc.assets)
|
||||
if r := f.request("PUT", "/api/settings", body); r.Code != 200 {
|
||||
t.Fatalf("%s failed %d: %s", body, r.Code, r.Body)
|
||||
}
|
||||
expect(body, tc.summary, tc.assets)
|
||||
for key, want := range map[string]int{settingNotifySummaryMaxRunes: tc.summary, settingNotifyAssetsMaxShown: tc.assets} {
|
||||
if v, ok, err := f.pg.GetSetting(key); err != nil || !ok || v != strconv.Itoa(want) {
|
||||
t.Fatalf("%s: settings row %s should hold %d, got %q (ok=%v err=%v)", body, key, want, v, ok, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, body := range []string{
|
||||
`{"notify_summary_max_runes":-1}`,
|
||||
`{"notify_summary_max_runes":4001}`,
|
||||
`{"notify_assets_max_shown":-1}`,
|
||||
`{"notify_assets_max_shown":51}`,
|
||||
} {
|
||||
if r := f.request("PUT", "/api/settings", body); r.Code != 400 {
|
||||
t.Errorf("%s should return 400, got %d", body, r.Code)
|
||||
}
|
||||
}
|
||||
expect("after rejected writes", 4000, 50)
|
||||
}
|
||||
|
||||
// TestNotifySummaryLimitReachesDelivery is the end-to-end evidence that the
|
||||
// stored limit reaches the message actually sent: with the key unset the summary
|
||||
// is cut at 120 characters as before, and once it is set to 0 the same summary
|
||||
// goes out whole.
|
||||
func TestNotifySummaryLimitReachesDelivery(t *testing.T) {
|
||||
f := newNotifyFixture(t)
|
||||
clear := func() { f.pg.Exec(`DELETE FROM settings WHERE key=$1`, settingNotifySummaryMaxRunes) }
|
||||
clear()
|
||||
t.Cleanup(clear)
|
||||
hook := newFakeWebhook(t)
|
||||
chID := f.createChannel(t, map[string]any{
|
||||
"name": "summary limit",
|
||||
"kind": notify.KindDingTalk,
|
||||
"config": map[string]any{"webhook": hook.URL},
|
||||
})
|
||||
summary := strings.Repeat("Z", 300)
|
||||
record := func(name string) {
|
||||
t.Helper()
|
||||
if _, err := f.s.evidenceStore().Record(context.Background(), db.RecordFindingInput{
|
||||
TaskID: f.taskID,
|
||||
ExplorationID: f.expID,
|
||||
Worker: "test",
|
||||
VulnClass: name,
|
||||
Name: name,
|
||||
Severity: "high",
|
||||
Summary: summary,
|
||||
Evidence: "poc",
|
||||
}, nil); err != nil {
|
||||
t.Fatalf("failed to record finding: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
record("summary limit unset")
|
||||
f.deliver(t, chID, "")
|
||||
text := markdownText(t, hook.last(t))
|
||||
if !strings.Contains(text, strings.Repeat("Z", 119)+"…") || strings.Contains(text, strings.Repeat("Z", 120)) {
|
||||
t.Fatalf("with the key unset the summary should be cut at 120 characters:\n%s", text)
|
||||
}
|
||||
|
||||
if r := f.request("PUT", "/api/settings", `{"notify_summary_max_runes":0}`); r.Code != 200 {
|
||||
t.Fatalf("failed to write settings %d: %s", r.Code, r.Body)
|
||||
}
|
||||
record("summary limit zero")
|
||||
f.deliver(t, chID, "")
|
||||
if hook.count() != 2 {
|
||||
t.Fatalf("expected 2 messages sent, got %d", hook.count())
|
||||
}
|
||||
if text := markdownText(t, hook.last(t)); !strings.Contains(text, summary) {
|
||||
t.Fatalf("with the limit at 0 the whole summary should be sent:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNotifyDeepLinkUsesPublicBaseURL covers back-link building: when
|
||||
// public_base_url is configured, single messages must use the ActionCard with a
|
||||
// button, and the link must point to the vulnerability detail page.
|
||||
@@ -1008,3 +1110,30 @@ func TestNotifyTickBudgetFitsWithinLease(t *testing.T) {
|
||||
worst, notifyLease, notifyMaxSendsPerChannelPerTick, notifySendTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNotifyLimitValue pins how a stored render limit is resolved. The case that
|
||||
// matters is "0": it is a real value (no truncation) and must not be read as
|
||||
// "unset", while anything unparseable or outside the bounds falls back to the
|
||||
// default instead of reaching the renderer.
|
||||
func TestNotifyLimitValue(t *testing.T) {
|
||||
const def, max = 120, 4000
|
||||
cases := []struct {
|
||||
v string
|
||||
ok bool
|
||||
want int
|
||||
}{
|
||||
{"", false, def},
|
||||
{"0", true, 0},
|
||||
{" 500 ", true, 500},
|
||||
{"4000", true, max},
|
||||
{"4001", true, def},
|
||||
{"-1", true, def},
|
||||
{"abc", true, def},
|
||||
{"", true, def},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := notifyLimitValue(tc.v, tc.ok, def, max); got != tc.want {
|
||||
t.Errorf("notifyLimitValue(%q, %v) = %d, want %d", tc.v, tc.ok, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -536,6 +536,7 @@ func (s *Server) seedOrchestrationTools() {
|
||||
s.seedWorkerReadToolsUnbind() // list_facts/list_companies/list_worker_traces unbound from worker's defaults (one-time)
|
||||
s.seedWorkerReadbackRebind() // repairs an old migration's wrong removal: re-binds search_all_worker_traces/get_worker_trace/node_detail back onto worker (one-time)
|
||||
s.seedTaskCredentialBindings() // save/list_task_credential onto worker + pentest (one-time, #3)
|
||||
s.healListTaskCredentialsSchema()
|
||||
s.seedAutoReportFindingBinding()
|
||||
s.unbindGoalMetDefault()
|
||||
s.reseedGoalsPrompt() // goals prompt gains the "extract operation constraints" step → old DBs append a new default version (one-time)
|
||||
@@ -944,6 +945,19 @@ func (s *Server) seedTaskCredentialBindings() {
|
||||
_ = s.m.pg.SetSetting(flag, "true")
|
||||
}
|
||||
|
||||
// healListTaskCredentialsSchema repairs the list_task_credentials row seeded by
|
||||
// older builds, whose schema carried "required": [""] — strict providers reject
|
||||
// it, which fails every worker run. SeedTool is first-insert-only, so the code fix
|
||||
// alone never reaches an already-seeded row. Only that exact malformed value on
|
||||
// that one key matches, and only the "required" key is dropped: the rest of the
|
||||
// schema, the description, bindings and enabled flag are untouched. A healed row
|
||||
// no longer matches, so this needs no settings flag and is a no-op afterwards.
|
||||
func (s *Server) healListTaskCredentialsSchema() {
|
||||
if _, err := s.m.pg.Exec(`UPDATE tools SET schema = schema - 'required', updated_at=now() WHERE key='list_task_credentials' AND schema->'required' = '[""]'::jsonb`); err != nil {
|
||||
log.Printf("[tools] heal list_task_credentials schema failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) seedWorkerReadbackRebind() {
|
||||
const flag = "worker_readback_rebind_v2" // v2: adds node_detail
|
||||
if v, _, _ := s.m.pg.GetSetting(flag); v == "true" {
|
||||
|
||||
@@ -3454,6 +3454,7 @@ func (s *Server) settingsPayload() map[string]any {
|
||||
if concLimit == 0 {
|
||||
concLimit = defaultConcurrencyLimit // when off, still echo a sane default to the UI
|
||||
}
|
||||
limits := notifyLimits(s.m.pg)
|
||||
return map[string]any{
|
||||
"traffic_capture": s.m.TrafficEnabled(),
|
||||
"agent_traffic_binding": s.m.pg.GetBool(settingAgentTrafficBinding, false),
|
||||
@@ -3483,6 +3484,9 @@ func (s *Server) settingsPayload() map[string]any {
|
||||
"notify_enabled": s.m.pg.GetBool(settingNotifyEnabled, true),
|
||||
"notify_public_base_url": notifyPublicBaseURL(s.m.pg),
|
||||
"notify_digest_interval_min": notifyDigestIntervalMin(s.m.pg),
|
||||
// Render limits of IM messages; 0 = no truncation.
|
||||
"notify_summary_max_runes": limits.SummaryRunes,
|
||||
"notify_assets_max_shown": limits.AssetsShown,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3555,6 +3559,9 @@ func (s *Server) putSettings(w http.ResponseWriter, r *http.Request) {
|
||||
NotifyEnabled *bool `json:"notify_enabled"`
|
||||
NotifyBaseURL *string `json:"notify_public_base_url"`
|
||||
NotifyDigestMins *int `json:"notify_digest_interval_min"`
|
||||
// Render limits of IM messages: summary characters (0-4000) and asset names listed (0-50); 0 = no truncation.
|
||||
NotifySummaryMaxRunes *int `json:"notify_summary_max_runes"`
|
||||
NotifyAssetsMaxShown *int `json:"notify_assets_max_shown"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeErr(w, 400, err.Error())
|
||||
@@ -3610,6 +3617,26 @@ func (s *Server) putSettings(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if req.NotifySummaryMaxRunes != nil {
|
||||
if *req.NotifySummaryMaxRunes < 0 || *req.NotifySummaryMaxRunes > notifySummaryMaxRunesCap {
|
||||
writeErr(w, 400, fmt.Sprintf("summary limit must be between 0 and %d characters (0 = no truncation)", notifySummaryMaxRunesCap))
|
||||
return
|
||||
}
|
||||
if err := s.m.pg.SetSetting(settingNotifySummaryMaxRunes, strconv.Itoa(*req.NotifySummaryMaxRunes)); err != nil {
|
||||
writeErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
if req.NotifyAssetsMaxShown != nil {
|
||||
if *req.NotifyAssetsMaxShown < 0 || *req.NotifyAssetsMaxShown > notifyAssetsMaxShownCap {
|
||||
writeErr(w, 400, fmt.Sprintf("asset limit must be between 0 and %d (0 = no truncation)", notifyAssetsMaxShownCap))
|
||||
return
|
||||
}
|
||||
if err := s.m.pg.SetSetting(settingNotifyAssetsMaxShown, strconv.Itoa(*req.NotifyAssetsMaxShown)); err != nil {
|
||||
writeErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
if req.Workers != nil {
|
||||
if err := s.m.SetWorkers(*req.Workers); err != nil {
|
||||
writeErr(w, 400, err.Error())
|
||||
|
||||
@@ -103,6 +103,96 @@ func TestWireTools(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestHealListTaskCredentialsSchema: a list_task_credentials row seeded by an older
|
||||
// build carries "required": [""]. The startup heal drops that key, leaves the rest of
|
||||
// the row alone, changes nothing on a second run, and touches no other tool.
|
||||
func TestHealListTaskCredentialsSchema(t *testing.T) {
|
||||
dsn, _, err := db.DSN()
|
||||
if err != nil {
|
||||
t.Skipf("no database config (%v) — skipping", err)
|
||||
}
|
||||
pg, err := db.Open(dsn)
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { pg.Close() })
|
||||
s := &Server{m: &Manager{pg: pg}}
|
||||
|
||||
const key = "list_task_credentials"
|
||||
// Other tools the heal must never touch: a normal required list, and the same
|
||||
// malformed value under a different key.
|
||||
const probeRequired, probeBlank = "heal_probe_required", "heal_probe_blank"
|
||||
const broken = `{"type":"object","properties":{},"required":[""]}`
|
||||
|
||||
// The package test DB is shared: put back whatever it held for the real key.
|
||||
old, err := pg.GetTool(key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if old == nil {
|
||||
pg.Exec(`DELETE FROM tools WHERE key=$1`, key)
|
||||
} else {
|
||||
pg.Exec(`UPDATE tools SET schema=$2::jsonb WHERE key=$1`, key, string(old.Schema))
|
||||
}
|
||||
pg.Exec(`DELETE FROM tools WHERE key IN ($1,$2)`, probeRequired, probeBlank)
|
||||
})
|
||||
for k, schema := range map[string]string{
|
||||
key: broken,
|
||||
probeRequired: `{"type":"object","properties":{"x":{"type":"string"}},"required":["x"]}`,
|
||||
probeBlank: broken,
|
||||
} {
|
||||
if _, err := pg.Exec(`INSERT INTO tools(key, description, schema, agents) VALUES ($1, 'legacy row', $2::jsonb, '["worker"]')
|
||||
ON CONFLICT (key) DO UPDATE SET schema=EXCLUDED.schema`, k, schema); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type row struct {
|
||||
schema, description, agents, updatedAt string
|
||||
enabled bool
|
||||
}
|
||||
read := func(k string) row {
|
||||
t.Helper()
|
||||
var r row
|
||||
if err := pg.QueryRow(`SELECT schema::text, description, agents::text, updated_at::text, enabled FROM tools WHERE key=$1`, k).
|
||||
Scan(&r.schema, &r.description, &r.agents, &r.updatedAt, &r.enabled); err != nil {
|
||||
t.Fatalf("read %s: %v", k, err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
before, beforeRequired, beforeBlank := read(key), read(probeRequired), read(probeBlank)
|
||||
|
||||
s.healListTaskCredentialsSchema()
|
||||
|
||||
healed := read(key)
|
||||
var schema map[string]any
|
||||
if err := json.Unmarshal([]byte(healed.schema), &schema); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := schema["required"]; ok {
|
||||
t.Fatalf("required survived the heal: %s", healed.schema)
|
||||
}
|
||||
if schema["type"] != "object" || schema["properties"] == nil {
|
||||
t.Errorf("heal dropped more than required: %s", healed.schema)
|
||||
}
|
||||
if healed.description != before.description || healed.agents != before.agents || healed.enabled != before.enabled {
|
||||
t.Errorf("heal changed description/bindings/enabled: %+v -> %+v", before, healed)
|
||||
}
|
||||
|
||||
s.healListTaskCredentialsSchema()
|
||||
|
||||
if again := read(key); again != healed {
|
||||
t.Errorf("second heal changed the row: %+v -> %+v", healed, again)
|
||||
}
|
||||
if got := read(probeRequired); got != beforeRequired {
|
||||
t.Errorf("heal touched a tool with a non-empty required: %+v -> %+v", beforeRequired, got)
|
||||
}
|
||||
if got := read(probeBlank); got != beforeBlank {
|
||||
t.Errorf("heal touched another tool key: %+v -> %+v", beforeBlank, got)
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(v any) json.RawMessage {
|
||||
b, _ := json.Marshal(v)
|
||||
return b
|
||||
|
||||
@@ -32,6 +32,18 @@ import { ConfigField, FilterSummary } from "./_components/channel-form";
|
||||
import { DeliveryList } from "./_components/delivery-list";
|
||||
import { formatBacklog, StatTile } from "./_components/stat-tile";
|
||||
|
||||
// Upper bounds of the two message render limits. They mirror the backend, which answers 400 outside [0, max].
|
||||
const SUMMARY_LIMIT_MAX = 4000;
|
||||
const ASSETS_LIMIT_MAX = 50;
|
||||
|
||||
// limitInvalid reports whether a limit input holds anything other than a whole number in [0, max].
|
||||
// Empty is valid: it is not submitted, so the stored value stays as it is.
|
||||
function limitInvalid(raw: string, max: number): boolean {
|
||||
if (raw.trim() === "") return false;
|
||||
const n = Number(raw);
|
||||
return !Number.isInteger(n) || n < 0 || n > max;
|
||||
}
|
||||
|
||||
// This page only orchestrates: loading data, holding form state, calling the API.
|
||||
// Field definitions and parsing live in _components/channel-fields.ts, controls and the filter summary in
|
||||
// _components/channel-form.tsx, and the delivery log in _components/delivery-list.tsx —
|
||||
@@ -50,6 +62,8 @@ export default function NotifyPage() {
|
||||
const [globalSaving, setGlobalSaving] = React.useState(false);
|
||||
const [baseURL, setBaseURL] = React.useState("");
|
||||
const [digestMin, setDigestMin] = React.useState("");
|
||||
const [summaryMax, setSummaryMax] = React.useState("");
|
||||
const [assetsMax, setAssetsMax] = React.useState("");
|
||||
|
||||
const load = React.useCallback(() => {
|
||||
api
|
||||
@@ -60,6 +74,14 @@ export default function NotifyPage() {
|
||||
setDigestMin(m.digest_interval_min);
|
||||
})
|
||||
.catch((e) => toast.error("Failed to load notification settings: " + (e as Error).message));
|
||||
// The two render limits are not part of /notify/meta; they are only exposed through /api/settings.
|
||||
api
|
||||
.settings()
|
||||
.then((s) => {
|
||||
setSummaryMax(String(s.notify_summary_max_runes ?? ""));
|
||||
setAssetsMax(String(s.notify_assets_max_shown ?? ""));
|
||||
})
|
||||
.catch((e) => toast.error(`Failed to load message limits: ${(e as Error).message}`));
|
||||
// Channel-list load failures must be reported: failing silently would render as "no channels at all",
|
||||
// and users would think their config was lost — scarier than a straight-up error.
|
||||
api
|
||||
@@ -244,11 +266,23 @@ export default function NotifyPage() {
|
||||
}
|
||||
|
||||
async function saveGlobal() {
|
||||
// Checked before anything is sent: the backend applies the fields of one PUT in order,
|
||||
// so a 400 on a limit would leave the URL and digest interval already saved.
|
||||
if (limitInvalid(summaryMax, SUMMARY_LIMIT_MAX)) {
|
||||
toast.error(`Summary length must be a whole number between 0 and ${SUMMARY_LIMIT_MAX}`);
|
||||
return;
|
||||
}
|
||||
if (limitInvalid(assetsMax, ASSETS_LIMIT_MAX)) {
|
||||
toast.error(`Assets listed must be a whole number between 0 and ${ASSETS_LIMIT_MAX}`);
|
||||
return;
|
||||
}
|
||||
setGlobalSaving(true);
|
||||
try {
|
||||
const patch: Record<string, unknown> = { notify_public_base_url: baseURL.trim() };
|
||||
const n = Number(digestMin);
|
||||
if (Number.isFinite(n) && n > 0) patch.notify_digest_interval_min = n;
|
||||
if (summaryMax.trim() !== "") patch.notify_summary_max_runes = Number(summaryMax);
|
||||
if (assetsMax.trim() !== "") patch.notify_assets_max_shown = Number(assetsMax);
|
||||
await api.setSettings(patch);
|
||||
toast.success("Saved");
|
||||
load();
|
||||
@@ -331,6 +365,40 @@ export default function NotifyPage() {
|
||||
/>
|
||||
<p className="text-muted-foreground text-xs">Only applies to channels in "digest" mode.</p>
|
||||
</div>
|
||||
<div className="grid gap-2">
|
||||
<Label htmlFor="n-summary-max">Summary length (characters)</Label>
|
||||
<Input
|
||||
id="n-summary-max"
|
||||
type="number"
|
||||
min={0}
|
||||
max={SUMMARY_LIMIT_MAX}
|
||||
step={1}
|
||||
placeholder="120"
|
||||
value={summaryMax}
|
||||
onChange={(e) => setSummaryMax(e.target.value)}
|
||||
aria-invalid={limitInvalid(summaryMax, SUMMARY_LIMIT_MAX)}
|
||||
/>
|
||||
<p className="text-muted-foreground text-xs">
|
||||
Longest finding summary shown in a message, 0–{SUMMARY_LIMIT_MAX}. 0 = no truncation.
|
||||
</p>
|
||||
</div>
|
||||
<div className="grid gap-2">
|
||||
<Label htmlFor="n-assets-max">Assets listed per finding</Label>
|
||||
<Input
|
||||
id="n-assets-max"
|
||||
type="number"
|
||||
min={0}
|
||||
max={ASSETS_LIMIT_MAX}
|
||||
step={1}
|
||||
placeholder="3"
|
||||
value={assetsMax}
|
||||
onChange={(e) => setAssetsMax(e.target.value)}
|
||||
aria-invalid={limitInvalid(assetsMax, ASSETS_LIMIT_MAX)}
|
||||
/>
|
||||
<p className="text-muted-foreground text-xs">
|
||||
Most asset names listed for a finding, 0–{ASSETS_LIMIT_MAX}. 0 = no truncation (all assets listed).
|
||||
</p>
|
||||
</div>
|
||||
<div className="sm:col-span-2">
|
||||
<Button onClick={saveGlobal} disabled={globalSaving}>
|
||||
Save global settings
|
||||
|
||||
@@ -957,6 +957,9 @@ export interface Settings {
|
||||
notify_enabled?: boolean; // master push switch, on by default; one-click kill switch for maintenance windows
|
||||
notify_public_base_url?: string; // external base URL for vulnerability detail links; empty = messages carry no link
|
||||
notify_digest_interval_min?: number; // digest mode period (minutes), default 30
|
||||
// Render limits of IM messages; 0 = no truncation. The backend answers 400 outside the bounds.
|
||||
notify_summary_max_runes?: number; // summary characters per finding, 0-4000, default 120
|
||||
notify_assets_max_shown?: number; // asset names listed per finding, 0-50, default 3
|
||||
}
|
||||
|
||||
// ---- Vulnerability IM push ----
|
||||
|
||||
Reference in New Issue
Block a user