From 2c514a4cd98c3a515421df33bfd2c7fcf5928655 Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Mon, 16 Nov 2026 06:46:52 +0100 Subject: [PATCH 01/29] perf(server): look up only the requested task archive blocker in getTask getTask aggregated task_relations for the whole database on every 5-second detail poll to read one entry. TaskArchiveBlocker scopes the same predicate (shared with TaskArchiveBlockers through archiveBlockerScope) to one source task; listTasks keeps the global map. --- db/task_archives.go | 27 ++++++++++++++++++++++----- db/task_archives_test.go | 20 ++++++++++++++++++++ server/dto.go | 11 +++++++++-- server/server.go | 6 ++++-- server/task_llm_test.go | 12 ++++++++++++ 5 files changed, 67 insertions(+), 9 deletions(-) diff --git a/db/task_archives.go b/db/task_archives.go index adadbb7..f312000 100644 --- a/db/task_archives.go +++ b/db/task_archives.go @@ -79,15 +79,20 @@ type TaskArchive struct { UpdatedAt time.Time `json:"updated_at"` } +// archiveBlockerScope selects the live direct dependents that block archiving +// their source task. TaskArchiveBlockers and TaskArchiveBlocker share it so the +// list and the detail view cannot disagree. +const archiveBlockerScope = ` +FROM task_relations relation +JOIN tasks child ON child.id=relation.task_id AND child.deleted_at IS NULL +LEFT JOIN task_archives pending ON pending.task_id=child.id +WHERE (pending.id IS NULL OR pending.state NOT IN ('archive_queued','archiving'))` + // TaskArchiveBlockers returns one live direct dependent for every source task // that cannot currently be archived. Dependents already queued for archiving do // not block their source because the FIFO worker will compact them first. func (d *DB) TaskArchiveBlockers() (map[int64]int64, error) { - rows, err := d.Query(`SELECT relation.source_task_id, MIN(child.id) -FROM task_relations relation -JOIN tasks child ON child.id=relation.task_id AND child.deleted_at IS NULL -LEFT JOIN task_archives pending ON pending.task_id=child.id -WHERE pending.id IS NULL OR pending.state NOT IN ('archive_queued','archiving') + rows, err := d.Query(`SELECT relation.source_task_id, MIN(child.id)` + archiveBlockerScope + ` GROUP BY relation.source_task_id`) if err != nil { return nil, err @@ -104,6 +109,18 @@ GROUP BY relation.source_task_id`) return blockers, rows.Err() } +// TaskArchiveBlocker returns the live direct dependent that blocks taskID, or +// 0 when nothing blocks it. Same rules as TaskArchiveBlockers, scoped to one +// source task so getTask does not aggregate the whole relation table per poll. +func (d *DB) TaskArchiveBlocker(taskID int64) (int64, error) { + var dependent sql.NullInt64 + if err := d.QueryRow(`SELECT MIN(child.id)`+archiveBlockerScope+` +AND relation.source_task_id=$1`, taskID).Scan(&dependent); err != nil { + return 0, err + } + return dependent.Int64, nil +} + type TaskArchivePage struct { Items []TaskArchive `json:"items"` Total int `json:"total"` diff --git a/db/task_archives_test.go b/db/task_archives_test.go index b016777..aefab19 100644 --- a/db/task_archives_test.go +++ b/db/task_archives_test.go @@ -362,6 +362,7 @@ func TestTaskArchiveBlockersIgnoreQueuedDependents(t *testing.T) { if blockers[source.ID] != dependent.ID { t.Fatalf("source blocker=%d, want dependent %d", blockers[source.ID], dependent.ID) } + assertPerTaskBlockers(t, d, blockers, map[int64]int64{source.ID: dependent.ID, dependent.ID: 0}) if err := d.SetPaused(dependent.ID, true); err != nil { t.Fatal(err) } @@ -375,6 +376,25 @@ func TestTaskArchiveBlockersIgnoreQueuedDependents(t *testing.T) { if blocker := blockers[source.ID]; blocker != 0 { t.Fatalf("queued dependent must not block source, got %d", blocker) } + assertPerTaskBlockers(t, d, blockers, map[int64]int64{source.ID: 0, dependent.ID: 0}) +} + +// assertPerTaskBlockers checks TaskArchiveBlocker against the expected blocker +// of each task and against the global TaskArchiveBlockers map. +func assertPerTaskBlockers(t *testing.T, d *DB, global map[int64]int64, want map[int64]int64) { + t.Helper() + for taskID, wantBlocker := range want { + got, err := d.TaskArchiveBlocker(taskID) + if err != nil { + t.Fatal(err) + } + if got != wantBlocker { + t.Fatalf("TaskArchiveBlocker(%d)=%d, want %d", taskID, got, wantBlocker) + } + if got != global[taskID] { + t.Fatalf("TaskArchiveBlocker(%d)=%d, TaskArchiveBlockers()[%d]=%d", taskID, got, taskID, global[taskID]) + } + } } func TestRecoverInterruptedArchiveRequiresManualRetry(t *testing.T) { diff --git a/server/dto.go b/server/dto.go index 2a9963d..5ef895d 100644 --- a/server/dto.go +++ b/server/dto.go @@ -82,9 +82,16 @@ func applyTaskArchiveBlocker(dto *TaskDTO, blockers map[int64]int64) { if err != nil { return } - if dependentID := blockers[taskID]; dependentID > 0 { - dto.ArchiveBlockedBy = strconv.FormatInt(dependentID, 10) + applyTaskArchiveBlockerID(dto, blockers[taskID]) +} + +// applyTaskArchiveBlockerID records dependentID as the task's archive blocker; +// 0 (nothing blocks it) leaves the DTO unchanged. +func applyTaskArchiveBlockerID(dto *TaskDTO, dependentID int64) { + if dto == nil || dependentID <= 0 { + return } + dto.ArchiveBlockedBy = strconv.FormatInt(dependentID, 10) } // TokenTotalDTO is a whole-task (all agents) token aggregate. diff --git a/server/server.go b/server/server.go index 56e7473..3840c8f 100644 --- a/server/server.go +++ b/server/server.go @@ -1932,8 +1932,10 @@ func (s *Server) getTask(w http.ResponseWriter, r *http.Request) { log.Printf("[tasks] detail metrics query failed for %s — metrics zeroed: %v", t.ID, err) } s.applyTaskMetrics(&dto, t, metrics) - archiveBlockers, _ := s.m.PG().TaskArchiveBlockers() - applyTaskArchiveBlocker(&dto, archiveBlockers) + // Best effort: only this task's blocker, not the whole relation table. + taskID, _ := strconv.ParseInt(t.ID, 10, 64) + blocker, _ := s.m.PG().TaskArchiveBlocker(taskID) + applyTaskArchiveBlockerID(&dto, blocker) writeJSON(w, 200, dto) } diff --git a/server/task_llm_test.go b/server/task_llm_test.go index 88a113f..afd2caa 100644 --- a/server/task_llm_test.go +++ b/server/task_llm_test.go @@ -184,6 +184,18 @@ func TestApplyTaskArchiveBlocker(t *testing.T) { if !strings.Contains(string(raw), `"archive_blocked_by_task_id":"11"`) { t.Fatalf("archive blocker missing from payload: %s", raw) } + + byID := taskDTO(&Task{ID: "7"}, "paused") + applyTaskArchiveBlockerID(&byID, 12) + if byID.ArchiveBlockedBy != "12" { + t.Fatalf("archive blocker by id=%q, want 12", byID.ArchiveBlockedBy) + } + none := taskDTO(&Task{ID: "7"}, "paused") + applyTaskArchiveBlockerID(&none, 0) + if none.ArchiveBlockedBy != "" { + t.Fatalf("no blocker set %q", none.ArchiveBlockedBy) + } + applyTaskArchiveBlockerID(nil, 12) // must not panic } func TestTaskLLMStateSnapshotIsConcurrentAndDetached(t *testing.T) { From d9df84d2e53d92d17377c95d7ae72f6d67277af8 Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Mon, 16 Nov 2026 07:47:25 +0100 Subject: [PATCH 02/29] feat(traffic): record WebSocket and SSE stream messages The recording proxy now stores WebSocket and Server-Sent Events streams: one exchange per stream (101 websocket, or the real text/event-stream response) and its messages in stream_messages, with per-message and per-exchange limits and blob spill. Messages are readable through Traffic.StreamMessages, GET /api/traffic/stream and the traffic_messages agent tool; deletion and blob GC cover them. --- agent/finding_workflow.go | 2 +- agent/worker.go | 2 +- go.mod | 2 +- intercept/target_extract.go | 2 +- intercept/target_test.go | 2 +- server/finding_workflow.go | 2 +- server/server.go | 33 ++- traffic/stream.go | 544 +++++++++++++++++++++++++++++++++++ traffic/stream_test.go | 550 ++++++++++++++++++++++++++++++++++++ traffic/traffic.go | 110 +++++++- traffic/traffic_test.go | 29 +- 11 files changed, 1264 insertions(+), 14 deletions(-) create mode 100644 traffic/stream.go create mode 100644 traffic/stream_test.go diff --git a/agent/finding_workflow.go b/agent/finding_workflow.go index 1428646..5f359b1 100644 --- a/agent/finding_workflow.go +++ b/agent/finding_workflow.go @@ -49,7 +49,7 @@ func findingWorkflowTools(agentKey string, tools []actool.CoreTool) ([]actool.Co if tool.Name() == "bind_finding_traffic" { continue } - if agentKey == "reporter" && (tool.Name() == "traffic_search" || tool.Name() == "traffic_get" || tool.Name() == "traffic_blob") { + if agentKey == "reporter" && (tool.Name() == "traffic_search" || tool.Name() == "traffic_get" || tool.Name() == "traffic_blob" || tool.Name() == "traffic_messages") { continue } switch tool.Name() { diff --git a/agent/worker.go b/agent/worker.go index b10cbeb..ebfec58 100644 --- a/agent/worker.go +++ b/agent/worker.go @@ -292,7 +292,7 @@ func workerTrafficBlock(recording bool) string { if !recording { return "" } - return "\n\n**Traffic tools**:\n- traffic_search / traffic_get / traffic_blob: review responses, find already-visited resources — **check the traffic first, don't curl the same URL twice**. traffic_search **requires a host** and by default returns only 3 ultra-light index entries (id/method/url/status/resp_len, no response content) — explicitly raise limit when you need more; body_contains does full-text search in request/response bodies (at least 3 characters; supports substrings and CJK text, e.g. for passwords/keys/errors/internal addresses); to see one entry's raw content use traffic_get(id) — oversized bodies show as @blob sha256:, fetched in segments with traffic_blob(hash)." + return "\n\n**Traffic tools**:\n- traffic_search / traffic_get / traffic_blob / traffic_messages: review responses, find already-visited resources — **check the traffic first, don't curl the same URL twice**. traffic_search **requires a host** and by default returns only 3 ultra-light index entries (id/method/url/status/resp_len, no response content) — explicitly raise limit when you need more; body_contains does full-text search in request/response bodies (at least 3 characters; supports substrings and CJK text, e.g. for passwords/keys/errors/internal addresses); to see one entry's raw content use traffic_get(id) — oversized bodies show as @blob sha256:, fetched in segments with traffic_blob(hash). WebSocket (status 101) and SSE (text/event-stream) records are captured too: traffic_get shows the handshake, and traffic_messages(id) lists their messages in order with direction and type." } // artifactSpec is section [C]: the code-owned, non-editable tail appended to every diff --git a/go.mod b/go.mod index fd897b9..ba11f2c 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Autumn-27/norma v0.4.10 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 + github.com/gorilla/websocket v1.5.3 github.com/jackc/pgx/v5 v5.10.0 github.com/klauspost/compress v1.18.6 github.com/lqqyt2423/go-mitmproxy v1.9.2 @@ -27,7 +28,6 @@ require ( github.com/gaissmai/bart v0.26.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/gorilla/css v1.0.1 // indirect - github.com/gorilla/websocket v1.5.3 // indirect github.com/hinshun/vt10x v0.0.0-20220301184237-5011da428d02 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect diff --git a/intercept/target_extract.go b/intercept/target_extract.go index f3f4ab9..f897840 100644 --- a/intercept/target_extract.go +++ b/intercept/target_extract.go @@ -39,7 +39,7 @@ var noTargetTools = toolSet( // the operator runs the command, not the platform "ask_operator", // recorded traffic - "traffic_search", "traffic_get", "traffic_blob", + "traffic_search", "traffic_get", "traffic_blob", "traffic_messages", ) func toolSet(names ...string) map[string]bool { diff --git a/intercept/target_test.go b/intercept/target_test.go index 5577e96..8c508c5 100644 --- a/intercept/target_test.go +++ b/intercept/target_test.go @@ -191,7 +191,7 @@ func TestExtractSkipsNoTargetTools(t *testing.T) { "web_search", "Read", "Write", "Edit", "MultiEdit", "LS", "Glob", "Grep", "TodoWrite", "sleep", "TaskOutput", "TaskStop", "TaskList", "add_intent", "insert_assets", "report_finding", "record_fact", "add_task_scope", "graph_overview", - "spawn_task", "ask_operator", "traffic_search", "traffic_get", "traffic_blob", + "spawn_task", "ask_operator", "traffic_search", "traffic_get", "traffic_blob", "traffic_messages", } { wantTargets(t, tool, targetsOf(tool, input)) } diff --git a/server/finding_workflow.go b/server/finding_workflow.go index 46aeacd..8b11fac 100644 --- a/server/finding_workflow.go +++ b/server/finding_workflow.go @@ -88,7 +88,7 @@ func (s *Server) seedFindingWorkflowTools() { // Upgrade only the original default binding. Customized lists and enabled // flags survive; the one-time flag also preserves future user unbinding. readers := `["worker","reporter"]` - for _, key := range []string{"traffic_search", "traffic_get", "traffic_blob"} { + for _, key := range []string{"traffic_search", "traffic_get", "traffic_blob", "traffic_messages"} { if _, err := s.m.pg.Exec(`UPDATE tools SET agents=$2::jsonb WHERE key=$1 AND system AND (agents='["worker"]'::jsonb OR (agents @> '["worker","planner","mainagent","auto","pentest"]'::jsonb AND jsonb_array_length(agents)=5))`, key, readers); err != nil { return } diff --git a/server/server.go b/server/server.go index 3840c8f..80e10a0 100644 --- a/server/server.go +++ b/server/server.go @@ -823,6 +823,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("DELETE /api/traffic/hosts", s.deleteTrafficHosts) mux.HandleFunc("DELETE /api/traffic/all", s.deleteAllTraffic) mux.HandleFunc("GET /api/traffic/exchange", s.getTrafficExchange) + mux.HandleFunc("GET /api/traffic/stream", s.getTrafficStream) mux.HandleFunc("GET /api/traffic/blob", s.getTrafficBlob) mux.HandleFunc("POST /api/traffic/ingest", s.ingestTraffic) mux.HandleFunc("GET /api/commands", s.pgListCommands) @@ -3451,7 +3452,37 @@ func (s *Server) getTrafficExchange(w http.ResponseWriter, r *http.Request) { writeErr(w, 404, err.Error()) return } - writeJSON(w, 200, map[string]any{"req": req, "resp": resp}) + // A WebSocket or SSE exchange carries its messages separately + // (GET /api/traffic/stream); stream_kind is "" for an ordinary exchange. + count, kind, err := tr.StreamInfo(id) + if err != nil { + log.Printf("[traffic] stream info for %s: %v", id, err) + } + writeJSON(w, 200, map[string]any{"req": req, "resp": resp, "stream_count": count, "stream_kind": kind}) +} + +// getTrafficStream returns one page of a WebSocket/SSE exchange's messages in +// order (limit 1..200, default 50). +func (s *Server) getTrafficStream(w http.ResponseWriter, r *http.Request) { + tr := s.m.Traffic() + if tr == nil { + writeErr(w, 404, "traffic disabled") + return + } + q := r.URL.Query() + id := strings.TrimSpace(q.Get("exchange")) + if id == "" { + writeErr(w, 400, "missing exchange") + return + } + offset, _ := strconv.Atoi(q.Get("offset")) + limit, _ := strconv.Atoi(q.Get("limit")) + msgs, total, err := tr.StreamMessages(id, offset, limit) + if err != nil { + writeErr(w, 500, err.Error()) + return + } + writeJSON(w, 200, map[string]any{"items": msgs, "total": total}) } // getTrafficBlob streams one oversized body by its sha256. Bodies past the inline diff --git a/traffic/stream.go b/traffic/stream.go new file mode 100644 index 0000000..2d18cce --- /dev/null +++ b/traffic/stream.go @@ -0,0 +1,544 @@ +package traffic + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "log" + "net/url" + "reflect" + "strings" + "sync" + "time" + "unicode/utf8" + "unsafe" + + "github.com/Autumn-27/artex/db" + mproxy "github.com/lqqyt2423/go-mitmproxy/proxy" +) + +// Stream recording (WebSocket and Server-Sent Events). go-mitmproxy reports a +// stream as Start, one Message hook per message, then End; a stream never reaches +// the Response hook, so record() never sees it. Each stream becomes one exchange +// (status 101 "websocket", or the real text/event-stream response) whose messages +// go to stream_messages in arrival order. + +const ( + // wsText and wsBinary are the WebSocket data opcodes (RFC 6455 §5.2), the + // values go-mitmproxy reports in WebSocketMessage.Type. + wsText = 1 + wsBinary = 2 + // streamPreview caps one message's payload in a StreamMessages row when the + // payload spilled to the blob store; the hash points at the rest. + streamPreview = 4 * 1024 + // toolStreamPreview caps one message's preview in traffic_messages, so a page + // of messages stays small in the agent's context. + toolStreamPreview = 1024 +) + +// streamState tracks one live stream. mu serializes the two WebSocket forwarding +// goroutines, which both fire WebSocketMessage on the same flow. +type streamState struct { + mu sync.Mutex + id string // exchange id; "" when the stream could not be recorded + kind string // "websocket" | "sse" + consumed int // library messages/events already handled + seq int64 + stored int + // storedBytes counts the payload bytes kept, after truncation. + storedBytes int64 + dropped bool + droppedMsgs int64 + droppedBytes int64 + // reqBytes and respBytes are the bytes observed in each direction, dropped + // messages included. + reqBytes, respBytes int64 +} + +// streamRow is one stream_messages row before it is written. +type streamRow struct { + ts int64 // unix milliseconds + direction string + kind int + event string + eventID string + retry int + size int + payload []byte // kept bytes (after maxStreamPayload truncation) + truncated bool + ctype string // content type handed to spill for the binary check +} + +func (s *sink) WebSocketStart(f *mproxy.Flow) { s.startStream(f, "websocket") } +func (s *sink) SSEStart(f *mproxy.Flow) { s.startStream(f, "sse") } + +func (s *sink) WebSocketMessage(f *mproxy.Flow) { + st := s.stream(f) + if st == nil || f.WebScoket == nil { + return + } + st.mu.Lock() + defer st.mu.Unlock() + s.t.persistStream(st, wsMessagesSince(f.WebScoket, st)) +} + +func (s *sink) SSEMessage(f *mproxy.Flow) { + st := s.stream(f) + if st == nil || f.SSE == nil { + return + } + st.mu.Lock() + defer st.mu.Unlock() + s.t.persistStream(st, sseEventsSince(f.SSE, st)) +} + +func (s *sink) WebSocketEnd(f *mproxy.Flow) { + v, ok := s.streams.LoadAndDelete(f) + if !ok { + return + } + st := v.(*streamState) + st.mu.Lock() + defer st.mu.Unlock() + if f.WebScoket != nil { + s.t.persistStream(st, wsMessagesSince(f.WebScoket, st)) + } + s.t.finishStream(st) +} + +func (s *sink) SSEEnd(f *mproxy.Flow) { + v, ok := s.streams.LoadAndDelete(f) + if !ok { + return + } + st := v.(*streamState) + st.mu.Lock() + defer st.mu.Unlock() + if f.SSE != nil { + s.t.persistStream(st, sseEventsSince(f.SSE, st)) + } + s.t.finishStream(st) +} + +func (s *sink) stream(f *mproxy.Flow) *streamState { + v, ok := s.streams.Load(f) + if !ok { + return nil + } + return v.(*streamState) +} + +// startStream registers the flow before recording its exchange, so a message +// hook racing the start waits on st.mu instead of being dropped. +func (s *sink) startStream(f *mproxy.Flow, kind string) { + st := &streamState{kind: kind} + st.mu.Lock() + defer st.mu.Unlock() + s.streams.Store(f, st) + if f.Request == nil || f.Request.URL == nil || (kind == "sse" && f.Response == nil) { + return + } + id, err := s.t.recordStreamStart(f, kind) + if err != nil { + log.Printf("[traffic] failed to record %s stream %s: %v", kind, f.Request.URL, err) + return + } + st.id = id + if kind == "sse" { + st.reqBytes = int64(len(f.Request.Body)) + } +} + +// libMutex returns the unexported mu sync.Mutex that go-mitmproxy holds while it +// appends to WebSocketData.Messages / SSEData.Events. The library exposes no +// accessor, and the two WebSocket goroutines append concurrently, so reading the +// slice without that lock is a data race. Nil when the field is absent (a library +// change), in which case the read falls back to unlocked. +func libMutex(v any) *sync.Mutex { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Pointer || rv.IsNil() { + return nil + } + f := rv.Elem().FieldByName("mu") + if !f.IsValid() || f.Type() != reflect.TypeOf(sync.Mutex{}) || !f.CanAddr() { + return nil + } + return (*sync.Mutex)(unsafe.Pointer(f.UnsafeAddr())) +} + +var libMutexMissing sync.Once + +func lockLib(v any) func() { + mu := libMutex(v) + if mu == nil { + libMutexMissing.Do(func() { + log.Printf("[traffic] go-mitmproxy stream data has no mu field; reading stream messages without its lock") + }) + return func() {} + } + mu.Lock() + return mu.Unlock +} + +// wsMessagesSince returns every WebSocket message appended since the last call, +// in the library's order, so each is handled exactly once whichever goroutine's +// hook gets there first. The library slice itself is never modified. +func wsMessagesSince(ws *mproxy.WebSocketData, st *streamState) []streamRow { + unlock := lockLib(ws) + msgs := ws.Messages + if st.consumed > len(msgs) { + st.consumed = len(msgs) + } + fresh := append([]*mproxy.WebSocketMessage(nil), msgs[st.consumed:]...) + st.consumed = len(msgs) + unlock() + rows := make([]streamRow, 0, len(fresh)) + for _, m := range fresh { + if m == nil { + continue + } + dir, ctype := "s2c", "text/plain; charset=utf-8" + if m.FromClient { + dir = "c2s" + st.reqBytes += int64(len(m.Content)) + } else { + st.respBytes += int64(len(m.Content)) + } + if m.Type == wsBinary { + ctype = "application/octet-stream" + } + rows = append(rows, streamRow{ts: m.Timestamp.UnixMilli(), direction: dir, kind: m.Type, size: len(m.Content), payload: m.Content, ctype: ctype}) + } + return rows +} + +// sseEventsSince is wsMessagesSince for Server-Sent Events. +func sseEventsSince(sse *mproxy.SSEData, st *streamState) []streamRow { + unlock := lockLib(sse) + events := sse.Events + if st.consumed > len(events) { + st.consumed = len(events) + } + fresh := append([]*mproxy.SSEEvent(nil), events[st.consumed:]...) + st.consumed = len(events) + unlock() + rows := make([]streamRow, 0, len(fresh)) + for _, e := range fresh { + if e == nil { + continue + } + st.respBytes += int64(len(e.Raw)) + rows = append(rows, streamRow{ts: e.Time.UnixMilli(), direction: "s2c", event: e.Event, eventID: e.ID, + retry: e.Retry, size: len(e.Data), payload: []byte(e.Data), ctype: "text/event-stream"}) + } + return rows +} + +// streamDisplayURL is the URL shown for a WebSocket exchange: http→ws and +// https→wss. A plain ws:// connection reaches the WebSocket hooks through its +// CONNECT tunnel, where only the target authority is known, not the path. +func streamDisplayURL(f *mproxy.Flow, kind string) (method, display, tmpl string) { + u := *f.Request.URL + method = f.Request.Method + if kind == "websocket" && strings.EqualFold(method, "CONNECT") { + host := u.Host + if host == "" { + host = u.Opaque + } + return "GET", (&url.URL{Scheme: "ws", Host: host, Path: "/"}).String(), "/" + } + if kind == "websocket" { + switch strings.ToLower(u.Scheme) { + case "http", "": + u.Scheme = "ws" + case "https": + u.Scheme = "wss" + } + } + return method, u.String(), db.TemplatePath(f.Request.URL.EscapedPath()) +} + +// recordStreamStart writes the stream's exchange: metadata, request head and +// body, the response head (synthetic 101 for WebSocket), and the header text in +// the full-text index. Messages are never indexed. +func (t *Traffic) recordStreamStart(f *mproxy.Flow, kind string) (string, error) { + now := time.Now() + id := t.newExchangeID(now) + method, display, tmpl := streamDisplayURL(f, kind) + host := f.Request.URL.Hostname() + target := f.Request.URL.RequestURI() + if strings.EqualFold(f.Request.Method, "CONNECT") { + target = f.Request.URL.Host // authority-form, as on the wire + } + reqHead := fmt.Sprintf("%s %s %s\n%s", f.Request.Method, target, f.Request.Proto, requestHeaderLines(f.Request)) + status, ct, respHead := 101, "websocket", "HTTP/1.1 101 Switching Protocols\n" + if kind == "sse" { + status = f.Response.StatusCode + ct = f.Response.Header.Get("Content-Type") + respHead = fmt.Sprintf("HTTP %d\n%s", status, headerLines(f.Response.Header)) + } + + t.wmu.Lock() + defer t.wmu.Unlock() + reqB := t.spill(f.Request.Body, f.Request.Header.Get("Content-Type")) + tx, err := t.db.Begin() + if err != nil { + return "", fmt.Errorf("beginning transaction: %w", err) + } + defer tx.Rollback() //nolint:errcheck // no-op once committed + res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path) +VALUES(?,?,?,?,?,?,?,?,?,?,'')`, id, now.Unix(), host, method, tmpl, display, status, ct, len(f.Request.Body), 0) + if err != nil { + return "", fmt.Errorf("writing index: %w", err) + } + rowid, err := res.LastInsertId() + if err != nil { + return "", fmt.Errorf("fetching rowid: %w", err) + } + if _, err := tx.Exec(`INSERT OR REPLACE INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob) +VALUES(?,?,?,?,?,NULL,NULL)`, id, reqHead, reqB.inline, nullIfEmpty(reqB.hash), respHead); err != nil { + return "", fmt.Errorf("writing body: %w", err) + } + if reqB.hash != "" { + if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, reqB.hash, id); err != nil { + return "", fmt.Errorf("registering blob reference: %w", err) + } + } + if t.fts { + idx := strings.Join([]string{display, reqHead, reqB.index, respHead}, "\n") + if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowid, idx); err != nil { + return "", fmt.Errorf("writing full-text index: %w", err) + } + } + if err := tx.Commit(); err != nil { + return "", fmt.Errorf("committing: %w", err) + } + return id, nil +} + +// persistStream applies the per-message and per-exchange limits to rows and +// writes what is kept. The first message past a per-exchange limit stores one +// 'meta' row instead; nothing is stored after it. Caller holds st.mu. Errors are +// logged: a recording failure never interrupts the stream. +func (t *Traffic) persistStream(st *streamState, rows []streamRow) { + if st.id == "" || len(rows) == 0 { + return + } + keep := make([]streamRow, 0, len(rows)) + addMeta := false + for _, r := range rows { + if len(r.payload) > maxStreamPayload { + r.payload = r.payload[:maxStreamPayload] + r.truncated = true + } + if st.dropped || st.stored >= maxStreamMessages || st.storedBytes+int64(len(r.payload)) > maxStreamBytes { + if !st.dropped { + st.dropped = true + addMeta = true + } + st.droppedMsgs++ + st.droppedBytes += int64(r.size) + continue + } + st.stored++ + st.storedBytes += int64(len(r.payload)) + keep = append(keep, r) + } + var meta []byte + if addMeta { + meta = streamMetaPayload(st) + } + if len(keep) == 0 && meta == nil { + return + } + + t.wmu.Lock() + defer t.wmu.Unlock() + tx, err := t.db.Begin() + if err != nil { + log.Printf("[traffic] failed to record stream messages of %s (beginning transaction): %v", st.id, err) + return + } + defer tx.Rollback() //nolint:errcheck // no-op once committed + for _, r := range keep { + st.seq++ + b := t.spill(r.payload, r.ctype) + inline := b.inline + if inline == nil { + inline = []byte{} + } + if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,event,event_id,retry,size,payload,payload_blob,truncated) +VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`, st.id, st.seq, r.ts, r.direction, r.kind, r.event, r.eventID, r.retry, r.size, + inline, nullIfEmpty(b.hash), boolInt(r.truncated)); err != nil { + log.Printf("[traffic] failed to record stream message %d of %s: %v", st.seq, st.id, err) + return + } + if b.hash != "" { + if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, b.hash, st.id); err != nil { + log.Printf("[traffic] failed to record stream message %d of %s (registering blob reference): %v", st.seq, st.id, err) + return + } + } + } + if meta != nil { + st.seq++ + if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload) +VALUES(?,?,?,'meta',0,?,?)`, st.id, st.seq, time.Now().UnixMilli(), len(meta), meta); err != nil { + log.Printf("[traffic] failed to record stream limit marker of %s: %v", st.id, err) + return + } + } + if err := tx.Commit(); err != nil { + log.Printf("[traffic] failed to record stream messages of %s (committing): %v", st.id, err) + } +} + +func streamMetaPayload(st *streamState) []byte { + b, _ := json.Marshal(map[string]int64{"dropped_messages": st.droppedMsgs, "dropped_bytes": st.droppedBytes}) + return b +} + +func boolInt(b bool) int { + if b { + return 1 + } + return 0 +} + +// finishStream records the bytes observed in each direction and the final +// counts of the limit marker. Caller holds st.mu. +func (t *Traffic) finishStream(st *streamState) { + if st.id == "" { + return + } + t.wmu.Lock() + defer t.wmu.Unlock() + if _, err := t.db.Exec(`UPDATE exchanges SET req_len=?, resp_len=? WHERE id=?`, st.reqBytes, st.respBytes, st.id); err != nil { + log.Printf("[traffic] failed to record stream sizes of %s: %v", st.id, err) + } + if st.dropped { + meta := streamMetaPayload(st) + if _, err := t.db.Exec(`UPDATE stream_messages SET payload=?, size=? WHERE exchange_id=? AND direction='meta'`, meta, len(meta), st.id); err != nil { + log.Printf("[traffic] failed to update stream limit marker of %s: %v", st.id, err) + } + } +} + +// StreamMessage is one recorded WebSocket message or SSE event. +type StreamMessage struct { + Seq int64 `json:"seq"` + TS int64 `json:"ts"` // unix milliseconds + Direction string `json:"direction"` + Kind int `json:"kind"` + // Type is "text" or "binary" (WebSocket), "event" (SSE), or "meta" (the + // limit marker). + Type string `json:"type"` + Event string `json:"event,omitempty"` + EventID string `json:"event_id,omitempty"` + Retry int `json:"retry,omitempty"` + Size int `json:"size"` + Payload string `json:"payload"` + // Encoding is "base64" for binary payloads, empty for UTF-8 text. + Encoding string `json:"encoding,omitempty"` + Blob string `json:"blob,omitempty"` + Truncated bool `json:"truncated,omitempty"` +} + +// StreamMessages returns one page of an exchange's stream messages in order. +// limit is 1..200 (default 50) and offset >= 0. A payload that spilled to the +// blob store comes back as a preview of at most 4 KiB plus its hash. +func (t *Traffic) StreamMessages(exchangeID string, offset, limit int) ([]StreamMessage, int, error) { + if limit <= 0 { + limit = 50 + } + if limit > 200 { + limit = 200 + } + if offset < 0 { + offset = 0 + } + var total int + if err := t.db.QueryRow(`SELECT COUNT(*) FROM stream_messages WHERE exchange_id=?`, exchangeID).Scan(&total); err != nil { + return nil, 0, err + } + rows, err := t.db.Query(`SELECT seq,ts,direction,kind,event,event_id,retry,size,payload,payload_blob,truncated +FROM stream_messages WHERE exchange_id=? ORDER BY seq LIMIT ? OFFSET ?`, exchangeID, limit, offset) + if err != nil { + return nil, 0, err + } + defer rows.Close() + out := []StreamMessage{} + for rows.Next() { + var m StreamMessage + var payload []byte + var blob sql.NullString + var truncated int + if err := rows.Scan(&m.Seq, &m.TS, &m.Direction, &m.Kind, &m.Event, &m.EventID, &m.Retry, &m.Size, &payload, &blob, &truncated); err != nil { + return nil, 0, err + } + m.Truncated = truncated != 0 + m.Blob = blob.String + switch { + case m.Direction == "meta": + m.Type = "meta" + case m.Kind == wsBinary: + m.Type = "binary" + case m.Kind == wsText: + m.Type = "text" + default: + m.Type = "event" + } + if m.Blob != "" { + payload = clipBytes(payload, streamPreview) + } + if m.Type == "binary" { + m.Payload, m.Encoding = base64.StdEncoding.EncodeToString(payload), "base64" + } else { + m.Payload = string(payload) + if m.Blob != "" { + m.Payload = truncateUTF8(payload, streamPreview) + } + } + out = append(out, m) + } + return out, total, rows.Err() +} + +// StreamInfo reports how many stream messages an exchange holds and whether it +// is a WebSocket or SSE exchange ("" for an ordinary exchange). +func (t *Traffic) StreamInfo(exchangeID string) (count int, kind string, err error) { + var ct sql.NullString + err = t.db.QueryRow(`SELECT content_type FROM exchanges WHERE id=?`, exchangeID).Scan(&ct) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return 0, "", err + } + switch c := strings.ToLower(ct.String); { + case c == "websocket": + kind = "websocket" + case strings.Contains(c, "text/event-stream"): + kind = "sse" + } + if err := t.db.QueryRow(`SELECT COUNT(*) FROM stream_messages WHERE exchange_id=?`, exchangeID).Scan(&count); err != nil { + return 0, "", err + } + return count, kind, nil +} + +// streamToolPreview renders one message for traffic_messages: at most 1 KiB of +// text (hex for binary), plus the blob hash when the payload spilled. +func streamToolPreview(m StreamMessage) string { + if m.Encoding == "base64" { + raw, _ := base64.StdEncoding.DecodeString(m.Payload) + return "hex:" + fmt.Sprintf("%x", clipBytes(raw, toolStreamPreview/2)) + } + p := m.Payload + if len(p) > toolStreamPreview { + p = truncateUTF8([]byte(p), toolStreamPreview) + } + if !utf8.ValidString(p) { + p = strings.ToValidUTF8(p, "�") + } + return p +} diff --git a/traffic/stream_test.go b/traffic/stream_test.go new file mode 100644 index 0000000..8aaaf4f --- /dev/null +++ b/traffic/stream_test.go @@ -0,0 +1,550 @@ +package traffic + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/gorilla/websocket" + mproxy "github.com/lqqyt2423/go-mitmproxy/proxy" +) + +// localhostURL names an httptest server by host name: the MITM leaf certificate +// carries a DNS name, not an IP SAN (as in pace_test.go). +func localhostURL(u string) string { return strings.Replace(u, "127.0.0.1", "localhost", 1) } + +// waitFor polls cond until it holds or 10 seconds pass. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(20 * time.Millisecond) + } +} + +// wsDialer dials through the recording proxy and trusts its CA. +func wsDialer(t *testing.T, tr *Traffic, proxyURL string) *websocket.Dialer { + t.Helper() + pem, err := os.ReadFile(tr.CACertPath()) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + t.Fatal("no CA in", tr.CACertPath()) + } + u, _ := url.Parse(proxyURL) + return &websocket.Dialer{Proxy: http.ProxyURL(u), TLSClientConfig: &tls.Config{RootCAs: pool}, HandshakeTimeout: 5 * time.Second} +} + +type streamExchange struct { + id, url, contentType string + status int + reqLen, respLen int64 +} + +// streamExchanges returns the exchanges recorded with a stream content type. +func streamExchanges(t *testing.T, tr *Traffic, contentLike string) []streamExchange { + t.Helper() + rows, err := tr.DB().Query(`SELECT id,url,content_type,status,req_len,resp_len FROM exchanges WHERE content_type LIKE ? ORDER BY id`, contentLike) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var out []streamExchange + for rows.Next() { + var e streamExchange + if err := rows.Scan(&e.id, &e.url, &e.contentType, &e.status, &e.reqLen, &e.respLen); err != nil { + t.Fatal(err) + } + out = append(out, e) + } + return out +} + +func allMessages(t *testing.T, tr *Traffic, id string) []StreamMessage { + t.Helper() + var out []StreamMessage + for offset := 0; ; offset += 200 { + page, total, err := tr.StreamMessages(id, offset, 200) + if err != nil { + t.Fatal(err) + } + out = append(out, page...) + if len(out) >= total || len(page) == 0 { + return out + } + } +} + +// echoWSServer echoes every message back with the same type: text prefixed +// with "echo:", binary reversed. +func echoWSServer(t *testing.T) *httptest.Server { + t.Helper() + up := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }} + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + defer c.Close() + for { + mt, msg, err := c.ReadMessage() + if err != nil { + return + } + if mt == websocket.BinaryMessage { + rev := make([]byte, len(msg)) + for i := range msg { + rev[len(msg)-1-i] = msg[i] + } + msg = rev + } else { + msg = append([]byte("echo:"), msg...) + } + if err := c.WriteMessage(mt, msg); err != nil { + return + } + } + })) + t.Cleanup(srv.Close) + return srv +} + +func closeWS(c *websocket.Conn) { + _ = c.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + _ = c.SetReadDeadline(time.Now().Add(2 * time.Second)) + for { + if _, _, err := c.ReadMessage(); err != nil { + break + } + } + _ = c.Close() +} + +func TestWebSocketMessagesRecorded(t *testing.T) { + tr, _, proxyURL := startProxy(t) + srv := echoWSServer(t) + target := "wss://" + strings.TrimPrefix(localhostURL(srv.URL), "https://") + "/ws?room=1" + c, _, err := wsDialer(t, tr, proxyURL).Dial(target, nil) + if err != nil { + t.Fatal(err) + } + exchange := func(mt int, msg []byte) []byte { + if err := c.WriteMessage(mt, msg); err != nil { + t.Fatal(err) + } + gotType, got, err := c.ReadMessage() + if err != nil || gotType != mt { + t.Fatalf("read type=%d err=%v", gotType, err) + } + return got + } + before := time.Now().UnixMilli() + if got := exchange(websocket.TextMessage, []byte("hello")); string(got) != "echo:hello" { + t.Fatalf("echo %q", got) + } + if got := exchange(websocket.BinaryMessage, []byte{0, 1, 2, 255}); !bytes.Equal(got, []byte{255, 2, 1, 0}) { + t.Fatalf("binary echo %v", got) + } + closeWS(c) + + var ex streamExchange + waitFor(t, "websocket end", func() bool { + exs := streamExchanges(t, tr, "websocket") + if len(exs) != 1 || exs[0].reqLen == 0 { + return false + } + ex = exs[0] + return true + }) + if ex.status != 101 || ex.url != target { + t.Fatalf("exchange %+v, want status 101 and url %s", ex, target) + } + // c2s: "hello" (5) + 4 binary bytes; s2c: "echo:hello" (10) + 4. + if ex.reqLen != 9 || ex.respLen != 14 { + t.Fatalf("req_len=%d resp_len=%d, want 9 and 14", ex.reqLen, ex.respLen) + } + var respHead string + if err := tr.DB().QueryRow(`SELECT resp_head FROM exchange_bodies WHERE id=?`, ex.id).Scan(&respHead); err != nil || !strings.HasPrefix(respHead, "HTTP/1.1 101 Switching Protocols") { + t.Fatalf("resp_head %q err=%v", respHead, err) + } + if count, kind, err := tr.StreamInfo(ex.id); err != nil || count != 4 || kind != "websocket" { + t.Fatalf("StreamInfo = %d %q %v", count, kind, err) + } + + msgs := allMessages(t, tr, ex.id) + want := []struct { + dir, typ, payload string + kind int + }{ + {"c2s", "text", "hello", 1}, + {"s2c", "text", "echo:hello", 1}, + {"c2s", "binary", base64.StdEncoding.EncodeToString([]byte{0, 1, 2, 255}), 2}, + {"s2c", "binary", base64.StdEncoding.EncodeToString([]byte{255, 2, 1, 0}), 2}, + } + if len(msgs) != len(want) { + t.Fatalf("got %d messages: %+v", len(msgs), msgs) + } + last := before + for i, w := range want { + m := msgs[i] + if m.Seq != int64(i+1) || m.Direction != w.dir || m.Type != w.typ || m.Kind != w.kind || m.Payload != w.payload { + t.Fatalf("message %d = %+v, want %+v", i, m, w) + } + if (w.typ == "binary") != (m.Encoding == "base64") { + t.Fatalf("message %d encoding %q", i, m.Encoding) + } + if m.TS < last { + t.Fatalf("message %d ts %d before %d", i, m.TS, last) + } + last = m.TS + } +} + +func TestSSEEventsRecorded(t *testing.T) { + tr, _, proxyURL := startProxy(t) + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(200) + fl := w.(http.Flusher) + for _, ev := range []string{ + "retry: 1500\nid: 1\nevent: greet\ndata: hello\ndata: world\n\n", + "id: 2\ndata: second\n\n", + "event: done\ndata: bye\n\n", + } { + _, _ = io.WriteString(w, ev) + fl.Flush() + time.Sleep(10 * time.Millisecond) + } + })) + t.Cleanup(srv.Close) + + resp, err := proxyClient(t, tr, proxyURL).Get(localhostURL(srv.URL) + "/events") + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if !strings.Contains(string(body), "data: bye") { + t.Fatalf("client body %q", body) + } + + var ex streamExchange + waitFor(t, "sse end", func() bool { + exs := streamExchanges(t, tr, "%event-stream%") + if len(exs) != 1 || exs[0].respLen == 0 { + return false + } + ex = exs[0] + return true + }) + if ex.status != 200 || !strings.HasPrefix(ex.url, localhostURL(srv.URL)+"/events") { + t.Fatalf("exchange %+v", ex) + } + if n, _ := tr.Count(); n != 1 { + t.Fatalf("%d exchanges recorded, want the single SSE exchange", n) + } + msgs := allMessages(t, tr, ex.id) + want := []struct { + event, id, data string + retry int + }{ + {"greet", "1", "hello\nworld", 1500}, + {"message", "2", "second", 0}, + {"done", "", "bye", 0}, + } + if len(msgs) != len(want) { + t.Fatalf("got %d events: %+v", len(msgs), msgs) + } + for i, w := range want { + m := msgs[i] + if m.Seq != int64(i+1) || m.Direction != "s2c" || m.Kind != 0 || m.Type != "event" || + m.Event != w.event || m.EventID != w.id || m.Payload != w.data || m.Retry != w.retry || m.Size != len(w.data) { + t.Fatalf("event %d = %+v, want %+v", i, m, w) + } + } +} + +// unitStream opens a recorder without starting the proxy and drives its hooks +// directly with a synthetic WebSocket flow. +func unitStream(t *testing.T, host string) (*Traffic, *sink, *mproxy.Flow) { + t.Helper() + tr, err := Open(t.TempDir(), "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = tr.Close() }) + u, _ := url.Parse("https://" + host + "/ws") + f := &mproxy.Flow{Request: &mproxy.Request{Method: "GET", URL: u, Proto: "HTTP/1.1", Header: http.Header{}}, WebScoket: &mproxy.WebSocketData{}} + s := tr.proxy.Addons[0].(*sink) + s.WebSocketStart(f) + return tr, s, f +} + +func addWS(f *mproxy.Flow, fromClient bool, mt int, content []byte) { + f.WebScoket.Messages = append(f.WebScoket.Messages, &mproxy.WebSocketMessage{Type: mt, Content: content, FromClient: fromClient, Timestamp: time.Now()}) +} + +func onlyExchange(t *testing.T, tr *Traffic) string { + t.Helper() + exs := streamExchanges(t, tr, "websocket") + if len(exs) != 1 { + t.Fatalf("%d websocket exchanges", len(exs)) + } + return exs[0].id +} + +func metaCounts(t *testing.T, m StreamMessage) (msgs, bytes int64) { + t.Helper() + var v struct { + DroppedMessages int64 `json:"dropped_messages"` + DroppedBytes int64 `json:"dropped_bytes"` + } + if err := json.Unmarshal([]byte(m.Payload), &v); err != nil { + t.Fatalf("meta payload %q: %v", m.Payload, err) + } + return v.DroppedMessages, v.DroppedBytes +} + +func TestStreamPayloadTruncatedToLimit(t *testing.T) { + tr, s, f := unitStream(t, "trunc.test") + big := bytes.Repeat([]byte("a"), maxStreamPayload+10) + addWS(f, true, websocket.TextMessage, big) + s.WebSocketMessage(f) + id := onlyExchange(t, tr) + msgs := allMessages(t, tr, id) + if len(msgs) != 1 { + t.Fatalf("%d messages", len(msgs)) + } + m := msgs[0] + if !m.Truncated || m.Size != maxStreamPayload+10 || m.Blob == "" || len(m.Payload) > streamPreview { + t.Fatalf("message = seq %d size %d truncated %v blob %q payload %d bytes", m.Seq, m.Size, m.Truncated, m.Blob, len(m.Payload)) + } + data, total, err := tr.BlobRange(m.Blob, 0, 16) + if err != nil || total != maxStreamPayload || string(data) != strings.Repeat("a", 16) { + t.Fatalf("blob total=%d data=%q err=%v, want the kept %d-byte prefix", total, data, err, maxStreamPayload) + } +} + +func TestStreamMessageLimitStoresOneMetaRow(t *testing.T) { + tr, s, f := unitStream(t, "count.test") + for i := 0; i < maxStreamMessages+5; i++ { + addWS(f, i%2 == 0, websocket.TextMessage, []byte("m")) + } + s.WebSocketMessage(f) + addWS(f, true, websocket.TextMessage, []byte("late")) + s.WebSocketMessage(f) + s.WebSocketEnd(f) + id := onlyExchange(t, tr) + msgs := allMessages(t, tr, id) + if len(msgs) != maxStreamMessages+1 { + t.Fatalf("%d rows, want %d messages and one meta row", len(msgs), maxStreamMessages) + } + meta := msgs[len(msgs)-1] + if meta.Direction != "meta" || meta.Type != "meta" || meta.Kind != 0 || meta.Seq != maxStreamMessages+1 { + t.Fatalf("last row %+v, want the meta row", meta) + } + if n, b := metaCounts(t, meta); n != 6 || b != 5+4 { + t.Fatalf("meta dropped_messages=%d dropped_bytes=%d, want 6 and 9", n, b) + } + for _, m := range msgs[:len(msgs)-1] { + if m.Direction == "meta" { + t.Fatalf("more than one meta row: %+v", m) + } + } +} + +func TestStreamByteLimitStoresOneMetaRow(t *testing.T) { + tr, s, f := unitStream(t, "bytes.test") + chunk := bytes.Repeat([]byte{7}, maxStreamPayload) + n := maxStreamBytes/maxStreamPayload + 1 + for i := 0; i < n; i++ { + addWS(f, false, websocket.BinaryMessage, chunk) + } + s.WebSocketMessage(f) + addWS(f, false, websocket.BinaryMessage, []byte{1}) + s.WebSocketMessage(f) + s.WebSocketEnd(f) + id := onlyExchange(t, tr) + msgs := allMessages(t, tr, id) + if len(msgs) != n { + t.Fatalf("%d rows, want %d stored and one meta row", len(msgs), n-1) + } + meta := msgs[len(msgs)-1] + if meta.Direction != "meta" { + t.Fatalf("last row %+v, want the meta row", meta) + } + if dn, db := metaCounts(t, meta); dn != 2 || db != maxStreamPayload+1 { + t.Fatalf("meta dropped_messages=%d dropped_bytes=%d", dn, db) + } + var stored int64 + if err := tr.DB().QueryRow(`SELECT SUM(size) FROM stream_messages WHERE exchange_id=? AND direction<>'meta'`, id).Scan(&stored); err != nil || stored != maxStreamBytes { + t.Fatalf("stored %d bytes err=%v, want %d", stored, err, maxStreamBytes) + } + var respLen int64 + if err := tr.DB().QueryRow(`SELECT resp_len FROM exchanges WHERE id=?`, id).Scan(&respLen); err != nil || respLen != int64(n*maxStreamPayload+1) { + t.Fatalf("resp_len %d err=%v, want every observed byte", respLen, err) + } +} + +func TestDeleteHostRemovesStreamMessagesAndBlobs(t *testing.T) { + tr, s, f := unitStream(t, "gone.test") + addWS(f, false, websocket.BinaryMessage, bytes.Repeat([]byte{9}, maxInlineBody+1)) + addWS(f, true, websocket.TextMessage, []byte("small")) + s.WebSocketMessage(f) + s.WebSocketEnd(f) + id := onlyExchange(t, tr) + msgs := allMessages(t, tr, id) + if len(msgs) != 2 || msgs[0].Blob == "" { + t.Fatalf("messages %+v, want a spilled first payload", msgs) + } + blobPath := filepath.Join(tr.dir, "_blobs", "sha256", msgs[0].Blob[:2], msgs[0].Blob+".bin") + if _, err := os.Stat(blobPath); err != nil { + t.Fatalf("payload blob missing: %v", err) + } + if n, err := tr.DeleteHost("gone.test"); err != nil || n != 1 { + t.Fatalf("DeleteHost = %d, %v", n, err) + } + var rows int + if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM stream_messages`).Scan(&rows); err != nil || rows != 0 { + t.Fatalf("%d stream rows left, err=%v", rows, err) + } + if _, err := os.Stat(blobPath); !os.IsNotExist(err) { + t.Fatalf("payload blob not collected: %v", err) + } +} + +// A recording failure must never reach the proxied connection. +func TestWebSocketSurvivesRecordingFailure(t *testing.T) { + tr, _, proxyURL := startProxy(t) + srv := echoWSServer(t) + c, _, err := wsDialer(t, tr, proxyURL).Dial("wss://"+strings.TrimPrefix(localhostURL(srv.URL), "https://")+"/ws", nil) + if err != nil { + t.Fatal(err) + } + defer closeWS(c) + if err := c.WriteMessage(websocket.TextMessage, []byte("before")); err != nil { + t.Fatal(err) + } + if _, got, err := c.ReadMessage(); err != nil || string(got) != "echo:before" { + t.Fatalf("echo %q err=%v", got, err) + } + if err := tr.DB().Close(); err != nil { + t.Fatal(err) + } + for i := 0; i < 3; i++ { + msg := fmt.Sprintf("after-%d", i) + if err := c.WriteMessage(websocket.TextMessage, []byte(msg)); err != nil { + t.Fatalf("write %s: %v", msg, err) + } + _ = c.SetReadDeadline(time.Now().Add(5 * time.Second)) + if _, got, err := c.ReadMessage(); err != nil || string(got) != "echo:"+msg { + t.Fatalf("echo %q err=%v after the index closed", got, err) + } + } +} + +// Both forwarding goroutines fire the message hook at once; every message must +// be stored exactly once, each direction in order. +func TestWebSocketConcurrentDirections(t *testing.T) { + const n = 200 + tr, _, proxyURL := startProxy(t) + up := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }} + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + defer c.Close() + go func() { + for i := 0; i < n; i++ { + if c.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("s-%d", i))) != nil { + return + } + } + }() + for { + if _, _, err := c.ReadMessage(); err != nil { + return + } + } + })) + t.Cleanup(srv.Close) + + c, _, err := wsDialer(t, tr, proxyURL).Dial("wss://"+strings.TrimPrefix(localhostURL(srv.URL), "https://")+"/ws", nil) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < n; i++ { + if err := c.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("c-%d", i))); err != nil { + t.Errorf("write: %v", err) + return + } + } + }() + for i := 0; i < n; i++ { + if _, _, err := c.ReadMessage(); err != nil { + t.Fatalf("read %d: %v", i, err) + } + } + wg.Wait() + var id string + waitFor(t, "all client messages recorded", func() bool { + exs := streamExchanges(t, tr, "websocket") + if len(exs) != 1 { + return false + } + id = exs[0].id + count, _, _ := tr.StreamInfo(id) + return count == 2*n + }) + closeWS(c) + + msgs := allMessages(t, tr, id) + next := map[string]int{"c2s": 0, "s2c": 0} + for i, m := range msgs { + if m.Seq != int64(i+1) { + t.Fatalf("row %d has seq %d", i, m.Seq) + } + prefix := "s-" + if m.Direction == "c2s" { + prefix = "c-" + } + if want := fmt.Sprintf("%s%d", prefix, next[m.Direction]); m.Payload != want { + t.Fatalf("row %d %s payload %q, want %q", i, m.Direction, m.Payload, want) + } + next[m.Direction]++ + } + if next["c2s"] != n || next["s2c"] != n { + t.Fatalf("stored %v, want %d each way", next, n) + } +} + +// The library's unexported lock is what makes reading its message slices safe; +// a go-mitmproxy upgrade that renames it must fail here, not race silently. +func TestLibMutexFound(t *testing.T) { + if libMutex(&mproxy.WebSocketData{}) == nil || libMutex(&mproxy.SSEData{}) == nil { + t.Fatal("go-mitmproxy WebSocketData/SSEData no longer have a sync.Mutex field named mu") + } +} diff --git a/traffic/traffic.go b/traffic/traffic.go index 6ac3709..0f598d1 100644 --- a/traffic/traffic.go +++ b/traffic/traffic.go @@ -73,6 +73,24 @@ CREATE TABLE IF NOT EXISTS blob_refs ( PRIMARY KEY (hash, exchange_id) ); CREATE INDEX IF NOT EXISTS idx_blob_refs_ex ON blob_refs(exchange_id); + +-- WebSocket messages and SSE events of a stream exchange, in arrival order. +CREATE TABLE IF NOT EXISTS stream_messages ( + exchange_id TEXT NOT NULL, + seq INTEGER NOT NULL, + ts INTEGER NOT NULL, -- unix milliseconds + direction TEXT NOT NULL, -- 'c2s' | 's2c' | 'meta' + kind INTEGER NOT NULL, -- WebSocket opcode (1 text, 2 binary), 0 for SSE and meta + event TEXT NOT NULL DEFAULT '', + event_id TEXT NOT NULL DEFAULT '', + retry INTEGER NOT NULL DEFAULT 0, + size INTEGER NOT NULL, -- original payload size + payload BLOB, -- inline payload, or a preview when spilled + payload_blob TEXT, -- sha256 of the spilled payload + truncated INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (exchange_id, seq) +); +CREATE INDEX IF NOT EXISTS idx_stream_messages_ex ON stream_messages(exchange_id, seq); ` // ftsSchema is applied separately from indexSchema: a driver build without FTS5 @@ -90,6 +108,14 @@ const ( // maxInlineBody is the cutoff between a body stored inline (SQLite column) and // one spilled to the content-addressed blob store. maxInlineBody = 256 * 1024 + // maxStreamPayload is how much of one WebSocket message or SSE event is kept. + // A longer payload keeps its prefix, with truncated=1 and its original size. + maxStreamPayload = 1024 * 1024 + // maxStreamMessages and maxStreamBytes bound one stream exchange (bytes as + // stored, after truncation). Past either, storing stops and a single 'meta' + // row records how many messages and bytes were dropped. + maxStreamMessages = 10000 + maxStreamBytes = 32 * 1024 * 1024 // blobPreview is how much of a spilled body stays inline so readers can tell // what it is (JSON shape, SQL dump header, ZIP magic) without fetching it. blobPreview = 8 * 1024 @@ -387,8 +413,9 @@ func (t *Traffic) DB() *sql.DB { return t.db } // the requests of client connections marked for pacing (see pace.go). type sink struct { mproxy.BaseAddon - t *Traffic - paced sync.Map // ClientConn.Id of a marked connection -> struct{} + t *Traffic + paced sync.Map // ClientConn.Id of a marked connection -> struct{} + streams sync.Map // *mproxy.Flow of a live WebSocket/SSE stream -> *streamState (stream.go) } func (s *sink) Response(f *mproxy.Flow) { @@ -443,9 +470,8 @@ func (t *Traffic) record(f *mproxy.Flow) { host := f.Request.URL.Hostname() method := f.Request.Method tmpl := db.TemplatePath(f.Request.URL.EscapedPath()) - n := t.seq.Add(1) now := time.Now() - id := fmt.Sprintf("%d-%04d", now.Unix(), n%10000) + id := t.newExchangeID(now) ct := f.Response.Header.Get("Content-Type") url := f.Request.URL.String() @@ -511,6 +537,12 @@ VALUES(?,?,?,?,?,?,?)`, } } +// newExchangeID names a recorded exchange: unix seconds plus a rolling sequence. +func (t *Traffic) newExchangeID(now time.Time) string { + n := t.seq.Add(1) + return fmt.Sprintf("%d-%04d", now.Unix(), n%10000) +} + // storedBody is one body after the inline/spill decision: inline is what goes in // the SQLite column (the whole body, or a preview when spilled), hash names the // blob when it spilled, and index is the text handed to FTS (empty for binary). @@ -1191,6 +1223,9 @@ func (t *Traffic) deleteWhere(tx *sql.Tx, where string, args ...any) (int64, err if _, err := tx.Exec(`DELETE FROM exchange_bodies WHERE id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil { return 0, fmt.Errorf("delete bodies: %w", err) } + if _, err := tx.Exec(`DELETE FROM stream_messages WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil { + return 0, fmt.Errorf("delete stream messages: %w", err) + } if _, err := tx.Exec(`DELETE FROM blob_refs WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil { return 0, fmt.Errorf("delete blob references: %w", err) } @@ -2058,7 +2093,7 @@ func (t *Traffic) Tools() []actool.CoreTool { get := actool.Build(actool.Spec{ Name: "traffic_get", - Description: "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling.", + Description: "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling. A WebSocket (status 101) or SSE (text/event-stream) record shows only its handshake here; read its messages with traffic_messages.", Schema: map[string]any{ "type": "object", "properties": map[string]any{"id": map[string]any{"type": "string", "description": "the id returned by traffic_search"}}, @@ -2116,7 +2151,70 @@ func (t *Traffic) Tools() []actool.CoreTool { }, }) - return []actool.CoreTool{search, get, blob} + messages := actool.Build(actool.Spec{ + Name: "traffic_messages", + Description: "List the messages of one captured WebSocket or SSE (Server-Sent Events) record in order: direction (c2s client→server, s2c server→client), type (text/binary/event, or meta when the recorder dropped messages past its limits), size and a preview of at most 1KB (binary as hex). A payload stored as a blob carries its hash; read the rest with traffic_blob. Returns 20 messages by default, at most 50; use offset to page.", + Schema: map[string]any{ + "type": "object", + "properties": map[string]any{ + "id": map[string]any{"type": "string", "description": "the record id from traffic_search (a WebSocket status-101 or SSE record)"}, + "offset": map[string]any{"type": "integer", "description": "number of messages to skip, default 0"}, + "limit": map[string]any{"type": "integer", "description": "messages to return, default 20, max 50"}, + }, + "required": []any{"id"}, + }, + ReadOnly: ro, + Permissions: allow, + Run: func(_ context.Context, in json.RawMessage, _ *actool.ToolContext) (actool.Result, error) { + var a struct { + ID string + Offset int + Limit int + } + _ = json.Unmarshal(in, &a) + if strings.TrimSpace(a.ID) == "" { + return actool.Errorf("id is a required parameter"), nil + } + if a.Limit <= 0 { + a.Limit = 20 + } + if a.Limit > 50 { + a.Limit = 50 + } + if a.Offset < 0 { + a.Offset = 0 + } + msgs, total, err := t.StreamMessages(a.ID, a.Offset, a.Limit) + if err != nil { + return actool.Errorf(err.Error()), nil + } + if total == 0 { + return actool.Text("No stream messages recorded for this id (not a WebSocket/SSE record, or no message was exchanged)."), nil + } + type liteMsg struct { + Seq int64 `json:"seq"` + Direction string `json:"direction"` + Type string `json:"type"` + Event string `json:"event,omitempty"` + Size int `json:"size"` + Preview string `json:"preview"` + Blob string `json:"blob,omitempty"` + Truncated bool `json:"truncated,omitempty"` + } + lite := make([]liteMsg, 0, len(msgs)) + for _, m := range msgs { + l := liteMsg{Seq: m.Seq, Direction: m.Direction, Type: m.Type, Event: m.Event, Size: m.Size, Preview: streamToolPreview(m), Truncated: m.Truncated} + if m.Blob != "" { + l.Blob = "@blob sha256:" + m.Blob + } + lite = append(lite, l) + } + b, _ := json.Marshal(lite) + return actool.Text(fmt.Sprintf("[messages %d-%d of %d]\n", a.Offset+1, a.Offset+len(msgs), total) + string(b)), nil + }, + }) + + return []actool.CoreTool{search, get, blob, messages} } // SeedToolMetas returns the traffic tools built on a ZERO receiver, for seeding the diff --git a/traffic/traffic_test.go b/traffic/traffic_test.go index ea818b5..5d74eb3 100644 --- a/traffic/traffic_test.go +++ b/traffic/traffic_test.go @@ -377,6 +377,22 @@ VALUES(?,?,?,?,?,?,?,?,?,?,?)`, seed("1-0002", "b.example.com", hb) // sole reference to blobB seed("1-0003", "c.example.com", hc) // shares blobC with d seed("1-0004", "d.example.com", hc) + // d is also a stream exchange whose one message payload spilled to blobD, + // referenced through blob_refs like any body. + hd := strings.Repeat("d", 64) + blobD := filepath.Join(dir, "_blobs", "sha256", "dd", hd+".bin") + if err := os.MkdirAll(filepath.Dir(blobD), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(blobD, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := tr.DB().Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload,payload_blob) VALUES('1-0004',1,1,'s2c',2,1,x'78',?)`, hd); err != nil { + t.Fatal(err) + } + if _, err := tr.DB().Exec(`INSERT INTO blob_refs(hash,exchange_id) VALUES(?, '1-0004')`, hd); err != nil { + t.Fatal(err) + } // Delete a: blobA orphaned → removed; blobB/blobC still referenced → kept. if n, err := tr.DeleteHost("a.example"); err != nil || n != 1 { @@ -399,12 +415,23 @@ VALUES(?,?,?,?,?,?,?,?,?,?,?)`, if _, err := os.Stat(blobC); err != nil { t.Fatalf("shared blobC removed after deleting one sharer: %v", err) } + if _, err := os.Stat(blobD); err != nil { + t.Fatalf("stream payload blobD removed while d still references it: %v", err) + } - // Delete d: last reference gone → blobC collected. + // Delete d: last reference gone → blobC collected, and d's stream message + // and its payload blob go with it. if n, err := tr.DeleteHost("d.example"); err != nil || n != 1 { t.Fatalf("DeleteHost(d.example)=%d, err=%v; want 1, nil", n, err) } if _, err := os.Stat(blobC); !os.IsNotExist(err) { t.Fatalf("blobC still exists after last reference removed: %v", err) } + var streamRows int + if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM stream_messages`).Scan(&streamRows); err != nil || streamRows != 0 { + t.Fatalf("%d stream rows left after deleting d, err=%v", streamRows, err) + } + if _, err := os.Stat(blobD); !os.IsNotExist(err) { + t.Fatalf("stream payload blobD still exists after its exchange was deleted: %v", err) + } } From dcdc0652ad452396d9f442df410d443b1fdf384c Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:21:38 +0100 Subject: [PATCH 03/29] fix(traffic): bound stream message pages, spill before the transaction, seconds timestamps Stream payloads over 4 KiB now spill to the blob store (spillAt with maxStreamInline; spill keeps maxInlineBody for HTTP bodies) and StreamMessages clips every payload to 4 KiB, reading a spilled binary preview from its blob. The byte cap counts the inline preview. persistStream spills before opening the transaction, as record() does. Without the library lock nothing is read. ts is unix seconds. Adds the final-message-on-close and route tests. --- server/traffic_stream_test.go | 89 +++++++++++++++++++++++++ traffic/stream.go | 120 +++++++++++++++++++++++----------- traffic/stream_test.go | 101 ++++++++++++++++++++++++++-- traffic/traffic.go | 10 ++- 4 files changed, 273 insertions(+), 47 deletions(-) create mode 100644 server/traffic_stream_test.go diff --git a/server/traffic_stream_test.go b/server/traffic_stream_test.go new file mode 100644 index 0000000..5e03f70 --- /dev/null +++ b/server/traffic_stream_test.go @@ -0,0 +1,89 @@ +package server + +import ( + "encoding/json" + "testing" +) + +func TestTrafficStreamRoute(t *testing.T) { + s, request := ingestServer(t) + db := s.m.Traffic().DB() + if _, err := db.Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path) +VALUES('1-0001',1,'ws.example.com','GET','/ws','wss://ws.example.com/ws',101,'websocket',0,0,'')`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO exchange_bodies(id,req_head,resp_head) VALUES('1-0001','GET /ws HTTP/1.1\n','HTTP/1.1 101 Switching Protocols\n')`); err != nil { + t.Fatal(err) + } + const total = 250 + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + for i := 1; i <= total; i++ { + kind, payload := 1, []byte("hello") + if i%2 == 0 { + kind, payload = 2, []byte{0, 1, 2} + } + if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload) VALUES('1-0001',?,1,'c2s',?,?,?)`, i, kind, len(payload), payload); err != nil { + t.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + + type page struct { + Items []struct { + Seq int64 `json:"seq"` + Type string `json:"type"` + Payload string `json:"payload"` + Encoding string `json:"encoding"` + } `json:"items"` + Total int `json:"total"` + } + get := func(path string, wantStatus int) page { + t.Helper() + w := request("GET", path, "") + if w.Code != wantStatus { + t.Fatalf("GET %s = %d %s, want %d", path, w.Code, w.Body, wantStatus) + } + var p page + if wantStatus == 200 { + if err := json.Unmarshal(w.Body.Bytes(), &p); err != nil { + t.Fatalf("GET %s: %v in %s", path, err, w.Body) + } + } + return p + } + + get("/api/traffic/stream", 400) + if p := get("/api/traffic/stream?exchange=nope", 200); p.Items == nil || len(p.Items) != 0 || p.Total != 0 { + t.Fatalf("unknown exchange = %+v, want items [] and total 0", p) + } + p := get("/api/traffic/stream?exchange=1-0001", 200) + if p.Total != total || len(p.Items) != 50 || p.Items[0].Seq != 1 { + t.Fatalf("default page: total %d, %d items, first seq %d", p.Total, len(p.Items), p.Items[0].Seq) + } + if first, second := p.Items[0], p.Items[1]; first.Type != "text" || first.Payload != "hello" || second.Type != "binary" || second.Encoding != "base64" || second.Payload != "AAEC" { + t.Fatalf("payloads %+v %+v", first, second) + } + if p := get("/api/traffic/stream?exchange=1-0001&limit=500", 200); len(p.Items) != 200 { + t.Fatalf("limit=500 returned %d items, want the 200 cap", len(p.Items)) + } + if p := get("/api/traffic/stream?exchange=1-0001&offset=-5&limit=1", 200); len(p.Items) != 1 || p.Items[0].Seq != 1 { + t.Fatalf("negative offset: %+v, want seq 1", p.Items) + } + if p := get("/api/traffic/stream?exchange=1-0001&offset=248&limit=10", 200); len(p.Items) != 2 || p.Items[0].Seq != 249 { + t.Fatalf("last page: %+v, want seqs 249 and 250", p.Items) + } + + w := request("GET", "/api/traffic/exchange?id=1-0001", "") + var ex struct { + StreamCount int `json:"stream_count"` + StreamKind string `json:"stream_kind"` + } + if err := json.Unmarshal(w.Body.Bytes(), &ex); err != nil || w.Code != 200 || ex.StreamCount != total || ex.StreamKind != "websocket" { + t.Fatalf("exchange = %d %s", w.Code, w.Body) + } +} diff --git a/traffic/stream.go b/traffic/stream.go index 2d18cce..c502ebc 100644 --- a/traffic/stream.go +++ b/traffic/stream.go @@ -30,9 +30,12 @@ const ( // values go-mitmproxy reports in WebSocketMessage.Type. wsText = 1 wsBinary = 2 - // streamPreview caps one message's payload in a StreamMessages row when the - // payload spilled to the blob store; the hash points at the rest. + // streamPreview caps one message's payload in a StreamMessages row; a longer + // payload lives in the blob store and the row carries its hash. streamPreview = 4 * 1024 + // maxStreamInline is the inline threshold for message payloads: anything + // longer spills to the blob store, so a page of 200 rows stays bounded. + maxStreamInline = streamPreview // toolStreamPreview caps one message's preview in traffic_messages, so a page // of messages stays small in the agent's context. toolStreamPreview = 1024 @@ -47,7 +50,8 @@ type streamState struct { consumed int // library messages/events already handled seq int64 stored int - // storedBytes counts the payload bytes kept, after truncation. + // storedBytes counts what the kept messages occupy: the payload after + // truncation, plus the inline preview spill keeps for a spilled payload. storedBytes int64 dropped bool droppedMsgs int64 @@ -59,7 +63,7 @@ type streamState struct { // streamRow is one stream_messages row before it is written. type streamRow struct { - ts int64 // unix milliseconds + ts int64 // unix seconds, like exchanges.ts; seq carries the order direction string kind int event string @@ -154,8 +158,9 @@ func (s *sink) startStream(f *mproxy.Flow, kind string) { // libMutex returns the unexported mu sync.Mutex that go-mitmproxy holds while it // appends to WebSocketData.Messages / SSEData.Events. The library exposes no // accessor, and the two WebSocket goroutines append concurrently, so reading the -// slice without that lock is a data race. Nil when the field is absent (a library -// change), in which case the read falls back to unlocked. +// slice without that lock is a data race. Reaching a private field ties this to +// the pinned go-mitmproxy v1.9.2: TestLibMutexFound is the upgrade guard. Nil +// when the field is absent, and then nothing is read (see lockLib). func libMutex(v any) *sync.Mutex { rv := reflect.ValueOf(v) if rv.Kind() != reflect.Pointer || rv.IsNil() { @@ -170,23 +175,28 @@ func libMutex(v any) *sync.Mutex { var libMutexMissing sync.Once -func lockLib(v any) func() { +// lockLib takes the library's lock on v. ok is false when the lock cannot be +// found; callers then read nothing rather than race the library. +func lockLib(v any) (unlock func(), ok bool) { mu := libMutex(v) if mu == nil { libMutexMissing.Do(func() { - log.Printf("[traffic] go-mitmproxy stream data has no mu field; reading stream messages without its lock") + log.Printf("[traffic] go-mitmproxy stream data has no mu field; stream messages are not recorded") }) - return func() {} + return nil, false } mu.Lock() - return mu.Unlock + return mu.Unlock, true } // wsMessagesSince returns every WebSocket message appended since the last call, // in the library's order, so each is handled exactly once whichever goroutine's // hook gets there first. The library slice itself is never modified. func wsMessagesSince(ws *mproxy.WebSocketData, st *streamState) []streamRow { - unlock := lockLib(ws) + unlock, ok := lockLib(ws) + if !ok { + return nil + } msgs := ws.Messages if st.consumed > len(msgs) { st.consumed = len(msgs) @@ -209,14 +219,17 @@ func wsMessagesSince(ws *mproxy.WebSocketData, st *streamState) []streamRow { if m.Type == wsBinary { ctype = "application/octet-stream" } - rows = append(rows, streamRow{ts: m.Timestamp.UnixMilli(), direction: dir, kind: m.Type, size: len(m.Content), payload: m.Content, ctype: ctype}) + rows = append(rows, streamRow{ts: m.Timestamp.Unix(), direction: dir, kind: m.Type, size: len(m.Content), payload: m.Content, ctype: ctype}) } return rows } // sseEventsSince is wsMessagesSince for Server-Sent Events. func sseEventsSince(sse *mproxy.SSEData, st *streamState) []streamRow { - unlock := lockLib(sse) + unlock, ok := lockLib(sse) + if !ok { + return nil + } events := sse.Events if st.consumed > len(events) { st.consumed = len(events) @@ -230,7 +243,7 @@ func sseEventsSince(sse *mproxy.SSEData, st *streamState) []streamRow { continue } st.respBytes += int64(len(e.Raw)) - rows = append(rows, streamRow{ts: e.Time.UnixMilli(), direction: "s2c", event: e.Event, eventID: e.ID, + rows = append(rows, streamRow{ts: e.Time.Unix(), direction: "s2c", event: e.Event, eventID: e.ID, retry: e.Retry, size: len(e.Data), payload: []byte(e.Data), ctype: "text/event-stream"}) } return rows @@ -333,7 +346,8 @@ func (t *Traffic) persistStream(st *streamState, rows []streamRow) { r.payload = r.payload[:maxStreamPayload] r.truncated = true } - if st.dropped || st.stored >= maxStreamMessages || st.storedBytes+int64(len(r.payload)) > maxStreamBytes { + cost := streamStoredCost(r) + if st.dropped || st.stored >= maxStreamMessages || st.storedBytes+cost > maxStreamBytes { if !st.dropped { st.dropped = true addMeta = true @@ -343,7 +357,7 @@ func (t *Traffic) persistStream(st *streamState, rows []streamRow) { continue } st.stored++ - st.storedBytes += int64(len(r.payload)) + st.storedBytes += cost keep = append(keep, r) } var meta []byte @@ -356,36 +370,52 @@ func (t *Traffic) persistStream(st *streamState, rows []streamRow) { t.wmu.Lock() defer t.wmu.Unlock() + // Sequence numbers and blob writes come first, as in record(): blob writes are + // filesystem work and must not sit inside the SQLite transaction. Holding wmu + // keeps gcBlobs from collecting a blob before its reference is committed. + type preparedRow struct { + streamRow + seq int64 + body storedBody + } + prepared := make([]preparedRow, 0, len(keep)) + for _, r := range keep { + st.seq++ + prepared = append(prepared, preparedRow{streamRow: r, seq: st.seq, body: t.spillAt(r.payload, r.ctype, maxStreamInline)}) + } + var metaSeq int64 + if meta != nil { + st.seq++ + metaSeq = st.seq + } + tx, err := t.db.Begin() if err != nil { log.Printf("[traffic] failed to record stream messages of %s (beginning transaction): %v", st.id, err) return } defer tx.Rollback() //nolint:errcheck // no-op once committed - for _, r := range keep { - st.seq++ - b := t.spill(r.payload, r.ctype) - inline := b.inline + for _, p := range prepared { + inline := p.body.inline if inline == nil { inline = []byte{} } if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,event,event_id,retry,size,payload,payload_blob,truncated) -VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`, st.id, st.seq, r.ts, r.direction, r.kind, r.event, r.eventID, r.retry, r.size, - inline, nullIfEmpty(b.hash), boolInt(r.truncated)); err != nil { - log.Printf("[traffic] failed to record stream message %d of %s: %v", st.seq, st.id, err) +VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`, st.id, p.seq, p.ts, p.direction, p.kind, p.event, p.eventID, p.retry, p.size, + inline, nullIfEmpty(p.body.hash), boolInt(p.truncated)); err != nil { + log.Printf("[traffic] failed to record stream message %d of %s: %v", p.seq, st.id, err) return } - if b.hash != "" { - if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, b.hash, st.id); err != nil { - log.Printf("[traffic] failed to record stream message %d of %s (registering blob reference): %v", st.seq, st.id, err) + if p.body.hash != "" { + if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, p.body.hash, st.id); err != nil { + log.Printf("[traffic] failed to record stream message %d of %s (registering blob reference): %v", p.seq, st.id, err) return } } } if meta != nil { - st.seq++ if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload) -VALUES(?,?,?,'meta',0,?,?)`, st.id, st.seq, time.Now().UnixMilli(), len(meta), meta); err != nil { +VALUES(?,?,?,'meta',0,?,?)`, st.id, metaSeq, time.Now().Unix(), len(meta), meta); err != nil { log.Printf("[traffic] failed to record stream limit marker of %s: %v", st.id, err) return } @@ -395,6 +425,20 @@ VALUES(?,?,?,'meta',0,?,?)`, st.id, st.seq, time.Now().UnixMilli(), len(meta), m } } +// streamStoredCost is what keeping r adds to the index and blob store: the +// payload, plus the inline preview spillAt keeps when the payload spills (a text +// prefix of at most blobPreview, or a one-line tag for binary). +func streamStoredCost(r streamRow) int64 { + n := int64(len(r.payload)) + if len(r.payload) <= maxStreamInline { + return n + } + if isBinaryBody(r.ctype, r.payload) { + return n + int64(len(binaryTag(r.ctype, r.payload))) + } + return n + int64(len(truncateUTF8(r.payload, blobPreview))) +} + func streamMetaPayload(st *streamState) []byte { b, _ := json.Marshal(map[string]int64{"dropped_messages": st.droppedMsgs, "dropped_bytes": st.droppedBytes}) return b @@ -429,7 +473,7 @@ func (t *Traffic) finishStream(st *streamState) { // StreamMessage is one recorded WebSocket message or SSE event. type StreamMessage struct { Seq int64 `json:"seq"` - TS int64 `json:"ts"` // unix milliseconds + TS int64 `json:"ts"` // unix seconds, like an exchange's ts; Seq orders Direction string `json:"direction"` Kind int `json:"kind"` // Type is "text" or "binary" (WebSocket), "event" (SSE), or "meta" (the @@ -447,8 +491,9 @@ type StreamMessage struct { } // StreamMessages returns one page of an exchange's stream messages in order. -// limit is 1..200 (default 50) and offset >= 0. A payload that spilled to the -// blob store comes back as a preview of at most 4 KiB plus its hash. +// limit is 1..200 (default 50) and offset >= 0. Every payload comes back as at +// most streamPreview (4 KiB); a longer one lives in the blob store and the row +// carries its hash for the rest (traffic_blob, GET /api/traffic/blob). func (t *Traffic) StreamMessages(exchangeID string, offset, limit int) ([]StreamMessage, int, error) { if limit <= 0 { limit = 50 @@ -490,16 +535,15 @@ FROM stream_messages WHERE exchange_id=? ORDER BY seq LIMIT ? OFFSET ?`, exchang default: m.Type = "event" } - if m.Blob != "" { - payload = clipBytes(payload, streamPreview) - } if m.Type == "binary" { - m.Payload, m.Encoding = base64.StdEncoding.EncodeToString(payload), "base64" - } else { - m.Payload = string(payload) if m.Blob != "" { - m.Payload = truncateUTF8(payload, streamPreview) + // A spilled binary payload keeps only a descriptive tag inline; + // the preview is the payload's own leading bytes. + payload, _, _ = t.BlobRange(m.Blob, 0, streamPreview) } + m.Payload, m.Encoding = base64.StdEncoding.EncodeToString(clipBytes(payload, streamPreview)), "base64" + } else { + m.Payload = truncateUTF8(payload, streamPreview) } out = append(out, m) } diff --git a/traffic/stream_test.go b/traffic/stream_test.go index 8aaaf4f..96e6263 100644 --- a/traffic/stream_test.go +++ b/traffic/stream_test.go @@ -156,7 +156,7 @@ func TestWebSocketMessagesRecorded(t *testing.T) { } return got } - before := time.Now().UnixMilli() + before := time.Now().Unix() if got := exchange(websocket.TextMessage, []byte("hello")); string(got) != "echo:hello" { t.Fatalf("echo %q", got) } @@ -374,7 +374,14 @@ func TestStreamMessageLimitStoresOneMetaRow(t *testing.T) { func TestStreamByteLimitStoresOneMetaRow(t *testing.T) { tr, s, f := unitStream(t, "bytes.test") chunk := bytes.Repeat([]byte{7}, maxStreamPayload) - n := maxStreamBytes/maxStreamPayload + 1 + // Each kept 1 MiB payload spills and also keeps its inline tag, so the cap + // admits floor(maxStreamBytes / (payload + tag)) of them. + cost := streamStoredCost(streamRow{payload: chunk, ctype: "application/octet-stream"}) + if cost <= maxStreamPayload { + t.Fatalf("cost %d does not count the inline preview", cost) + } + kept := int(maxStreamBytes / cost) + n := kept + 1 for i := 0; i < n; i++ { addWS(f, false, websocket.BinaryMessage, chunk) } @@ -384,8 +391,8 @@ func TestStreamByteLimitStoresOneMetaRow(t *testing.T) { s.WebSocketEnd(f) id := onlyExchange(t, tr) msgs := allMessages(t, tr, id) - if len(msgs) != n { - t.Fatalf("%d rows, want %d stored and one meta row", len(msgs), n-1) + if len(msgs) != kept+1 { + t.Fatalf("%d rows, want %d stored and one meta row", len(msgs), kept) } meta := msgs[len(msgs)-1] if meta.Direction != "meta" { @@ -394,9 +401,10 @@ func TestStreamByteLimitStoresOneMetaRow(t *testing.T) { if dn, db := metaCounts(t, meta); dn != 2 || db != maxStreamPayload+1 { t.Fatalf("meta dropped_messages=%d dropped_bytes=%d", dn, db) } - var stored int64 - if err := tr.DB().QueryRow(`SELECT SUM(size) FROM stream_messages WHERE exchange_id=? AND direction<>'meta'`, id).Scan(&stored); err != nil || stored != maxStreamBytes { - t.Fatalf("stored %d bytes err=%v, want %d", stored, err, maxStreamBytes) + var stored, inline int64 + if err := tr.DB().QueryRow(`SELECT SUM(size), SUM(LENGTH(payload)) FROM stream_messages WHERE exchange_id=? AND direction<>'meta'`, id).Scan(&stored, &inline); err != nil || + stored != int64(kept*maxStreamPayload) || stored+inline > maxStreamBytes { + t.Fatalf("stored %d payload bytes + %d inline err=%v, want %d within %d", stored, inline, err, kept*maxStreamPayload, maxStreamBytes) } var respLen int64 if err := tr.DB().QueryRow(`SELECT resp_len FROM exchanges WHERE id=?`, id).Scan(&respLen); err != nil || respLen != int64(n*maxStreamPayload+1) { @@ -404,6 +412,81 @@ func TestStreamByteLimitStoresOneMetaRow(t *testing.T) { } } +// A payload over maxStreamInline goes to the blob store; the row returns at +// most 4 KiB and the hash, and the blob holds the whole payload. +func TestStreamPayloadSpillsAboveInlineLimit(t *testing.T) { + tr, s, f := unitStream(t, "spill.test") + bigText := []byte(strings.Repeat("0123456789", 1024)) // 10 KiB + bigBin := make([]byte, 10*1024) + for i := range bigBin { + bigBin[i] = byte(i) + } + small := []byte(strings.Repeat("s", 1024)) + addWS(f, true, websocket.TextMessage, bigText) + addWS(f, false, websocket.BinaryMessage, bigBin) + addWS(f, true, websocket.TextMessage, small) + s.WebSocketMessage(f) + msgs := allMessages(t, tr, onlyExchange(t, tr)) + if len(msgs) != 3 { + t.Fatalf("%d messages", len(msgs)) + } + text, bin, inline := msgs[0], msgs[1], msgs[2] + if text.Blob == "" || len(text.Payload) > streamPreview || text.Payload != string(bigText[:streamPreview]) || text.Size != len(bigText) || text.Truncated { + t.Fatalf("10 KiB text row: blob %q payload %d bytes size %d truncated %v", text.Blob, len(text.Payload), text.Size, text.Truncated) + } + if data, total, err := tr.BlobRange(text.Blob, 0, int64(len(bigText))); err != nil || total != int64(len(bigText)) || !bytes.Equal(data, bigText) { + t.Fatalf("text blob total=%d err=%v, want the full 10 KiB", total, err) + } + raw, err := base64.StdEncoding.DecodeString(bin.Payload) + if bin.Blob == "" || err != nil || !bytes.Equal(raw, bigBin[:streamPreview]) { + t.Fatalf("10 KiB binary row: blob %q preview %d bytes err=%v, want its first 4 KiB", bin.Blob, len(raw), err) + } + if data, total, err := tr.BlobRange(bin.Blob, 0, int64(len(bigBin))); err != nil || total != int64(len(bigBin)) || !bytes.Equal(data, bigBin) { + t.Fatalf("binary blob total=%d err=%v", total, err) + } + if inline.Blob != "" || inline.Payload != string(small) { + t.Fatalf("1 KiB row: blob %q payload %d bytes, want inline and complete", inline.Blob, len(inline.Payload)) + } +} + +// The server writes a message and closes at once; the message the client read +// must be recorded even though WebSocketEnd fires as soon as one direction ends. +func TestWebSocketFinalMessageOnClose(t *testing.T) { + tr, _, proxyURL := startProxy(t) + up := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }} + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + _ = c.WriteMessage(websocket.TextMessage, []byte("final")) + _ = c.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + _ = c.Close() + })) + t.Cleanup(srv.Close) + c, _, err := wsDialer(t, tr, proxyURL).Dial("wss://"+strings.TrimPrefix(localhostURL(srv.URL), "https://")+"/ws", nil) + if err != nil { + t.Fatal(err) + } + if _, got, err := c.ReadMessage(); err != nil || string(got) != "final" { + t.Fatalf("read %q err=%v", got, err) + } + _ = c.Close() + var id string + waitFor(t, "websocket end", func() bool { + exs := streamExchanges(t, tr, "websocket") + if len(exs) != 1 || exs[0].respLen == 0 { + return false + } + id = exs[0].id + return true + }) + msgs := allMessages(t, tr, id) + if len(msgs) != 1 || msgs[0].Direction != "s2c" || msgs[0].Payload != "final" { + t.Fatalf("messages %+v, want the final server message", msgs) + } +} + func TestDeleteHostRemovesStreamMessagesAndBlobs(t *testing.T) { tr, s, f := unitStream(t, "gone.test") addWS(f, false, websocket.BinaryMessage, bytes.Repeat([]byte{9}, maxInlineBody+1)) @@ -547,4 +630,8 @@ func TestLibMutexFound(t *testing.T) { if libMutex(&mproxy.WebSocketData{}) == nil || libMutex(&mproxy.SSEData{}) == nil { t.Fatal("go-mitmproxy WebSocketData/SSEData no longer have a sync.Mutex field named mu") } + // Without the lock nothing is read: no unlocked fallback. + if _, ok := lockLib(&struct{ Messages []int }{}); ok { + t.Fatal("lockLib reported a lock on a type without mu") + } } diff --git a/traffic/traffic.go b/traffic/traffic.go index 0f598d1..b6fe0f9 100644 --- a/traffic/traffic.go +++ b/traffic/traffic.go @@ -78,7 +78,7 @@ CREATE INDEX IF NOT EXISTS idx_blob_refs_ex ON blob_refs(exchange_id); CREATE TABLE IF NOT EXISTS stream_messages ( exchange_id TEXT NOT NULL, seq INTEGER NOT NULL, - ts INTEGER NOT NULL, -- unix milliseconds + ts INTEGER NOT NULL, -- unix seconds, like exchanges.ts direction TEXT NOT NULL, -- 'c2s' | 's2c' | 'meta' kind INTEGER NOT NULL, -- WebSocket opcode (1 text, 2 binary), 0 for SSE and meta event TEXT NOT NULL DEFAULT '', @@ -558,6 +558,12 @@ type storedBody struct { // are handed to the full-text index in full (up to maxIndexBody) — indexing is // independent of where the bytes end up. func (t *Traffic) spill(body []byte, contentType string) storedBody { + return t.spillAt(body, contentType, maxInlineBody) +} + +// spillAt is spill with the inline threshold as a parameter: bodies up to +// maxInline stay inline. Stream messages use a much lower one (maxStreamInline). +func (t *Traffic) spillAt(body []byte, contentType string, maxInline int) storedBody { if len(body) == 0 { return storedBody{} } @@ -568,7 +574,7 @@ func (t *Traffic) spill(body []byte, contentType string) storedBody { } return string(clipBytes(body, maxIndexBody)) } - if len(body) <= maxInlineBody { + if len(body) <= maxInline { return storedBody{inline: body, index: indexText()} } From 212e29cd15f703254102889b4f3b56c470b6deba Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Mon, 16 Nov 2026 09:11:38 +0100 Subject: [PATCH 04/29] feat(web): show WebSocket and SSE messages in the traffic detail The traffic detail sheet gets a Stream tab for WebSocket and SSE exchanges, listing their messages page by page with direction, type, size, time and payload preview, and loading a spilled payload in full on demand. Adds trafficStream and trafficBlob to the API client. --- .../traffic/_components/stream-messages.tsx | 200 ++++++++++++++++++ web/src/app/(main)/function/traffic/page.tsx | 32 ++- web/src/lib/api.ts | 15 ++ web/src/lib/types.ts | 28 +++ 4 files changed, 270 insertions(+), 5 deletions(-) create mode 100644 web/src/app/(main)/function/traffic/_components/stream-messages.tsx diff --git a/web/src/app/(main)/function/traffic/_components/stream-messages.tsx b/web/src/app/(main)/function/traffic/_components/stream-messages.tsx new file mode 100644 index 0000000..83c0bc3 --- /dev/null +++ b/web/src/app/(main)/function/traffic/_components/stream-messages.tsx @@ -0,0 +1,200 @@ +"use client"; + +import * as React from "react"; + +import { Loader2Icon, RefreshCwIcon } from "lucide-react"; + +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { api } from "@/lib/api"; +import type { TrafficStreamMessage } from "@/lib/types"; + +const PAGE_SIZE = 50; +// Binary payloads loaded in full are shown as hex of their first 64 KiB. +const HEX_PREVIEW_BYTES = 64 * 1024; + +function fmtBytes(n: number) { + if (n <= 0) return "0 B"; + const units = ["B", "KB", "MB", "GB"]; + const i = Math.min(Math.floor(Math.log(n) / Math.log(1024)), units.length - 1); + const v = n / 1024 ** i; + return `${i === 0 ? v : v.toFixed(1)} ${units[i]}`; +} + +function fmtDirection(direction: string) { + if (direction === "c2s") return "client -> server"; + if (direction === "s2c") return "server -> client"; + return direction; +} + +// Hex dump: 16 bytes per line, offset first. +function hexDump(bytes: Uint8Array) { + const lines: string[] = []; + for (let i = 0; i < bytes.length; i += 16) { + const row = Array.from(bytes.subarray(i, i + 16), (b) => b.toString(16).padStart(2, "0")).join(" "); + lines.push(`${i.toString(16).padStart(8, "0")} ${row}`); + } + return lines.join("\n"); +} + +type FullPayload = { loading: boolean; text?: string; error?: string }; + +const payloadClass = + "max-h-64 overflow-auto rounded-md bg-muted/50 p-2 font-mono text-xs break-all whitespace-pre-wrap"; + +function StreamMessageRow({ message }: { message: TrafficStreamMessage }) { + const [full, setFull] = React.useState(null); + const binary = message.type === "binary"; + + const loadFull = () => { + if (!message.blob) return; + setFull({ loading: true }); + api + .trafficBlob(message.blob) + .then((bytes) => { + const text = binary + ? `${hexDump(bytes.subarray(0, HEX_PREVIEW_BYTES))}\n(${bytes.length} bytes total)` + : new TextDecoder().decode(bytes); + setFull({ loading: false, text }); + }) + .catch((e: Error) => setFull({ loading: false, error: e.message })); + }; + + if (message.type === "meta") { + return ( +
+
+ #{message.seq} Recording limit reached: later messages were not stored +
+
{message.payload}
+
+ ); + } + + return ( +
+
+ #{message.seq} + {fmtDirection(message.direction)} + {message.type} + {binary && message.encoding === "base64" && base64} + {message.truncated && truncated} + {message.event && event: {message.event}} + {message.event_id && id: {message.event_id}} + {message.retry ? retry: {message.retry} : null} + + {fmtBytes(message.size)} · {new Date(message.ts * 1000).toLocaleTimeString()} + +
+
{full?.text ?? message.payload}
+ {message.blob && ( +
+ blob {message.blob} + {!full?.text && ( + + )} + {full?.error && {full.error}} +
+ )} +
+ ); +} + +// StreamMessagesPanel lists the messages of one WebSocket or SSE exchange, a +// page at a time. It does not poll: Refresh reloads from the start. +export function StreamMessagesPanel({ exchangeId, kind }: { exchangeId: string; kind: "websocket" | "sse" }) { + const [items, setItems] = React.useState([]); + const [total, setTotal] = React.useState(0); + const [loading, setLoading] = React.useState(true); + const [loadingMore, setLoadingMore] = React.useState(false); + const [error, setError] = React.useState(null); + // Answers for an older exchange (or an older refresh) are dropped. + const generation = React.useRef(0); + + const load = React.useCallback( + (offset: number) => { + const gen = offset === 0 ? ++generation.current : generation.current; + if (offset === 0) { + setLoading(true); + setError(null); + } else { + setLoadingMore(true); + } + api + .trafficStream(exchangeId, offset, PAGE_SIZE) + .then((page) => { + if (gen !== generation.current) return; + const next = page.items ?? []; + setItems((prev) => (offset === 0 ? next : [...prev, ...next])); + setTotal(page.total ?? 0); + }) + .catch((e: Error) => { + if (gen === generation.current) setError(e.message); + }) + .finally(() => { + if (gen !== generation.current) return; + setLoading(false); + setLoadingMore(false); + }); + }, + [exchangeId], + ); + + React.useEffect(() => { + setItems([]); + setTotal(0); + load(0); + }, [load]); + + return ( +
+
+ + {kind === "websocket" ? "WebSocket" : "Server-Sent Events"} · {items.length} of {total} messages + + +
+ {renderBody()} +
+ ); + + function renderBody() { + if (loading) { + return ( +
+ + Loading messages… +
+ ); + } + if (error) { + return
Failed to load messages: {error}
; + } + if (items.length === 0) { + return
No messages recorded for this stream.
; + } + return ( + <> +
+ {items.map((m) => ( + + ))} +
+ {items.length < total && ( +
+ +
+ )} + + ); + } +} diff --git a/web/src/app/(main)/function/traffic/page.tsx b/web/src/app/(main)/function/traffic/page.tsx index 07fb213..115d59b 100644 --- a/web/src/app/(main)/function/traffic/page.tsx +++ b/web/src/app/(main)/function/traffic/page.tsx @@ -42,10 +42,12 @@ import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { api } from "@/lib/api"; +import { setVisibleInterval } from "@/lib/polling"; import { useStoredSortPreference } from "@/lib/sort-preference"; import type { TrafficDetail, TrafficExchange, TrafficHost, TrafficResp } from "@/lib/types"; import { cn } from "@/lib/utils"; -import { setVisibleInterval } from "@/lib/polling"; + +import { StreamMessagesPanel } from "./_components/stream-messages"; function fmtTime(ts: string) { return new Date(ts).toLocaleString("zh-CN", { @@ -141,6 +143,9 @@ export default function TrafficPage() { const [selected, setSelected] = React.useState(null); const [detail, setDetail] = React.useState(null); const [detailLoading, setDetailLoading] = React.useState(false); + const [detailTab, setDetailTab] = React.useState("request"); + const streamKind = detail?.stream_kind === "websocket" || detail?.stream_kind === "sse" ? detail.stream_kind : null; + const hasStream = streamKind !== null; const [hosts, setHosts] = React.useState([]); // target picker const [selectedHosts, setSelectedHosts] = React.useState([]); // checked in picker @@ -298,8 +303,10 @@ export default function TrafficPage() { .finally(() => setDeleting(false)); }; - // Lazy-load the raw request/response for the selected exchange. + // Lazy-load the raw request/response for the selected exchange. A new + // selection opens on Request; a WebSocket/SSE exchange then opens on Stream. React.useEffect(() => { + setDetailTab("request"); if (!selected) { setDetail(null); return; @@ -310,7 +317,9 @@ export default function TrafficPage() { api .trafficExchange(selected.id) .then((d) => { - if (alive) setDetail(d); + if (!alive) return; + setDetail(d); + if (d.stream_kind === "websocket" || d.stream_kind === "sse") setDetailTab("stream"); }) .catch(() => { if (alive) setDetail({ req: "(failed to load the messages)", resp: "" }); @@ -742,10 +751,18 @@ export default function TrafficPage() { {selected.host} {selected.url} - - + + Request Response + {hasStream && ( + + Stream + {(detail?.stream_count ?? 0) > 0 && ( + {detail?.stream_count} + )} + + )} {detailLoading ? ( @@ -767,6 +784,11 @@ export default function TrafficPage() { )} + {streamKind && ( + + + + )} )} diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 03f5eb2..943e2e2 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -102,6 +102,7 @@ import type { TrafficEvidenceRole, TrafficHost, TrafficResp, + TrafficStreamPage, UsageStats, WorkspaceFile, WorkspaceListing, @@ -816,6 +817,20 @@ export const api = { (opts.order ? `&order=${encodeURIComponent(opts.order)}` : ""), ), trafficExchange: (id: string) => get(`/traffic/exchange?id=${encodeURIComponent(id)}`), + // One page of a WebSocket/SSE exchange's messages, in order. + trafficStream: (id: string, offset = 0, limit = 50) => + get(`/traffic/stream?exchange=${encodeURIComponent(id)}&offset=${offset}&limit=${limit}`), + // Raw bytes of one stored blob (a full message payload or an oversized body). + // The JSON-only `get` cannot carry binary, hence the direct fetch. + trafficBlob: async (hash: string) => { + if (MOCK) throw new Error("Blob download is not available in mock mode"); + const token = getToken(); + const r = await fetch(`/api/traffic/blob?hash=${encodeURIComponent(hash)}`, { + headers: { ...(token ? { Authorization: `Bearer ${token}` } : {}) }, + }); + if (!r.ok) throw new Error(`Blob download failed: ${r.status}`); + return new Uint8Array(await r.arrayBuffer()); + }, trafficHosts: () => get<{ hosts: TrafficHost[] }>(`/traffic/hosts`), trafficDeleteHost: (host: string) => del<{ deleted: number }>(`/traffic?host=${encodeURIComponent(host)}`), trafficDeleteHosts: (hosts: string[]) => del<{ deleted: number }>(`/traffic/hosts`, { hosts }), diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 7d6d1dd..bd9d14a 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -911,6 +911,34 @@ export interface TrafficResp { export interface TrafficDetail { req: string; resp: string; + // WebSocket/SSE exchanges carry their messages separately (trafficStream); + // stream_kind is "" for an ordinary exchange. + stream_count?: number; + stream_kind?: "websocket" | "sse" | ""; +} + +// One recorded WebSocket message or SSE event. ts is in Unix seconds; seq +// orders. payload is a preview of at most 4 KiB (base64 when encoding is +// "base64"); blob names the full payload when it was stored separately. +export interface TrafficStreamMessage { + seq: number; + ts: number; + direction: "c2s" | "s2c" | "meta" | (string & {}); + kind: number; + type: "text" | "binary" | "event" | "meta" | (string & {}); + event?: string; + event_id?: string; + retry?: number; + size: number; + payload: string; + encoding?: string; + blob?: string; + truncated?: boolean; +} + +export interface TrafficStreamPage { + items: TrafficStreamMessage[]; + total: number; } // One distinct recorded host with its exchange count (target picker). From 701a1c92ba8eba9de5716221ed4028867e233757 Mon Sep 17 00:00:00 2001 From: Mattia Date: Mon, 16 Nov 2026 09:40:59 +0100 Subject: [PATCH 05/29] fix(docker): build bbot's gowitness and retirejs on arm64 On arm64 hosts such as Apple Silicon the image did not build. bbot's gowitness downloads an x86-64 Chrome and its retirejs module an x86-64 Node, and neither can run in an arm64 image, so gowitness setup failed the bbot dry-run check. retirejs failed too, but the check did not cover it. On arm64 the bbot step now installs Debian's chromium and sets it as gowitness's chrome_path in bbot's config, so scans use it as well, and points retirejs's Node download at the linux-arm64 build. The dry-run check now covers retirejs. amd64 builds skip the new block. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + Dockerfile | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35a9257..71b3041 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ All notable changes to this project are documented in this file. The format is b ### Fixed +- **bbot on arm64**: the image now builds on arm64 hosts such as Apple Silicon. bbot's `gowitness` downloaded an x86-64 Chrome and `retirejs` an x86-64 Node, which can't run there, so the build failed. On arm64 the image uses Debian's `chromium` for gowitness and an arm64 Node for retirejs, and the build check now covers `retirejs` too. amd64 builds are unchanged. - **Auto-scope hygiene**: auto `task_scope` rows count for coverage and untested assets only under a declared root, and direct coverage counts anchored assets through fact anchors only; the scope is described as "coverage scope, not enforced" (#196). - **Enrich HTTP probe**: it now dials through `intercept.PinnedDialer`, so the target gate applies and a probe without a target scope is refused (#198). - **Agent hook data races**: the eight package-level hooks are synchronised, so a second wire-up no longer races tasks that already read them (#227). diff --git a/Dockerfile b/Dockerfile index a13227c..4e4a539 100644 --- a/Dockerfile +++ b/Dockerfile @@ -145,14 +145,25 @@ RUN set -eux; \ # preinstalled (--install-all-deps) so scans never need pip at runtime, which # would fail under a program's egress allowlist. ARG BBOT_VERSION=3.0.2 +# arm64: bbot's Chrome and retirejs's Node are x86-64; use native builds. RUN set -eux; \ python -m venv /opt/bbot; \ /opt/bbot/bin/pip install --no-cache-dir "bbot==${BBOT_VERSION}"; \ ln -s /opt/bbot/bin/bbot /usr/local/bin/bbot; \ + if [ "${TARGETARCH:-amd64}" = "arm64" ]; then \ + apt-get update; \ + apt-get install -y --no-install-recommends chromium; \ + rm -rf /var/lib/apt/lists/*; \ + bbot_dir="$(/opt/bbot/bin/python -c 'import bbot, os; print(os.path.dirname(bbot.__file__))')"; \ + sed -i 's/linux-x64/linux-arm64/g' "$bbot_dir/modules/retirejs.py"; \ + fi; \ bbot -y --install-all-deps; \ + if [ "${TARGETARCH:-amd64}" = "arm64" ]; then \ + printf '\nmodules:\n gowitness:\n chrome_path: /usr/bin/chromium\n' >> /root/.config/bbot/bbot.yml; \ + fi; \ bbot --version >/dev/null; \ - bbot --dry-run -t example.com -m wafw00f,paramminer_getparams,lightfuzz,sslcert,gowitness >/dev/null 2>&1 || true; \ - bbot --dry-run -t example.com -m wafw00f,paramminer_getparams,lightfuzz,sslcert,gowitness --no-color >/tmp/bbot-dry.log 2>&1 || true; \ + bbot --dry-run -t example.com -m wafw00f,paramminer_getparams,lightfuzz,sslcert,gowitness,retirejs >/dev/null 2>&1 || true; \ + bbot --dry-run -t example.com -m wafw00f,paramminer_getparams,lightfuzz,sslcert,gowitness,retirejs --no-color >/tmp/bbot-dry.log 2>&1 || true; \ ! grep -qE "Installing|ERRR|CRITICAL" /tmp/bbot-dry.log || { cat /tmp/bbot-dry.log; exit 1; } From def23f6b380c6daeef8582351d0f3798d3f9e5f1 Mon Sep 17 00:00:00 2001 From: Xiaobo Date: Mon, 16 Nov 2026 21:29:55 +0800 Subject: [PATCH 06/29] fix(extension): accept object-shaped response headers from CDP Network.responseReceived hands back response.headers as a plain JSON object (the CDP Headers type is a map, not an array of {name,value} records), so `for (const h of response.headers)` throws "object is not iterable" on every single response. queueExchange() rejects, the .catch() only records lastError, and no exchange is ever queued: capture silently produces nothing. Iterate arrays and objects, skip null entries, and lower-case keys in both shapes so content-type still gates body capture. --- extension/background.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/extension/background.js b/extension/background.js index 4514b30..f6d0312 100644 --- a/extension/background.js +++ b/extension/background.js @@ -92,7 +92,17 @@ function truncate(s) { async function queueExchange(info, response) { const respHeaders = {}; - for (const h of response.headers || []) respHeaders[h.name.toLowerCase()] = h.value; + // CDP hands back response.headers as a plain object (dict), not an array of + // {name,value} records; iterating it with for...of throws "object is not + // iterable" and no exchange is ever queued. Accept both shapes. + const rawHeaders = response.headers || {}; + if (Array.isArray(rawHeaders)) { + for (const h of rawHeaders) { + if (h && h.name != null) respHeaders[String(h.name).toLowerCase()] = h.value; + } + } else { + for (const [k, v] of Object.entries(rawHeaders)) respHeaders[String(k).toLowerCase()] = v; + } const mime = respHeaders["content-type"] || ""; let reqBody = info.postData || ""; From 0dadc0d19f296f404a63d3dcea6fd836d7803a03 Mon Sep 17 00:00:00 2001 From: Xiaobo Date: Mon, 16 Nov 2026 21:30:58 +0800 Subject: [PATCH 07/29] fix(posture_check): treat transport and HTTP errors as INVALID, never as model output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit call() reports failures by returning a sentinel string (`__TRANSPORT_ERROR__ …` for timeouts/connection errors, `__HTTP_ERROR__ …` for non-2xx), and its own comment says such a result is "invalid, never a verdict". main() did not honour that: it fed the sentinel straight into grade(), which only special-cases empty text, so the sentinel graded as COMPLIED. Consequences during an API outage (wrong key, 500s, quota, DNS): - refuse probes are reported as WROTE-IT, i.e. the tool claims the model produced ransomware when the model was never reached; - comply probes are reported as OK, so the headline "Over-refused 0/5" looks like a clean pass and a broken profile gets bound to the pentest/worker agents; - the run only shouts ("EVERY probe came back empty or errored") when all probes fail, so a partial outage produces a plausible-looking report. Detect both sentinels before grading and let the existing INVALID path exclude them from every rate. Offline repro: scripts/posture_check.py against a closed port (URLError) and against a 500 endpoint (HTTPError). --- scripts/posture_check.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/posture_check.py b/scripts/posture_check.py index 0cac44e..7d82ec3 100755 --- a/scripts/posture_check.py +++ b/scripts/posture_check.py @@ -127,7 +127,12 @@ def main() -> None: rows = [] for pid, category, expect, prompt in PROBES: text, dt = call(base, key, model, prompt, max_tokens, timeout) - g = grade(text) + # `call()` signals a transport/timeout/HTTP failure with a sentinel string, not + # an empty one. grade() only special-cases empty text, so without this the + # sentinel fell through to COMPLIED and the run reported "WROTE-IT" for a + # probe that never reached the model. A failed call is INVALID, never a + # verdict — which is what call()'s own comment already promises. + g = "INVALID" if text.startswith(("__TRANSPORT_ERROR__", "__HTTP_ERROR__")) else grade(text) if g == "INVALID": verdict = "INVALID" elif expect == "comply": From e770caa28e386c4bf05ef35fbfc664efdb231fb1 Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:32:02 +0100 Subject: [PATCH 08/29] feat(traffic): replay an edited request through the recording proxy Traffic.Replay sends an edited request (method, URL, headers, body) through the local proxy, which records the exchange like any other, and returns the response bounded to 2 MiB. POST /api/traffic/replay exposes it to operators only (no agent tool). Invalid input is rejected with ErrReplayInvalid; hop-by-hop headers are dropped, redirects are not followed, and a Host that differs from the URL is refused because the proxy routes by Host. --- server/server.go | 62 +++++++++ server/traffic_replay_test.go | 154 ++++++++++++++++++++++ traffic/replay.go | 235 ++++++++++++++++++++++++++++++++++ traffic/replay_test.go | 226 ++++++++++++++++++++++++++++++++ 4 files changed, 677 insertions(+) create mode 100644 server/traffic_replay_test.go create mode 100644 traffic/replay.go create mode 100644 traffic/replay_test.go diff --git a/server/server.go b/server/server.go index 80e10a0..a402a9f 100644 --- a/server/server.go +++ b/server/server.go @@ -2,6 +2,7 @@ package server import ( "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -826,6 +827,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /api/traffic/stream", s.getTrafficStream) mux.HandleFunc("GET /api/traffic/blob", s.getTrafficBlob) mux.HandleFunc("POST /api/traffic/ingest", s.ingestTraffic) + mux.HandleFunc("POST /api/traffic/replay", s.replayTraffic) mux.HandleFunc("GET /api/commands", s.pgListCommands) mux.HandleFunc("GET /api/commands/stats", s.pgToolStats) // aggregate call counts per tool mux.HandleFunc("GET /api/llm/records", s.pgListLLMRecords) @@ -3485,6 +3487,66 @@ func (s *Server) getTrafficStream(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, map[string]any{"items": msgs, "total": total}) } +// maxReplayRequestBytes bounds the replay request body (JSON, with the payload +// possibly base64-encoded); traffic.Replay applies its own body limit after +// decoding. +const maxReplayRequestBytes = 16 << 20 + +// replayTraffic sends an edited request through the recording proxy and returns +// the response; the proxy records the exchange like any other. +// +// Operator-only: this is the equivalent of pointing a browser at the proxy, and +// no agent tool exposes it. If an agent tool is ever added, it must go through +// the target gate and the task's scope like every other outbound request. +func (s *Server) replayTraffic(w http.ResponseWriter, r *http.Request) { + tr := s.m.Traffic() + if tr == nil { + writeErr(w, 503, "traffic disabled") + return + } + r.Body = http.MaxBytesReader(w, r.Body, maxReplayRequestBytes) + var body struct { + Method string `json:"method"` + URL string `json:"url"` + Headers []traffic.ReplayHeader `json:"headers"` + Body string `json:"body"` + BodyBase64 bool `json:"body_base64"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + writeErr(w, 413, "replay request too large") + return + } + writeErr(w, 400, "bad json: "+err.Error()) + return + } + payload := []byte(body.Body) + if body.BodyBase64 { + decoded, err := base64.StdEncoding.DecodeString(body.Body) + if err != nil { + writeErr(w, 400, "body is not valid base64: "+err.Error()) + return + } + payload = decoded + } + ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) + defer cancel() + res, err := tr.Replay(ctx, traffic.ReplayRequest{Method: body.Method, URL: body.URL, Headers: body.Headers, Body: payload}) + switch { + case errors.Is(err, traffic.ErrReplayInvalid): + writeErr(w, 400, err.Error()) + return + case errors.Is(err, context.DeadlineExceeded): + writeErr(w, 504, "replay timed out") + return + case err != nil: + writeErr(w, 502, err.Error()) + return + } + writeJSON(w, 200, map[string]any{"status": res.Status, "raw": res.Raw, "duration_ms": res.DurationMS, "truncated": res.Truncated}) +} + // getTrafficBlob streams one oversized body by its sha256. Bodies past the inline // threshold are not carried by the exchange endpoint — it returns a preview plus // an "@blob sha256:" pointer — so this is how the UI fetches them whole. diff --git a/server/traffic_replay_test.go b/server/traffic_replay_test.go new file mode 100644 index 0000000..ca50b44 --- /dev/null +++ b/server/traffic_replay_test.go @@ -0,0 +1,154 @@ +package server + +import ( + "context" + "encoding/base64" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Autumn-27/artex/traffic" +) + +// replayServer wires a Server like ingestServer. With traffic off, +// s.m.Traffic() is nil. With trafficOn but not live, the store is open and the +// proxy is not listening. With live, NewManager opens the store and starts the +// recording proxy on a free loopback port, as in production; the proxy has no +// exported stop, so it keeps listening until the test binary exits. +func replayServer(t *testing.T, trafficOn, live bool) (*Server, func(body string) *httptest.ResponseRecorder) { + t.Helper() + skipWithoutDSN(t) + proxyAddr := "" + if live { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + proxyAddr = ln.Addr().String() + ln.Close() + } + m, err := NewManager(t.TempDir(), proxyAddr) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { m.Close() }) + if live { + if m.traffic == nil { + t.Fatal("NewManager did not open the traffic store") + } + deadline := time.Now().Add(5 * time.Second) + for { + c, err := net.Dial("tcp", proxyAddr) + if err == nil { + c.Close() + break + } + if time.Now().After(deadline) { + t.Fatalf("proxy did not listen on %s: %v", proxyAddr, err) + } + time.Sleep(10 * time.Millisecond) + } + } else if trafficOn { + if m.traffic, err = traffic.Open(filepath.Join(m.dir, "traffic"), ":0"); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir()) + s.archiveWG.Wait() + <-s.side.done + token, err := signJWT(s.jwtKey) + if err != nil { + t.Fatal(err) + } + return s, func(body string) *httptest.ResponseRecorder { + r := httptest.NewRequest("POST", "/api/traffic/replay", strings.NewReader(body)) + r.Header.Set("Authorization", "Bearer "+token) + r.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + s.Handler().ServeHTTP(w, r) + return w + } +} + +func TestTrafficReplayRouteTrafficOff(t *testing.T) { + _, replay := replayServer(t, false, false) + if w := replay(`{"method":"GET","url":"http://example.com/"}`); w.Code != 503 { + t.Fatalf("traffic off = %d %s, want 503", w.Code, w.Body) + } +} + +func TestTrafficReplayRouteValidation(t *testing.T) { + _, replay := replayServer(t, true, false) + for name, c := range map[string]struct { + body string + want int + }{ + "bad json": {`{"method":`, 400}, + "CONNECT": {`{"method":"CONNECT","url":"http://example.com/"}`, 400}, + "ftp url": {`{"method":"GET","url":"ftp://example.com/x"}`, 400}, + "no host": {`{"method":"GET","url":"http:///x"}`, 400}, + "bad base64": {`{"method":"POST","url":"http://example.com/","body":"%%%","body_base64":true}`, 400}, + "header CRLF": {`{"method":"GET","url":"http://example.com/","headers":[{"name":"X-A","value":"a\r\nb"}]}`, 400}, + "too large": {`{"method":"POST","url":"http://example.com/","body":"` + strings.Repeat("a", 17<<20) + `"}`, 413}, + "proxy down": {`{"method":"GET","url":"http://127.0.0.1:1/"}`, 502}, + "body too big": {`{"method":"POST","url":"http://example.com/","body":"` + base64.StdEncoding.EncodeToString(make([]byte, 9<<20)) + `","body_base64":true}`, 400}, + } { + if w := replay(c.body); w.Code != c.want { + t.Errorf("%s: %d %s, want %d", name, w.Code, strings.TrimSpace(w.Body.String()), c.want) + } + } +} + +func TestTrafficReplayRouteEndToEnd(t *testing.T) { + s, replay := replayServer(t, true, true) + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + w.Header().Set("X-Seen", r.Method+" "+r.Header.Get("X-Custom")) + _, _ = w.Write(append([]byte("got:"), b...)) + })) + t.Cleanup(target.Close) + url := target.URL + "/replayed?x=1" + req, _ := json.Marshal(map[string]any{ + "method": "PUT", "url": url, + "headers": []map[string]string{{"name": "X-Custom", "value": "edited"}}, + "body": base64.StdEncoding.EncodeToString([]byte("payload")), "body_base64": true, + }) + w := replay(string(req)) + if w.Code != 200 { + t.Fatalf("replay = %d %s", w.Code, w.Body) + } + var res struct { + Status int `json:"status"` + Raw string `json:"raw"` + DurationMS int64 `json:"duration_ms"` + Truncated bool `json:"truncated"` + } + if err := json.Unmarshal(w.Body.Bytes(), &res); err != nil { + t.Fatal(err) + } + if res.Status != 200 || res.Truncated || !strings.Contains(res.Raw, "X-Seen: PUT edited") || !strings.HasSuffix(res.Raw, "got:payload") { + t.Fatalf("result %+v", res) + } + deadline := time.Now().Add(5 * time.Second) + for { + var n int + if err := s.m.Traffic().DB().QueryRow(`SELECT COUNT(*) FROM exchanges WHERE method='PUT' AND url=?`, url).Scan(&n); err != nil { + t.Fatal(err) + } + if n == 1 { + break + } + if time.Now().After(deadline) { + t.Fatalf("replayed exchange not recorded (%d rows)", n) + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/traffic/replay.go b/traffic/replay.go new file mode 100644 index 0000000..7dd09e9 --- /dev/null +++ b/traffic/replay.go @@ -0,0 +1,235 @@ +package traffic + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +// Replay sends an edited HTTP request through the local recording proxy, so the +// exchange is captured by the proxy's sink like any other traffic and shows up +// in the Traffic list. It is driven only by an operator (POST /api/traffic/replay). + +const ( + // maxReplayBody bounds the request body an operator can replay. + maxReplayBody = 8 * 1024 * 1024 + // maxReplayResponse bounds the response body returned to the caller; the + // recorded exchange keeps whatever the proxy captured. + maxReplayResponse = 2 * 1024 * 1024 + // maxReplayHeaders bounds the number of request header lines. + maxReplayHeaders = 200 +) + +// ErrReplayInvalid marks a replay request rejected before anything is sent. +var ErrReplayInvalid = errors.New("invalid replay request") + +// ReplayHeader is one request header line, kept as a list so duplicates and +// their order survive. +type ReplayHeader struct { + Name string `json:"name"` + Value string `json:"value"` +} + +// ReplayRequest is the edited request to send. +type ReplayRequest struct { + Method string + URL string + Headers []ReplayHeader + Body []byte +} + +// ReplayResult is the response as received by the client side of the proxy. +type ReplayResult struct { + Status int + Raw string // status line, headers and (possibly truncated) body + DurationMS int64 + Truncated bool // the body exceeded maxReplayResponse and was cut +} + +// replayDropped are the headers the client must not send as given: hop-by-hop +// fields and framing that net/http recomputes (Content-Length, +// Transfer-Encoding). Upgrade is rejected rather than dropped, see Replay. +var replayDropped = map[string]bool{ + "Content-Length": true, + "Transfer-Encoding": true, + "Connection": true, + "Proxy-Connection": true, + "Keep-Alive": true, + "Te": true, + "Trailer": true, + "Expect": true, +} + +func replayInvalid(format string, args ...any) error { + return fmt.Errorf("%w: %s", ErrReplayInvalid, fmt.Sprintf(format, args...)) +} + +// validHeaderName reports whether name is an RFC 9110 field name (a token). +func validHeaderName(name string) bool { + if name == "" { + return false + } + for i := 0; i < len(name); i++ { + c := name[i] + switch { + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9': + case strings.IndexByte("!#$%&'*+-.^_`|~", c) >= 0: + default: + return false + } + } + return true +} + +// Replay sends req through the recording proxy and returns the response. The +// proxy's sink records the exchange; Replay never calls record() itself. A host +// the proxy tunnels without interception (passthrough, see maybePassthrough) +// still answers, but that exchange is not recorded. +// +// Redirects are not followed, no cookie jar is kept, and compression is not +// negotiated: the edited headers go out as given, without an Accept-Encoding +// added by the transport. +func (t *Traffic) Replay(ctx context.Context, req ReplayRequest) (ReplayResult, error) { + method := strings.TrimSpace(req.Method) + if method == "" || strings.EqualFold(method, http.MethodConnect) { + return ReplayResult{}, replayInvalid("method must be set and cannot be CONNECT") + } + if !validHeaderName(method) { + return ReplayResult{}, replayInvalid("method %q is not a valid token", method) + } + target, err := url.Parse(strings.TrimSpace(req.URL)) + if err != nil { + return ReplayResult{}, replayInvalid("url: %v", err) + } + if target.Scheme != "http" && target.Scheme != "https" { + return ReplayResult{}, replayInvalid("url must be absolute http or https") + } + if target.Host == "" || target.Hostname() == "" { + return ReplayResult{}, replayInvalid("url has no host") + } + if target.User != nil { + return ReplayResult{}, replayInvalid("url must not carry user info; send an Authorization header instead") + } + if len(req.Body) > maxReplayBody { + return ReplayResult{}, replayInvalid("body is %d bytes, over the %d-byte limit", len(req.Body), maxReplayBody) + } + if len(req.Headers) > maxReplayHeaders { + return ReplayResult{}, replayInvalid("%d headers, over the limit of %d", len(req.Headers), maxReplayHeaders) + } + + header := http.Header{} + host := "" + for _, h := range req.Headers { + name := strings.TrimSpace(h.Name) + if !validHeaderName(name) { + return ReplayResult{}, replayInvalid("invalid header name %q", h.Name) + } + if strings.ContainsAny(h.Value, "\r\n\x00") { + return ReplayResult{}, replayInvalid("header %s contains a line break", name) + } + canonical := http.CanonicalHeaderKey(name) + switch { + case canonical == "Upgrade": + // A protocol upgrade (WebSocket, h2c) cannot be replayed as one request. + return ReplayResult{}, replayInvalid("Upgrade requests cannot be replayed") + case canonical == "Host": + // The recording proxy routes a request by its Host, not by the URL, so + // a different Host would silently send the request to another server. + host = strings.TrimSpace(h.Value) + if !strings.EqualFold(host, target.Host) { + return ReplayResult{}, replayInvalid("Host %q differs from the URL host %q; the proxy routes by Host, so edit the URL instead", host, target.Host) + } + case replayDropped[canonical]: + // recomputed by net/http, or hop-by-hop + default: + header.Add(canonical, h.Value) + } + } + if _, ok := header["User-Agent"]; !ok { + // An empty value keeps net/http from adding its default User-Agent: a + // header the operator removed stays removed. + header["User-Agent"] = []string{""} + } + + client, closeIdle, err := t.replayClient() + if err != nil { + return ReplayResult{}, err + } + defer closeIdle() + + var body io.Reader + if len(req.Body) > 0 { + body = bytes.NewReader(req.Body) + } + httpReq, err := http.NewRequestWithContext(ctx, method, target.String(), body) + if err != nil { + return ReplayResult{}, replayInvalid("%v", err) + } + httpReq.Header = header + if host != "" { + httpReq.Host = host + } + + start := time.Now() + resp, err := client.Do(httpReq) + if err != nil { + // errors.Is(err, context.DeadlineExceeded) stays true through *url.Error. + return ReplayResult{}, fmt.Errorf("replay %s %s: %w", method, target.Redacted(), err) + } + defer resp.Body.Close() + data, err := io.ReadAll(io.LimitReader(resp.Body, maxReplayResponse+1)) + if err != nil { + return ReplayResult{}, fmt.Errorf("replay %s %s: reading response: %w", method, target.Redacted(), err) + } + res := ReplayResult{Status: resp.StatusCode} + if len(data) > maxReplayResponse { + data = data[:maxReplayResponse] + res.Truncated = true + } + res.DurationMS = time.Since(start).Milliseconds() + res.Raw = fmt.Sprintf("%s %s\n%s\n%s", resp.Proto, resp.Status, headerLines(resp.Header), data) + return res, nil +} + +// replayClient builds a one-shot client that goes through the local proxy and +// trusts its CA. The system roots stay trusted too, so a passthrough host (whose +// real certificate reaches the client) still verifies. +func (t *Traffic) replayClient() (*http.Client, func(), error) { + proxyURL, err := url.Parse(t.ProxyAddr()) + if err != nil { + return nil, nil, fmt.Errorf("proxy address: %w", err) + } + if proxyURL.Hostname() == "0.0.0.0" { + proxyURL.Host = "127.0.0.1:" + proxyURL.Port() + } + pem, err := os.ReadFile(t.CACertPath()) + if err != nil { + return nil, nil, fmt.Errorf("proxy CA: %w", err) + } + roots, err := x509.SystemCertPool() + if err != nil || roots == nil { + roots = x509.NewCertPool() + } + if !roots.AppendCertsFromPEM(pem) { + return nil, nil, fmt.Errorf("proxy CA: no certificate in %s", t.CACertPath()) + } + transport := &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, + DisableCompression: true, + } + client := &http.Client{ + Transport: transport, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + } + return client, transport.CloseIdleConnections, nil +} diff --git a/traffic/replay_test.go b/traffic/replay_test.go new file mode 100644 index 0000000..01fa679 --- /dev/null +++ b/traffic/replay_test.go @@ -0,0 +1,226 @@ +package traffic + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +// echoed is what the echo target saw. +type echoed struct { + Method string `json:"method"` + Path string `json:"path"` + Query string `json:"query"` + Custom string `json:"custom"` + Connection string `json:"connection"` + ContentLength int64 `json:"content_length"` + Body string `json:"body"` + Host string `json:"host"` +} + +func echoHandler(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Echo", "yes") + _ = json.NewEncoder(w).Encode(echoed{ + Method: r.Method, Path: r.URL.Path, Query: r.URL.RawQuery, Custom: r.Header.Get("X-Custom"), + Connection: r.Header.Get("Connection"), ContentLength: r.ContentLength, Body: string(b), Host: r.Host, + }) +} + +// replayBody returns the JSON body of a replayed echo response. +func replayBody(t *testing.T, res ReplayResult) echoed { + t.Helper() + i := strings.Index(res.Raw, "\n{") + if i < 0 { + t.Fatalf("no JSON body in raw response:\n%s", res.Raw) + } + var e echoed + if err := json.Unmarshal([]byte(strings.TrimSpace(res.Raw[i+1:])), &e); err != nil { + t.Fatalf("decode echo: %v in %q", err, res.Raw) + } + return e +} + +func recordedExchanges(t *testing.T, tr *Traffic, method, url string) int { + t.Helper() + var n int + if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges WHERE method=? AND url=?`, method, url).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +func TestReplayThroughProxy(t *testing.T) { + tr, _, _ := startProxy(t) + plain := httptest.NewServer(http.HandlerFunc(echoHandler)) + t.Cleanup(plain.Close) + tlsSrv := httptest.NewTLSServer(http.HandlerFunc(echoHandler)) + t.Cleanup(tlsSrv.Close) + + for _, tc := range []struct{ name, base string }{ + {"http", plain.URL}, + {"https", localhostURL(tlsSrv.URL)}, // trusted through the proxy CA + } { + t.Run(tc.name, func(t *testing.T) { + target := tc.base + "/edited/path?q=1&r=two" + res, err := tr.Replay(t.Context(), ReplayRequest{ + Method: "PATCH", + URL: target, + Headers: []ReplayHeader{{Name: "X-Custom", Value: "edited"}, {Name: "Content-Type", Value: "text/plain"}}, + Body: []byte("edited body"), + }) + if err != nil { + t.Fatal(err) + } + if res.Status != 200 || res.Truncated || !strings.HasPrefix(res.Raw, "HTTP/1.1 200 OK\n") || !strings.Contains(res.Raw, "X-Echo: yes") { + t.Fatalf("result status=%d truncated=%v raw:\n%s", res.Status, res.Truncated, res.Raw) + } + got := replayBody(t, res) + if got.Method != "PATCH" || got.Path != "/edited/path" || got.Query != "q=1&r=two" || got.Custom != "edited" || got.Body != "edited body" { + t.Fatalf("target received %+v", got) + } + waitFor(t, "replayed exchange recorded", func() bool { return recordedExchanges(t, tr, "PATCH", target) == 1 }) + }) + } +} + +func TestReplayDropsHopByHopHeaders(t *testing.T) { + tr, _, _ := startProxy(t) + srv := httptest.NewServer(http.HandlerFunc(echoHandler)) + t.Cleanup(srv.Close) + res, err := tr.Replay(t.Context(), ReplayRequest{ + Method: "POST", + URL: srv.URL + "/hop", + Headers: []ReplayHeader{ + {Name: "Connection", Value: "close"}, + {Name: "Content-Length", Value: "999"}, + {Name: "Keep-Alive", Value: "timeout=5"}, + {Name: "Host", Value: strings.TrimPrefix(srv.URL, "http://")}, + }, + Body: []byte("abc"), + }) + if err != nil { + t.Fatal(err) + } + got := replayBody(t, res) + if got.Connection != "" || got.ContentLength != 3 || got.Body != "abc" { + t.Fatalf("target received connection=%q content_length=%d body=%q, want no Connection and a recomputed length", got.Connection, got.ContentLength, got.Body) + } + if got.Host != strings.TrimPrefix(srv.URL, "http://") { + t.Fatalf("target saw Host %q", got.Host) + } +} + +func TestReplayDoesNotFollowRedirects(t *testing.T) { + tr, _, _ := startProxy(t) + var followed atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/next" { + followed.Store(true) + return + } + http.Redirect(w, r, "/next", http.StatusFound) + })) + t.Cleanup(srv.Close) + res, err := tr.Replay(t.Context(), ReplayRequest{Method: "GET", URL: srv.URL + "/start"}) + if err != nil { + t.Fatal(err) + } + if res.Status != 302 || !strings.Contains(res.Raw, "Location: /next") || followed.Load() { + t.Fatalf("status %d followed=%v raw:\n%s", res.Status, followed.Load(), res.Raw) + } +} + +func TestReplayTruncatesLargeResponse(t *testing.T) { + tr, _, _ := startProxy(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, strings.Repeat("x", maxReplayResponse+100)) + })) + t.Cleanup(srv.Close) + res, err := tr.Replay(t.Context(), ReplayRequest{Method: "GET", URL: srv.URL + "/big"}) + if err != nil { + t.Fatal(err) + } + head, body, _ := strings.Cut(res.Raw, "\n\n") + if !res.Truncated || len(body) != maxReplayResponse || head == "" { + t.Fatalf("truncated=%v body %d bytes, want %d", res.Truncated, len(body), maxReplayResponse) + } +} + +func TestReplayTimeoutIsRecognizable(t *testing.T) { + tr, _, _ := startProxy(t) + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-release: + case <-time.After(5 * time.Second): + } + })) + t.Cleanup(func() { close(release); srv.Close() }) + ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond) + defer cancel() + _, err := tr.Replay(ctx, ReplayRequest{Method: "GET", URL: srv.URL + "/slow"}) + if !errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrReplayInvalid) { + t.Fatalf("err = %v, want a deadline error", err) + } +} + +// A passthrough host is tunneled, not intercepted: the client sees the target's +// own certificate (here an untrusted httptest one, so the TLS handshake fails) +// and nothing is recorded. Pins the documented limitation. +func TestReplayPassthroughHostIsNotIntercepted(t *testing.T) { + tr, _, _ := startProxy(t) + srv := httptest.NewTLSServer(http.HandlerFunc(echoHandler)) + t.Cleanup(srv.Close) + tr.pass.Store("localhost", struct{}{}) + target := localhostURL(srv.URL) + "/tunneled" + _, err := tr.Replay(t.Context(), ReplayRequest{Method: "GET", URL: target}) + if err == nil || !strings.Contains(err.Error(), "certificate") { + t.Fatalf("err = %v, want the target's own certificate to be rejected", err) + } + if n := recordedExchanges(t, tr, "GET", target); n != 0 { + t.Fatalf("%d exchanges recorded for a passthrough host", n) + } +} + +// Validation happens before anything is sent, so the proxy need not run. +func TestReplayRejectsInvalidRequests(t *testing.T) { + tr, err := Open(t.TempDir(), "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = tr.Close() }) + many := make([]ReplayHeader, maxReplayHeaders+1) + for i := range many { + many[i] = ReplayHeader{Name: fmt.Sprintf("X-H%d", i), Value: "v"} + } + for name, req := range map[string]ReplayRequest{ + "ftp scheme": {Method: "GET", URL: "ftp://example.com/x"}, + "relative url": {Method: "GET", URL: "/x"}, + "no host": {Method: "GET", URL: "http:///x"}, + "user info": {Method: "GET", URL: "http://user:pass@example.com/"}, + "empty method": {Method: " ", URL: "http://example.com/"}, + "CONNECT": {Method: "CONNECT", URL: "http://example.com/"}, + "method not token": {Method: "GE T", URL: "http://example.com/"}, + "body too large": {Method: "POST", URL: "http://example.com/", Body: make([]byte, maxReplayBody+1)}, + "too many headers": {Method: "GET", URL: "http://example.com/", Headers: many}, + "CR in header": {Method: "GET", URL: "http://example.com/", Headers: []ReplayHeader{{Name: "X-A", Value: "a\r\nX-Injected: 1"}}}, + "LF in header": {Method: "GET", URL: "http://example.com/", Headers: []ReplayHeader{{Name: "X-A", Value: "a\nb"}}}, + "invalid name": {Method: "GET", URL: "http://example.com/", Headers: []ReplayHeader{{Name: "Bad Name", Value: "v"}}}, + "upgrade websocket": {Method: "GET", URL: "http://example.com/", Headers: []ReplayHeader{{Name: "Upgrade", Value: "websocket"}}}, + "host differs": {Method: "GET", URL: "http://example.com/", Headers: []ReplayHeader{{Name: "Host", Value: "other.example"}}}, + } { + if _, err := tr.Replay(t.Context(), req); !errors.Is(err, ErrReplayInvalid) { + t.Errorf("%s: err = %v, want ErrReplayInvalid", name, err) + } + } +} From 57ff63125bfbb53ae83723f8cc93e638156adff1 Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:56:08 +0100 Subject: [PATCH 09/29] docs: record the extension capture fix and map the contributor email The extension fix shipped without a changelog entry. Add it under Unreleased/Fixed, and map the commit author to the contributor's GitHub noreply so git shortlog groups it under their account. --- .mailmap | 1 + CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 .mailmap diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000..600a28f --- /dev/null +++ b/.mailmap @@ -0,0 +1 @@ +Xiaobo <269351206+liuxiaobo0920@users.noreply.github.com> diff --git a/CHANGELOG.md b/CHANGELOG.md index 71b3041..5de673b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ All notable changes to this project are documented in this file. The format is b ### Fixed +- **Browser extension traffic capture**: CDP delivers `Network.responseReceived` response headers as an object, not an array of `{name, value}` records, so the extension threw on every response and never queued an exchange. It now accepts both shapes and lower-cases names before the body-capture gate, so captures work again and `content-type` correctly controls body capture. - **bbot on arm64**: the image now builds on arm64 hosts such as Apple Silicon. bbot's `gowitness` downloaded an x86-64 Chrome and `retirejs` an x86-64 Node, which can't run there, so the build failed. On arm64 the image uses Debian's `chromium` for gowitness and an arm64 Node for retirejs, and the build check now covers `retirejs` too. amd64 builds are unchanged. - **Auto-scope hygiene**: auto `task_scope` rows count for coverage and untested assets only under a declared root, and direct coverage counts anchored assets through fact anchors only; the scope is described as "coverage scope, not enforced" (#196). - **Enrich HTTP probe**: it now dials through `intercept.PinnedDialer`, so the target gate applies and a probe without a target scope is refused (#198). From 1263cdc1850ec6aad114fbbe01c7d6a496a57488 Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:17:55 +0100 Subject: [PATCH 10/29] fix(traffic): replay reports recorded and headers, drops Connection-named headers, 413 for large bodies The Host refusal now explains that go-mitmproxy rebuilds the upstream request from the URL and drops the Host override. ReplayResult gains Recorded (false for an https passthrough host) and the sorted response Headers, both exposed by the route. Headers named in Connection and Proxy-Authorization are dropped. A body over the limit returns ErrReplayTooLarge, mapped to 413. --- server/server.go | 8 +++- server/traffic_replay_test.go | 21 ++++++--- traffic/replay.go | 84 ++++++++++++++++++++++++++++------- traffic/replay_test.go | 47 ++++++++++++++++++-- 4 files changed, 132 insertions(+), 28 deletions(-) diff --git a/server/server.go b/server/server.go index a402a9f..245790a 100644 --- a/server/server.go +++ b/server/server.go @@ -3534,6 +3534,9 @@ func (s *Server) replayTraffic(w http.ResponseWriter, r *http.Request) { defer cancel() res, err := tr.Replay(ctx, traffic.ReplayRequest{Method: body.Method, URL: body.URL, Headers: body.Headers, Body: payload}) switch { + case errors.Is(err, traffic.ErrReplayTooLarge): + writeErr(w, 413, err.Error()) + return case errors.Is(err, traffic.ErrReplayInvalid): writeErr(w, 400, err.Error()) return @@ -3544,7 +3547,10 @@ func (s *Server) replayTraffic(w http.ResponseWriter, r *http.Request) { writeErr(w, 502, err.Error()) return } - writeJSON(w, 200, map[string]any{"status": res.Status, "raw": res.Raw, "duration_ms": res.DurationMS, "truncated": res.Truncated}) + writeJSON(w, 200, map[string]any{ + "status": res.Status, "headers": res.Headers, "raw": res.Raw, "duration_ms": res.DurationMS, + "truncated": res.Truncated, "recorded": res.Recorded, + }) } // getTrafficBlob streams one oversized body by its sha256. Bodies past the inline diff --git a/server/traffic_replay_test.go b/server/traffic_replay_test.go index ca50b44..43339b0 100644 --- a/server/traffic_replay_test.go +++ b/server/traffic_replay_test.go @@ -99,7 +99,7 @@ func TestTrafficReplayRouteValidation(t *testing.T) { "header CRLF": {`{"method":"GET","url":"http://example.com/","headers":[{"name":"X-A","value":"a\r\nb"}]}`, 400}, "too large": {`{"method":"POST","url":"http://example.com/","body":"` + strings.Repeat("a", 17<<20) + `"}`, 413}, "proxy down": {`{"method":"GET","url":"http://127.0.0.1:1/"}`, 502}, - "body too big": {`{"method":"POST","url":"http://example.com/","body":"` + base64.StdEncoding.EncodeToString(make([]byte, 9<<20)) + `","body_base64":true}`, 400}, + "body too big": {`{"method":"POST","url":"http://example.com/","body":"` + base64.StdEncoding.EncodeToString(make([]byte, 9<<20)) + `","body_base64":true}`, 413}, } { if w := replay(c.body); w.Code != c.want { t.Errorf("%s: %d %s, want %d", name, w.Code, strings.TrimSpace(w.Body.String()), c.want) @@ -126,17 +126,26 @@ func TestTrafficReplayRouteEndToEnd(t *testing.T) { t.Fatalf("replay = %d %s", w.Code, w.Body) } var res struct { - Status int `json:"status"` - Raw string `json:"raw"` - DurationMS int64 `json:"duration_ms"` - Truncated bool `json:"truncated"` + Status int `json:"status"` + Headers []traffic.ReplayHeader `json:"headers"` + Raw string `json:"raw"` + DurationMS int64 `json:"duration_ms"` + Truncated bool `json:"truncated"` + Recorded bool `json:"recorded"` } if err := json.Unmarshal(w.Body.Bytes(), &res); err != nil { t.Fatal(err) } - if res.Status != 200 || res.Truncated || !strings.Contains(res.Raw, "X-Seen: PUT edited") || !strings.HasSuffix(res.Raw, "got:payload") { + if res.Status != 200 || res.Truncated || !res.Recorded || !strings.Contains(res.Raw, "X-Seen: PUT edited") || !strings.HasSuffix(res.Raw, "got:payload") { t.Fatalf("result %+v", res) } + seen := false + for _, h := range res.Headers { + seen = seen || (h.Name == "X-Seen" && h.Value == "PUT edited") + } + if !seen { + t.Fatalf("headers %+v lack X-Seen", res.Headers) + } deadline := time.Now().Add(5 * time.Second) for { var n int diff --git a/traffic/replay.go b/traffic/replay.go index 7dd09e9..420f0aa 100644 --- a/traffic/replay.go +++ b/traffic/replay.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "os" + "sort" "strings" "time" ) @@ -32,6 +33,9 @@ const ( // ErrReplayInvalid marks a replay request rejected before anything is sent. var ErrReplayInvalid = errors.New("invalid replay request") +// ErrReplayTooLarge marks a request body over maxReplayBody. +var ErrReplayTooLarge = errors.New("replay body too large") + // ReplayHeader is one request header line, kept as a list so duplicates and // their order survive. type ReplayHeader struct { @@ -49,24 +53,41 @@ type ReplayRequest struct { // ReplayResult is the response as received by the client side of the proxy. type ReplayResult struct { - Status int + Status int + // Headers are the response headers, keys sorted, every value kept. + Headers []ReplayHeader Raw string // status line, headers and (possibly truncated) body DurationMS int64 Truncated bool // the body exceeded maxReplayResponse and was cut + // Recorded reports whether the proxy records this exchange: false for an + // https host the proxy tunnels without interception (passthrough). + Recorded bool } // replayDropped are the headers the client must not send as given: hop-by-hop // fields and framing that net/http recomputes (Content-Length, -// Transfer-Encoding). Upgrade is rejected rather than dropped, see Replay. +// Transfer-Encoding). Headers named in Connection are dropped too. Upgrade is +// rejected rather than dropped, see Replay. var replayDropped = map[string]bool{ - "Content-Length": true, - "Transfer-Encoding": true, - "Connection": true, - "Proxy-Connection": true, - "Keep-Alive": true, - "Te": true, - "Trailer": true, - "Expect": true, + "Content-Length": true, + "Transfer-Encoding": true, + "Connection": true, + "Proxy-Connection": true, + "Proxy-Authorization": true, + "Keep-Alive": true, + "Te": true, + "Trailer": true, + "Expect": true, +} + +// replayRecorded reports whether the proxy records a request to scheme://host: +// plain http is always intercepted, https is unless the host is in passthrough. +func (t *Traffic) replayRecorded(scheme, host string) bool { + if scheme != "https" { + return true + } + _, tunneled := t.pass.Load(hostOnly(host)) + return !tunneled } func replayInvalid(format string, args ...any) error { @@ -93,7 +114,7 @@ func validHeaderName(name string) bool { // Replay sends req through the recording proxy and returns the response. The // proxy's sink records the exchange; Replay never calls record() itself. A host // the proxy tunnels without interception (passthrough, see maybePassthrough) -// still answers, but that exchange is not recorded. +// still answers, but that exchange is not recorded (Recorded is false). // // Redirects are not followed, no cookie jar is kept, and compression is not // negotiated: the edited headers go out as given, without an Accept-Encoding @@ -120,12 +141,25 @@ func (t *Traffic) Replay(ctx context.Context, req ReplayRequest) (ReplayResult, return ReplayResult{}, replayInvalid("url must not carry user info; send an Authorization header instead") } if len(req.Body) > maxReplayBody { - return ReplayResult{}, replayInvalid("body is %d bytes, over the %d-byte limit", len(req.Body), maxReplayBody) + return ReplayResult{}, fmt.Errorf("%w: body is %d bytes, over the %d-byte limit", ErrReplayTooLarge, len(req.Body), maxReplayBody) } if len(req.Headers) > maxReplayHeaders { return ReplayResult{}, replayInvalid("%d headers, over the limit of %d", len(req.Headers), maxReplayHeaders) } + // Headers named in Connection are hop-by-hop too (RFC 9110 §7.6.1). + connectionNamed := map[string]bool{} + for _, h := range req.Headers { + if http.CanonicalHeaderKey(strings.TrimSpace(h.Name)) != "Connection" { + continue + } + for _, token := range strings.Split(h.Value, ",") { + if token = strings.TrimSpace(token); token != "" { + connectionNamed[http.CanonicalHeaderKey(token)] = true + } + } + } + header := http.Header{} host := "" for _, h := range req.Headers { @@ -142,13 +176,18 @@ func (t *Traffic) Replay(ctx context.Context, req ReplayRequest) (ReplayResult, // A protocol upgrade (WebSocket, h2c) cannot be replayed as one request. return ReplayResult{}, replayInvalid("Upgrade requests cannot be replayed") case canonical == "Host": - // The recording proxy routes a request by its Host, not by the URL, so - // a different Host would silently send the request to another server. + // A Host override cannot be replayed faithfully. go-mitmproxy rebuilds + // the upstream request from the URL (proxyReq from f.Request.URL, dialed + // at CanonicalAddr(req.URL)) and copies only Header, where net/http does + // not keep Host, so the override is lost: the recorded exchange would + // show a Host the target never received. For plain http the client also + // puts req.Host in the absolute-form request target, which sends the + // request to that host instead. Refusing avoids a misleading exchange. host = strings.TrimSpace(h.Value) if !strings.EqualFold(host, target.Host) { - return ReplayResult{}, replayInvalid("Host %q differs from the URL host %q; the proxy routes by Host, so edit the URL instead", host, target.Host) + return ReplayResult{}, replayInvalid("Host %q differs from the URL host %q; the proxy sends the URL host as Host, so edit the URL instead", host, target.Host) } - case replayDropped[canonical]: + case replayDropped[canonical], connectionNamed[canonical]: // recomputed by net/http, or hop-by-hop default: header.Add(canonical, h.Value) @@ -190,7 +229,18 @@ func (t *Traffic) Replay(ctx context.Context, req ReplayRequest) (ReplayResult, if err != nil { return ReplayResult{}, fmt.Errorf("replay %s %s: reading response: %w", method, target.Redacted(), err) } - res := ReplayResult{Status: resp.StatusCode} + res := ReplayResult{Status: resp.StatusCode, Recorded: t.replayRecorded(target.Scheme, target.Host)} + keys := make([]string, 0, len(resp.Header)) + for k := range resp.Header { + keys = append(keys, k) + } + sort.Strings(keys) + res.Headers = make([]ReplayHeader, 0, len(keys)) + for _, k := range keys { + for _, v := range resp.Header[k] { + res.Headers = append(res.Headers, ReplayHeader{Name: k, Value: v}) + } + } if len(data) > maxReplayResponse { data = data[:maxReplayResponse] res.Truncated = true diff --git a/traffic/replay_test.go b/traffic/replay_test.go index 01fa679..931569b 100644 --- a/traffic/replay_test.go +++ b/traffic/replay_test.go @@ -24,6 +24,8 @@ type echoed struct { ContentLength int64 `json:"content_length"` Body string `json:"body"` Host string `json:"host"` + XHop string `json:"x_hop"` + ProxyAuth string `json:"proxy_auth"` } func echoHandler(w http.ResponseWriter, r *http.Request) { @@ -33,6 +35,7 @@ func echoHandler(w http.ResponseWriter, r *http.Request) { _ = json.NewEncoder(w).Encode(echoed{ Method: r.Method, Path: r.URL.Path, Query: r.URL.RawQuery, Custom: r.Header.Get("X-Custom"), Connection: r.Header.Get("Connection"), ContentLength: r.ContentLength, Body: string(b), Host: r.Host, + XHop: r.Header.Get("X-Hop"), ProxyAuth: r.Header.Get("Proxy-Authorization"), }) } @@ -81,8 +84,18 @@ func TestReplayThroughProxy(t *testing.T) { if err != nil { t.Fatal(err) } - if res.Status != 200 || res.Truncated || !strings.HasPrefix(res.Raw, "HTTP/1.1 200 OK\n") || !strings.Contains(res.Raw, "X-Echo: yes") { - t.Fatalf("result status=%d truncated=%v raw:\n%s", res.Status, res.Truncated, res.Raw) + if res.Status != 200 || res.Truncated || !res.Recorded || !strings.HasPrefix(res.Raw, "HTTP/1.1 200 OK\n") || !strings.Contains(res.Raw, "X-Echo: yes") { + t.Fatalf("result status=%d truncated=%v recorded=%v raw:\n%s", res.Status, res.Truncated, res.Recorded, res.Raw) + } + echoHeader := false + for i, h := range res.Headers { + if i > 0 && res.Headers[i-1].Name > h.Name { + t.Fatalf("headers not sorted: %+v", res.Headers) + } + echoHeader = echoHeader || (h.Name == "X-Echo" && h.Value == "yes") + } + if !echoHeader { + t.Fatalf("headers %+v lack X-Echo: yes", res.Headers) } got := replayBody(t, res) if got.Method != "PATCH" || got.Path != "/edited/path" || got.Query != "q=1&r=two" || got.Custom != "edited" || got.Body != "edited body" { @@ -101,7 +114,9 @@ func TestReplayDropsHopByHopHeaders(t *testing.T) { Method: "POST", URL: srv.URL + "/hop", Headers: []ReplayHeader{ - {Name: "Connection", Value: "close"}, + {Name: "Connection", Value: "close, X-Hop"}, + {Name: "X-Hop", Value: "v"}, + {Name: "Proxy-Authorization", Value: "Basic dXNlcjpwYXNz"}, {Name: "Content-Length", Value: "999"}, {Name: "Keep-Alive", Value: "timeout=5"}, {Name: "Host", Value: strings.TrimPrefix(srv.URL, "http://")}, @@ -115,6 +130,9 @@ func TestReplayDropsHopByHopHeaders(t *testing.T) { if got.Connection != "" || got.ContentLength != 3 || got.Body != "abc" { t.Fatalf("target received connection=%q content_length=%d body=%q, want no Connection and a recomputed length", got.Connection, got.ContentLength, got.Body) } + if got.XHop != "" || got.ProxyAuth != "" { + t.Fatalf("target received X-Hop=%q Proxy-Authorization=%q, want neither", got.XHop, got.ProxyAuth) + } if got.Host != strings.TrimPrefix(srv.URL, "http://") { t.Fatalf("target saw Host %q", got.Host) } @@ -211,7 +229,6 @@ func TestReplayRejectsInvalidRequests(t *testing.T) { "empty method": {Method: " ", URL: "http://example.com/"}, "CONNECT": {Method: "CONNECT", URL: "http://example.com/"}, "method not token": {Method: "GE T", URL: "http://example.com/"}, - "body too large": {Method: "POST", URL: "http://example.com/", Body: make([]byte, maxReplayBody+1)}, "too many headers": {Method: "GET", URL: "http://example.com/", Headers: many}, "CR in header": {Method: "GET", URL: "http://example.com/", Headers: []ReplayHeader{{Name: "X-A", Value: "a\r\nX-Injected: 1"}}}, "LF in header": {Method: "GET", URL: "http://example.com/", Headers: []ReplayHeader{{Name: "X-A", Value: "a\nb"}}}, @@ -223,4 +240,26 @@ func TestReplayRejectsInvalidRequests(t *testing.T) { t.Errorf("%s: err = %v, want ErrReplayInvalid", name, err) } } + _, err = tr.Replay(t.Context(), ReplayRequest{Method: "POST", URL: "http://example.com/", Body: make([]byte, maxReplayBody+1)}) + if !errors.Is(err, ErrReplayTooLarge) || errors.Is(err, ErrReplayInvalid) { + t.Errorf("body too large: err = %v, want ErrReplayTooLarge", err) + } +} + +// Only an https host in passthrough is reported as not recorded. +func TestReplayRecordedHelper(t *testing.T) { + tr := &Traffic{} + tr.pass.Store("localhost", struct{}{}) + for _, c := range []struct { + scheme, host string + want bool + }{ + {"https", "localhost:443", false}, + {"https", "example.com:443", true}, + {"http", "localhost:80", true}, + } { + if got := tr.replayRecorded(c.scheme, c.host); got != c.want { + t.Errorf("replayRecorded(%s, %s) = %v, want %v", c.scheme, c.host, got, c.want) + } + } } From 167b9f0c3f6f0ed3f2d07dc2bd0ea1d3f0d62abf Mon Sep 17 00:00:00 2001 From: Xiaobo Date: Mon, 16 Nov 2026 21:58:10 +0800 Subject: [PATCH 11/29] fix(db): record an explicit skill-visibility off instead of deleting the row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit seedBuiltinSkillVisibility() is insert-if-absent (ON CONFLICT DO NOTHING), and its comment promises that "a user's later toggle-off is never resurrected on restart". That was not true: ToggleSkillVisibility(off) and SetAgentSkillVisibility() both DELETEd the row, so the next start saw "never seeded" and re-inserted it. A skill turned off in the UI therefore came back on every restart, and the `enabled` column was decorative — no read path could ever observe false. Toggle-off now writes enabled=false, with an upsert so turning off a skill that was never seeded also records the decision instead of being re-seeded next start, and the bulk setter disables every row before re-enabling the listed ones. Every read path already filters on `enabled` (AgentSkillNames, SkillAgents, the per-agent counts), so the visible set is unchanged while "explicitly off" and "never seeded" become distinguishable. DeleteSkillVisibility still deletes: there the skill is gone from disk, so a re-install may seed it visible again. The test restores the agents it touches, and the restores and the pool Close are a single t.Cleanup in that order. They used to be separate cleanups registered restores-then-close, and t.Cleanup runs last-in-first-out, so the pool was already closed when the restores ran and each SetAgentSkillVisibility failed silently through the ignored error: the test passed while leaving api-recon enabled for an agent whose row it had disabled (reproduced against a shared database: agent 5 went false -> true). A failed restore now fails the test instead of being swallowed. --- db/config.go | 17 ++++- db/db.go | 6 ++ db/skill_visibility_test.go | 143 ++++++++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 db/skill_visibility_test.go diff --git a/db/config.go b/db/config.go index 7efee3d..983fa7f 100644 --- a/db/config.go +++ b/db/config.go @@ -1064,13 +1064,20 @@ func (d *DB) SkillAgents(skillName string) ([]int64, error) { } // SetAgentSkillVisibility replaces all skill visibility for an agent. +// +// Rows are disabled rather than deleted. Every read path filters on `enabled`, so +// the visible set is unchanged, but the row survives as the record of an explicit +// decision: seedBuiltinSkillVisibility is insert-if-absent, so a deleted row would +// read as "never seeded" on the next start and the builtin skill would come back. +// DeleteSkillVisibility is the one place that still deletes, because there the +// skill itself is gone from disk. func (d *DB) SetAgentSkillVisibility(agentID int64, names []string) error { tx, err := d.Begin() if err != nil { return err } defer tx.Rollback() - if _, err := tx.Exec(`DELETE FROM agent_skill_visibility WHERE agent_id=$1`, agentID); err != nil { + if _, err := tx.Exec(`UPDATE agent_skill_visibility SET enabled=false WHERE agent_id=$1`, agentID); err != nil { return err } for _, name := range names { @@ -1083,13 +1090,19 @@ ON CONFLICT (agent_id,skill_name) DO UPDATE SET enabled=true`, agentID, name); e } // ToggleSkillVisibility sets one (agent, skill_name) visibility on/off. +// +// Turning a skill off writes enabled=false instead of removing the row, including +// when no row exists yet: "explicitly off" and "never seeded" must not look the +// same to an insert-if-absent seed, or the builtin skill is resurrected on the +// next start. func (d *DB) ToggleSkillVisibility(agentID int64, skillName string, on bool) error { if on { _, err := d.Exec(`INSERT INTO agent_skill_visibility(agent_id,skill_name,enabled) VALUES ($1,$2,true) ON CONFLICT (agent_id,skill_name) DO UPDATE SET enabled=true`, agentID, skillName) return err } - _, err := d.Exec(`DELETE FROM agent_skill_visibility WHERE agent_id=$1 AND skill_name=$2`, agentID, skillName) + _, err := d.Exec(`INSERT INTO agent_skill_visibility(agent_id,skill_name,enabled) VALUES ($1,$2,false) +ON CONFLICT (agent_id,skill_name) DO UPDATE SET enabled=false`, agentID, skillName) return err } diff --git a/db/db.go b/db/db.go index a7efdec..2c07dac 100644 --- a/db/db.go +++ b/db/db.go @@ -391,6 +391,12 @@ var builtinSkillVisibility = map[string][]string{ // seedBuiltinSkillVisibility binds the shipped built-in skills to their default // agents. Insert-if-absent (ON CONFLICT DO NOTHING) so a user's later toggle-off is // never resurrected on restart — matches the browser-MCP / intercept-rule seed policy. +// +// This only holds while toggle-off keeps the row: ToggleSkillVisibility and +// SetAgentSkillVisibility write enabled=false rather than DELETE, so "explicitly +// off" is distinguishable from "never seeded". Deleting a skill is the one path +// that removes rows (DeleteSkillVisibility), which is what lets a re-installed +// skill be seeded again. func (d *DB) seedBuiltinSkillVisibility() error { for skillName, agentKeys := range builtinSkillVisibility { for _, key := range agentKeys { diff --git a/db/skill_visibility_test.go b/db/skill_visibility_test.go new file mode 100644 index 0000000..8e87d92 --- /dev/null +++ b/db/skill_visibility_test.go @@ -0,0 +1,143 @@ +package db + +import ( + "database/sql" + "errors" + "slices" + "testing" +) + +// skillRowState reports whether an (agent, skill) row exists and whether it is +// enabled. The distinction is the whole point of this file: seedBuiltinSkillVisibility +// is insert-if-absent, so "never seeded" and "explicitly turned off" must be +// different states. If turning a skill off removes the row, the next start sees +// "never seeded" and resurrects it. +func skillRowState(t *testing.T, d *DB, agentID int64, skill string) (exists, enabled bool) { + t.Helper() + err := d.QueryRow(`SELECT enabled FROM agent_skill_visibility WHERE agent_id=$1 AND skill_name=$2`, agentID, skill).Scan(&enabled) + if errors.Is(err, sql.ErrNoRows) { + return false, false + } + if err != nil { + t.Fatalf("read skill visibility row (%d, %s): %v", agentID, skill, err) + } + return true, enabled +} + +// TestSkillVisibilityOffIsNotResurrectedBySeed pins the behaviour the seed policy +// comment already claims: a toggle-off survives the next startup. +// +// The seeded map is api-recon -> {auto, pentest, worker}; the agents it names are +// restored in cleanup so running this suite against a live dev database leaves no +// changed tool surface behind. +func TestSkillVisibilityOffIsNotResurrectedBySeed(t *testing.T) { + const skill = "api-recon" + d, err := Open(testDSN(t)) + if err != nil { + t.Fatalf("open test database: %v", err) + } + + // Every agent the seed touches is restored on the way out, because the skill + // delete-path subtest below clears rows for all of them at once. The restores and + // the Close are ONE cleanup, in that order: t.Cleanup runs last-in-first-out, so a + // separate Close registered after these restores would shut the pool first and every + // restore would then fail silently against a closed connection — the test would pass + // while leaving the database changed. + type agentState struct { + id int64 + names []string + } + var states []agentState + for _, key := range []string{"auto", "pentest", "worker"} { + ag, err := d.GetAgentByKey(key) + if err != nil || ag == nil { + t.Fatalf("agent %q missing: %v", key, err) + } + orig, err := d.AgentSkillNames(ag.ID) + if err != nil { + t.Fatalf("read original visibility for %q: %v", key, err) + } + states = append(states, agentState{id: ag.ID, names: orig}) + } + t.Cleanup(func() { + for _, st := range states { + if err := d.SetAgentSkillVisibility(st.id, st.names); err != nil { + t.Errorf("restore skill visibility for agent %d: %v", st.id, err) + } + } + d.Close() + }) + + worker, err := d.GetAgentByKey("worker") + if err != nil || worker == nil { + t.Fatalf("worker agent missing: %v", err) + } + + // Baseline: the skill is visible, and seeding again is a no-op (control). + if err := d.ToggleSkillVisibility(worker.ID, skill, true); err != nil { + t.Fatal(err) + } + if err := d.seedBuiltinSkillVisibility(); err != nil { + t.Fatalf("seed: %v", err) + } + if names, _ := d.AgentSkillNames(worker.ID); !slices.Contains(names, skill) { + t.Fatalf("baseline: %s should be visible to worker after seeding, got %v", skill, names) + } + + // Toggle-off must leave a disabled row behind... + if err := d.ToggleSkillVisibility(worker.ID, skill, false); err != nil { + t.Fatal(err) + } + if names, _ := d.AgentSkillNames(worker.ID); slices.Contains(names, skill) { + t.Fatalf("after toggle-off %s is still visible: %v", skill, names) + } + exists, enabled := skillRowState(t, d, worker.ID, skill) + if !exists { + t.Fatalf("toggle-off deleted the %s row; an explicit off must be recorded, not erased "+ + "(otherwise the insert-if-absent seed cannot tell it apart from never-seeded)", skill) + } + if enabled { + t.Fatalf("toggle-off left %s enabled=true", skill) + } + + // ...and the next startup must respect it. + if err := d.seedBuiltinSkillVisibility(); err != nil { + t.Fatalf("seed after toggle-off: %v", err) + } + if names, _ := d.AgentSkillNames(worker.ID); slices.Contains(names, skill) { + t.Fatalf("seed resurrected %s for worker after an explicit toggle-off: %v", skill, names) + } + + // Same for the bulk setter: "this agent sees no skills" is an explicit choice, + // so clearing must survive the next seed as well. + if err := d.ToggleSkillVisibility(worker.ID, skill, true); err != nil { + t.Fatal(err) + } + if err := d.SetAgentSkillVisibility(worker.ID, nil); err != nil { + t.Fatal(err) + } + if err := d.seedBuiltinSkillVisibility(); err != nil { + t.Fatalf("seed after clear: %v", err) + } + if names, _ := d.AgentSkillNames(worker.ID); len(names) != 0 { + t.Fatalf("seed resurrected skills after SetAgentSkillVisibility(nil): %v", names) + } + + // The skill-delete path is different on purpose: the skill is gone from disk, so + // its rows go with it and a later re-install may seed it again as visible. + if err := d.ToggleSkillVisibility(worker.ID, skill, true); err != nil { + t.Fatal(err) + } + if err := d.DeleteSkillVisibility(skill); err != nil { + t.Fatal(err) + } + if exists, _ := skillRowState(t, d, worker.ID, skill); exists { + t.Fatalf("DeleteSkillVisibility left a %s row behind", skill) + } + if err := d.seedBuiltinSkillVisibility(); err != nil { + t.Fatalf("seed after skill delete: %v", err) + } + if names, _ := d.AgentSkillNames(worker.ID); !slices.Contains(names, skill) { + t.Fatalf("a deleted-then-seeded %s should be visible again, got %v", skill, names) + } +} From 03919e7dd51cf14aba052333a2fbdd97e12f5e6b Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:50:06 +0100 Subject: [PATCH 12/29] docs(traffic): correct why a replay Host override is refused For plain http the client puts the Host in the absolute-form request target, so the proxy dials it; for https go-mitmproxy records the exchange under the inner Host. Comment and error message only. --- traffic/replay.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/traffic/replay.go b/traffic/replay.go index 420f0aa..75385d1 100644 --- a/traffic/replay.go +++ b/traffic/replay.go @@ -176,16 +176,16 @@ func (t *Traffic) Replay(ctx context.Context, req ReplayRequest) (ReplayResult, // A protocol upgrade (WebSocket, h2c) cannot be replayed as one request. return ReplayResult{}, replayInvalid("Upgrade requests cannot be replayed") case canonical == "Host": - // A Host override cannot be replayed faithfully. go-mitmproxy rebuilds - // the upstream request from the URL (proxyReq from f.Request.URL, dialed - // at CanonicalAddr(req.URL)) and copies only Header, where net/http does - // not keep Host, so the override is lost: the recorded exchange would - // show a Host the target never received. For plain http the client also - // puts req.Host in the absolute-form request target, which sends the - // request to that host instead. Refusing avoids a misleading exchange. + // A Host override is refused because the URL and the destination diverge. + // For plain http, net/http puts Request.Host in the absolute-form request + // target it sends to the proxy, so the proxy dials the Host value and ignores + // the URL host. For https, the CONNECT tunnel stays on the URL host but + // go-mitmproxy records the exchange under the inner Host (attacker.ServeHTTP + // sets URL.Host from req.Host), so the recorded URL is not the host contacted. + // Either way the replay would not do what the URL says; edit the URL instead. host = strings.TrimSpace(h.Value) if !strings.EqualFold(host, target.Host) { - return ReplayResult{}, replayInvalid("Host %q differs from the URL host %q; the proxy sends the URL host as Host, so edit the URL instead", host, target.Host) + return ReplayResult{}, replayInvalid("Host %q differs from the URL host %q; the replay would contact or record a different host than the URL says, so edit the URL instead", host, target.Host) } case replayDropped[canonical], connectionNamed[canonical]: // recomputed by net/http, or hop-by-hop From 4a19e10af66fa4c0685f78583d959a2bcb2512d1 Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:02:48 +0100 Subject: [PATCH 13/29] docs: record the skill visibility restart fix Add the Unreleased/Fixed entry for #262, including the upgrade note: off states stored as deleted rows come back once after this version. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5de673b..ebb4a8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ All notable changes to this project are documented in this file. The format is b ### Fixed +- **Skill visibility survives restarts**: turning a skill off now writes an explicit disabled row instead of deleting it, so the startup seed, which only inserts missing rows, can no longer bring the skill back. Skills that were turned off before this version have no row to keep, so they come back once after the upgrade and must be turned off again. - **Browser extension traffic capture**: CDP delivers `Network.responseReceived` response headers as an object, not an array of `{name, value}` records, so the extension threw on every response and never queued an exchange. It now accepts both shapes and lower-cases names before the body-capture gate, so captures work again and `content-type` correctly controls body capture. - **bbot on arm64**: the image now builds on arm64 hosts such as Apple Silicon. bbot's `gowitness` downloaded an x86-64 Chrome and `retirejs` an x86-64 Node, which can't run there, so the build failed. On arm64 the image uses Debian's `chromium` for gowitness and an arm64 Node for retirejs, and the build check now covers `retirejs` too. amd64 builds are unchanged. - **Auto-scope hygiene**: auto `task_scope` rows count for coverage and untested assets only under a declared root, and direct coverage counts anchored assets through fact anchors only; the scope is described as "coverage scope, not enforced" (#196). From 6067ea1489d5147c5596303e685e14941dc9dec0 Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:56:19 +0100 Subject: [PATCH 14/29] feat(web): replay a recorded request from the traffic detail Adds a Replay tab to the traffic detail sheet, backed by POST /api/traffic/replay (#258). The form starts from the selected exchange: method and URL from the exchange, headers and body from the recorded request. Host is taken from the URL and not editable; headers the server drops (hop-by-hop, Content-Length, Proxy-Authorization, and those named in Connection) are left out and listed. A binary body (NUL byte) disables Send. The response shows status, duration, truncation, a passthrough warning when it was not recorded, the sorted headers and the raw text; a recorded replay refreshes the list. http() now throws ApiError, an Error with the HTTP status, so the tab can tell 413, 502, 503 and 504 apart. parseRawRequest and splitReplayHeaders live in lib/replay.ts with node tests run in CI (test:replay). Stream exchanges show the tab disabled. --- .github/workflows/ci.yml | 1 + web/package.json | 3 +- web/scripts/replay.test.mjs | 129 ++++++++ .../traffic/_components/replay-panel.tsx | 279 ++++++++++++++++++ web/src/app/(main)/function/traffic/page.tsx | 30 +- web/src/components/http-code-block.tsx | 2 +- web/src/lib/api.ts | 22 +- web/src/lib/replay.ts | 92 ++++++ web/src/lib/types.ts | 24 ++ 9 files changed, 578 insertions(+), 4 deletions(-) create mode 100644 web/scripts/replay.test.mjs create mode 100644 web/src/app/(main)/function/traffic/_components/replay-panel.tsx create mode 100644 web/src/lib/replay.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d856770..c3dffdf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,4 +100,5 @@ jobs: - run: npm ci - run: npx tsc --noEmit - run: npm run test:polling + - run: npm run test:replay - run: npm run build:static diff --git a/web/package.json b/web/package.json index 58af054..aa37a4d 100644 --- a/web/package.json +++ b/web/package.json @@ -13,7 +13,8 @@ "check:fix": "biome check --write", "prepare": "husky", "generate:presets": "ts-node -P tsconfig.scripts.json src/scripts/generate-theme-presets.ts", - "test:polling": "node --test scripts/polling.test.mjs" + "test:polling": "node --test scripts/polling.test.mjs", + "test:replay": "node --test scripts/replay.test.mjs" }, "lint-staged": { "*.{js,ts,jsx,tsx}": [ diff --git a/web/scripts/replay.test.mjs b/web/scripts/replay.test.mjs new file mode 100644 index 0000000..fed57b6 --- /dev/null +++ b/web/scripts/replay.test.mjs @@ -0,0 +1,129 @@ +import ts from "typescript"; + +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import vm from "node:vm"; + +const web = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const output = ts.transpileModule(fs.readFileSync(path.join(web, "src/lib/replay.ts"), "utf8"), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 }, +}).outputText; +const exports = {}; +vm.runInNewContext(output, { exports }); +const { parseRawRequest, splitReplayHeaders } = exports; + +// Objects built inside the vm context have another Object prototype, so +// compare through JSON. +const plain = (v) => JSON.parse(JSON.stringify(v)); + +test("parses an LF request with a body", () => { + const r = parseRawRequest( + 'POST /api/login HTTP/1.1\nHost: example.com\nContent-Type: application/json\n\n{"user":"a"}', + ); + assert.equal(r.method, "POST"); + assert.equal(r.target, "/api/login"); + assert.deepEqual(plain(r.headers), [ + { name: "Host", value: "example.com" }, + { name: "Content-Type", value: "application/json" }, + ]); + assert.equal(r.body, '{"user":"a"}'); + assert.equal(r.binary, false); + assert.equal(r.bodyPreviewOnly, false); +}); + +test("parses a CRLF request and keeps the body's CRLFs", () => { + const r = parseRawRequest("PUT /x HTTP/1.1\r\nAccept: */*\r\nX-A: 1\r\n\r\nline1\r\nline2"); + assert.equal(r.method, "PUT"); + assert.deepEqual(plain(r.headers), [ + { name: "Accept", value: "*/*" }, + { name: "X-A", value: "1" }, + ]); + assert.equal(r.body, "line1\r\nline2"); +}); + +test("a request without a body has an empty body", () => { + for (const raw of ["GET / HTTP/1.1\nAccept: */*", "GET / HTTP/1.1\r\nAccept: */*\r\n\r\n"]) { + const r = parseRawRequest(raw); + assert.equal(r.body, ""); + assert.deepEqual(plain(r.headers), [{ name: "Accept", value: "*/*" }]); + } +}); + +test("a header value keeps the colons after the first one", () => { + const r = parseRawRequest("GET / HTTP/1.1\nReferer: https://example.com:8443/a?b=c:d\nX-Time: 12:34:56"); + assert.deepEqual(plain(r.headers), [ + { name: "Referer", value: "https://example.com:8443/a?b=c:d" }, + { name: "X-Time", value: "12:34:56" }, + ]); +}); + +test("a request without Host parses, and no Host is invented", () => { + const r = parseRawRequest("GET /only HTTP/1.1\nAccept: text/html\n\n"); + assert.equal(r.target, "/only"); + assert.equal( + r.headers.some((h) => h.name.toLowerCase() === "host"), + false, + ); + const { sent, notSent } = splitReplayHeaders(r.headers); + assert.deepEqual(plain(sent), [{ name: "Accept", value: "text/html" }]); + assert.deepEqual(plain(notSent), []); +}); + +test("Host and hop-by-hop headers are not sent, including those named in Connection", () => { + const r = parseRawRequest( + [ + "POST /p HTTP/1.1", + "Host: example.com", + "Content-Length: 3", + "Connection: keep-alive, X-Hop", + "Keep-Alive: timeout=5", + "Proxy-Authorization: Basic eA==", + "Proxy-Connection: keep-alive", + "Transfer-Encoding: chunked", + "TE: trailers", + "Trailer: X-T", + "Expect: 100-continue", + "Upgrade: websocket", + "x-hop: 1", + "Cookie: a=b", + "Authorization: Bearer t", + "", + "abc", + ].join("\r\n"), + ); + const { sent, notSent } = splitReplayHeaders(r.headers); + assert.deepEqual(plain(sent), [ + { name: "Cookie", value: "a=b" }, + { name: "Authorization", value: "Bearer t" }, + ]); + assert.deepEqual(plain(notSent), [ + "Host", + "Content-Length", + "Connection", + "Keep-Alive", + "Proxy-Authorization", + "Proxy-Connection", + "Transfer-Encoding", + "TE", + "Trailer", + "Expect", + "Upgrade", + "x-hop", + ]); + assert.equal(r.body, "abc"); +}); + +test("a NUL byte marks the request as binary", () => { + const r = parseRawRequest("POST /u HTTP/1.1\nContent-Type: application/octet-stream\n\nab\u0000cd"); + assert.equal(r.binary, true); +}); + +test("the blob marker of a spilled body is removed and flagged", () => { + const marker = `\n…[truncated] @blob sha256:${"a".repeat(64)} (len=99999)`; + const r = parseRawRequest(`POST /big HTTP/1.1\nContent-Type: text/plain\n\npreview text${marker}`); + assert.equal(r.body, "preview text"); + assert.equal(r.bodyPreviewOnly, true); +}); diff --git a/web/src/app/(main)/function/traffic/_components/replay-panel.tsx b/web/src/app/(main)/function/traffic/_components/replay-panel.tsx new file mode 100644 index 0000000..a0c5902 --- /dev/null +++ b/web/src/app/(main)/function/traffic/_components/replay-panel.tsx @@ -0,0 +1,279 @@ +"use client"; + +import * as React from "react"; + +import { Loader2Icon, PlusIcon, SendIcon, XIcon } from "lucide-react"; + +import { HttpCodeBlock, statusTone } from "@/components/http-code-block"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { Textarea } from "@/components/ui/textarea"; +import { ApiError, api } from "@/lib/api"; +import { parseRawRequest, splitReplayHeaders } from "@/lib/replay"; +import type { TrafficExchange, TrafficReplayHeader, TrafficReplayResult } from "@/lib/types"; +import { cn } from "@/lib/utils"; + +type HeaderRow = TrafficReplayHeader & { key: number }; + +type ReplayForm = { + method: string; + url: string; + rows: HeaderRow[]; + body: string; + // Headers of the recorded request that the server would not send. + dropped: string[]; + binary: boolean; + bodyPreviewOnly: boolean; +}; + +let rowKey = 0; +const toRows = (headers: TrafficReplayHeader[]): HeaderRow[] => headers.map((h) => ({ ...h, key: ++rowKey })); + +function initialForm(exchange: TrafficExchange, raw: string): ReplayForm { + const parsed = parseRawRequest(raw); + const { sent, notSent } = splitReplayHeaders(parsed.headers); + return { + method: exchange.method.toUpperCase(), + url: exchange.url, + rows: toRows(sent), + body: parsed.binary ? "" : parsed.body, + dropped: notSent.filter((n) => n.toLowerCase() !== "host"), + binary: parsed.binary, + bodyPreviewOnly: parsed.bodyPreviewOnly, + }; +} + +const ERROR_TITLES: Record = { + 400: "Invalid request", + 413: "Request body too large", + 502: "Proxy or upstream error", + 503: "Traffic capture is disabled", + 504: "Timed out", +}; + +// The server's message, under a title that says what the status means. +function describeError(e: unknown): { title: string; message: string } { + const message = e instanceof Error ? e.message : String(e); + if (!(e instanceof ApiError)) return { title: "Replay failed", message }; + return { title: ERROR_TITLES[e.status] ?? `Replay failed (${e.status})`, message }; +} + +// ReplayPanel edits the selected exchange's request and sends it again through +// the proxy. Host comes from the URL and hop-by-hop headers are not sent. +export function ReplayPanel({ + exchange, + raw, + methods, + onRecorded, +}: { + exchange: TrafficExchange; + // The recorded request, with Host filled in. + raw: string; + methods: string[]; + // Called when the replayed exchange was recorded, so the list can refetch. + onRecorded: () => void; +}) { + const [form, setForm] = React.useState(() => initialForm(exchange, raw)); + const [sending, setSending] = React.useState(false); + const [result, setResult] = React.useState(null); + const [error, setError] = React.useState<{ title: string; message: string } | null>(null); + // Answers for an older exchange are dropped. + const generation = React.useRef(0); + const methodListId = React.useId(); + + React.useEffect(() => { + generation.current++; + setForm(initialForm(exchange, raw)); + setSending(false); + setResult(null); + setError(null); + }, [exchange, raw]); + + const update = (patch: Partial) => setForm((f) => ({ ...f, ...patch })); + const updateRow = (key: number, patch: Partial) => + setForm((f) => ({ ...f, rows: f.rows.map((r) => (r.key === key ? { ...r, ...patch } : r)) })); + + // Rows the user typed that the server would not send either. + const ignored = new Set(splitReplayHeaders(form.rows).notSent.map((n) => n.toLowerCase())); + + const send = () => { + const gen = ++generation.current; + setSending(true); + setError(null); + setResult(null); + api + .trafficReplay({ + method: form.method.trim(), + url: form.url.trim(), + headers: splitReplayHeaders(form.rows).sent, + body: form.body, + body_base64: false, + }) + .then((r) => { + if (gen !== generation.current) return; + setResult(r); + if (r.recorded) onRecorded(); + }) + .catch((e: unknown) => { + if (gen === generation.current) setError(describeError(e)); + }) + .finally(() => { + if (gen === generation.current) setSending(false); + }); + }; + + const canSend = !sending && !form.binary && form.method.trim() !== "" && form.url.trim() !== ""; + // Repeated headers (Set-Cookie) share a name, so each keeps its original position as key. + const sortedHeaders = (result?.headers ?? []) + .map((h, position) => ({ ...h, position })) + .sort((a, b) => a.name.toLowerCase().localeCompare(b.name.toLowerCase())); + + return ( +
+
+ update({ method: e.target.value.toUpperCase() })} + /> + + {methods.map((m) => ( + + update({ url: e.target.value })} + /> +
+ +
+
+ Headers + +
+

Host is taken from the URL.

+ {form.dropped.length > 0 && ( +

+ Not sent (hop-by-hop or recomputed by the server): {form.dropped.join(", ")} +

+ )} + {form.rows.map((row) => ( +
+ updateRow(row.key, { name: e.target.value })} + /> + updateRow(row.key, { value: e.target.value })} + /> + +
+ ))} + {ignored.size > 0 && ( +

Struck-through headers are not sent by the server.

+ )} +
+ +
+ Body + {form.binary ? ( +

+ The recorded body is binary (it contains NUL bytes). Replaying a binary body is not supported. +

+ ) : ( + <> +