diff --git a/agent/finding_workflow.go b/agent/finding_workflow.go index 1428646..5f359b1 100644 --- a/agent/finding_workflow.go +++ b/agent/finding_workflow.go @@ -49,7 +49,7 @@ func findingWorkflowTools(agentKey string, tools []actool.CoreTool) ([]actool.Co if tool.Name() == "bind_finding_traffic" { continue } - if agentKey == "reporter" && (tool.Name() == "traffic_search" || tool.Name() == "traffic_get" || tool.Name() == "traffic_blob") { + if agentKey == "reporter" && (tool.Name() == "traffic_search" || tool.Name() == "traffic_get" || tool.Name() == "traffic_blob" || tool.Name() == "traffic_messages") { continue } switch tool.Name() { diff --git a/agent/worker.go b/agent/worker.go index b10cbeb..ebfec58 100644 --- a/agent/worker.go +++ b/agent/worker.go @@ -292,7 +292,7 @@ func workerTrafficBlock(recording bool) string { if !recording { return "" } - return "\n\n**Traffic tools**:\n- traffic_search / traffic_get / traffic_blob: review responses, find already-visited resources — **check the traffic first, don't curl the same URL twice**. traffic_search **requires a host** and by default returns only 3 ultra-light index entries (id/method/url/status/resp_len, no response content) — explicitly raise limit when you need more; body_contains does full-text search in request/response bodies (at least 3 characters; supports substrings and CJK text, e.g. for passwords/keys/errors/internal addresses); to see one entry's raw content use traffic_get(id) — oversized bodies show as @blob sha256:, fetched in segments with traffic_blob(hash)." + return "\n\n**Traffic tools**:\n- traffic_search / traffic_get / traffic_blob / traffic_messages: review responses, find already-visited resources — **check the traffic first, don't curl the same URL twice**. traffic_search **requires a host** and by default returns only 3 ultra-light index entries (id/method/url/status/resp_len, no response content) — explicitly raise limit when you need more; body_contains does full-text search in request/response bodies (at least 3 characters; supports substrings and CJK text, e.g. for passwords/keys/errors/internal addresses); to see one entry's raw content use traffic_get(id) — oversized bodies show as @blob sha256:, fetched in segments with traffic_blob(hash). WebSocket (status 101) and SSE (text/event-stream) records are captured too: traffic_get shows the handshake, and traffic_messages(id) lists their messages in order with direction and type." } // artifactSpec is section [C]: the code-owned, non-editable tail appended to every diff --git a/go.mod b/go.mod index fd897b9..ba11f2c 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Autumn-27/norma v0.4.10 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 + github.com/gorilla/websocket v1.5.3 github.com/jackc/pgx/v5 v5.10.0 github.com/klauspost/compress v1.18.6 github.com/lqqyt2423/go-mitmproxy v1.9.2 @@ -27,7 +28,6 @@ require ( github.com/gaissmai/bart v0.26.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/gorilla/css v1.0.1 // indirect - github.com/gorilla/websocket v1.5.3 // indirect github.com/hinshun/vt10x v0.0.0-20220301184237-5011da428d02 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect diff --git a/intercept/target_extract.go b/intercept/target_extract.go index f3f4ab9..f897840 100644 --- a/intercept/target_extract.go +++ b/intercept/target_extract.go @@ -39,7 +39,7 @@ var noTargetTools = toolSet( // the operator runs the command, not the platform "ask_operator", // recorded traffic - "traffic_search", "traffic_get", "traffic_blob", + "traffic_search", "traffic_get", "traffic_blob", "traffic_messages", ) func toolSet(names ...string) map[string]bool { diff --git a/intercept/target_test.go b/intercept/target_test.go index 5577e96..8c508c5 100644 --- a/intercept/target_test.go +++ b/intercept/target_test.go @@ -191,7 +191,7 @@ func TestExtractSkipsNoTargetTools(t *testing.T) { "web_search", "Read", "Write", "Edit", "MultiEdit", "LS", "Glob", "Grep", "TodoWrite", "sleep", "TaskOutput", "TaskStop", "TaskList", "add_intent", "insert_assets", "report_finding", "record_fact", "add_task_scope", "graph_overview", - "spawn_task", "ask_operator", "traffic_search", "traffic_get", "traffic_blob", + "spawn_task", "ask_operator", "traffic_search", "traffic_get", "traffic_blob", "traffic_messages", } { wantTargets(t, tool, targetsOf(tool, input)) } diff --git a/server/finding_workflow.go b/server/finding_workflow.go index 46aeacd..8b11fac 100644 --- a/server/finding_workflow.go +++ b/server/finding_workflow.go @@ -88,7 +88,7 @@ func (s *Server) seedFindingWorkflowTools() { // Upgrade only the original default binding. Customized lists and enabled // flags survive; the one-time flag also preserves future user unbinding. readers := `["worker","reporter"]` - for _, key := range []string{"traffic_search", "traffic_get", "traffic_blob"} { + for _, key := range []string{"traffic_search", "traffic_get", "traffic_blob", "traffic_messages"} { if _, err := s.m.pg.Exec(`UPDATE tools SET agents=$2::jsonb WHERE key=$1 AND system AND (agents='["worker"]'::jsonb OR (agents @> '["worker","planner","mainagent","auto","pentest"]'::jsonb AND jsonb_array_length(agents)=5))`, key, readers); err != nil { return } diff --git a/server/server.go b/server/server.go index 3840c8f..80e10a0 100644 --- a/server/server.go +++ b/server/server.go @@ -823,6 +823,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("DELETE /api/traffic/hosts", s.deleteTrafficHosts) mux.HandleFunc("DELETE /api/traffic/all", s.deleteAllTraffic) mux.HandleFunc("GET /api/traffic/exchange", s.getTrafficExchange) + mux.HandleFunc("GET /api/traffic/stream", s.getTrafficStream) mux.HandleFunc("GET /api/traffic/blob", s.getTrafficBlob) mux.HandleFunc("POST /api/traffic/ingest", s.ingestTraffic) mux.HandleFunc("GET /api/commands", s.pgListCommands) @@ -3451,7 +3452,37 @@ func (s *Server) getTrafficExchange(w http.ResponseWriter, r *http.Request) { writeErr(w, 404, err.Error()) return } - writeJSON(w, 200, map[string]any{"req": req, "resp": resp}) + // A WebSocket or SSE exchange carries its messages separately + // (GET /api/traffic/stream); stream_kind is "" for an ordinary exchange. + count, kind, err := tr.StreamInfo(id) + if err != nil { + log.Printf("[traffic] stream info for %s: %v", id, err) + } + writeJSON(w, 200, map[string]any{"req": req, "resp": resp, "stream_count": count, "stream_kind": kind}) +} + +// getTrafficStream returns one page of a WebSocket/SSE exchange's messages in +// order (limit 1..200, default 50). +func (s *Server) getTrafficStream(w http.ResponseWriter, r *http.Request) { + tr := s.m.Traffic() + if tr == nil { + writeErr(w, 404, "traffic disabled") + return + } + q := r.URL.Query() + id := strings.TrimSpace(q.Get("exchange")) + if id == "" { + writeErr(w, 400, "missing exchange") + return + } + offset, _ := strconv.Atoi(q.Get("offset")) + limit, _ := strconv.Atoi(q.Get("limit")) + msgs, total, err := tr.StreamMessages(id, offset, limit) + if err != nil { + writeErr(w, 500, err.Error()) + return + } + writeJSON(w, 200, map[string]any{"items": msgs, "total": total}) } // getTrafficBlob streams one oversized body by its sha256. Bodies past the inline diff --git a/server/traffic_stream_test.go b/server/traffic_stream_test.go new file mode 100644 index 0000000..5e03f70 --- /dev/null +++ b/server/traffic_stream_test.go @@ -0,0 +1,89 @@ +package server + +import ( + "encoding/json" + "testing" +) + +func TestTrafficStreamRoute(t *testing.T) { + s, request := ingestServer(t) + db := s.m.Traffic().DB() + if _, err := db.Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path) +VALUES('1-0001',1,'ws.example.com','GET','/ws','wss://ws.example.com/ws',101,'websocket',0,0,'')`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO exchange_bodies(id,req_head,resp_head) VALUES('1-0001','GET /ws HTTP/1.1\n','HTTP/1.1 101 Switching Protocols\n')`); err != nil { + t.Fatal(err) + } + const total = 250 + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + for i := 1; i <= total; i++ { + kind, payload := 1, []byte("hello") + if i%2 == 0 { + kind, payload = 2, []byte{0, 1, 2} + } + if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload) VALUES('1-0001',?,1,'c2s',?,?,?)`, i, kind, len(payload), payload); err != nil { + t.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + + type page struct { + Items []struct { + Seq int64 `json:"seq"` + Type string `json:"type"` + Payload string `json:"payload"` + Encoding string `json:"encoding"` + } `json:"items"` + Total int `json:"total"` + } + get := func(path string, wantStatus int) page { + t.Helper() + w := request("GET", path, "") + if w.Code != wantStatus { + t.Fatalf("GET %s = %d %s, want %d", path, w.Code, w.Body, wantStatus) + } + var p page + if wantStatus == 200 { + if err := json.Unmarshal(w.Body.Bytes(), &p); err != nil { + t.Fatalf("GET %s: %v in %s", path, err, w.Body) + } + } + return p + } + + get("/api/traffic/stream", 400) + if p := get("/api/traffic/stream?exchange=nope", 200); p.Items == nil || len(p.Items) != 0 || p.Total != 0 { + t.Fatalf("unknown exchange = %+v, want items [] and total 0", p) + } + p := get("/api/traffic/stream?exchange=1-0001", 200) + if p.Total != total || len(p.Items) != 50 || p.Items[0].Seq != 1 { + t.Fatalf("default page: total %d, %d items, first seq %d", p.Total, len(p.Items), p.Items[0].Seq) + } + if first, second := p.Items[0], p.Items[1]; first.Type != "text" || first.Payload != "hello" || second.Type != "binary" || second.Encoding != "base64" || second.Payload != "AAEC" { + t.Fatalf("payloads %+v %+v", first, second) + } + if p := get("/api/traffic/stream?exchange=1-0001&limit=500", 200); len(p.Items) != 200 { + t.Fatalf("limit=500 returned %d items, want the 200 cap", len(p.Items)) + } + if p := get("/api/traffic/stream?exchange=1-0001&offset=-5&limit=1", 200); len(p.Items) != 1 || p.Items[0].Seq != 1 { + t.Fatalf("negative offset: %+v, want seq 1", p.Items) + } + if p := get("/api/traffic/stream?exchange=1-0001&offset=248&limit=10", 200); len(p.Items) != 2 || p.Items[0].Seq != 249 { + t.Fatalf("last page: %+v, want seqs 249 and 250", p.Items) + } + + w := request("GET", "/api/traffic/exchange?id=1-0001", "") + var ex struct { + StreamCount int `json:"stream_count"` + StreamKind string `json:"stream_kind"` + } + if err := json.Unmarshal(w.Body.Bytes(), &ex); err != nil || w.Code != 200 || ex.StreamCount != total || ex.StreamKind != "websocket" { + t.Fatalf("exchange = %d %s", w.Code, w.Body) + } +} diff --git a/traffic/stream.go b/traffic/stream.go new file mode 100644 index 0000000..c502ebc --- /dev/null +++ b/traffic/stream.go @@ -0,0 +1,588 @@ +package traffic + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "log" + "net/url" + "reflect" + "strings" + "sync" + "time" + "unicode/utf8" + "unsafe" + + "github.com/Autumn-27/artex/db" + mproxy "github.com/lqqyt2423/go-mitmproxy/proxy" +) + +// Stream recording (WebSocket and Server-Sent Events). go-mitmproxy reports a +// stream as Start, one Message hook per message, then End; a stream never reaches +// the Response hook, so record() never sees it. Each stream becomes one exchange +// (status 101 "websocket", or the real text/event-stream response) whose messages +// go to stream_messages in arrival order. + +const ( + // wsText and wsBinary are the WebSocket data opcodes (RFC 6455 §5.2), the + // values go-mitmproxy reports in WebSocketMessage.Type. + wsText = 1 + wsBinary = 2 + // streamPreview caps one message's payload in a StreamMessages row; a longer + // payload lives in the blob store and the row carries its hash. + streamPreview = 4 * 1024 + // maxStreamInline is the inline threshold for message payloads: anything + // longer spills to the blob store, so a page of 200 rows stays bounded. + maxStreamInline = streamPreview + // toolStreamPreview caps one message's preview in traffic_messages, so a page + // of messages stays small in the agent's context. + toolStreamPreview = 1024 +) + +// streamState tracks one live stream. mu serializes the two WebSocket forwarding +// goroutines, which both fire WebSocketMessage on the same flow. +type streamState struct { + mu sync.Mutex + id string // exchange id; "" when the stream could not be recorded + kind string // "websocket" | "sse" + consumed int // library messages/events already handled + seq int64 + stored int + // storedBytes counts what the kept messages occupy: the payload after + // truncation, plus the inline preview spill keeps for a spilled payload. + storedBytes int64 + dropped bool + droppedMsgs int64 + droppedBytes int64 + // reqBytes and respBytes are the bytes observed in each direction, dropped + // messages included. + reqBytes, respBytes int64 +} + +// streamRow is one stream_messages row before it is written. +type streamRow struct { + ts int64 // unix seconds, like exchanges.ts; seq carries the order + direction string + kind int + event string + eventID string + retry int + size int + payload []byte // kept bytes (after maxStreamPayload truncation) + truncated bool + ctype string // content type handed to spill for the binary check +} + +func (s *sink) WebSocketStart(f *mproxy.Flow) { s.startStream(f, "websocket") } +func (s *sink) SSEStart(f *mproxy.Flow) { s.startStream(f, "sse") } + +func (s *sink) WebSocketMessage(f *mproxy.Flow) { + st := s.stream(f) + if st == nil || f.WebScoket == nil { + return + } + st.mu.Lock() + defer st.mu.Unlock() + s.t.persistStream(st, wsMessagesSince(f.WebScoket, st)) +} + +func (s *sink) SSEMessage(f *mproxy.Flow) { + st := s.stream(f) + if st == nil || f.SSE == nil { + return + } + st.mu.Lock() + defer st.mu.Unlock() + s.t.persistStream(st, sseEventsSince(f.SSE, st)) +} + +func (s *sink) WebSocketEnd(f *mproxy.Flow) { + v, ok := s.streams.LoadAndDelete(f) + if !ok { + return + } + st := v.(*streamState) + st.mu.Lock() + defer st.mu.Unlock() + if f.WebScoket != nil { + s.t.persistStream(st, wsMessagesSince(f.WebScoket, st)) + } + s.t.finishStream(st) +} + +func (s *sink) SSEEnd(f *mproxy.Flow) { + v, ok := s.streams.LoadAndDelete(f) + if !ok { + return + } + st := v.(*streamState) + st.mu.Lock() + defer st.mu.Unlock() + if f.SSE != nil { + s.t.persistStream(st, sseEventsSince(f.SSE, st)) + } + s.t.finishStream(st) +} + +func (s *sink) stream(f *mproxy.Flow) *streamState { + v, ok := s.streams.Load(f) + if !ok { + return nil + } + return v.(*streamState) +} + +// startStream registers the flow before recording its exchange, so a message +// hook racing the start waits on st.mu instead of being dropped. +func (s *sink) startStream(f *mproxy.Flow, kind string) { + st := &streamState{kind: kind} + st.mu.Lock() + defer st.mu.Unlock() + s.streams.Store(f, st) + if f.Request == nil || f.Request.URL == nil || (kind == "sse" && f.Response == nil) { + return + } + id, err := s.t.recordStreamStart(f, kind) + if err != nil { + log.Printf("[traffic] failed to record %s stream %s: %v", kind, f.Request.URL, err) + return + } + st.id = id + if kind == "sse" { + st.reqBytes = int64(len(f.Request.Body)) + } +} + +// libMutex returns the unexported mu sync.Mutex that go-mitmproxy holds while it +// appends to WebSocketData.Messages / SSEData.Events. The library exposes no +// accessor, and the two WebSocket goroutines append concurrently, so reading the +// slice without that lock is a data race. Reaching a private field ties this to +// the pinned go-mitmproxy v1.9.2: TestLibMutexFound is the upgrade guard. Nil +// when the field is absent, and then nothing is read (see lockLib). +func libMutex(v any) *sync.Mutex { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Pointer || rv.IsNil() { + return nil + } + f := rv.Elem().FieldByName("mu") + if !f.IsValid() || f.Type() != reflect.TypeOf(sync.Mutex{}) || !f.CanAddr() { + return nil + } + return (*sync.Mutex)(unsafe.Pointer(f.UnsafeAddr())) +} + +var libMutexMissing sync.Once + +// lockLib takes the library's lock on v. ok is false when the lock cannot be +// found; callers then read nothing rather than race the library. +func lockLib(v any) (unlock func(), ok bool) { + mu := libMutex(v) + if mu == nil { + libMutexMissing.Do(func() { + log.Printf("[traffic] go-mitmproxy stream data has no mu field; stream messages are not recorded") + }) + return nil, false + } + mu.Lock() + return mu.Unlock, true +} + +// wsMessagesSince returns every WebSocket message appended since the last call, +// in the library's order, so each is handled exactly once whichever goroutine's +// hook gets there first. The library slice itself is never modified. +func wsMessagesSince(ws *mproxy.WebSocketData, st *streamState) []streamRow { + unlock, ok := lockLib(ws) + if !ok { + return nil + } + msgs := ws.Messages + if st.consumed > len(msgs) { + st.consumed = len(msgs) + } + fresh := append([]*mproxy.WebSocketMessage(nil), msgs[st.consumed:]...) + st.consumed = len(msgs) + unlock() + rows := make([]streamRow, 0, len(fresh)) + for _, m := range fresh { + if m == nil { + continue + } + dir, ctype := "s2c", "text/plain; charset=utf-8" + if m.FromClient { + dir = "c2s" + st.reqBytes += int64(len(m.Content)) + } else { + st.respBytes += int64(len(m.Content)) + } + if m.Type == wsBinary { + ctype = "application/octet-stream" + } + rows = append(rows, streamRow{ts: m.Timestamp.Unix(), direction: dir, kind: m.Type, size: len(m.Content), payload: m.Content, ctype: ctype}) + } + return rows +} + +// sseEventsSince is wsMessagesSince for Server-Sent Events. +func sseEventsSince(sse *mproxy.SSEData, st *streamState) []streamRow { + unlock, ok := lockLib(sse) + if !ok { + return nil + } + events := sse.Events + if st.consumed > len(events) { + st.consumed = len(events) + } + fresh := append([]*mproxy.SSEEvent(nil), events[st.consumed:]...) + st.consumed = len(events) + unlock() + rows := make([]streamRow, 0, len(fresh)) + for _, e := range fresh { + if e == nil { + continue + } + st.respBytes += int64(len(e.Raw)) + rows = append(rows, streamRow{ts: e.Time.Unix(), direction: "s2c", event: e.Event, eventID: e.ID, + retry: e.Retry, size: len(e.Data), payload: []byte(e.Data), ctype: "text/event-stream"}) + } + return rows +} + +// streamDisplayURL is the URL shown for a WebSocket exchange: http→ws and +// https→wss. A plain ws:// connection reaches the WebSocket hooks through its +// CONNECT tunnel, where only the target authority is known, not the path. +func streamDisplayURL(f *mproxy.Flow, kind string) (method, display, tmpl string) { + u := *f.Request.URL + method = f.Request.Method + if kind == "websocket" && strings.EqualFold(method, "CONNECT") { + host := u.Host + if host == "" { + host = u.Opaque + } + return "GET", (&url.URL{Scheme: "ws", Host: host, Path: "/"}).String(), "/" + } + if kind == "websocket" { + switch strings.ToLower(u.Scheme) { + case "http", "": + u.Scheme = "ws" + case "https": + u.Scheme = "wss" + } + } + return method, u.String(), db.TemplatePath(f.Request.URL.EscapedPath()) +} + +// recordStreamStart writes the stream's exchange: metadata, request head and +// body, the response head (synthetic 101 for WebSocket), and the header text in +// the full-text index. Messages are never indexed. +func (t *Traffic) recordStreamStart(f *mproxy.Flow, kind string) (string, error) { + now := time.Now() + id := t.newExchangeID(now) + method, display, tmpl := streamDisplayURL(f, kind) + host := f.Request.URL.Hostname() + target := f.Request.URL.RequestURI() + if strings.EqualFold(f.Request.Method, "CONNECT") { + target = f.Request.URL.Host // authority-form, as on the wire + } + reqHead := fmt.Sprintf("%s %s %s\n%s", f.Request.Method, target, f.Request.Proto, requestHeaderLines(f.Request)) + status, ct, respHead := 101, "websocket", "HTTP/1.1 101 Switching Protocols\n" + if kind == "sse" { + status = f.Response.StatusCode + ct = f.Response.Header.Get("Content-Type") + respHead = fmt.Sprintf("HTTP %d\n%s", status, headerLines(f.Response.Header)) + } + + t.wmu.Lock() + defer t.wmu.Unlock() + reqB := t.spill(f.Request.Body, f.Request.Header.Get("Content-Type")) + tx, err := t.db.Begin() + if err != nil { + return "", fmt.Errorf("beginning transaction: %w", err) + } + defer tx.Rollback() //nolint:errcheck // no-op once committed + res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path) +VALUES(?,?,?,?,?,?,?,?,?,?,'')`, id, now.Unix(), host, method, tmpl, display, status, ct, len(f.Request.Body), 0) + if err != nil { + return "", fmt.Errorf("writing index: %w", err) + } + rowid, err := res.LastInsertId() + if err != nil { + return "", fmt.Errorf("fetching rowid: %w", err) + } + if _, err := tx.Exec(`INSERT OR REPLACE INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob) +VALUES(?,?,?,?,?,NULL,NULL)`, id, reqHead, reqB.inline, nullIfEmpty(reqB.hash), respHead); err != nil { + return "", fmt.Errorf("writing body: %w", err) + } + if reqB.hash != "" { + if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, reqB.hash, id); err != nil { + return "", fmt.Errorf("registering blob reference: %w", err) + } + } + if t.fts { + idx := strings.Join([]string{display, reqHead, reqB.index, respHead}, "\n") + if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowid, idx); err != nil { + return "", fmt.Errorf("writing full-text index: %w", err) + } + } + if err := tx.Commit(); err != nil { + return "", fmt.Errorf("committing: %w", err) + } + return id, nil +} + +// persistStream applies the per-message and per-exchange limits to rows and +// writes what is kept. The first message past a per-exchange limit stores one +// 'meta' row instead; nothing is stored after it. Caller holds st.mu. Errors are +// logged: a recording failure never interrupts the stream. +func (t *Traffic) persistStream(st *streamState, rows []streamRow) { + if st.id == "" || len(rows) == 0 { + return + } + keep := make([]streamRow, 0, len(rows)) + addMeta := false + for _, r := range rows { + if len(r.payload) > maxStreamPayload { + r.payload = r.payload[:maxStreamPayload] + r.truncated = true + } + cost := streamStoredCost(r) + if st.dropped || st.stored >= maxStreamMessages || st.storedBytes+cost > maxStreamBytes { + if !st.dropped { + st.dropped = true + addMeta = true + } + st.droppedMsgs++ + st.droppedBytes += int64(r.size) + continue + } + st.stored++ + st.storedBytes += cost + keep = append(keep, r) + } + var meta []byte + if addMeta { + meta = streamMetaPayload(st) + } + if len(keep) == 0 && meta == nil { + return + } + + t.wmu.Lock() + defer t.wmu.Unlock() + // Sequence numbers and blob writes come first, as in record(): blob writes are + // filesystem work and must not sit inside the SQLite transaction. Holding wmu + // keeps gcBlobs from collecting a blob before its reference is committed. + type preparedRow struct { + streamRow + seq int64 + body storedBody + } + prepared := make([]preparedRow, 0, len(keep)) + for _, r := range keep { + st.seq++ + prepared = append(prepared, preparedRow{streamRow: r, seq: st.seq, body: t.spillAt(r.payload, r.ctype, maxStreamInline)}) + } + var metaSeq int64 + if meta != nil { + st.seq++ + metaSeq = st.seq + } + + tx, err := t.db.Begin() + if err != nil { + log.Printf("[traffic] failed to record stream messages of %s (beginning transaction): %v", st.id, err) + return + } + defer tx.Rollback() //nolint:errcheck // no-op once committed + for _, p := range prepared { + inline := p.body.inline + if inline == nil { + inline = []byte{} + } + if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,event,event_id,retry,size,payload,payload_blob,truncated) +VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`, st.id, p.seq, p.ts, p.direction, p.kind, p.event, p.eventID, p.retry, p.size, + inline, nullIfEmpty(p.body.hash), boolInt(p.truncated)); err != nil { + log.Printf("[traffic] failed to record stream message %d of %s: %v", p.seq, st.id, err) + return + } + if p.body.hash != "" { + if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, p.body.hash, st.id); err != nil { + log.Printf("[traffic] failed to record stream message %d of %s (registering blob reference): %v", p.seq, st.id, err) + return + } + } + } + if meta != nil { + if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload) +VALUES(?,?,?,'meta',0,?,?)`, st.id, metaSeq, time.Now().Unix(), len(meta), meta); err != nil { + log.Printf("[traffic] failed to record stream limit marker of %s: %v", st.id, err) + return + } + } + if err := tx.Commit(); err != nil { + log.Printf("[traffic] failed to record stream messages of %s (committing): %v", st.id, err) + } +} + +// streamStoredCost is what keeping r adds to the index and blob store: the +// payload, plus the inline preview spillAt keeps when the payload spills (a text +// prefix of at most blobPreview, or a one-line tag for binary). +func streamStoredCost(r streamRow) int64 { + n := int64(len(r.payload)) + if len(r.payload) <= maxStreamInline { + return n + } + if isBinaryBody(r.ctype, r.payload) { + return n + int64(len(binaryTag(r.ctype, r.payload))) + } + return n + int64(len(truncateUTF8(r.payload, blobPreview))) +} + +func streamMetaPayload(st *streamState) []byte { + b, _ := json.Marshal(map[string]int64{"dropped_messages": st.droppedMsgs, "dropped_bytes": st.droppedBytes}) + return b +} + +func boolInt(b bool) int { + if b { + return 1 + } + return 0 +} + +// finishStream records the bytes observed in each direction and the final +// counts of the limit marker. Caller holds st.mu. +func (t *Traffic) finishStream(st *streamState) { + if st.id == "" { + return + } + t.wmu.Lock() + defer t.wmu.Unlock() + if _, err := t.db.Exec(`UPDATE exchanges SET req_len=?, resp_len=? WHERE id=?`, st.reqBytes, st.respBytes, st.id); err != nil { + log.Printf("[traffic] failed to record stream sizes of %s: %v", st.id, err) + } + if st.dropped { + meta := streamMetaPayload(st) + if _, err := t.db.Exec(`UPDATE stream_messages SET payload=?, size=? WHERE exchange_id=? AND direction='meta'`, meta, len(meta), st.id); err != nil { + log.Printf("[traffic] failed to update stream limit marker of %s: %v", st.id, err) + } + } +} + +// StreamMessage is one recorded WebSocket message or SSE event. +type StreamMessage struct { + Seq int64 `json:"seq"` + TS int64 `json:"ts"` // unix seconds, like an exchange's ts; Seq orders + Direction string `json:"direction"` + Kind int `json:"kind"` + // Type is "text" or "binary" (WebSocket), "event" (SSE), or "meta" (the + // limit marker). + Type string `json:"type"` + Event string `json:"event,omitempty"` + EventID string `json:"event_id,omitempty"` + Retry int `json:"retry,omitempty"` + Size int `json:"size"` + Payload string `json:"payload"` + // Encoding is "base64" for binary payloads, empty for UTF-8 text. + Encoding string `json:"encoding,omitempty"` + Blob string `json:"blob,omitempty"` + Truncated bool `json:"truncated,omitempty"` +} + +// StreamMessages returns one page of an exchange's stream messages in order. +// limit is 1..200 (default 50) and offset >= 0. Every payload comes back as at +// most streamPreview (4 KiB); a longer one lives in the blob store and the row +// carries its hash for the rest (traffic_blob, GET /api/traffic/blob). +func (t *Traffic) StreamMessages(exchangeID string, offset, limit int) ([]StreamMessage, int, error) { + if limit <= 0 { + limit = 50 + } + if limit > 200 { + limit = 200 + } + if offset < 0 { + offset = 0 + } + var total int + if err := t.db.QueryRow(`SELECT COUNT(*) FROM stream_messages WHERE exchange_id=?`, exchangeID).Scan(&total); err != nil { + return nil, 0, err + } + rows, err := t.db.Query(`SELECT seq,ts,direction,kind,event,event_id,retry,size,payload,payload_blob,truncated +FROM stream_messages WHERE exchange_id=? ORDER BY seq LIMIT ? OFFSET ?`, exchangeID, limit, offset) + if err != nil { + return nil, 0, err + } + defer rows.Close() + out := []StreamMessage{} + for rows.Next() { + var m StreamMessage + var payload []byte + var blob sql.NullString + var truncated int + if err := rows.Scan(&m.Seq, &m.TS, &m.Direction, &m.Kind, &m.Event, &m.EventID, &m.Retry, &m.Size, &payload, &blob, &truncated); err != nil { + return nil, 0, err + } + m.Truncated = truncated != 0 + m.Blob = blob.String + switch { + case m.Direction == "meta": + m.Type = "meta" + case m.Kind == wsBinary: + m.Type = "binary" + case m.Kind == wsText: + m.Type = "text" + default: + m.Type = "event" + } + if m.Type == "binary" { + if m.Blob != "" { + // A spilled binary payload keeps only a descriptive tag inline; + // the preview is the payload's own leading bytes. + payload, _, _ = t.BlobRange(m.Blob, 0, streamPreview) + } + m.Payload, m.Encoding = base64.StdEncoding.EncodeToString(clipBytes(payload, streamPreview)), "base64" + } else { + m.Payload = truncateUTF8(payload, streamPreview) + } + out = append(out, m) + } + return out, total, rows.Err() +} + +// StreamInfo reports how many stream messages an exchange holds and whether it +// is a WebSocket or SSE exchange ("" for an ordinary exchange). +func (t *Traffic) StreamInfo(exchangeID string) (count int, kind string, err error) { + var ct sql.NullString + err = t.db.QueryRow(`SELECT content_type FROM exchanges WHERE id=?`, exchangeID).Scan(&ct) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return 0, "", err + } + switch c := strings.ToLower(ct.String); { + case c == "websocket": + kind = "websocket" + case strings.Contains(c, "text/event-stream"): + kind = "sse" + } + if err := t.db.QueryRow(`SELECT COUNT(*) FROM stream_messages WHERE exchange_id=?`, exchangeID).Scan(&count); err != nil { + return 0, "", err + } + return count, kind, nil +} + +// streamToolPreview renders one message for traffic_messages: at most 1 KiB of +// text (hex for binary), plus the blob hash when the payload spilled. +func streamToolPreview(m StreamMessage) string { + if m.Encoding == "base64" { + raw, _ := base64.StdEncoding.DecodeString(m.Payload) + return "hex:" + fmt.Sprintf("%x", clipBytes(raw, toolStreamPreview/2)) + } + p := m.Payload + if len(p) > toolStreamPreview { + p = truncateUTF8([]byte(p), toolStreamPreview) + } + if !utf8.ValidString(p) { + p = strings.ToValidUTF8(p, "�") + } + return p +} diff --git a/traffic/stream_test.go b/traffic/stream_test.go new file mode 100644 index 0000000..96e6263 --- /dev/null +++ b/traffic/stream_test.go @@ -0,0 +1,637 @@ +package traffic + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/gorilla/websocket" + mproxy "github.com/lqqyt2423/go-mitmproxy/proxy" +) + +// localhostURL names an httptest server by host name: the MITM leaf certificate +// carries a DNS name, not an IP SAN (as in pace_test.go). +func localhostURL(u string) string { return strings.Replace(u, "127.0.0.1", "localhost", 1) } + +// waitFor polls cond until it holds or 10 seconds pass. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(20 * time.Millisecond) + } +} + +// wsDialer dials through the recording proxy and trusts its CA. +func wsDialer(t *testing.T, tr *Traffic, proxyURL string) *websocket.Dialer { + t.Helper() + pem, err := os.ReadFile(tr.CACertPath()) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + t.Fatal("no CA in", tr.CACertPath()) + } + u, _ := url.Parse(proxyURL) + return &websocket.Dialer{Proxy: http.ProxyURL(u), TLSClientConfig: &tls.Config{RootCAs: pool}, HandshakeTimeout: 5 * time.Second} +} + +type streamExchange struct { + id, url, contentType string + status int + reqLen, respLen int64 +} + +// streamExchanges returns the exchanges recorded with a stream content type. +func streamExchanges(t *testing.T, tr *Traffic, contentLike string) []streamExchange { + t.Helper() + rows, err := tr.DB().Query(`SELECT id,url,content_type,status,req_len,resp_len FROM exchanges WHERE content_type LIKE ? ORDER BY id`, contentLike) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var out []streamExchange + for rows.Next() { + var e streamExchange + if err := rows.Scan(&e.id, &e.url, &e.contentType, &e.status, &e.reqLen, &e.respLen); err != nil { + t.Fatal(err) + } + out = append(out, e) + } + return out +} + +func allMessages(t *testing.T, tr *Traffic, id string) []StreamMessage { + t.Helper() + var out []StreamMessage + for offset := 0; ; offset += 200 { + page, total, err := tr.StreamMessages(id, offset, 200) + if err != nil { + t.Fatal(err) + } + out = append(out, page...) + if len(out) >= total || len(page) == 0 { + return out + } + } +} + +// echoWSServer echoes every message back with the same type: text prefixed +// with "echo:", binary reversed. +func echoWSServer(t *testing.T) *httptest.Server { + t.Helper() + up := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }} + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + defer c.Close() + for { + mt, msg, err := c.ReadMessage() + if err != nil { + return + } + if mt == websocket.BinaryMessage { + rev := make([]byte, len(msg)) + for i := range msg { + rev[len(msg)-1-i] = msg[i] + } + msg = rev + } else { + msg = append([]byte("echo:"), msg...) + } + if err := c.WriteMessage(mt, msg); err != nil { + return + } + } + })) + t.Cleanup(srv.Close) + return srv +} + +func closeWS(c *websocket.Conn) { + _ = c.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + _ = c.SetReadDeadline(time.Now().Add(2 * time.Second)) + for { + if _, _, err := c.ReadMessage(); err != nil { + break + } + } + _ = c.Close() +} + +func TestWebSocketMessagesRecorded(t *testing.T) { + tr, _, proxyURL := startProxy(t) + srv := echoWSServer(t) + target := "wss://" + strings.TrimPrefix(localhostURL(srv.URL), "https://") + "/ws?room=1" + c, _, err := wsDialer(t, tr, proxyURL).Dial(target, nil) + if err != nil { + t.Fatal(err) + } + exchange := func(mt int, msg []byte) []byte { + if err := c.WriteMessage(mt, msg); err != nil { + t.Fatal(err) + } + gotType, got, err := c.ReadMessage() + if err != nil || gotType != mt { + t.Fatalf("read type=%d err=%v", gotType, err) + } + return got + } + before := time.Now().Unix() + if got := exchange(websocket.TextMessage, []byte("hello")); string(got) != "echo:hello" { + t.Fatalf("echo %q", got) + } + if got := exchange(websocket.BinaryMessage, []byte{0, 1, 2, 255}); !bytes.Equal(got, []byte{255, 2, 1, 0}) { + t.Fatalf("binary echo %v", got) + } + closeWS(c) + + var ex streamExchange + waitFor(t, "websocket end", func() bool { + exs := streamExchanges(t, tr, "websocket") + if len(exs) != 1 || exs[0].reqLen == 0 { + return false + } + ex = exs[0] + return true + }) + if ex.status != 101 || ex.url != target { + t.Fatalf("exchange %+v, want status 101 and url %s", ex, target) + } + // c2s: "hello" (5) + 4 binary bytes; s2c: "echo:hello" (10) + 4. + if ex.reqLen != 9 || ex.respLen != 14 { + t.Fatalf("req_len=%d resp_len=%d, want 9 and 14", ex.reqLen, ex.respLen) + } + var respHead string + if err := tr.DB().QueryRow(`SELECT resp_head FROM exchange_bodies WHERE id=?`, ex.id).Scan(&respHead); err != nil || !strings.HasPrefix(respHead, "HTTP/1.1 101 Switching Protocols") { + t.Fatalf("resp_head %q err=%v", respHead, err) + } + if count, kind, err := tr.StreamInfo(ex.id); err != nil || count != 4 || kind != "websocket" { + t.Fatalf("StreamInfo = %d %q %v", count, kind, err) + } + + msgs := allMessages(t, tr, ex.id) + want := []struct { + dir, typ, payload string + kind int + }{ + {"c2s", "text", "hello", 1}, + {"s2c", "text", "echo:hello", 1}, + {"c2s", "binary", base64.StdEncoding.EncodeToString([]byte{0, 1, 2, 255}), 2}, + {"s2c", "binary", base64.StdEncoding.EncodeToString([]byte{255, 2, 1, 0}), 2}, + } + if len(msgs) != len(want) { + t.Fatalf("got %d messages: %+v", len(msgs), msgs) + } + last := before + for i, w := range want { + m := msgs[i] + if m.Seq != int64(i+1) || m.Direction != w.dir || m.Type != w.typ || m.Kind != w.kind || m.Payload != w.payload { + t.Fatalf("message %d = %+v, want %+v", i, m, w) + } + if (w.typ == "binary") != (m.Encoding == "base64") { + t.Fatalf("message %d encoding %q", i, m.Encoding) + } + if m.TS < last { + t.Fatalf("message %d ts %d before %d", i, m.TS, last) + } + last = m.TS + } +} + +func TestSSEEventsRecorded(t *testing.T) { + tr, _, proxyURL := startProxy(t) + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(200) + fl := w.(http.Flusher) + for _, ev := range []string{ + "retry: 1500\nid: 1\nevent: greet\ndata: hello\ndata: world\n\n", + "id: 2\ndata: second\n\n", + "event: done\ndata: bye\n\n", + } { + _, _ = io.WriteString(w, ev) + fl.Flush() + time.Sleep(10 * time.Millisecond) + } + })) + t.Cleanup(srv.Close) + + resp, err := proxyClient(t, tr, proxyURL).Get(localhostURL(srv.URL) + "/events") + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if !strings.Contains(string(body), "data: bye") { + t.Fatalf("client body %q", body) + } + + var ex streamExchange + waitFor(t, "sse end", func() bool { + exs := streamExchanges(t, tr, "%event-stream%") + if len(exs) != 1 || exs[0].respLen == 0 { + return false + } + ex = exs[0] + return true + }) + if ex.status != 200 || !strings.HasPrefix(ex.url, localhostURL(srv.URL)+"/events") { + t.Fatalf("exchange %+v", ex) + } + if n, _ := tr.Count(); n != 1 { + t.Fatalf("%d exchanges recorded, want the single SSE exchange", n) + } + msgs := allMessages(t, tr, ex.id) + want := []struct { + event, id, data string + retry int + }{ + {"greet", "1", "hello\nworld", 1500}, + {"message", "2", "second", 0}, + {"done", "", "bye", 0}, + } + if len(msgs) != len(want) { + t.Fatalf("got %d events: %+v", len(msgs), msgs) + } + for i, w := range want { + m := msgs[i] + if m.Seq != int64(i+1) || m.Direction != "s2c" || m.Kind != 0 || m.Type != "event" || + m.Event != w.event || m.EventID != w.id || m.Payload != w.data || m.Retry != w.retry || m.Size != len(w.data) { + t.Fatalf("event %d = %+v, want %+v", i, m, w) + } + } +} + +// unitStream opens a recorder without starting the proxy and drives its hooks +// directly with a synthetic WebSocket flow. +func unitStream(t *testing.T, host string) (*Traffic, *sink, *mproxy.Flow) { + t.Helper() + tr, err := Open(t.TempDir(), "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = tr.Close() }) + u, _ := url.Parse("https://" + host + "/ws") + f := &mproxy.Flow{Request: &mproxy.Request{Method: "GET", URL: u, Proto: "HTTP/1.1", Header: http.Header{}}, WebScoket: &mproxy.WebSocketData{}} + s := tr.proxy.Addons[0].(*sink) + s.WebSocketStart(f) + return tr, s, f +} + +func addWS(f *mproxy.Flow, fromClient bool, mt int, content []byte) { + f.WebScoket.Messages = append(f.WebScoket.Messages, &mproxy.WebSocketMessage{Type: mt, Content: content, FromClient: fromClient, Timestamp: time.Now()}) +} + +func onlyExchange(t *testing.T, tr *Traffic) string { + t.Helper() + exs := streamExchanges(t, tr, "websocket") + if len(exs) != 1 { + t.Fatalf("%d websocket exchanges", len(exs)) + } + return exs[0].id +} + +func metaCounts(t *testing.T, m StreamMessage) (msgs, bytes int64) { + t.Helper() + var v struct { + DroppedMessages int64 `json:"dropped_messages"` + DroppedBytes int64 `json:"dropped_bytes"` + } + if err := json.Unmarshal([]byte(m.Payload), &v); err != nil { + t.Fatalf("meta payload %q: %v", m.Payload, err) + } + return v.DroppedMessages, v.DroppedBytes +} + +func TestStreamPayloadTruncatedToLimit(t *testing.T) { + tr, s, f := unitStream(t, "trunc.test") + big := bytes.Repeat([]byte("a"), maxStreamPayload+10) + addWS(f, true, websocket.TextMessage, big) + s.WebSocketMessage(f) + id := onlyExchange(t, tr) + msgs := allMessages(t, tr, id) + if len(msgs) != 1 { + t.Fatalf("%d messages", len(msgs)) + } + m := msgs[0] + if !m.Truncated || m.Size != maxStreamPayload+10 || m.Blob == "" || len(m.Payload) > streamPreview { + t.Fatalf("message = seq %d size %d truncated %v blob %q payload %d bytes", m.Seq, m.Size, m.Truncated, m.Blob, len(m.Payload)) + } + data, total, err := tr.BlobRange(m.Blob, 0, 16) + if err != nil || total != maxStreamPayload || string(data) != strings.Repeat("a", 16) { + t.Fatalf("blob total=%d data=%q err=%v, want the kept %d-byte prefix", total, data, err, maxStreamPayload) + } +} + +func TestStreamMessageLimitStoresOneMetaRow(t *testing.T) { + tr, s, f := unitStream(t, "count.test") + for i := 0; i < maxStreamMessages+5; i++ { + addWS(f, i%2 == 0, websocket.TextMessage, []byte("m")) + } + s.WebSocketMessage(f) + addWS(f, true, websocket.TextMessage, []byte("late")) + s.WebSocketMessage(f) + s.WebSocketEnd(f) + id := onlyExchange(t, tr) + msgs := allMessages(t, tr, id) + if len(msgs) != maxStreamMessages+1 { + t.Fatalf("%d rows, want %d messages and one meta row", len(msgs), maxStreamMessages) + } + meta := msgs[len(msgs)-1] + if meta.Direction != "meta" || meta.Type != "meta" || meta.Kind != 0 || meta.Seq != maxStreamMessages+1 { + t.Fatalf("last row %+v, want the meta row", meta) + } + if n, b := metaCounts(t, meta); n != 6 || b != 5+4 { + t.Fatalf("meta dropped_messages=%d dropped_bytes=%d, want 6 and 9", n, b) + } + for _, m := range msgs[:len(msgs)-1] { + if m.Direction == "meta" { + t.Fatalf("more than one meta row: %+v", m) + } + } +} + +func TestStreamByteLimitStoresOneMetaRow(t *testing.T) { + tr, s, f := unitStream(t, "bytes.test") + chunk := bytes.Repeat([]byte{7}, maxStreamPayload) + // Each kept 1 MiB payload spills and also keeps its inline tag, so the cap + // admits floor(maxStreamBytes / (payload + tag)) of them. + cost := streamStoredCost(streamRow{payload: chunk, ctype: "application/octet-stream"}) + if cost <= maxStreamPayload { + t.Fatalf("cost %d does not count the inline preview", cost) + } + kept := int(maxStreamBytes / cost) + n := kept + 1 + for i := 0; i < n; i++ { + addWS(f, false, websocket.BinaryMessage, chunk) + } + s.WebSocketMessage(f) + addWS(f, false, websocket.BinaryMessage, []byte{1}) + s.WebSocketMessage(f) + s.WebSocketEnd(f) + id := onlyExchange(t, tr) + msgs := allMessages(t, tr, id) + if len(msgs) != kept+1 { + t.Fatalf("%d rows, want %d stored and one meta row", len(msgs), kept) + } + meta := msgs[len(msgs)-1] + if meta.Direction != "meta" { + t.Fatalf("last row %+v, want the meta row", meta) + } + if dn, db := metaCounts(t, meta); dn != 2 || db != maxStreamPayload+1 { + t.Fatalf("meta dropped_messages=%d dropped_bytes=%d", dn, db) + } + var stored, inline int64 + if err := tr.DB().QueryRow(`SELECT SUM(size), SUM(LENGTH(payload)) FROM stream_messages WHERE exchange_id=? AND direction<>'meta'`, id).Scan(&stored, &inline); err != nil || + stored != int64(kept*maxStreamPayload) || stored+inline > maxStreamBytes { + t.Fatalf("stored %d payload bytes + %d inline err=%v, want %d within %d", stored, inline, err, kept*maxStreamPayload, maxStreamBytes) + } + var respLen int64 + if err := tr.DB().QueryRow(`SELECT resp_len FROM exchanges WHERE id=?`, id).Scan(&respLen); err != nil || respLen != int64(n*maxStreamPayload+1) { + t.Fatalf("resp_len %d err=%v, want every observed byte", respLen, err) + } +} + +// A payload over maxStreamInline goes to the blob store; the row returns at +// most 4 KiB and the hash, and the blob holds the whole payload. +func TestStreamPayloadSpillsAboveInlineLimit(t *testing.T) { + tr, s, f := unitStream(t, "spill.test") + bigText := []byte(strings.Repeat("0123456789", 1024)) // 10 KiB + bigBin := make([]byte, 10*1024) + for i := range bigBin { + bigBin[i] = byte(i) + } + small := []byte(strings.Repeat("s", 1024)) + addWS(f, true, websocket.TextMessage, bigText) + addWS(f, false, websocket.BinaryMessage, bigBin) + addWS(f, true, websocket.TextMessage, small) + s.WebSocketMessage(f) + msgs := allMessages(t, tr, onlyExchange(t, tr)) + if len(msgs) != 3 { + t.Fatalf("%d messages", len(msgs)) + } + text, bin, inline := msgs[0], msgs[1], msgs[2] + if text.Blob == "" || len(text.Payload) > streamPreview || text.Payload != string(bigText[:streamPreview]) || text.Size != len(bigText) || text.Truncated { + t.Fatalf("10 KiB text row: blob %q payload %d bytes size %d truncated %v", text.Blob, len(text.Payload), text.Size, text.Truncated) + } + if data, total, err := tr.BlobRange(text.Blob, 0, int64(len(bigText))); err != nil || total != int64(len(bigText)) || !bytes.Equal(data, bigText) { + t.Fatalf("text blob total=%d err=%v, want the full 10 KiB", total, err) + } + raw, err := base64.StdEncoding.DecodeString(bin.Payload) + if bin.Blob == "" || err != nil || !bytes.Equal(raw, bigBin[:streamPreview]) { + t.Fatalf("10 KiB binary row: blob %q preview %d bytes err=%v, want its first 4 KiB", bin.Blob, len(raw), err) + } + if data, total, err := tr.BlobRange(bin.Blob, 0, int64(len(bigBin))); err != nil || total != int64(len(bigBin)) || !bytes.Equal(data, bigBin) { + t.Fatalf("binary blob total=%d err=%v", total, err) + } + if inline.Blob != "" || inline.Payload != string(small) { + t.Fatalf("1 KiB row: blob %q payload %d bytes, want inline and complete", inline.Blob, len(inline.Payload)) + } +} + +// The server writes a message and closes at once; the message the client read +// must be recorded even though WebSocketEnd fires as soon as one direction ends. +func TestWebSocketFinalMessageOnClose(t *testing.T) { + tr, _, proxyURL := startProxy(t) + up := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }} + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + _ = c.WriteMessage(websocket.TextMessage, []byte("final")) + _ = c.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + _ = c.Close() + })) + t.Cleanup(srv.Close) + c, _, err := wsDialer(t, tr, proxyURL).Dial("wss://"+strings.TrimPrefix(localhostURL(srv.URL), "https://")+"/ws", nil) + if err != nil { + t.Fatal(err) + } + if _, got, err := c.ReadMessage(); err != nil || string(got) != "final" { + t.Fatalf("read %q err=%v", got, err) + } + _ = c.Close() + var id string + waitFor(t, "websocket end", func() bool { + exs := streamExchanges(t, tr, "websocket") + if len(exs) != 1 || exs[0].respLen == 0 { + return false + } + id = exs[0].id + return true + }) + msgs := allMessages(t, tr, id) + if len(msgs) != 1 || msgs[0].Direction != "s2c" || msgs[0].Payload != "final" { + t.Fatalf("messages %+v, want the final server message", msgs) + } +} + +func TestDeleteHostRemovesStreamMessagesAndBlobs(t *testing.T) { + tr, s, f := unitStream(t, "gone.test") + addWS(f, false, websocket.BinaryMessage, bytes.Repeat([]byte{9}, maxInlineBody+1)) + addWS(f, true, websocket.TextMessage, []byte("small")) + s.WebSocketMessage(f) + s.WebSocketEnd(f) + id := onlyExchange(t, tr) + msgs := allMessages(t, tr, id) + if len(msgs) != 2 || msgs[0].Blob == "" { + t.Fatalf("messages %+v, want a spilled first payload", msgs) + } + blobPath := filepath.Join(tr.dir, "_blobs", "sha256", msgs[0].Blob[:2], msgs[0].Blob+".bin") + if _, err := os.Stat(blobPath); err != nil { + t.Fatalf("payload blob missing: %v", err) + } + if n, err := tr.DeleteHost("gone.test"); err != nil || n != 1 { + t.Fatalf("DeleteHost = %d, %v", n, err) + } + var rows int + if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM stream_messages`).Scan(&rows); err != nil || rows != 0 { + t.Fatalf("%d stream rows left, err=%v", rows, err) + } + if _, err := os.Stat(blobPath); !os.IsNotExist(err) { + t.Fatalf("payload blob not collected: %v", err) + } +} + +// A recording failure must never reach the proxied connection. +func TestWebSocketSurvivesRecordingFailure(t *testing.T) { + tr, _, proxyURL := startProxy(t) + srv := echoWSServer(t) + c, _, err := wsDialer(t, tr, proxyURL).Dial("wss://"+strings.TrimPrefix(localhostURL(srv.URL), "https://")+"/ws", nil) + if err != nil { + t.Fatal(err) + } + defer closeWS(c) + if err := c.WriteMessage(websocket.TextMessage, []byte("before")); err != nil { + t.Fatal(err) + } + if _, got, err := c.ReadMessage(); err != nil || string(got) != "echo:before" { + t.Fatalf("echo %q err=%v", got, err) + } + if err := tr.DB().Close(); err != nil { + t.Fatal(err) + } + for i := 0; i < 3; i++ { + msg := fmt.Sprintf("after-%d", i) + if err := c.WriteMessage(websocket.TextMessage, []byte(msg)); err != nil { + t.Fatalf("write %s: %v", msg, err) + } + _ = c.SetReadDeadline(time.Now().Add(5 * time.Second)) + if _, got, err := c.ReadMessage(); err != nil || string(got) != "echo:"+msg { + t.Fatalf("echo %q err=%v after the index closed", got, err) + } + } +} + +// Both forwarding goroutines fire the message hook at once; every message must +// be stored exactly once, each direction in order. +func TestWebSocketConcurrentDirections(t *testing.T) { + const n = 200 + tr, _, proxyURL := startProxy(t) + up := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }} + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + defer c.Close() + go func() { + for i := 0; i < n; i++ { + if c.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("s-%d", i))) != nil { + return + } + } + }() + for { + if _, _, err := c.ReadMessage(); err != nil { + return + } + } + })) + t.Cleanup(srv.Close) + + c, _, err := wsDialer(t, tr, proxyURL).Dial("wss://"+strings.TrimPrefix(localhostURL(srv.URL), "https://")+"/ws", nil) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < n; i++ { + if err := c.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("c-%d", i))); err != nil { + t.Errorf("write: %v", err) + return + } + } + }() + for i := 0; i < n; i++ { + if _, _, err := c.ReadMessage(); err != nil { + t.Fatalf("read %d: %v", i, err) + } + } + wg.Wait() + var id string + waitFor(t, "all client messages recorded", func() bool { + exs := streamExchanges(t, tr, "websocket") + if len(exs) != 1 { + return false + } + id = exs[0].id + count, _, _ := tr.StreamInfo(id) + return count == 2*n + }) + closeWS(c) + + msgs := allMessages(t, tr, id) + next := map[string]int{"c2s": 0, "s2c": 0} + for i, m := range msgs { + if m.Seq != int64(i+1) { + t.Fatalf("row %d has seq %d", i, m.Seq) + } + prefix := "s-" + if m.Direction == "c2s" { + prefix = "c-" + } + if want := fmt.Sprintf("%s%d", prefix, next[m.Direction]); m.Payload != want { + t.Fatalf("row %d %s payload %q, want %q", i, m.Direction, m.Payload, want) + } + next[m.Direction]++ + } + if next["c2s"] != n || next["s2c"] != n { + t.Fatalf("stored %v, want %d each way", next, n) + } +} + +// The library's unexported lock is what makes reading its message slices safe; +// a go-mitmproxy upgrade that renames it must fail here, not race silently. +func TestLibMutexFound(t *testing.T) { + if libMutex(&mproxy.WebSocketData{}) == nil || libMutex(&mproxy.SSEData{}) == nil { + t.Fatal("go-mitmproxy WebSocketData/SSEData no longer have a sync.Mutex field named mu") + } + // Without the lock nothing is read: no unlocked fallback. + if _, ok := lockLib(&struct{ Messages []int }{}); ok { + t.Fatal("lockLib reported a lock on a type without mu") + } +} diff --git a/traffic/traffic.go b/traffic/traffic.go index 6ac3709..b6fe0f9 100644 --- a/traffic/traffic.go +++ b/traffic/traffic.go @@ -73,6 +73,24 @@ CREATE TABLE IF NOT EXISTS blob_refs ( PRIMARY KEY (hash, exchange_id) ); CREATE INDEX IF NOT EXISTS idx_blob_refs_ex ON blob_refs(exchange_id); + +-- WebSocket messages and SSE events of a stream exchange, in arrival order. +CREATE TABLE IF NOT EXISTS stream_messages ( + exchange_id TEXT NOT NULL, + seq INTEGER NOT NULL, + ts INTEGER NOT NULL, -- unix seconds, like exchanges.ts + direction TEXT NOT NULL, -- 'c2s' | 's2c' | 'meta' + kind INTEGER NOT NULL, -- WebSocket opcode (1 text, 2 binary), 0 for SSE and meta + event TEXT NOT NULL DEFAULT '', + event_id TEXT NOT NULL DEFAULT '', + retry INTEGER NOT NULL DEFAULT 0, + size INTEGER NOT NULL, -- original payload size + payload BLOB, -- inline payload, or a preview when spilled + payload_blob TEXT, -- sha256 of the spilled payload + truncated INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (exchange_id, seq) +); +CREATE INDEX IF NOT EXISTS idx_stream_messages_ex ON stream_messages(exchange_id, seq); ` // ftsSchema is applied separately from indexSchema: a driver build without FTS5 @@ -90,6 +108,14 @@ const ( // maxInlineBody is the cutoff between a body stored inline (SQLite column) and // one spilled to the content-addressed blob store. maxInlineBody = 256 * 1024 + // maxStreamPayload is how much of one WebSocket message or SSE event is kept. + // A longer payload keeps its prefix, with truncated=1 and its original size. + maxStreamPayload = 1024 * 1024 + // maxStreamMessages and maxStreamBytes bound one stream exchange (bytes as + // stored, after truncation). Past either, storing stops and a single 'meta' + // row records how many messages and bytes were dropped. + maxStreamMessages = 10000 + maxStreamBytes = 32 * 1024 * 1024 // blobPreview is how much of a spilled body stays inline so readers can tell // what it is (JSON shape, SQL dump header, ZIP magic) without fetching it. blobPreview = 8 * 1024 @@ -387,8 +413,9 @@ func (t *Traffic) DB() *sql.DB { return t.db } // the requests of client connections marked for pacing (see pace.go). type sink struct { mproxy.BaseAddon - t *Traffic - paced sync.Map // ClientConn.Id of a marked connection -> struct{} + t *Traffic + paced sync.Map // ClientConn.Id of a marked connection -> struct{} + streams sync.Map // *mproxy.Flow of a live WebSocket/SSE stream -> *streamState (stream.go) } func (s *sink) Response(f *mproxy.Flow) { @@ -443,9 +470,8 @@ func (t *Traffic) record(f *mproxy.Flow) { host := f.Request.URL.Hostname() method := f.Request.Method tmpl := db.TemplatePath(f.Request.URL.EscapedPath()) - n := t.seq.Add(1) now := time.Now() - id := fmt.Sprintf("%d-%04d", now.Unix(), n%10000) + id := t.newExchangeID(now) ct := f.Response.Header.Get("Content-Type") url := f.Request.URL.String() @@ -511,6 +537,12 @@ VALUES(?,?,?,?,?,?,?)`, } } +// newExchangeID names a recorded exchange: unix seconds plus a rolling sequence. +func (t *Traffic) newExchangeID(now time.Time) string { + n := t.seq.Add(1) + return fmt.Sprintf("%d-%04d", now.Unix(), n%10000) +} + // storedBody is one body after the inline/spill decision: inline is what goes in // the SQLite column (the whole body, or a preview when spilled), hash names the // blob when it spilled, and index is the text handed to FTS (empty for binary). @@ -526,6 +558,12 @@ type storedBody struct { // are handed to the full-text index in full (up to maxIndexBody) — indexing is // independent of where the bytes end up. func (t *Traffic) spill(body []byte, contentType string) storedBody { + return t.spillAt(body, contentType, maxInlineBody) +} + +// spillAt is spill with the inline threshold as a parameter: bodies up to +// maxInline stay inline. Stream messages use a much lower one (maxStreamInline). +func (t *Traffic) spillAt(body []byte, contentType string, maxInline int) storedBody { if len(body) == 0 { return storedBody{} } @@ -536,7 +574,7 @@ func (t *Traffic) spill(body []byte, contentType string) storedBody { } return string(clipBytes(body, maxIndexBody)) } - if len(body) <= maxInlineBody { + if len(body) <= maxInline { return storedBody{inline: body, index: indexText()} } @@ -1191,6 +1229,9 @@ func (t *Traffic) deleteWhere(tx *sql.Tx, where string, args ...any) (int64, err if _, err := tx.Exec(`DELETE FROM exchange_bodies WHERE id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil { return 0, fmt.Errorf("delete bodies: %w", err) } + if _, err := tx.Exec(`DELETE FROM stream_messages WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil { + return 0, fmt.Errorf("delete stream messages: %w", err) + } if _, err := tx.Exec(`DELETE FROM blob_refs WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil { return 0, fmt.Errorf("delete blob references: %w", err) } @@ -2058,7 +2099,7 @@ func (t *Traffic) Tools() []actool.CoreTool { get := actool.Build(actool.Spec{ Name: "traffic_get", - Description: "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling.", + Description: "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling. A WebSocket (status 101) or SSE (text/event-stream) record shows only its handshake here; read its messages with traffic_messages.", Schema: map[string]any{ "type": "object", "properties": map[string]any{"id": map[string]any{"type": "string", "description": "the id returned by traffic_search"}}, @@ -2116,7 +2157,70 @@ func (t *Traffic) Tools() []actool.CoreTool { }, }) - return []actool.CoreTool{search, get, blob} + messages := actool.Build(actool.Spec{ + Name: "traffic_messages", + Description: "List the messages of one captured WebSocket or SSE (Server-Sent Events) record in order: direction (c2s client→server, s2c server→client), type (text/binary/event, or meta when the recorder dropped messages past its limits), size and a preview of at most 1KB (binary as hex). A payload stored as a blob carries its hash; read the rest with traffic_blob. Returns 20 messages by default, at most 50; use offset to page.", + Schema: map[string]any{ + "type": "object", + "properties": map[string]any{ + "id": map[string]any{"type": "string", "description": "the record id from traffic_search (a WebSocket status-101 or SSE record)"}, + "offset": map[string]any{"type": "integer", "description": "number of messages to skip, default 0"}, + "limit": map[string]any{"type": "integer", "description": "messages to return, default 20, max 50"}, + }, + "required": []any{"id"}, + }, + ReadOnly: ro, + Permissions: allow, + Run: func(_ context.Context, in json.RawMessage, _ *actool.ToolContext) (actool.Result, error) { + var a struct { + ID string + Offset int + Limit int + } + _ = json.Unmarshal(in, &a) + if strings.TrimSpace(a.ID) == "" { + return actool.Errorf("id is a required parameter"), nil + } + if a.Limit <= 0 { + a.Limit = 20 + } + if a.Limit > 50 { + a.Limit = 50 + } + if a.Offset < 0 { + a.Offset = 0 + } + msgs, total, err := t.StreamMessages(a.ID, a.Offset, a.Limit) + if err != nil { + return actool.Errorf(err.Error()), nil + } + if total == 0 { + return actool.Text("No stream messages recorded for this id (not a WebSocket/SSE record, or no message was exchanged)."), nil + } + type liteMsg struct { + Seq int64 `json:"seq"` + Direction string `json:"direction"` + Type string `json:"type"` + Event string `json:"event,omitempty"` + Size int `json:"size"` + Preview string `json:"preview"` + Blob string `json:"blob,omitempty"` + Truncated bool `json:"truncated,omitempty"` + } + lite := make([]liteMsg, 0, len(msgs)) + for _, m := range msgs { + l := liteMsg{Seq: m.Seq, Direction: m.Direction, Type: m.Type, Event: m.Event, Size: m.Size, Preview: streamToolPreview(m), Truncated: m.Truncated} + if m.Blob != "" { + l.Blob = "@blob sha256:" + m.Blob + } + lite = append(lite, l) + } + b, _ := json.Marshal(lite) + return actool.Text(fmt.Sprintf("[messages %d-%d of %d]\n", a.Offset+1, a.Offset+len(msgs), total) + string(b)), nil + }, + }) + + return []actool.CoreTool{search, get, blob, messages} } // SeedToolMetas returns the traffic tools built on a ZERO receiver, for seeding the diff --git a/traffic/traffic_test.go b/traffic/traffic_test.go index ea818b5..5d74eb3 100644 --- a/traffic/traffic_test.go +++ b/traffic/traffic_test.go @@ -377,6 +377,22 @@ VALUES(?,?,?,?,?,?,?,?,?,?,?)`, seed("1-0002", "b.example.com", hb) // sole reference to blobB seed("1-0003", "c.example.com", hc) // shares blobC with d seed("1-0004", "d.example.com", hc) + // d is also a stream exchange whose one message payload spilled to blobD, + // referenced through blob_refs like any body. + hd := strings.Repeat("d", 64) + blobD := filepath.Join(dir, "_blobs", "sha256", "dd", hd+".bin") + if err := os.MkdirAll(filepath.Dir(blobD), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(blobD, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := tr.DB().Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload,payload_blob) VALUES('1-0004',1,1,'s2c',2,1,x'78',?)`, hd); err != nil { + t.Fatal(err) + } + if _, err := tr.DB().Exec(`INSERT INTO blob_refs(hash,exchange_id) VALUES(?, '1-0004')`, hd); err != nil { + t.Fatal(err) + } // Delete a: blobA orphaned → removed; blobB/blobC still referenced → kept. if n, err := tr.DeleteHost("a.example"); err != nil || n != 1 { @@ -399,12 +415,23 @@ VALUES(?,?,?,?,?,?,?,?,?,?,?)`, if _, err := os.Stat(blobC); err != nil { t.Fatalf("shared blobC removed after deleting one sharer: %v", err) } + if _, err := os.Stat(blobD); err != nil { + t.Fatalf("stream payload blobD removed while d still references it: %v", err) + } - // Delete d: last reference gone → blobC collected. + // Delete d: last reference gone → blobC collected, and d's stream message + // and its payload blob go with it. if n, err := tr.DeleteHost("d.example"); err != nil || n != 1 { t.Fatalf("DeleteHost(d.example)=%d, err=%v; want 1, nil", n, err) } if _, err := os.Stat(blobC); !os.IsNotExist(err) { t.Fatalf("blobC still exists after last reference removed: %v", err) } + var streamRows int + if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM stream_messages`).Scan(&streamRows); err != nil || streamRows != 0 { + t.Fatalf("%d stream rows left after deleting d, err=%v", streamRows, err) + } + if _, err := os.Stat(blobD); !os.IsNotExist(err) { + t.Fatalf("stream payload blobD still exists after its exchange was deleted: %v", err) + } }