Compare commits

...
Author SHA1 Message Date
BaaRai 212e29cd15 feat(web): show WebSocket and SSE messages in the traffic detail
The traffic detail sheet gets a Stream tab for WebSocket and SSE exchanges, listing their messages page by page with direction, type, size, time and payload preview, and loading a spilled payload in full on demand. Adds trafficStream and trafficBlob to the API client.
2026-11-16 09:11:38 +01:00
BaaRai ec98975bf3 Merge pull request #254 from BaaRai/feat/traffic-stream-messages
feat(traffic): record WebSocket and SSE stream messages
2026-11-16 08:52:15 +01:00
BaaRai dcdc0652ad fix(traffic): bound stream message pages, spill before the transaction, seconds timestamps
Stream payloads over 4 KiB now spill to the blob store (spillAt with maxStreamInline; spill keeps maxInlineBody for HTTP bodies) and StreamMessages clips every payload to 4 KiB, reading a spilled binary preview from its blob. The byte cap counts the inline preview. persistStream spills before opening the transaction, as record() does. Without the library lock nothing is read. ts is unix seconds. Adds the final-message-on-close and route tests.
2026-11-16 08:21:38 +01:00
BaaRai d9df84d2e5 feat(traffic): record WebSocket and SSE stream messages
The recording proxy now stores WebSocket and Server-Sent Events streams: one exchange per stream (101 websocket, or the real text/event-stream response) and its messages in stream_messages, with per-message and per-exchange limits and blob spill. Messages are readable through Traffic.StreamMessages, GET /api/traffic/stream and the traffic_messages agent tool; deletion and blob GC cover them.
2026-11-16 07:47:25 +01:00
BaaRai 959ec538ad Merge pull request #253 from BaaRai/perf/task-archive-blocker
perf(server): look up only the requested task archive blocker in getTask
2026-11-16 07:26:50 +01:00
16 changed files with 1761 additions and 20 deletions
+1 -1
View File
@@ -49,7 +49,7 @@ func findingWorkflowTools(agentKey string, tools []actool.CoreTool) ([]actool.Co
if tool.Name() == "bind_finding_traffic" {
continue
}
if agentKey == "reporter" && (tool.Name() == "traffic_search" || tool.Name() == "traffic_get" || tool.Name() == "traffic_blob") {
if agentKey == "reporter" && (tool.Name() == "traffic_search" || tool.Name() == "traffic_get" || tool.Name() == "traffic_blob" || tool.Name() == "traffic_messages") {
continue
}
switch tool.Name() {
+1 -1
View File
@@ -292,7 +292,7 @@ func workerTrafficBlock(recording bool) string {
if !recording {
return ""
}
return "\n\n**Traffic tools**:\n- traffic_search / traffic_get / traffic_blob: review responses, find already-visited resources — **check the traffic first, don't curl the same URL twice**. traffic_search **requires a host** and by default returns only 3 ultra-light index entries (id/method/url/status/resp_len, no response content) — explicitly raise limit when you need more; body_contains does full-text search in request/response bodies (at least 3 characters; supports substrings and CJK text, e.g. for passwords/keys/errors/internal addresses); to see one entry's raw content use traffic_get(id) — oversized bodies show as @blob sha256:<hash>, fetched in segments with traffic_blob(hash)."
return "\n\n**Traffic tools**:\n- traffic_search / traffic_get / traffic_blob / traffic_messages: review responses, find already-visited resources — **check the traffic first, don't curl the same URL twice**. traffic_search **requires a host** and by default returns only 3 ultra-light index entries (id/method/url/status/resp_len, no response content) — explicitly raise limit when you need more; body_contains does full-text search in request/response bodies (at least 3 characters; supports substrings and CJK text, e.g. for passwords/keys/errors/internal addresses); to see one entry's raw content use traffic_get(id) — oversized bodies show as @blob sha256:<hash>, fetched in segments with traffic_blob(hash). WebSocket (status 101) and SSE (text/event-stream) records are captured too: traffic_get shows the handshake, and traffic_messages(id) lists their messages in order with direction and type."
}
// artifactSpec is section [C]: the code-owned, non-editable tail appended to every
+1 -1
View File
@@ -6,6 +6,7 @@ require (
github.com/Autumn-27/norma v0.4.10
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.5.3
github.com/jackc/pgx/v5 v5.10.0
github.com/klauspost/compress v1.18.6
github.com/lqqyt2423/go-mitmproxy v1.9.2
@@ -27,7 +28,6 @@ require (
github.com/gaissmai/bart v0.26.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/hinshun/vt10x v0.0.0-20220301184237-5011da428d02 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+1 -1
View File
@@ -39,7 +39,7 @@ var noTargetTools = toolSet(
// the operator runs the command, not the platform
"ask_operator",
// recorded traffic
"traffic_search", "traffic_get", "traffic_blob",
"traffic_search", "traffic_get", "traffic_blob", "traffic_messages",
)
func toolSet(names ...string) map[string]bool {
+1 -1
View File
@@ -191,7 +191,7 @@ func TestExtractSkipsNoTargetTools(t *testing.T) {
"web_search", "Read", "Write", "Edit", "MultiEdit", "LS", "Glob", "Grep", "TodoWrite", "sleep",
"TaskOutput", "TaskStop", "TaskList",
"add_intent", "insert_assets", "report_finding", "record_fact", "add_task_scope", "graph_overview",
"spawn_task", "ask_operator", "traffic_search", "traffic_get", "traffic_blob",
"spawn_task", "ask_operator", "traffic_search", "traffic_get", "traffic_blob", "traffic_messages",
} {
wantTargets(t, tool, targetsOf(tool, input))
}
+1 -1
View File
@@ -88,7 +88,7 @@ func (s *Server) seedFindingWorkflowTools() {
// Upgrade only the original default binding. Customized lists and enabled
// flags survive; the one-time flag also preserves future user unbinding.
readers := `["worker","reporter"]`
for _, key := range []string{"traffic_search", "traffic_get", "traffic_blob"} {
for _, key := range []string{"traffic_search", "traffic_get", "traffic_blob", "traffic_messages"} {
if _, err := s.m.pg.Exec(`UPDATE tools SET agents=$2::jsonb WHERE key=$1 AND system AND (agents='["worker"]'::jsonb OR (agents @> '["worker","planner","mainagent","auto","pentest"]'::jsonb AND jsonb_array_length(agents)=5))`, key, readers); err != nil {
return
}
+32 -1
View File
@@ -823,6 +823,7 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("DELETE /api/traffic/hosts", s.deleteTrafficHosts)
mux.HandleFunc("DELETE /api/traffic/all", s.deleteAllTraffic)
mux.HandleFunc("GET /api/traffic/exchange", s.getTrafficExchange)
mux.HandleFunc("GET /api/traffic/stream", s.getTrafficStream)
mux.HandleFunc("GET /api/traffic/blob", s.getTrafficBlob)
mux.HandleFunc("POST /api/traffic/ingest", s.ingestTraffic)
mux.HandleFunc("GET /api/commands", s.pgListCommands)
@@ -3451,7 +3452,37 @@ func (s *Server) getTrafficExchange(w http.ResponseWriter, r *http.Request) {
writeErr(w, 404, err.Error())
return
}
writeJSON(w, 200, map[string]any{"req": req, "resp": resp})
// A WebSocket or SSE exchange carries its messages separately
// (GET /api/traffic/stream); stream_kind is "" for an ordinary exchange.
count, kind, err := tr.StreamInfo(id)
if err != nil {
log.Printf("[traffic] stream info for %s: %v", id, err)
}
writeJSON(w, 200, map[string]any{"req": req, "resp": resp, "stream_count": count, "stream_kind": kind})
}
// getTrafficStream returns one page of a WebSocket/SSE exchange's messages in
// order (limit 1..200, default 50).
func (s *Server) getTrafficStream(w http.ResponseWriter, r *http.Request) {
tr := s.m.Traffic()
if tr == nil {
writeErr(w, 404, "traffic disabled")
return
}
q := r.URL.Query()
id := strings.TrimSpace(q.Get("exchange"))
if id == "" {
writeErr(w, 400, "missing exchange")
return
}
offset, _ := strconv.Atoi(q.Get("offset"))
limit, _ := strconv.Atoi(q.Get("limit"))
msgs, total, err := tr.StreamMessages(id, offset, limit)
if err != nil {
writeErr(w, 500, err.Error())
return
}
writeJSON(w, 200, map[string]any{"items": msgs, "total": total})
}
// getTrafficBlob streams one oversized body by its sha256. Bodies past the inline
+89
View File
@@ -0,0 +1,89 @@
package server
import (
"encoding/json"
"testing"
)
func TestTrafficStreamRoute(t *testing.T) {
s, request := ingestServer(t)
db := s.m.Traffic().DB()
if _, err := db.Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
VALUES('1-0001',1,'ws.example.com','GET','/ws','wss://ws.example.com/ws',101,'websocket',0,0,'')`); err != nil {
t.Fatal(err)
}
if _, err := db.Exec(`INSERT INTO exchange_bodies(id,req_head,resp_head) VALUES('1-0001','GET /ws HTTP/1.1\n','HTTP/1.1 101 Switching Protocols\n')`); err != nil {
t.Fatal(err)
}
const total = 250
tx, err := db.Begin()
if err != nil {
t.Fatal(err)
}
for i := 1; i <= total; i++ {
kind, payload := 1, []byte("hello")
if i%2 == 0 {
kind, payload = 2, []byte{0, 1, 2}
}
if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload) VALUES('1-0001',?,1,'c2s',?,?,?)`, i, kind, len(payload), payload); err != nil {
t.Fatal(err)
}
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
type page struct {
Items []struct {
Seq int64 `json:"seq"`
Type string `json:"type"`
Payload string `json:"payload"`
Encoding string `json:"encoding"`
} `json:"items"`
Total int `json:"total"`
}
get := func(path string, wantStatus int) page {
t.Helper()
w := request("GET", path, "")
if w.Code != wantStatus {
t.Fatalf("GET %s = %d %s, want %d", path, w.Code, w.Body, wantStatus)
}
var p page
if wantStatus == 200 {
if err := json.Unmarshal(w.Body.Bytes(), &p); err != nil {
t.Fatalf("GET %s: %v in %s", path, err, w.Body)
}
}
return p
}
get("/api/traffic/stream", 400)
if p := get("/api/traffic/stream?exchange=nope", 200); p.Items == nil || len(p.Items) != 0 || p.Total != 0 {
t.Fatalf("unknown exchange = %+v, want items [] and total 0", p)
}
p := get("/api/traffic/stream?exchange=1-0001", 200)
if p.Total != total || len(p.Items) != 50 || p.Items[0].Seq != 1 {
t.Fatalf("default page: total %d, %d items, first seq %d", p.Total, len(p.Items), p.Items[0].Seq)
}
if first, second := p.Items[0], p.Items[1]; first.Type != "text" || first.Payload != "hello" || second.Type != "binary" || second.Encoding != "base64" || second.Payload != "AAEC" {
t.Fatalf("payloads %+v %+v", first, second)
}
if p := get("/api/traffic/stream?exchange=1-0001&limit=500", 200); len(p.Items) != 200 {
t.Fatalf("limit=500 returned %d items, want the 200 cap", len(p.Items))
}
if p := get("/api/traffic/stream?exchange=1-0001&offset=-5&limit=1", 200); len(p.Items) != 1 || p.Items[0].Seq != 1 {
t.Fatalf("negative offset: %+v, want seq 1", p.Items)
}
if p := get("/api/traffic/stream?exchange=1-0001&offset=248&limit=10", 200); len(p.Items) != 2 || p.Items[0].Seq != 249 {
t.Fatalf("last page: %+v, want seqs 249 and 250", p.Items)
}
w := request("GET", "/api/traffic/exchange?id=1-0001", "")
var ex struct {
StreamCount int `json:"stream_count"`
StreamKind string `json:"stream_kind"`
}
if err := json.Unmarshal(w.Body.Bytes(), &ex); err != nil || w.Code != 200 || ex.StreamCount != total || ex.StreamKind != "websocket" {
t.Fatalf("exchange = %d %s", w.Code, w.Body)
}
}
+588
View File
@@ -0,0 +1,588 @@
package traffic
import (
"database/sql"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"log"
"net/url"
"reflect"
"strings"
"sync"
"time"
"unicode/utf8"
"unsafe"
"github.com/Autumn-27/artex/db"
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
)
// Stream recording (WebSocket and Server-Sent Events). go-mitmproxy reports a
// stream as Start, one Message hook per message, then End; a stream never reaches
// the Response hook, so record() never sees it. Each stream becomes one exchange
// (status 101 "websocket", or the real text/event-stream response) whose messages
// go to stream_messages in arrival order.
const (
// wsText and wsBinary are the WebSocket data opcodes (RFC 6455 §5.2), the
// values go-mitmproxy reports in WebSocketMessage.Type.
wsText = 1
wsBinary = 2
// streamPreview caps one message's payload in a StreamMessages row; a longer
// payload lives in the blob store and the row carries its hash.
streamPreview = 4 * 1024
// maxStreamInline is the inline threshold for message payloads: anything
// longer spills to the blob store, so a page of 200 rows stays bounded.
maxStreamInline = streamPreview
// toolStreamPreview caps one message's preview in traffic_messages, so a page
// of messages stays small in the agent's context.
toolStreamPreview = 1024
)
// streamState tracks one live stream. mu serializes the two WebSocket forwarding
// goroutines, which both fire WebSocketMessage on the same flow.
type streamState struct {
mu sync.Mutex
id string // exchange id; "" when the stream could not be recorded
kind string // "websocket" | "sse"
consumed int // library messages/events already handled
seq int64
stored int
// storedBytes counts what the kept messages occupy: the payload after
// truncation, plus the inline preview spill keeps for a spilled payload.
storedBytes int64
dropped bool
droppedMsgs int64
droppedBytes int64
// reqBytes and respBytes are the bytes observed in each direction, dropped
// messages included.
reqBytes, respBytes int64
}
// streamRow is one stream_messages row before it is written.
type streamRow struct {
ts int64 // unix seconds, like exchanges.ts; seq carries the order
direction string
kind int
event string
eventID string
retry int
size int
payload []byte // kept bytes (after maxStreamPayload truncation)
truncated bool
ctype string // content type handed to spill for the binary check
}
func (s *sink) WebSocketStart(f *mproxy.Flow) { s.startStream(f, "websocket") }
func (s *sink) SSEStart(f *mproxy.Flow) { s.startStream(f, "sse") }
func (s *sink) WebSocketMessage(f *mproxy.Flow) {
st := s.stream(f)
if st == nil || f.WebScoket == nil {
return
}
st.mu.Lock()
defer st.mu.Unlock()
s.t.persistStream(st, wsMessagesSince(f.WebScoket, st))
}
func (s *sink) SSEMessage(f *mproxy.Flow) {
st := s.stream(f)
if st == nil || f.SSE == nil {
return
}
st.mu.Lock()
defer st.mu.Unlock()
s.t.persistStream(st, sseEventsSince(f.SSE, st))
}
func (s *sink) WebSocketEnd(f *mproxy.Flow) {
v, ok := s.streams.LoadAndDelete(f)
if !ok {
return
}
st := v.(*streamState)
st.mu.Lock()
defer st.mu.Unlock()
if f.WebScoket != nil {
s.t.persistStream(st, wsMessagesSince(f.WebScoket, st))
}
s.t.finishStream(st)
}
func (s *sink) SSEEnd(f *mproxy.Flow) {
v, ok := s.streams.LoadAndDelete(f)
if !ok {
return
}
st := v.(*streamState)
st.mu.Lock()
defer st.mu.Unlock()
if f.SSE != nil {
s.t.persistStream(st, sseEventsSince(f.SSE, st))
}
s.t.finishStream(st)
}
func (s *sink) stream(f *mproxy.Flow) *streamState {
v, ok := s.streams.Load(f)
if !ok {
return nil
}
return v.(*streamState)
}
// startStream registers the flow before recording its exchange, so a message
// hook racing the start waits on st.mu instead of being dropped.
func (s *sink) startStream(f *mproxy.Flow, kind string) {
st := &streamState{kind: kind}
st.mu.Lock()
defer st.mu.Unlock()
s.streams.Store(f, st)
if f.Request == nil || f.Request.URL == nil || (kind == "sse" && f.Response == nil) {
return
}
id, err := s.t.recordStreamStart(f, kind)
if err != nil {
log.Printf("[traffic] failed to record %s stream %s: %v", kind, f.Request.URL, err)
return
}
st.id = id
if kind == "sse" {
st.reqBytes = int64(len(f.Request.Body))
}
}
// libMutex returns the unexported mu sync.Mutex that go-mitmproxy holds while it
// appends to WebSocketData.Messages / SSEData.Events. The library exposes no
// accessor, and the two WebSocket goroutines append concurrently, so reading the
// slice without that lock is a data race. Reaching a private field ties this to
// the pinned go-mitmproxy v1.9.2: TestLibMutexFound is the upgrade guard. Nil
// when the field is absent, and then nothing is read (see lockLib).
func libMutex(v any) *sync.Mutex {
rv := reflect.ValueOf(v)
if rv.Kind() != reflect.Pointer || rv.IsNil() {
return nil
}
f := rv.Elem().FieldByName("mu")
if !f.IsValid() || f.Type() != reflect.TypeOf(sync.Mutex{}) || !f.CanAddr() {
return nil
}
return (*sync.Mutex)(unsafe.Pointer(f.UnsafeAddr()))
}
var libMutexMissing sync.Once
// lockLib takes the library's lock on v. ok is false when the lock cannot be
// found; callers then read nothing rather than race the library.
func lockLib(v any) (unlock func(), ok bool) {
mu := libMutex(v)
if mu == nil {
libMutexMissing.Do(func() {
log.Printf("[traffic] go-mitmproxy stream data has no mu field; stream messages are not recorded")
})
return nil, false
}
mu.Lock()
return mu.Unlock, true
}
// wsMessagesSince returns every WebSocket message appended since the last call,
// in the library's order, so each is handled exactly once whichever goroutine's
// hook gets there first. The library slice itself is never modified.
func wsMessagesSince(ws *mproxy.WebSocketData, st *streamState) []streamRow {
unlock, ok := lockLib(ws)
if !ok {
return nil
}
msgs := ws.Messages
if st.consumed > len(msgs) {
st.consumed = len(msgs)
}
fresh := append([]*mproxy.WebSocketMessage(nil), msgs[st.consumed:]...)
st.consumed = len(msgs)
unlock()
rows := make([]streamRow, 0, len(fresh))
for _, m := range fresh {
if m == nil {
continue
}
dir, ctype := "s2c", "text/plain; charset=utf-8"
if m.FromClient {
dir = "c2s"
st.reqBytes += int64(len(m.Content))
} else {
st.respBytes += int64(len(m.Content))
}
if m.Type == wsBinary {
ctype = "application/octet-stream"
}
rows = append(rows, streamRow{ts: m.Timestamp.Unix(), direction: dir, kind: m.Type, size: len(m.Content), payload: m.Content, ctype: ctype})
}
return rows
}
// sseEventsSince is wsMessagesSince for Server-Sent Events.
func sseEventsSince(sse *mproxy.SSEData, st *streamState) []streamRow {
unlock, ok := lockLib(sse)
if !ok {
return nil
}
events := sse.Events
if st.consumed > len(events) {
st.consumed = len(events)
}
fresh := append([]*mproxy.SSEEvent(nil), events[st.consumed:]...)
st.consumed = len(events)
unlock()
rows := make([]streamRow, 0, len(fresh))
for _, e := range fresh {
if e == nil {
continue
}
st.respBytes += int64(len(e.Raw))
rows = append(rows, streamRow{ts: e.Time.Unix(), direction: "s2c", event: e.Event, eventID: e.ID,
retry: e.Retry, size: len(e.Data), payload: []byte(e.Data), ctype: "text/event-stream"})
}
return rows
}
// streamDisplayURL is the URL shown for a WebSocket exchange: http→ws and
// https→wss. A plain ws:// connection reaches the WebSocket hooks through its
// CONNECT tunnel, where only the target authority is known, not the path.
func streamDisplayURL(f *mproxy.Flow, kind string) (method, display, tmpl string) {
u := *f.Request.URL
method = f.Request.Method
if kind == "websocket" && strings.EqualFold(method, "CONNECT") {
host := u.Host
if host == "" {
host = u.Opaque
}
return "GET", (&url.URL{Scheme: "ws", Host: host, Path: "/"}).String(), "/"
}
if kind == "websocket" {
switch strings.ToLower(u.Scheme) {
case "http", "":
u.Scheme = "ws"
case "https":
u.Scheme = "wss"
}
}
return method, u.String(), db.TemplatePath(f.Request.URL.EscapedPath())
}
// recordStreamStart writes the stream's exchange: metadata, request head and
// body, the response head (synthetic 101 for WebSocket), and the header text in
// the full-text index. Messages are never indexed.
func (t *Traffic) recordStreamStart(f *mproxy.Flow, kind string) (string, error) {
now := time.Now()
id := t.newExchangeID(now)
method, display, tmpl := streamDisplayURL(f, kind)
host := f.Request.URL.Hostname()
target := f.Request.URL.RequestURI()
if strings.EqualFold(f.Request.Method, "CONNECT") {
target = f.Request.URL.Host // authority-form, as on the wire
}
reqHead := fmt.Sprintf("%s %s %s\n%s", f.Request.Method, target, f.Request.Proto, requestHeaderLines(f.Request))
status, ct, respHead := 101, "websocket", "HTTP/1.1 101 Switching Protocols\n"
if kind == "sse" {
status = f.Response.StatusCode
ct = f.Response.Header.Get("Content-Type")
respHead = fmt.Sprintf("HTTP %d\n%s", status, headerLines(f.Response.Header))
}
t.wmu.Lock()
defer t.wmu.Unlock()
reqB := t.spill(f.Request.Body, f.Request.Header.Get("Content-Type"))
tx, err := t.db.Begin()
if err != nil {
return "", fmt.Errorf("beginning transaction: %w", err)
}
defer tx.Rollback() //nolint:errcheck // no-op once committed
res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
VALUES(?,?,?,?,?,?,?,?,?,?,'')`, id, now.Unix(), host, method, tmpl, display, status, ct, len(f.Request.Body), 0)
if err != nil {
return "", fmt.Errorf("writing index: %w", err)
}
rowid, err := res.LastInsertId()
if err != nil {
return "", fmt.Errorf("fetching rowid: %w", err)
}
if _, err := tx.Exec(`INSERT OR REPLACE INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob)
VALUES(?,?,?,?,?,NULL,NULL)`, id, reqHead, reqB.inline, nullIfEmpty(reqB.hash), respHead); err != nil {
return "", fmt.Errorf("writing body: %w", err)
}
if reqB.hash != "" {
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, reqB.hash, id); err != nil {
return "", fmt.Errorf("registering blob reference: %w", err)
}
}
if t.fts {
idx := strings.Join([]string{display, reqHead, reqB.index, respHead}, "\n")
if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowid, idx); err != nil {
return "", fmt.Errorf("writing full-text index: %w", err)
}
}
if err := tx.Commit(); err != nil {
return "", fmt.Errorf("committing: %w", err)
}
return id, nil
}
// persistStream applies the per-message and per-exchange limits to rows and
// writes what is kept. The first message past a per-exchange limit stores one
// 'meta' row instead; nothing is stored after it. Caller holds st.mu. Errors are
// logged: a recording failure never interrupts the stream.
func (t *Traffic) persistStream(st *streamState, rows []streamRow) {
if st.id == "" || len(rows) == 0 {
return
}
keep := make([]streamRow, 0, len(rows))
addMeta := false
for _, r := range rows {
if len(r.payload) > maxStreamPayload {
r.payload = r.payload[:maxStreamPayload]
r.truncated = true
}
cost := streamStoredCost(r)
if st.dropped || st.stored >= maxStreamMessages || st.storedBytes+cost > maxStreamBytes {
if !st.dropped {
st.dropped = true
addMeta = true
}
st.droppedMsgs++
st.droppedBytes += int64(r.size)
continue
}
st.stored++
st.storedBytes += cost
keep = append(keep, r)
}
var meta []byte
if addMeta {
meta = streamMetaPayload(st)
}
if len(keep) == 0 && meta == nil {
return
}
t.wmu.Lock()
defer t.wmu.Unlock()
// Sequence numbers and blob writes come first, as in record(): blob writes are
// filesystem work and must not sit inside the SQLite transaction. Holding wmu
// keeps gcBlobs from collecting a blob before its reference is committed.
type preparedRow struct {
streamRow
seq int64
body storedBody
}
prepared := make([]preparedRow, 0, len(keep))
for _, r := range keep {
st.seq++
prepared = append(prepared, preparedRow{streamRow: r, seq: st.seq, body: t.spillAt(r.payload, r.ctype, maxStreamInline)})
}
var metaSeq int64
if meta != nil {
st.seq++
metaSeq = st.seq
}
tx, err := t.db.Begin()
if err != nil {
log.Printf("[traffic] failed to record stream messages of %s (beginning transaction): %v", st.id, err)
return
}
defer tx.Rollback() //nolint:errcheck // no-op once committed
for _, p := range prepared {
inline := p.body.inline
if inline == nil {
inline = []byte{}
}
if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,event,event_id,retry,size,payload,payload_blob,truncated)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`, st.id, p.seq, p.ts, p.direction, p.kind, p.event, p.eventID, p.retry, p.size,
inline, nullIfEmpty(p.body.hash), boolInt(p.truncated)); err != nil {
log.Printf("[traffic] failed to record stream message %d of %s: %v", p.seq, st.id, err)
return
}
if p.body.hash != "" {
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, p.body.hash, st.id); err != nil {
log.Printf("[traffic] failed to record stream message %d of %s (registering blob reference): %v", p.seq, st.id, err)
return
}
}
}
if meta != nil {
if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload)
VALUES(?,?,?,'meta',0,?,?)`, st.id, metaSeq, time.Now().Unix(), len(meta), meta); err != nil {
log.Printf("[traffic] failed to record stream limit marker of %s: %v", st.id, err)
return
}
}
if err := tx.Commit(); err != nil {
log.Printf("[traffic] failed to record stream messages of %s (committing): %v", st.id, err)
}
}
// streamStoredCost is what keeping r adds to the index and blob store: the
// payload, plus the inline preview spillAt keeps when the payload spills (a text
// prefix of at most blobPreview, or a one-line tag for binary).
func streamStoredCost(r streamRow) int64 {
n := int64(len(r.payload))
if len(r.payload) <= maxStreamInline {
return n
}
if isBinaryBody(r.ctype, r.payload) {
return n + int64(len(binaryTag(r.ctype, r.payload)))
}
return n + int64(len(truncateUTF8(r.payload, blobPreview)))
}
func streamMetaPayload(st *streamState) []byte {
b, _ := json.Marshal(map[string]int64{"dropped_messages": st.droppedMsgs, "dropped_bytes": st.droppedBytes})
return b
}
func boolInt(b bool) int {
if b {
return 1
}
return 0
}
// finishStream records the bytes observed in each direction and the final
// counts of the limit marker. Caller holds st.mu.
func (t *Traffic) finishStream(st *streamState) {
if st.id == "" {
return
}
t.wmu.Lock()
defer t.wmu.Unlock()
if _, err := t.db.Exec(`UPDATE exchanges SET req_len=?, resp_len=? WHERE id=?`, st.reqBytes, st.respBytes, st.id); err != nil {
log.Printf("[traffic] failed to record stream sizes of %s: %v", st.id, err)
}
if st.dropped {
meta := streamMetaPayload(st)
if _, err := t.db.Exec(`UPDATE stream_messages SET payload=?, size=? WHERE exchange_id=? AND direction='meta'`, meta, len(meta), st.id); err != nil {
log.Printf("[traffic] failed to update stream limit marker of %s: %v", st.id, err)
}
}
}
// StreamMessage is one recorded WebSocket message or SSE event.
type StreamMessage struct {
Seq int64 `json:"seq"`
TS int64 `json:"ts"` // unix seconds, like an exchange's ts; Seq orders
Direction string `json:"direction"`
Kind int `json:"kind"`
// Type is "text" or "binary" (WebSocket), "event" (SSE), or "meta" (the
// limit marker).
Type string `json:"type"`
Event string `json:"event,omitempty"`
EventID string `json:"event_id,omitempty"`
Retry int `json:"retry,omitempty"`
Size int `json:"size"`
Payload string `json:"payload"`
// Encoding is "base64" for binary payloads, empty for UTF-8 text.
Encoding string `json:"encoding,omitempty"`
Blob string `json:"blob,omitempty"`
Truncated bool `json:"truncated,omitempty"`
}
// StreamMessages returns one page of an exchange's stream messages in order.
// limit is 1..200 (default 50) and offset >= 0. Every payload comes back as at
// most streamPreview (4 KiB); a longer one lives in the blob store and the row
// carries its hash for the rest (traffic_blob, GET /api/traffic/blob).
func (t *Traffic) StreamMessages(exchangeID string, offset, limit int) ([]StreamMessage, int, error) {
if limit <= 0 {
limit = 50
}
if limit > 200 {
limit = 200
}
if offset < 0 {
offset = 0
}
var total int
if err := t.db.QueryRow(`SELECT COUNT(*) FROM stream_messages WHERE exchange_id=?`, exchangeID).Scan(&total); err != nil {
return nil, 0, err
}
rows, err := t.db.Query(`SELECT seq,ts,direction,kind,event,event_id,retry,size,payload,payload_blob,truncated
FROM stream_messages WHERE exchange_id=? ORDER BY seq LIMIT ? OFFSET ?`, exchangeID, limit, offset)
if err != nil {
return nil, 0, err
}
defer rows.Close()
out := []StreamMessage{}
for rows.Next() {
var m StreamMessage
var payload []byte
var blob sql.NullString
var truncated int
if err := rows.Scan(&m.Seq, &m.TS, &m.Direction, &m.Kind, &m.Event, &m.EventID, &m.Retry, &m.Size, &payload, &blob, &truncated); err != nil {
return nil, 0, err
}
m.Truncated = truncated != 0
m.Blob = blob.String
switch {
case m.Direction == "meta":
m.Type = "meta"
case m.Kind == wsBinary:
m.Type = "binary"
case m.Kind == wsText:
m.Type = "text"
default:
m.Type = "event"
}
if m.Type == "binary" {
if m.Blob != "" {
// A spilled binary payload keeps only a descriptive tag inline;
// the preview is the payload's own leading bytes.
payload, _, _ = t.BlobRange(m.Blob, 0, streamPreview)
}
m.Payload, m.Encoding = base64.StdEncoding.EncodeToString(clipBytes(payload, streamPreview)), "base64"
} else {
m.Payload = truncateUTF8(payload, streamPreview)
}
out = append(out, m)
}
return out, total, rows.Err()
}
// StreamInfo reports how many stream messages an exchange holds and whether it
// is a WebSocket or SSE exchange ("" for an ordinary exchange).
func (t *Traffic) StreamInfo(exchangeID string) (count int, kind string, err error) {
var ct sql.NullString
err = t.db.QueryRow(`SELECT content_type FROM exchanges WHERE id=?`, exchangeID).Scan(&ct)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return 0, "", err
}
switch c := strings.ToLower(ct.String); {
case c == "websocket":
kind = "websocket"
case strings.Contains(c, "text/event-stream"):
kind = "sse"
}
if err := t.db.QueryRow(`SELECT COUNT(*) FROM stream_messages WHERE exchange_id=?`, exchangeID).Scan(&count); err != nil {
return 0, "", err
}
return count, kind, nil
}
// streamToolPreview renders one message for traffic_messages: at most 1 KiB of
// text (hex for binary), plus the blob hash when the payload spilled.
func streamToolPreview(m StreamMessage) string {
if m.Encoding == "base64" {
raw, _ := base64.StdEncoding.DecodeString(m.Payload)
return "hex:" + fmt.Sprintf("%x", clipBytes(raw, toolStreamPreview/2))
}
p := m.Payload
if len(p) > toolStreamPreview {
p = truncateUTF8([]byte(p), toolStreamPreview)
}
if !utf8.ValidString(p) {
p = strings.ToValidUTF8(p, "�")
}
return p
}
+637
View File
@@ -0,0 +1,637 @@
package traffic
import (
"bytes"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/gorilla/websocket"
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
)
// localhostURL names an httptest server by host name: the MITM leaf certificate
// carries a DNS name, not an IP SAN (as in pace_test.go).
func localhostURL(u string) string { return strings.Replace(u, "127.0.0.1", "localhost", 1) }
// waitFor polls cond until it holds or 10 seconds pass.
func waitFor(t *testing.T, what string, cond func() bool) {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for !cond() {
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for %s", what)
}
time.Sleep(20 * time.Millisecond)
}
}
// wsDialer dials through the recording proxy and trusts its CA.
func wsDialer(t *testing.T, tr *Traffic, proxyURL string) *websocket.Dialer {
t.Helper()
pem, err := os.ReadFile(tr.CACertPath())
if err != nil {
t.Fatal(err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(pem) {
t.Fatal("no CA in", tr.CACertPath())
}
u, _ := url.Parse(proxyURL)
return &websocket.Dialer{Proxy: http.ProxyURL(u), TLSClientConfig: &tls.Config{RootCAs: pool}, HandshakeTimeout: 5 * time.Second}
}
type streamExchange struct {
id, url, contentType string
status int
reqLen, respLen int64
}
// streamExchanges returns the exchanges recorded with a stream content type.
func streamExchanges(t *testing.T, tr *Traffic, contentLike string) []streamExchange {
t.Helper()
rows, err := tr.DB().Query(`SELECT id,url,content_type,status,req_len,resp_len FROM exchanges WHERE content_type LIKE ? ORDER BY id`, contentLike)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
var out []streamExchange
for rows.Next() {
var e streamExchange
if err := rows.Scan(&e.id, &e.url, &e.contentType, &e.status, &e.reqLen, &e.respLen); err != nil {
t.Fatal(err)
}
out = append(out, e)
}
return out
}
func allMessages(t *testing.T, tr *Traffic, id string) []StreamMessage {
t.Helper()
var out []StreamMessage
for offset := 0; ; offset += 200 {
page, total, err := tr.StreamMessages(id, offset, 200)
if err != nil {
t.Fatal(err)
}
out = append(out, page...)
if len(out) >= total || len(page) == 0 {
return out
}
}
}
// echoWSServer echoes every message back with the same type: text prefixed
// with "echo:", binary reversed.
func echoWSServer(t *testing.T) *httptest.Server {
t.Helper()
up := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer c.Close()
for {
mt, msg, err := c.ReadMessage()
if err != nil {
return
}
if mt == websocket.BinaryMessage {
rev := make([]byte, len(msg))
for i := range msg {
rev[len(msg)-1-i] = msg[i]
}
msg = rev
} else {
msg = append([]byte("echo:"), msg...)
}
if err := c.WriteMessage(mt, msg); err != nil {
return
}
}
}))
t.Cleanup(srv.Close)
return srv
}
func closeWS(c *websocket.Conn) {
_ = c.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
_ = c.SetReadDeadline(time.Now().Add(2 * time.Second))
for {
if _, _, err := c.ReadMessage(); err != nil {
break
}
}
_ = c.Close()
}
func TestWebSocketMessagesRecorded(t *testing.T) {
tr, _, proxyURL := startProxy(t)
srv := echoWSServer(t)
target := "wss://" + strings.TrimPrefix(localhostURL(srv.URL), "https://") + "/ws?room=1"
c, _, err := wsDialer(t, tr, proxyURL).Dial(target, nil)
if err != nil {
t.Fatal(err)
}
exchange := func(mt int, msg []byte) []byte {
if err := c.WriteMessage(mt, msg); err != nil {
t.Fatal(err)
}
gotType, got, err := c.ReadMessage()
if err != nil || gotType != mt {
t.Fatalf("read type=%d err=%v", gotType, err)
}
return got
}
before := time.Now().Unix()
if got := exchange(websocket.TextMessage, []byte("hello")); string(got) != "echo:hello" {
t.Fatalf("echo %q", got)
}
if got := exchange(websocket.BinaryMessage, []byte{0, 1, 2, 255}); !bytes.Equal(got, []byte{255, 2, 1, 0}) {
t.Fatalf("binary echo %v", got)
}
closeWS(c)
var ex streamExchange
waitFor(t, "websocket end", func() bool {
exs := streamExchanges(t, tr, "websocket")
if len(exs) != 1 || exs[0].reqLen == 0 {
return false
}
ex = exs[0]
return true
})
if ex.status != 101 || ex.url != target {
t.Fatalf("exchange %+v, want status 101 and url %s", ex, target)
}
// c2s: "hello" (5) + 4 binary bytes; s2c: "echo:hello" (10) + 4.
if ex.reqLen != 9 || ex.respLen != 14 {
t.Fatalf("req_len=%d resp_len=%d, want 9 and 14", ex.reqLen, ex.respLen)
}
var respHead string
if err := tr.DB().QueryRow(`SELECT resp_head FROM exchange_bodies WHERE id=?`, ex.id).Scan(&respHead); err != nil || !strings.HasPrefix(respHead, "HTTP/1.1 101 Switching Protocols") {
t.Fatalf("resp_head %q err=%v", respHead, err)
}
if count, kind, err := tr.StreamInfo(ex.id); err != nil || count != 4 || kind != "websocket" {
t.Fatalf("StreamInfo = %d %q %v", count, kind, err)
}
msgs := allMessages(t, tr, ex.id)
want := []struct {
dir, typ, payload string
kind int
}{
{"c2s", "text", "hello", 1},
{"s2c", "text", "echo:hello", 1},
{"c2s", "binary", base64.StdEncoding.EncodeToString([]byte{0, 1, 2, 255}), 2},
{"s2c", "binary", base64.StdEncoding.EncodeToString([]byte{255, 2, 1, 0}), 2},
}
if len(msgs) != len(want) {
t.Fatalf("got %d messages: %+v", len(msgs), msgs)
}
last := before
for i, w := range want {
m := msgs[i]
if m.Seq != int64(i+1) || m.Direction != w.dir || m.Type != w.typ || m.Kind != w.kind || m.Payload != w.payload {
t.Fatalf("message %d = %+v, want %+v", i, m, w)
}
if (w.typ == "binary") != (m.Encoding == "base64") {
t.Fatalf("message %d encoding %q", i, m.Encoding)
}
if m.TS < last {
t.Fatalf("message %d ts %d before %d", i, m.TS, last)
}
last = m.TS
}
}
func TestSSEEventsRecorded(t *testing.T) {
tr, _, proxyURL := startProxy(t)
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
w.WriteHeader(200)
fl := w.(http.Flusher)
for _, ev := range []string{
"retry: 1500\nid: 1\nevent: greet\ndata: hello\ndata: world\n\n",
"id: 2\ndata: second\n\n",
"event: done\ndata: bye\n\n",
} {
_, _ = io.WriteString(w, ev)
fl.Flush()
time.Sleep(10 * time.Millisecond)
}
}))
t.Cleanup(srv.Close)
resp, err := proxyClient(t, tr, proxyURL).Get(localhostURL(srv.URL) + "/events")
if err != nil {
t.Fatal(err)
}
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if !strings.Contains(string(body), "data: bye") {
t.Fatalf("client body %q", body)
}
var ex streamExchange
waitFor(t, "sse end", func() bool {
exs := streamExchanges(t, tr, "%event-stream%")
if len(exs) != 1 || exs[0].respLen == 0 {
return false
}
ex = exs[0]
return true
})
if ex.status != 200 || !strings.HasPrefix(ex.url, localhostURL(srv.URL)+"/events") {
t.Fatalf("exchange %+v", ex)
}
if n, _ := tr.Count(); n != 1 {
t.Fatalf("%d exchanges recorded, want the single SSE exchange", n)
}
msgs := allMessages(t, tr, ex.id)
want := []struct {
event, id, data string
retry int
}{
{"greet", "1", "hello\nworld", 1500},
{"message", "2", "second", 0},
{"done", "", "bye", 0},
}
if len(msgs) != len(want) {
t.Fatalf("got %d events: %+v", len(msgs), msgs)
}
for i, w := range want {
m := msgs[i]
if m.Seq != int64(i+1) || m.Direction != "s2c" || m.Kind != 0 || m.Type != "event" ||
m.Event != w.event || m.EventID != w.id || m.Payload != w.data || m.Retry != w.retry || m.Size != len(w.data) {
t.Fatalf("event %d = %+v, want %+v", i, m, w)
}
}
}
// unitStream opens a recorder without starting the proxy and drives its hooks
// directly with a synthetic WebSocket flow.
func unitStream(t *testing.T, host string) (*Traffic, *sink, *mproxy.Flow) {
t.Helper()
tr, err := Open(t.TempDir(), "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = tr.Close() })
u, _ := url.Parse("https://" + host + "/ws")
f := &mproxy.Flow{Request: &mproxy.Request{Method: "GET", URL: u, Proto: "HTTP/1.1", Header: http.Header{}}, WebScoket: &mproxy.WebSocketData{}}
s := tr.proxy.Addons[0].(*sink)
s.WebSocketStart(f)
return tr, s, f
}
func addWS(f *mproxy.Flow, fromClient bool, mt int, content []byte) {
f.WebScoket.Messages = append(f.WebScoket.Messages, &mproxy.WebSocketMessage{Type: mt, Content: content, FromClient: fromClient, Timestamp: time.Now()})
}
func onlyExchange(t *testing.T, tr *Traffic) string {
t.Helper()
exs := streamExchanges(t, tr, "websocket")
if len(exs) != 1 {
t.Fatalf("%d websocket exchanges", len(exs))
}
return exs[0].id
}
func metaCounts(t *testing.T, m StreamMessage) (msgs, bytes int64) {
t.Helper()
var v struct {
DroppedMessages int64 `json:"dropped_messages"`
DroppedBytes int64 `json:"dropped_bytes"`
}
if err := json.Unmarshal([]byte(m.Payload), &v); err != nil {
t.Fatalf("meta payload %q: %v", m.Payload, err)
}
return v.DroppedMessages, v.DroppedBytes
}
func TestStreamPayloadTruncatedToLimit(t *testing.T) {
tr, s, f := unitStream(t, "trunc.test")
big := bytes.Repeat([]byte("a"), maxStreamPayload+10)
addWS(f, true, websocket.TextMessage, big)
s.WebSocketMessage(f)
id := onlyExchange(t, tr)
msgs := allMessages(t, tr, id)
if len(msgs) != 1 {
t.Fatalf("%d messages", len(msgs))
}
m := msgs[0]
if !m.Truncated || m.Size != maxStreamPayload+10 || m.Blob == "" || len(m.Payload) > streamPreview {
t.Fatalf("message = seq %d size %d truncated %v blob %q payload %d bytes", m.Seq, m.Size, m.Truncated, m.Blob, len(m.Payload))
}
data, total, err := tr.BlobRange(m.Blob, 0, 16)
if err != nil || total != maxStreamPayload || string(data) != strings.Repeat("a", 16) {
t.Fatalf("blob total=%d data=%q err=%v, want the kept %d-byte prefix", total, data, err, maxStreamPayload)
}
}
func TestStreamMessageLimitStoresOneMetaRow(t *testing.T) {
tr, s, f := unitStream(t, "count.test")
for i := 0; i < maxStreamMessages+5; i++ {
addWS(f, i%2 == 0, websocket.TextMessage, []byte("m"))
}
s.WebSocketMessage(f)
addWS(f, true, websocket.TextMessage, []byte("late"))
s.WebSocketMessage(f)
s.WebSocketEnd(f)
id := onlyExchange(t, tr)
msgs := allMessages(t, tr, id)
if len(msgs) != maxStreamMessages+1 {
t.Fatalf("%d rows, want %d messages and one meta row", len(msgs), maxStreamMessages)
}
meta := msgs[len(msgs)-1]
if meta.Direction != "meta" || meta.Type != "meta" || meta.Kind != 0 || meta.Seq != maxStreamMessages+1 {
t.Fatalf("last row %+v, want the meta row", meta)
}
if n, b := metaCounts(t, meta); n != 6 || b != 5+4 {
t.Fatalf("meta dropped_messages=%d dropped_bytes=%d, want 6 and 9", n, b)
}
for _, m := range msgs[:len(msgs)-1] {
if m.Direction == "meta" {
t.Fatalf("more than one meta row: %+v", m)
}
}
}
func TestStreamByteLimitStoresOneMetaRow(t *testing.T) {
tr, s, f := unitStream(t, "bytes.test")
chunk := bytes.Repeat([]byte{7}, maxStreamPayload)
// Each kept 1 MiB payload spills and also keeps its inline tag, so the cap
// admits floor(maxStreamBytes / (payload + tag)) of them.
cost := streamStoredCost(streamRow{payload: chunk, ctype: "application/octet-stream"})
if cost <= maxStreamPayload {
t.Fatalf("cost %d does not count the inline preview", cost)
}
kept := int(maxStreamBytes / cost)
n := kept + 1
for i := 0; i < n; i++ {
addWS(f, false, websocket.BinaryMessage, chunk)
}
s.WebSocketMessage(f)
addWS(f, false, websocket.BinaryMessage, []byte{1})
s.WebSocketMessage(f)
s.WebSocketEnd(f)
id := onlyExchange(t, tr)
msgs := allMessages(t, tr, id)
if len(msgs) != kept+1 {
t.Fatalf("%d rows, want %d stored and one meta row", len(msgs), kept)
}
meta := msgs[len(msgs)-1]
if meta.Direction != "meta" {
t.Fatalf("last row %+v, want the meta row", meta)
}
if dn, db := metaCounts(t, meta); dn != 2 || db != maxStreamPayload+1 {
t.Fatalf("meta dropped_messages=%d dropped_bytes=%d", dn, db)
}
var stored, inline int64
if err := tr.DB().QueryRow(`SELECT SUM(size), SUM(LENGTH(payload)) FROM stream_messages WHERE exchange_id=? AND direction<>'meta'`, id).Scan(&stored, &inline); err != nil ||
stored != int64(kept*maxStreamPayload) || stored+inline > maxStreamBytes {
t.Fatalf("stored %d payload bytes + %d inline err=%v, want %d within %d", stored, inline, err, kept*maxStreamPayload, maxStreamBytes)
}
var respLen int64
if err := tr.DB().QueryRow(`SELECT resp_len FROM exchanges WHERE id=?`, id).Scan(&respLen); err != nil || respLen != int64(n*maxStreamPayload+1) {
t.Fatalf("resp_len %d err=%v, want every observed byte", respLen, err)
}
}
// A payload over maxStreamInline goes to the blob store; the row returns at
// most 4 KiB and the hash, and the blob holds the whole payload.
func TestStreamPayloadSpillsAboveInlineLimit(t *testing.T) {
tr, s, f := unitStream(t, "spill.test")
bigText := []byte(strings.Repeat("0123456789", 1024)) // 10 KiB
bigBin := make([]byte, 10*1024)
for i := range bigBin {
bigBin[i] = byte(i)
}
small := []byte(strings.Repeat("s", 1024))
addWS(f, true, websocket.TextMessage, bigText)
addWS(f, false, websocket.BinaryMessage, bigBin)
addWS(f, true, websocket.TextMessage, small)
s.WebSocketMessage(f)
msgs := allMessages(t, tr, onlyExchange(t, tr))
if len(msgs) != 3 {
t.Fatalf("%d messages", len(msgs))
}
text, bin, inline := msgs[0], msgs[1], msgs[2]
if text.Blob == "" || len(text.Payload) > streamPreview || text.Payload != string(bigText[:streamPreview]) || text.Size != len(bigText) || text.Truncated {
t.Fatalf("10 KiB text row: blob %q payload %d bytes size %d truncated %v", text.Blob, len(text.Payload), text.Size, text.Truncated)
}
if data, total, err := tr.BlobRange(text.Blob, 0, int64(len(bigText))); err != nil || total != int64(len(bigText)) || !bytes.Equal(data, bigText) {
t.Fatalf("text blob total=%d err=%v, want the full 10 KiB", total, err)
}
raw, err := base64.StdEncoding.DecodeString(bin.Payload)
if bin.Blob == "" || err != nil || !bytes.Equal(raw, bigBin[:streamPreview]) {
t.Fatalf("10 KiB binary row: blob %q preview %d bytes err=%v, want its first 4 KiB", bin.Blob, len(raw), err)
}
if data, total, err := tr.BlobRange(bin.Blob, 0, int64(len(bigBin))); err != nil || total != int64(len(bigBin)) || !bytes.Equal(data, bigBin) {
t.Fatalf("binary blob total=%d err=%v", total, err)
}
if inline.Blob != "" || inline.Payload != string(small) {
t.Fatalf("1 KiB row: blob %q payload %d bytes, want inline and complete", inline.Blob, len(inline.Payload))
}
}
// The server writes a message and closes at once; the message the client read
// must be recorded even though WebSocketEnd fires as soon as one direction ends.
func TestWebSocketFinalMessageOnClose(t *testing.T) {
tr, _, proxyURL := startProxy(t)
up := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
_ = c.WriteMessage(websocket.TextMessage, []byte("final"))
_ = c.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
_ = c.Close()
}))
t.Cleanup(srv.Close)
c, _, err := wsDialer(t, tr, proxyURL).Dial("wss://"+strings.TrimPrefix(localhostURL(srv.URL), "https://")+"/ws", nil)
if err != nil {
t.Fatal(err)
}
if _, got, err := c.ReadMessage(); err != nil || string(got) != "final" {
t.Fatalf("read %q err=%v", got, err)
}
_ = c.Close()
var id string
waitFor(t, "websocket end", func() bool {
exs := streamExchanges(t, tr, "websocket")
if len(exs) != 1 || exs[0].respLen == 0 {
return false
}
id = exs[0].id
return true
})
msgs := allMessages(t, tr, id)
if len(msgs) != 1 || msgs[0].Direction != "s2c" || msgs[0].Payload != "final" {
t.Fatalf("messages %+v, want the final server message", msgs)
}
}
func TestDeleteHostRemovesStreamMessagesAndBlobs(t *testing.T) {
tr, s, f := unitStream(t, "gone.test")
addWS(f, false, websocket.BinaryMessage, bytes.Repeat([]byte{9}, maxInlineBody+1))
addWS(f, true, websocket.TextMessage, []byte("small"))
s.WebSocketMessage(f)
s.WebSocketEnd(f)
id := onlyExchange(t, tr)
msgs := allMessages(t, tr, id)
if len(msgs) != 2 || msgs[0].Blob == "" {
t.Fatalf("messages %+v, want a spilled first payload", msgs)
}
blobPath := filepath.Join(tr.dir, "_blobs", "sha256", msgs[0].Blob[:2], msgs[0].Blob+".bin")
if _, err := os.Stat(blobPath); err != nil {
t.Fatalf("payload blob missing: %v", err)
}
if n, err := tr.DeleteHost("gone.test"); err != nil || n != 1 {
t.Fatalf("DeleteHost = %d, %v", n, err)
}
var rows int
if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM stream_messages`).Scan(&rows); err != nil || rows != 0 {
t.Fatalf("%d stream rows left, err=%v", rows, err)
}
if _, err := os.Stat(blobPath); !os.IsNotExist(err) {
t.Fatalf("payload blob not collected: %v", err)
}
}
// A recording failure must never reach the proxied connection.
func TestWebSocketSurvivesRecordingFailure(t *testing.T) {
tr, _, proxyURL := startProxy(t)
srv := echoWSServer(t)
c, _, err := wsDialer(t, tr, proxyURL).Dial("wss://"+strings.TrimPrefix(localhostURL(srv.URL), "https://")+"/ws", nil)
if err != nil {
t.Fatal(err)
}
defer closeWS(c)
if err := c.WriteMessage(websocket.TextMessage, []byte("before")); err != nil {
t.Fatal(err)
}
if _, got, err := c.ReadMessage(); err != nil || string(got) != "echo:before" {
t.Fatalf("echo %q err=%v", got, err)
}
if err := tr.DB().Close(); err != nil {
t.Fatal(err)
}
for i := 0; i < 3; i++ {
msg := fmt.Sprintf("after-%d", i)
if err := c.WriteMessage(websocket.TextMessage, []byte(msg)); err != nil {
t.Fatalf("write %s: %v", msg, err)
}
_ = c.SetReadDeadline(time.Now().Add(5 * time.Second))
if _, got, err := c.ReadMessage(); err != nil || string(got) != "echo:"+msg {
t.Fatalf("echo %q err=%v after the index closed", got, err)
}
}
}
// Both forwarding goroutines fire the message hook at once; every message must
// be stored exactly once, each direction in order.
func TestWebSocketConcurrentDirections(t *testing.T) {
const n = 200
tr, _, proxyURL := startProxy(t)
up := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
c, err := up.Upgrade(w, r, nil)
if err != nil {
return
}
defer c.Close()
go func() {
for i := 0; i < n; i++ {
if c.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("s-%d", i))) != nil {
return
}
}
}()
for {
if _, _, err := c.ReadMessage(); err != nil {
return
}
}
}))
t.Cleanup(srv.Close)
c, _, err := wsDialer(t, tr, proxyURL).Dial("wss://"+strings.TrimPrefix(localhostURL(srv.URL), "https://")+"/ws", nil)
if err != nil {
t.Fatal(err)
}
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < n; i++ {
if err := c.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("c-%d", i))); err != nil {
t.Errorf("write: %v", err)
return
}
}
}()
for i := 0; i < n; i++ {
if _, _, err := c.ReadMessage(); err != nil {
t.Fatalf("read %d: %v", i, err)
}
}
wg.Wait()
var id string
waitFor(t, "all client messages recorded", func() bool {
exs := streamExchanges(t, tr, "websocket")
if len(exs) != 1 {
return false
}
id = exs[0].id
count, _, _ := tr.StreamInfo(id)
return count == 2*n
})
closeWS(c)
msgs := allMessages(t, tr, id)
next := map[string]int{"c2s": 0, "s2c": 0}
for i, m := range msgs {
if m.Seq != int64(i+1) {
t.Fatalf("row %d has seq %d", i, m.Seq)
}
prefix := "s-"
if m.Direction == "c2s" {
prefix = "c-"
}
if want := fmt.Sprintf("%s%d", prefix, next[m.Direction]); m.Payload != want {
t.Fatalf("row %d %s payload %q, want %q", i, m.Direction, m.Payload, want)
}
next[m.Direction]++
}
if next["c2s"] != n || next["s2c"] != n {
t.Fatalf("stored %v, want %d each way", next, n)
}
}
// The library's unexported lock is what makes reading its message slices safe;
// a go-mitmproxy upgrade that renames it must fail here, not race silently.
func TestLibMutexFound(t *testing.T) {
if libMutex(&mproxy.WebSocketData{}) == nil || libMutex(&mproxy.SSEData{}) == nil {
t.Fatal("go-mitmproxy WebSocketData/SSEData no longer have a sync.Mutex field named mu")
}
// Without the lock nothing is read: no unlocked fallback.
if _, ok := lockLib(&struct{ Messages []int }{}); ok {
t.Fatal("lockLib reported a lock on a type without mu")
}
}
+111 -7
View File
@@ -73,6 +73,24 @@ CREATE TABLE IF NOT EXISTS blob_refs (
PRIMARY KEY (hash, exchange_id)
);
CREATE INDEX IF NOT EXISTS idx_blob_refs_ex ON blob_refs(exchange_id);
-- WebSocket messages and SSE events of a stream exchange, in arrival order.
CREATE TABLE IF NOT EXISTS stream_messages (
exchange_id TEXT NOT NULL,
seq INTEGER NOT NULL,
ts INTEGER NOT NULL, -- unix seconds, like exchanges.ts
direction TEXT NOT NULL, -- 'c2s' | 's2c' | 'meta'
kind INTEGER NOT NULL, -- WebSocket opcode (1 text, 2 binary), 0 for SSE and meta
event TEXT NOT NULL DEFAULT '',
event_id TEXT NOT NULL DEFAULT '',
retry INTEGER NOT NULL DEFAULT 0,
size INTEGER NOT NULL, -- original payload size
payload BLOB, -- inline payload, or a preview when spilled
payload_blob TEXT, -- sha256 of the spilled payload
truncated INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (exchange_id, seq)
);
CREATE INDEX IF NOT EXISTS idx_stream_messages_ex ON stream_messages(exchange_id, seq);
`
// ftsSchema is applied separately from indexSchema: a driver build without FTS5
@@ -90,6 +108,14 @@ const (
// maxInlineBody is the cutoff between a body stored inline (SQLite column) and
// one spilled to the content-addressed blob store.
maxInlineBody = 256 * 1024
// maxStreamPayload is how much of one WebSocket message or SSE event is kept.
// A longer payload keeps its prefix, with truncated=1 and its original size.
maxStreamPayload = 1024 * 1024
// maxStreamMessages and maxStreamBytes bound one stream exchange (bytes as
// stored, after truncation). Past either, storing stops and a single 'meta'
// row records how many messages and bytes were dropped.
maxStreamMessages = 10000
maxStreamBytes = 32 * 1024 * 1024
// blobPreview is how much of a spilled body stays inline so readers can tell
// what it is (JSON shape, SQL dump header, ZIP magic) without fetching it.
blobPreview = 8 * 1024
@@ -387,8 +413,9 @@ func (t *Traffic) DB() *sql.DB { return t.db }
// the requests of client connections marked for pacing (see pace.go).
type sink struct {
mproxy.BaseAddon
t *Traffic
paced sync.Map // ClientConn.Id of a marked connection -> struct{}
t *Traffic
paced sync.Map // ClientConn.Id of a marked connection -> struct{}
streams sync.Map // *mproxy.Flow of a live WebSocket/SSE stream -> *streamState (stream.go)
}
func (s *sink) Response(f *mproxy.Flow) {
@@ -443,9 +470,8 @@ func (t *Traffic) record(f *mproxy.Flow) {
host := f.Request.URL.Hostname()
method := f.Request.Method
tmpl := db.TemplatePath(f.Request.URL.EscapedPath())
n := t.seq.Add(1)
now := time.Now()
id := fmt.Sprintf("%d-%04d", now.Unix(), n%10000)
id := t.newExchangeID(now)
ct := f.Response.Header.Get("Content-Type")
url := f.Request.URL.String()
@@ -511,6 +537,12 @@ VALUES(?,?,?,?,?,?,?)`,
}
}
// newExchangeID names a recorded exchange: unix seconds plus a rolling sequence.
func (t *Traffic) newExchangeID(now time.Time) string {
n := t.seq.Add(1)
return fmt.Sprintf("%d-%04d", now.Unix(), n%10000)
}
// storedBody is one body after the inline/spill decision: inline is what goes in
// the SQLite column (the whole body, or a preview when spilled), hash names the
// blob when it spilled, and index is the text handed to FTS (empty for binary).
@@ -526,6 +558,12 @@ type storedBody struct {
// are handed to the full-text index in full (up to maxIndexBody) — indexing is
// independent of where the bytes end up.
func (t *Traffic) spill(body []byte, contentType string) storedBody {
return t.spillAt(body, contentType, maxInlineBody)
}
// spillAt is spill with the inline threshold as a parameter: bodies up to
// maxInline stay inline. Stream messages use a much lower one (maxStreamInline).
func (t *Traffic) spillAt(body []byte, contentType string, maxInline int) storedBody {
if len(body) == 0 {
return storedBody{}
}
@@ -536,7 +574,7 @@ func (t *Traffic) spill(body []byte, contentType string) storedBody {
}
return string(clipBytes(body, maxIndexBody))
}
if len(body) <= maxInlineBody {
if len(body) <= maxInline {
return storedBody{inline: body, index: indexText()}
}
@@ -1191,6 +1229,9 @@ func (t *Traffic) deleteWhere(tx *sql.Tx, where string, args ...any) (int64, err
if _, err := tx.Exec(`DELETE FROM exchange_bodies WHERE id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil {
return 0, fmt.Errorf("delete bodies: %w", err)
}
if _, err := tx.Exec(`DELETE FROM stream_messages WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil {
return 0, fmt.Errorf("delete stream messages: %w", err)
}
if _, err := tx.Exec(`DELETE FROM blob_refs WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil {
return 0, fmt.Errorf("delete blob references: %w", err)
}
@@ -2058,7 +2099,7 @@ func (t *Traffic) Tools() []actool.CoreTool {
get := actool.Build(actool.Spec{
Name: "traffic_get",
Description: "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling.",
Description: "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling. A WebSocket (status 101) or SSE (text/event-stream) record shows only its handshake here; read its messages with traffic_messages.",
Schema: map[string]any{
"type": "object",
"properties": map[string]any{"id": map[string]any{"type": "string", "description": "the id returned by traffic_search"}},
@@ -2116,7 +2157,70 @@ func (t *Traffic) Tools() []actool.CoreTool {
},
})
return []actool.CoreTool{search, get, blob}
messages := actool.Build(actool.Spec{
Name: "traffic_messages",
Description: "List the messages of one captured WebSocket or SSE (Server-Sent Events) record in order: direction (c2s client→server, s2c server→client), type (text/binary/event, or meta when the recorder dropped messages past its limits), size and a preview of at most 1KB (binary as hex). A payload stored as a blob carries its hash; read the rest with traffic_blob. Returns 20 messages by default, at most 50; use offset to page.",
Schema: map[string]any{
"type": "object",
"properties": map[string]any{
"id": map[string]any{"type": "string", "description": "the record id from traffic_search (a WebSocket status-101 or SSE record)"},
"offset": map[string]any{"type": "integer", "description": "number of messages to skip, default 0"},
"limit": map[string]any{"type": "integer", "description": "messages to return, default 20, max 50"},
},
"required": []any{"id"},
},
ReadOnly: ro,
Permissions: allow,
Run: func(_ context.Context, in json.RawMessage, _ *actool.ToolContext) (actool.Result, error) {
var a struct {
ID string
Offset int
Limit int
}
_ = json.Unmarshal(in, &a)
if strings.TrimSpace(a.ID) == "" {
return actool.Errorf("id is a required parameter"), nil
}
if a.Limit <= 0 {
a.Limit = 20
}
if a.Limit > 50 {
a.Limit = 50
}
if a.Offset < 0 {
a.Offset = 0
}
msgs, total, err := t.StreamMessages(a.ID, a.Offset, a.Limit)
if err != nil {
return actool.Errorf(err.Error()), nil
}
if total == 0 {
return actool.Text("No stream messages recorded for this id (not a WebSocket/SSE record, or no message was exchanged)."), nil
}
type liteMsg struct {
Seq int64 `json:"seq"`
Direction string `json:"direction"`
Type string `json:"type"`
Event string `json:"event,omitempty"`
Size int `json:"size"`
Preview string `json:"preview"`
Blob string `json:"blob,omitempty"`
Truncated bool `json:"truncated,omitempty"`
}
lite := make([]liteMsg, 0, len(msgs))
for _, m := range msgs {
l := liteMsg{Seq: m.Seq, Direction: m.Direction, Type: m.Type, Event: m.Event, Size: m.Size, Preview: streamToolPreview(m), Truncated: m.Truncated}
if m.Blob != "" {
l.Blob = "@blob sha256:" + m.Blob
}
lite = append(lite, l)
}
b, _ := json.Marshal(lite)
return actool.Text(fmt.Sprintf("[messages %d-%d of %d]\n", a.Offset+1, a.Offset+len(msgs), total) + string(b)), nil
},
})
return []actool.CoreTool{search, get, blob, messages}
}
// SeedToolMetas returns the traffic tools built on a ZERO receiver, for seeding the
+28 -1
View File
@@ -377,6 +377,22 @@ VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
seed("1-0002", "b.example.com", hb) // sole reference to blobB
seed("1-0003", "c.example.com", hc) // shares blobC with d
seed("1-0004", "d.example.com", hc)
// d is also a stream exchange whose one message payload spilled to blobD,
// referenced through blob_refs like any body.
hd := strings.Repeat("d", 64)
blobD := filepath.Join(dir, "_blobs", "sha256", "dd", hd+".bin")
if err := os.MkdirAll(filepath.Dir(blobD), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(blobD, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := tr.DB().Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload,payload_blob) VALUES('1-0004',1,1,'s2c',2,1,x'78',?)`, hd); err != nil {
t.Fatal(err)
}
if _, err := tr.DB().Exec(`INSERT INTO blob_refs(hash,exchange_id) VALUES(?, '1-0004')`, hd); err != nil {
t.Fatal(err)
}
// Delete a: blobA orphaned → removed; blobB/blobC still referenced → kept.
if n, err := tr.DeleteHost("a.example"); err != nil || n != 1 {
@@ -399,12 +415,23 @@ VALUES(?,?,?,?,?,?,?,?,?,?,?)`,
if _, err := os.Stat(blobC); err != nil {
t.Fatalf("shared blobC removed after deleting one sharer: %v", err)
}
if _, err := os.Stat(blobD); err != nil {
t.Fatalf("stream payload blobD removed while d still references it: %v", err)
}
// Delete d: last reference gone → blobC collected.
// Delete d: last reference gone → blobC collected, and d's stream message
// and its payload blob go with it.
if n, err := tr.DeleteHost("d.example"); err != nil || n != 1 {
t.Fatalf("DeleteHost(d.example)=%d, err=%v; want 1, nil", n, err)
}
if _, err := os.Stat(blobC); !os.IsNotExist(err) {
t.Fatalf("blobC still exists after last reference removed: %v", err)
}
var streamRows int
if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM stream_messages`).Scan(&streamRows); err != nil || streamRows != 0 {
t.Fatalf("%d stream rows left after deleting d, err=%v", streamRows, err)
}
if _, err := os.Stat(blobD); !os.IsNotExist(err) {
t.Fatalf("stream payload blobD still exists after its exchange was deleted: %v", err)
}
}
@@ -0,0 +1,200 @@
"use client";
import * as React from "react";
import { Loader2Icon, RefreshCwIcon } from "lucide-react";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { api } from "@/lib/api";
import type { TrafficStreamMessage } from "@/lib/types";
const PAGE_SIZE = 50;
// Binary payloads loaded in full are shown as hex of their first 64 KiB.
const HEX_PREVIEW_BYTES = 64 * 1024;
function fmtBytes(n: number) {
if (n <= 0) return "0 B";
const units = ["B", "KB", "MB", "GB"];
const i = Math.min(Math.floor(Math.log(n) / Math.log(1024)), units.length - 1);
const v = n / 1024 ** i;
return `${i === 0 ? v : v.toFixed(1)} ${units[i]}`;
}
function fmtDirection(direction: string) {
if (direction === "c2s") return "client -> server";
if (direction === "s2c") return "server -> client";
return direction;
}
// Hex dump: 16 bytes per line, offset first.
function hexDump(bytes: Uint8Array) {
const lines: string[] = [];
for (let i = 0; i < bytes.length; i += 16) {
const row = Array.from(bytes.subarray(i, i + 16), (b) => b.toString(16).padStart(2, "0")).join(" ");
lines.push(`${i.toString(16).padStart(8, "0")} ${row}`);
}
return lines.join("\n");
}
type FullPayload = { loading: boolean; text?: string; error?: string };
const payloadClass =
"max-h-64 overflow-auto rounded-md bg-muted/50 p-2 font-mono text-xs break-all whitespace-pre-wrap";
function StreamMessageRow({ message }: { message: TrafficStreamMessage }) {
const [full, setFull] = React.useState<FullPayload | null>(null);
const binary = message.type === "binary";
const loadFull = () => {
if (!message.blob) return;
setFull({ loading: true });
api
.trafficBlob(message.blob)
.then((bytes) => {
const text = binary
? `${hexDump(bytes.subarray(0, HEX_PREVIEW_BYTES))}\n(${bytes.length} bytes total)`
: new TextDecoder().decode(bytes);
setFull({ loading: false, text });
})
.catch((e: Error) => setFull({ loading: false, error: e.message }));
};
if (message.type === "meta") {
return (
<div className="mx-5 my-3 space-y-1 rounded-md border border-amber-500/40 bg-amber-500/10 p-3">
<div className="font-medium text-amber-700 text-xs dark:text-amber-400">
#{message.seq} Recording limit reached: later messages were not stored
</div>
<pre className={payloadClass}>{message.payload}</pre>
</div>
);
}
return (
<div className="space-y-1.5 border-b px-5 py-3 last:border-b-0">
<div className="flex flex-wrap items-center gap-2 text-xs">
<span className="font-mono text-muted-foreground tabular-nums">#{message.seq}</span>
<Badge variant="outline">{fmtDirection(message.direction)}</Badge>
<Badge variant="secondary">{message.type}</Badge>
{binary && message.encoding === "base64" && <Badge variant="outline">base64</Badge>}
{message.truncated && <Badge variant="destructive">truncated</Badge>}
{message.event && <span className="font-mono">event: {message.event}</span>}
{message.event_id && <span className="font-mono">id: {message.event_id}</span>}
{message.retry ? <span className="font-mono">retry: {message.retry}</span> : null}
<span className="ml-auto text-muted-foreground tabular-nums">
{fmtBytes(message.size)} · {new Date(message.ts * 1000).toLocaleTimeString()}
</span>
</div>
<pre className={payloadClass}>{full?.text ?? message.payload}</pre>
{message.blob && (
<div className="flex flex-wrap items-center gap-2 text-muted-foreground text-xs">
<span className="break-all font-mono">blob {message.blob}</span>
{!full?.text && (
<Button variant="outline" size="xs" disabled={full?.loading} onClick={loadFull}>
{full?.loading && <Loader2Icon className="animate-spin" />}
Load full payload
</Button>
)}
{full?.error && <span className="text-destructive">{full.error}</span>}
</div>
)}
</div>
);
}
// StreamMessagesPanel lists the messages of one WebSocket or SSE exchange, a
// page at a time. It does not poll: Refresh reloads from the start.
export function StreamMessagesPanel({ exchangeId, kind }: { exchangeId: string; kind: "websocket" | "sse" }) {
const [items, setItems] = React.useState<TrafficStreamMessage[]>([]);
const [total, setTotal] = React.useState(0);
const [loading, setLoading] = React.useState(true);
const [loadingMore, setLoadingMore] = React.useState(false);
const [error, setError] = React.useState<string | null>(null);
// Answers for an older exchange (or an older refresh) are dropped.
const generation = React.useRef(0);
const load = React.useCallback(
(offset: number) => {
const gen = offset === 0 ? ++generation.current : generation.current;
if (offset === 0) {
setLoading(true);
setError(null);
} else {
setLoadingMore(true);
}
api
.trafficStream(exchangeId, offset, PAGE_SIZE)
.then((page) => {
if (gen !== generation.current) return;
const next = page.items ?? [];
setItems((prev) => (offset === 0 ? next : [...prev, ...next]));
setTotal(page.total ?? 0);
})
.catch((e: Error) => {
if (gen === generation.current) setError(e.message);
})
.finally(() => {
if (gen !== generation.current) return;
setLoading(false);
setLoadingMore(false);
});
},
[exchangeId],
);
React.useEffect(() => {
setItems([]);
setTotal(0);
load(0);
}, [load]);
return (
<div className="flex flex-col">
<div className="flex items-center gap-2 px-5 py-3 text-muted-foreground text-xs">
<span>
{kind === "websocket" ? "WebSocket" : "Server-Sent Events"} · {items.length} of {total} messages
</span>
<Button variant="outline" size="xs" className="ml-auto" disabled={loading} onClick={() => load(0)}>
<RefreshCwIcon className={loading ? "animate-spin" : undefined} />
Refresh
</Button>
</div>
{renderBody()}
</div>
);
function renderBody() {
if (loading) {
return (
<div className="flex items-center gap-2 p-5 text-muted-foreground text-xs">
<Loader2Icon className="size-3.5 animate-spin" />
Loading messages…
</div>
);
}
if (error) {
return <div className="p-5 text-destructive text-xs">Failed to load messages: {error}</div>;
}
if (items.length === 0) {
return <div className="p-5 text-muted-foreground text-xs">No messages recorded for this stream.</div>;
}
return (
<>
<div className="border-t">
{items.map((m) => (
<StreamMessageRow key={`${exchangeId}:${m.seq}`} message={m} />
))}
</div>
{items.length < total && (
<div className="flex justify-center p-4">
<Button variant="outline" size="sm" disabled={loadingMore} onClick={() => load(items.length)}>
{loadingMore && <Loader2Icon className="animate-spin" />}
Load more
</Button>
</div>
)}
</>
);
}
}
+27 -5
View File
@@ -42,10 +42,12 @@ import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { api } from "@/lib/api";
import { setVisibleInterval } from "@/lib/polling";
import { useStoredSortPreference } from "@/lib/sort-preference";
import type { TrafficDetail, TrafficExchange, TrafficHost, TrafficResp } from "@/lib/types";
import { cn } from "@/lib/utils";
import { setVisibleInterval } from "@/lib/polling";
import { StreamMessagesPanel } from "./_components/stream-messages";
function fmtTime(ts: string) {
return new Date(ts).toLocaleString("zh-CN", {
@@ -141,6 +143,9 @@ export default function TrafficPage() {
const [selected, setSelected] = React.useState<TrafficExchange | null>(null);
const [detail, setDetail] = React.useState<TrafficDetail | null>(null);
const [detailLoading, setDetailLoading] = React.useState(false);
const [detailTab, setDetailTab] = React.useState("request");
const streamKind = detail?.stream_kind === "websocket" || detail?.stream_kind === "sse" ? detail.stream_kind : null;
const hasStream = streamKind !== null;
const [hosts, setHosts] = React.useState<TrafficHost[]>([]); // target picker
const [selectedHosts, setSelectedHosts] = React.useState<string[]>([]); // checked in picker
@@ -298,8 +303,10 @@ export default function TrafficPage() {
.finally(() => setDeleting(false));
};
// Lazy-load the raw request/response for the selected exchange.
// Lazy-load the raw request/response for the selected exchange. A new
// selection opens on Request; a WebSocket/SSE exchange then opens on Stream.
React.useEffect(() => {
setDetailTab("request");
if (!selected) {
setDetail(null);
return;
@@ -310,7 +317,9 @@ export default function TrafficPage() {
api
.trafficExchange(selected.id)
.then((d) => {
if (alive) setDetail(d);
if (!alive) return;
setDetail(d);
if (d.stream_kind === "websocket" || d.stream_kind === "sse") setDetailTab("stream");
})
.catch(() => {
if (alive) setDetail({ req: "(failed to load the messages)", resp: "" });
@@ -742,10 +751,18 @@ export default function TrafficPage() {
<SheetTitle className="break-all font-mono">{selected.host}</SheetTitle>
<SheetDescription className="break-all font-mono">{selected.url}</SheetDescription>
</SheetHeader>
<Tabs defaultValue="request" className="min-h-0 flex-1 gap-0">
<TabsList className="mx-5 mt-4 grid w-auto grid-cols-2">
<Tabs value={detailTab} onValueChange={setDetailTab} className="min-h-0 flex-1 gap-0">
<TabsList className={cn("mx-5 mt-4 grid w-auto", hasStream ? "grid-cols-3" : "grid-cols-2")}>
<TabsTrigger value="request">Request</TabsTrigger>
<TabsTrigger value="response">Response</TabsTrigger>
{hasStream && (
<TabsTrigger value="stream">
Stream
{(detail?.stream_count ?? 0) > 0 && (
<span className="text-muted-foreground text-xs tabular-nums">{detail?.stream_count}</span>
)}
</TabsTrigger>
)}
</TabsList>
<TabsContent value="request" className="min-h-0 overflow-auto">
{detailLoading ? (
@@ -767,6 +784,11 @@ export default function TrafficPage() {
<HttpCodeBlock raw={detail?.resp ?? ""} />
)}
</TabsContent>
{streamKind && (
<TabsContent value="stream" className="min-h-0 overflow-auto">
<StreamMessagesPanel exchangeId={selected.id} kind={streamKind} />
</TabsContent>
)}
</Tabs>
</>
)}
+15
View File
@@ -102,6 +102,7 @@ import type {
TrafficEvidenceRole,
TrafficHost,
TrafficResp,
TrafficStreamPage,
UsageStats,
WorkspaceFile,
WorkspaceListing,
@@ -816,6 +817,20 @@ export const api = {
(opts.order ? `&order=${encodeURIComponent(opts.order)}` : ""),
),
trafficExchange: (id: string) => get<TrafficDetail>(`/traffic/exchange?id=${encodeURIComponent(id)}`),
// One page of a WebSocket/SSE exchange's messages, in order.
trafficStream: (id: string, offset = 0, limit = 50) =>
get<TrafficStreamPage>(`/traffic/stream?exchange=${encodeURIComponent(id)}&offset=${offset}&limit=${limit}`),
// Raw bytes of one stored blob (a full message payload or an oversized body).
// The JSON-only `get` cannot carry binary, hence the direct fetch.
trafficBlob: async (hash: string) => {
if (MOCK) throw new Error("Blob download is not available in mock mode");
const token = getToken();
const r = await fetch(`/api/traffic/blob?hash=${encodeURIComponent(hash)}`, {
headers: { ...(token ? { Authorization: `Bearer ${token}` } : {}) },
});
if (!r.ok) throw new Error(`Blob download failed: ${r.status}`);
return new Uint8Array(await r.arrayBuffer());
},
trafficHosts: () => get<{ hosts: TrafficHost[] }>(`/traffic/hosts`),
trafficDeleteHost: (host: string) => del<{ deleted: number }>(`/traffic?host=${encodeURIComponent(host)}`),
trafficDeleteHosts: (hosts: string[]) => del<{ deleted: number }>(`/traffic/hosts`, { hosts }),
+28
View File
@@ -911,6 +911,34 @@ export interface TrafficResp {
export interface TrafficDetail {
req: string;
resp: string;
// WebSocket/SSE exchanges carry their messages separately (trafficStream);
// stream_kind is "" for an ordinary exchange.
stream_count?: number;
stream_kind?: "websocket" | "sse" | "";
}
// One recorded WebSocket message or SSE event. ts is in Unix seconds; seq
// orders. payload is a preview of at most 4 KiB (base64 when encoding is
// "base64"); blob names the full payload when it was stored separately.
export interface TrafficStreamMessage {
seq: number;
ts: number;
direction: "c2s" | "s2c" | "meta" | (string & {});
kind: number;
type: "text" | "binary" | "event" | "meta" | (string & {});
event?: string;
event_id?: string;
retry?: number;
size: number;
payload: string;
encoding?: string;
blob?: string;
truncated?: boolean;
}
export interface TrafficStreamPage {
items: TrafficStreamMessage[];
total: number;
}
// One distinct recorded host with its exchange count (target picker).