Files
ARTEX/traffic/stream.go
T
BaaRai fa07d69f1d feat(traffic): apply rewrite rules in the capture proxy (P3.3b1)
The sink runs the request phase in the Request hook and the response phase
at the start of the Response hook; at SSEStart only status and header
actions apply. WebSocketStart comes after the client is upgraded, so the
handshake is traced "not rewritten: stream" and recorded as sent.

Exchanges keep what the client sent and what the target answered. The
original of a rewritten request lives in a per-flow snapshot slot (entry or
tombstone, 1024 entries / 64 MiB, 4096 tombstones), reserved before the
request is changed, released on Response, RequestError, SSEStart, a 5 MiB
response, or by a sweep of finished flows (ClientDisconnected, 1 s
debounce, 1 min period). Under pressure an entry older than 15 minutes is
turned into a tombstone in place, only when that frees room for the new
reservation; its exchange is recorded "original request unavailable".

record(), recordStreamStart() and insertExchange() clear any trace under
the id before INSERT OR REPLACE; the trace (exchange_rewrites, created
with CREATE TABLE IF NOT EXISTS) is written in the exchange's transaction,
its spilled values registered in blob_refs, and purged with the exchange.
The full-text index covers the originals only. Logs of a rewritten flow
name the host, the step or error class and the rule ids, never the URL or
the error text. A rewritten request's transport error never switches its
host to passthrough. traffic_get shows the rewritten flag and rule ids.

Without a rule that applies, what is recorded is unchanged byte for byte.
2026-11-17 22:23:12 +01:00

622 lines
20 KiB
Go

package traffic
import (
"database/sql"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"log"
"net/url"
"reflect"
"strings"
"sync"
"time"
"unicode/utf8"
"unsafe"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/traffic/rewrite"
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
)
// Stream recording (WebSocket and Server-Sent Events). go-mitmproxy reports a
// stream as Start, one Message hook per message, then End; a stream never reaches
// the Response hook, so record() never sees it. Each stream becomes one exchange
// (status 101 "websocket", or the real text/event-stream response) whose messages
// go to stream_messages in arrival order.
const (
// wsText and wsBinary are the WebSocket data opcodes (RFC 6455 §5.2), the
// values go-mitmproxy reports in WebSocketMessage.Type.
wsText = 1
wsBinary = 2
// streamPreview caps one message's payload in a StreamMessages row; a longer
// payload lives in the blob store and the row carries its hash.
streamPreview = 4 * 1024
// maxStreamInline is the inline threshold for message payloads: anything
// longer spills to the blob store, so a page of 200 rows stays bounded.
maxStreamInline = streamPreview
// toolStreamPreview caps one message's preview in traffic_messages, so a page
// of messages stays small in the agent's context.
toolStreamPreview = 1024
)
// streamState tracks one live stream. mu serializes the two WebSocket forwarding
// goroutines, which both fire WebSocketMessage on the same flow.
type streamState struct {
mu sync.Mutex
id string // exchange id; "" when the stream could not be recorded
kind string // "websocket" | "sse"
consumed int // library messages/events already handled
seq int64
stored int
// storedBytes counts what the kept messages occupy: the payload after
// truncation, plus the inline preview spill keeps for a spilled payload.
storedBytes int64
dropped bool
droppedMsgs int64
droppedBytes int64
// reqBytes and respBytes are the bytes observed in each direction, dropped
// messages included.
reqBytes, respBytes int64
}
// streamRow is one stream_messages row before it is written.
type streamRow struct {
ts int64 // unix seconds, like exchanges.ts; seq carries the order
direction string
kind int
event string
eventID string
retry int
size int
payload []byte // kept bytes (after maxStreamPayload truncation)
truncated bool
ctype string // content type handed to spill for the binary check
}
func (s *sink) WebSocketStart(f *mproxy.Flow) { s.startStream(f, "websocket") }
func (s *sink) SSEStart(f *mproxy.Flow) { s.startStream(f, "sse") }
func (s *sink) WebSocketMessage(f *mproxy.Flow) {
st := s.stream(f)
if st == nil || f.WebScoket == nil {
return
}
st.mu.Lock()
defer st.mu.Unlock()
s.t.persistStream(st, wsMessagesSince(f.WebScoket, st))
}
func (s *sink) SSEMessage(f *mproxy.Flow) {
st := s.stream(f)
if st == nil || f.SSE == nil {
return
}
st.mu.Lock()
defer st.mu.Unlock()
s.t.persistStream(st, sseEventsSince(f.SSE, st))
}
func (s *sink) WebSocketEnd(f *mproxy.Flow) {
v, ok := s.streams.LoadAndDelete(f)
if !ok {
return
}
st := v.(*streamState)
st.mu.Lock()
defer st.mu.Unlock()
if f.WebScoket != nil {
s.t.persistStream(st, wsMessagesSince(f.WebScoket, st))
}
s.t.finishStream(st)
}
func (s *sink) SSEEnd(f *mproxy.Flow) {
v, ok := s.streams.LoadAndDelete(f)
if !ok {
return
}
st := v.(*streamState)
st.mu.Lock()
defer st.mu.Unlock()
if f.SSE != nil {
s.t.persistStream(st, sseEventsSince(f.SSE, st))
}
s.t.finishStream(st)
}
func (s *sink) stream(f *mproxy.Flow) *streamState {
v, ok := s.streams.Load(f)
if !ok {
return nil
}
return v.(*streamState)
}
// startStream registers the flow before recording its exchange, so a message
// hook racing the start waits on st.mu instead of being dropped.
func (s *sink) startStream(f *mproxy.Flow, kind string) {
st := &streamState{kind: kind}
st.mu.Lock()
defer st.mu.Unlock()
s.streams.Store(f, st)
if f.Request == nil || f.Request.URL == nil || (kind == "sse" && f.Response == nil) {
return
}
rq, rs, lines, res := s.rewriteStream(f, kind)
id, err := s.t.recordStreamStart(rq, rs, kind, lines)
s.applyStreamResponse(f, res)
if err != nil {
if rewritten, ruleIDs := appliedRules(lines); rewritten {
var re *recordError
errors.As(err, &re)
log.Printf("[traffic] failed to record a rewritten %s stream of %s (%s; rules %v)", kind, f.Request.URL.Hostname(), re.step, ruleIDs)
return
}
log.Printf("[traffic] failed to record %s stream %s: %v", kind, f.Request.URL, err)
return
}
st.id = id
if kind == "sse" {
st.reqBytes = int64(len(rq.body))
}
}
// recordError names the step of a recording that failed. Its text is the step
// and the cause, as recordStreamStart always reported.
type recordError struct {
step string
err error
}
func (e *recordError) Error() string { return e.step + ": " + e.err.Error() }
func (e *recordError) Unwrap() error { return e.err }
// libMutex returns the unexported mu sync.Mutex that go-mitmproxy holds while it
// appends to WebSocketData.Messages / SSEData.Events. The library exposes no
// accessor, and the two WebSocket goroutines append concurrently, so reading the
// slice without that lock is a data race. Reaching a private field ties this to
// the pinned go-mitmproxy v1.9.2: TestLibMutexFound is the upgrade guard. Nil
// when the field is absent, and then nothing is read (see lockLib).
func libMutex(v any) *sync.Mutex {
rv := reflect.ValueOf(v)
if rv.Kind() != reflect.Pointer || rv.IsNil() {
return nil
}
f := rv.Elem().FieldByName("mu")
if !f.IsValid() || f.Type() != reflect.TypeOf(sync.Mutex{}) || !f.CanAddr() {
return nil
}
return (*sync.Mutex)(unsafe.Pointer(f.UnsafeAddr()))
}
var libMutexMissing sync.Once
// lockLib takes the library's lock on v. ok is false when the lock cannot be
// found; callers then read nothing rather than race the library.
func lockLib(v any) (unlock func(), ok bool) {
mu := libMutex(v)
if mu == nil {
libMutexMissing.Do(func() {
log.Printf("[traffic] go-mitmproxy stream data has no mu field; stream messages are not recorded")
})
return nil, false
}
mu.Lock()
return mu.Unlock, true
}
// wsMessagesSince returns every WebSocket message appended since the last call,
// in the library's order, so each is handled exactly once whichever goroutine's
// hook gets there first. The library slice itself is never modified.
func wsMessagesSince(ws *mproxy.WebSocketData, st *streamState) []streamRow {
unlock, ok := lockLib(ws)
if !ok {
return nil
}
msgs := ws.Messages
if st.consumed > len(msgs) {
st.consumed = len(msgs)
}
fresh := append([]*mproxy.WebSocketMessage(nil), msgs[st.consumed:]...)
st.consumed = len(msgs)
unlock()
rows := make([]streamRow, 0, len(fresh))
for _, m := range fresh {
if m == nil {
continue
}
dir, ctype := "s2c", "text/plain; charset=utf-8"
if m.FromClient {
dir = "c2s"
st.reqBytes += int64(len(m.Content))
} else {
st.respBytes += int64(len(m.Content))
}
if m.Type == wsBinary {
ctype = "application/octet-stream"
}
rows = append(rows, streamRow{ts: m.Timestamp.Unix(), direction: dir, kind: m.Type, size: len(m.Content), payload: m.Content, ctype: ctype})
}
return rows
}
// sseEventsSince is wsMessagesSince for Server-Sent Events.
func sseEventsSince(sse *mproxy.SSEData, st *streamState) []streamRow {
unlock, ok := lockLib(sse)
if !ok {
return nil
}
events := sse.Events
if st.consumed > len(events) {
st.consumed = len(events)
}
fresh := append([]*mproxy.SSEEvent(nil), events[st.consumed:]...)
st.consumed = len(events)
unlock()
rows := make([]streamRow, 0, len(fresh))
for _, e := range fresh {
if e == nil {
continue
}
st.respBytes += int64(len(e.Raw))
rows = append(rows, streamRow{ts: e.Time.Unix(), direction: "s2c", event: e.Event, eventID: e.ID,
retry: e.Retry, size: len(e.Data), payload: []byte(e.Data), ctype: "text/event-stream"})
}
return rows
}
// streamDisplayURL is the URL shown for a WebSocket exchange: http→ws and
// https→wss. A plain ws:// connection reaches the WebSocket hooks through its
// CONNECT tunnel, where only the target authority is known, not the path.
func streamDisplayURL(reqMethod string, reqURL *url.URL, kind string) (method, display, tmpl string) {
u := *reqURL
method = reqMethod
if kind == "websocket" && strings.EqualFold(method, "CONNECT") {
host := u.Host
if host == "" {
host = u.Opaque
}
return "GET", (&url.URL{Scheme: "ws", Host: host, Path: "/"}).String(), "/"
}
if kind == "websocket" {
switch strings.ToLower(u.Scheme) {
case "http", "":
u.Scheme = "ws"
case "https":
u.Scheme = "wss"
}
}
return method, u.String(), db.TemplatePath(reqURL.EscapedPath())
}
// recordStreamStart writes the stream's exchange: metadata, request head and
// body, the response head (synthetic 101 for WebSocket), the rewrite trace,
// and the header text in the full-text index. Messages are never indexed. rq
// is the request the client sent and rs (SSE only) the status and headers the
// target sent, whatever the rules changed.
func (t *Traffic) recordStreamStart(rq requestView, rs responseView, kind string, lines []rewrite.TraceLine) (string, error) {
now := time.Now()
id := t.newExchangeID(now)
method, display, tmpl, host, target := rq.method, "", "", "", ""
if rq.url != nil {
method, display, tmpl = streamDisplayURL(rq.method, rq.url, kind)
host = rq.url.Hostname()
target = rq.url.RequestURI()
if strings.EqualFold(rq.method, "CONNECT") {
target = rq.url.Host // authority-form, as on the wire
}
}
reqHead := requestHead(rq, target)
status, ct, respHead := 101, "websocket", "HTTP/1.1 101 Switching Protocols\n"
if kind == "sse" {
status = rs.status
ct = rs.header.Get("Content-Type")
respHead = fmt.Sprintf("HTTP %d\n%s", status, headerLines(rs.header))
}
t.wmu.Lock()
defer t.wmu.Unlock()
reqB := t.spill(rq.body, rq.header.Get("Content-Type"))
trace := t.spillTrace(lines)
tx, err := t.db.Begin()
if err != nil {
return "", &recordError{"beginning transaction", err}
}
defer tx.Rollback() //nolint:errcheck // no-op once committed
// An id can repeat (seconds plus a rolling counter); the trace of the
// exchange it replaces must not stay attached to this one.
if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id=?`, id); err != nil {
return "", &recordError{"clearing rewrite trace", err}
}
res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
VALUES(?,?,?,?,?,?,?,?,?,?,'')`, id, now.Unix(), host, method, tmpl, display, status, ct, len(rq.body), 0)
if err != nil {
return "", &recordError{"writing index", err}
}
rowid, err := res.LastInsertId()
if err != nil {
return "", &recordError{"fetching rowid", err}
}
if _, err := tx.Exec(`INSERT OR REPLACE INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob)
VALUES(?,?,?,?,?,NULL,NULL)`, id, reqHead, reqB.inline, nullIfEmpty(reqB.hash), respHead); err != nil {
return "", &recordError{"writing body", err}
}
if reqB.hash != "" {
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, reqB.hash, id); err != nil {
return "", &recordError{"registering blob reference", err}
}
}
if err := writeTrace(tx, id, lines, trace); err != nil {
return "", err
}
if t.fts {
idx := strings.Join([]string{display, reqHead, reqB.index, respHead}, "\n")
if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowid, idx); err != nil {
return "", &recordError{"writing full-text index", err}
}
}
if err := tx.Commit(); err != nil {
return "", &recordError{"committing", err}
}
return id, nil
}
// persistStream applies the per-message and per-exchange limits to rows and
// writes what is kept. The first message past a per-exchange limit stores one
// 'meta' row instead; nothing is stored after it. Caller holds st.mu. Errors are
// logged: a recording failure never interrupts the stream.
func (t *Traffic) persistStream(st *streamState, rows []streamRow) {
if st.id == "" || len(rows) == 0 {
return
}
keep := make([]streamRow, 0, len(rows))
addMeta := false
for _, r := range rows {
if len(r.payload) > maxStreamPayload {
r.payload = r.payload[:maxStreamPayload]
r.truncated = true
}
cost := streamStoredCost(r)
if st.dropped || st.stored >= maxStreamMessages || st.storedBytes+cost > maxStreamBytes {
if !st.dropped {
st.dropped = true
addMeta = true
}
st.droppedMsgs++
st.droppedBytes += int64(r.size)
continue
}
st.stored++
st.storedBytes += cost
keep = append(keep, r)
}
var meta []byte
if addMeta {
meta = streamMetaPayload(st)
}
if len(keep) == 0 && meta == nil {
return
}
t.wmu.Lock()
defer t.wmu.Unlock()
// Sequence numbers and blob writes come first, as in record(): blob writes are
// filesystem work and must not sit inside the SQLite transaction. Holding wmu
// keeps gcBlobs from collecting a blob before its reference is committed.
type preparedRow struct {
streamRow
seq int64
body storedBody
}
prepared := make([]preparedRow, 0, len(keep))
for _, r := range keep {
st.seq++
prepared = append(prepared, preparedRow{streamRow: r, seq: st.seq, body: t.spillAt(r.payload, r.ctype, maxStreamInline)})
}
var metaSeq int64
if meta != nil {
st.seq++
metaSeq = st.seq
}
tx, err := t.db.Begin()
if err != nil {
log.Printf("[traffic] failed to record stream messages of %s (beginning transaction): %v", st.id, err)
return
}
defer tx.Rollback() //nolint:errcheck // no-op once committed
for _, p := range prepared {
inline := p.body.inline
if inline == nil {
inline = []byte{}
}
if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,event,event_id,retry,size,payload,payload_blob,truncated)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`, st.id, p.seq, p.ts, p.direction, p.kind, p.event, p.eventID, p.retry, p.size,
inline, nullIfEmpty(p.body.hash), boolInt(p.truncated)); err != nil {
log.Printf("[traffic] failed to record stream message %d of %s: %v", p.seq, st.id, err)
return
}
if p.body.hash != "" {
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, p.body.hash, st.id); err != nil {
log.Printf("[traffic] failed to record stream message %d of %s (registering blob reference): %v", p.seq, st.id, err)
return
}
}
}
if meta != nil {
if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload)
VALUES(?,?,?,'meta',0,?,?)`, st.id, metaSeq, time.Now().Unix(), len(meta), meta); err != nil {
log.Printf("[traffic] failed to record stream limit marker of %s: %v", st.id, err)
return
}
}
if err := tx.Commit(); err != nil {
log.Printf("[traffic] failed to record stream messages of %s (committing): %v", st.id, err)
}
}
// streamStoredCost is what keeping r adds to the index and blob store: the
// payload, plus the inline preview spillAt keeps when the payload spills (a text
// prefix of at most blobPreview, or a one-line tag for binary).
func streamStoredCost(r streamRow) int64 {
n := int64(len(r.payload))
if len(r.payload) <= maxStreamInline {
return n
}
if isBinaryBody(r.ctype, r.payload) {
return n + int64(len(binaryTag(r.ctype, r.payload)))
}
return n + int64(len(truncateUTF8(r.payload, blobPreview)))
}
func streamMetaPayload(st *streamState) []byte {
b, _ := json.Marshal(map[string]int64{"dropped_messages": st.droppedMsgs, "dropped_bytes": st.droppedBytes})
return b
}
func boolInt(b bool) int {
if b {
return 1
}
return 0
}
// finishStream records the bytes observed in each direction and the final
// counts of the limit marker. Caller holds st.mu.
func (t *Traffic) finishStream(st *streamState) {
if st.id == "" {
return
}
t.wmu.Lock()
defer t.wmu.Unlock()
if _, err := t.db.Exec(`UPDATE exchanges SET req_len=?, resp_len=? WHERE id=?`, st.reqBytes, st.respBytes, st.id); err != nil {
log.Printf("[traffic] failed to record stream sizes of %s: %v", st.id, err)
}
if st.dropped {
meta := streamMetaPayload(st)
if _, err := t.db.Exec(`UPDATE stream_messages SET payload=?, size=? WHERE exchange_id=? AND direction='meta'`, meta, len(meta), st.id); err != nil {
log.Printf("[traffic] failed to update stream limit marker of %s: %v", st.id, err)
}
}
}
// StreamMessage is one recorded WebSocket message or SSE event.
type StreamMessage struct {
Seq int64 `json:"seq"`
TS int64 `json:"ts"` // unix seconds, like an exchange's ts; Seq orders
Direction string `json:"direction"`
Kind int `json:"kind"`
// Type is "text" or "binary" (WebSocket), "event" (SSE), or "meta" (the
// limit marker).
Type string `json:"type"`
Event string `json:"event,omitempty"`
EventID string `json:"event_id,omitempty"`
Retry int `json:"retry,omitempty"`
Size int `json:"size"`
Payload string `json:"payload"`
// Encoding is "base64" for binary payloads, empty for UTF-8 text.
Encoding string `json:"encoding,omitempty"`
Blob string `json:"blob,omitempty"`
Truncated bool `json:"truncated,omitempty"`
}
// StreamMessages returns one page of an exchange's stream messages in order.
// limit is 1..200 (default 50) and offset >= 0. Every payload comes back as at
// most streamPreview (4 KiB); a longer one lives in the blob store and the row
// carries its hash for the rest (traffic_blob, GET /api/traffic/blob).
func (t *Traffic) StreamMessages(exchangeID string, offset, limit int) ([]StreamMessage, int, error) {
if limit <= 0 {
limit = 50
}
if limit > 200 {
limit = 200
}
if offset < 0 {
offset = 0
}
var total int
if err := t.db.QueryRow(`SELECT COUNT(*) FROM stream_messages WHERE exchange_id=?`, exchangeID).Scan(&total); err != nil {
return nil, 0, err
}
rows, err := t.db.Query(`SELECT seq,ts,direction,kind,event,event_id,retry,size,payload,payload_blob,truncated
FROM stream_messages WHERE exchange_id=? ORDER BY seq LIMIT ? OFFSET ?`, exchangeID, limit, offset)
if err != nil {
return nil, 0, err
}
defer rows.Close()
out := []StreamMessage{}
for rows.Next() {
var m StreamMessage
var payload []byte
var blob sql.NullString
var truncated int
if err := rows.Scan(&m.Seq, &m.TS, &m.Direction, &m.Kind, &m.Event, &m.EventID, &m.Retry, &m.Size, &payload, &blob, &truncated); err != nil {
return nil, 0, err
}
m.Truncated = truncated != 0
m.Blob = blob.String
switch {
case m.Direction == "meta":
m.Type = "meta"
case m.Kind == wsBinary:
m.Type = "binary"
case m.Kind == wsText:
m.Type = "text"
default:
m.Type = "event"
}
if m.Type == "binary" {
if m.Blob != "" {
// A spilled binary payload keeps only a descriptive tag inline;
// the preview is the payload's own leading bytes.
payload, _, _ = t.BlobRange(m.Blob, 0, streamPreview)
}
m.Payload, m.Encoding = base64.StdEncoding.EncodeToString(clipBytes(payload, streamPreview)), "base64"
} else {
m.Payload = truncateUTF8(payload, streamPreview)
}
out = append(out, m)
}
return out, total, rows.Err()
}
// StreamInfo reports how many stream messages an exchange holds and whether it
// is a WebSocket or SSE exchange ("" for an ordinary exchange).
func (t *Traffic) StreamInfo(exchangeID string) (count int, kind string, err error) {
var ct sql.NullString
err = t.db.QueryRow(`SELECT content_type FROM exchanges WHERE id=?`, exchangeID).Scan(&ct)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return 0, "", err
}
switch c := strings.ToLower(ct.String); {
case c == "websocket":
kind = "websocket"
case strings.Contains(c, "text/event-stream"):
kind = "sse"
}
if err := t.db.QueryRow(`SELECT COUNT(*) FROM stream_messages WHERE exchange_id=?`, exchangeID).Scan(&count); err != nil {
return 0, "", err
}
return count, kind, nil
}
// streamToolPreview renders one message for traffic_messages: at most 1 KiB of
// text (hex for binary), plus the blob hash when the payload spilled.
func streamToolPreview(m StreamMessage) string {
if m.Encoding == "base64" {
raw, _ := base64.StdEncoding.DecodeString(m.Payload)
return "hex:" + fmt.Sprintf("%x", clipBytes(raw, toolStreamPreview/2))
}
p := m.Payload
if len(p) > toolStreamPreview {
p = truncateUTF8([]byte(p), toolStreamPreview)
}
if !utf8.ValidString(p) {
p = strings.ToValidUTF8(p, "�")
}
return p
}