mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-19 06:30:54 +08:00
The sink runs the request phase in the Request hook and the response phase at the start of the Response hook; at SSEStart only status and header actions apply. WebSocketStart comes after the client is upgraded, so the handshake is traced "not rewritten: stream" and recorded as sent. Exchanges keep what the client sent and what the target answered. The original of a rewritten request lives in a per-flow snapshot slot (entry or tombstone, 1024 entries / 64 MiB, 4096 tombstones), reserved before the request is changed, released on Response, RequestError, SSEStart, a 5 MiB response, or by a sweep of finished flows (ClientDisconnected, 1 s debounce, 1 min period). Under pressure an entry older than 15 minutes is turned into a tombstone in place, only when that frees room for the new reservation; its exchange is recorded "original request unavailable". record(), recordStreamStart() and insertExchange() clear any trace under the id before INSERT OR REPLACE; the trace (exchange_rewrites, created with CREATE TABLE IF NOT EXISTS) is written in the exchange's transaction, its spilled values registered in blob_refs, and purged with the exchange. The full-text index covers the originals only. Logs of a rewritten flow name the host, the step or error class and the rule ids, never the URL or the error text. A rewritten request's transport error never switches its host to passthrough. traffic_get shows the rewritten flag and rule ids. Without a rule that applies, what is recorded is unchanged byte for byte.
622 lines
20 KiB
Go
622 lines
20 KiB
Go
package traffic
|
|
|
|
import (
|
|
"database/sql"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"net/url"
|
|
"reflect"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
"unicode/utf8"
|
|
"unsafe"
|
|
|
|
"github.com/Autumn-27/artex/db"
|
|
"github.com/Autumn-27/artex/traffic/rewrite"
|
|
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
|
|
)
|
|
|
|
// Stream recording (WebSocket and Server-Sent Events). go-mitmproxy reports a
|
|
// stream as Start, one Message hook per message, then End; a stream never reaches
|
|
// the Response hook, so record() never sees it. Each stream becomes one exchange
|
|
// (status 101 "websocket", or the real text/event-stream response) whose messages
|
|
// go to stream_messages in arrival order.
|
|
|
|
const (
|
|
// wsText and wsBinary are the WebSocket data opcodes (RFC 6455 §5.2), the
|
|
// values go-mitmproxy reports in WebSocketMessage.Type.
|
|
wsText = 1
|
|
wsBinary = 2
|
|
// streamPreview caps one message's payload in a StreamMessages row; a longer
|
|
// payload lives in the blob store and the row carries its hash.
|
|
streamPreview = 4 * 1024
|
|
// maxStreamInline is the inline threshold for message payloads: anything
|
|
// longer spills to the blob store, so a page of 200 rows stays bounded.
|
|
maxStreamInline = streamPreview
|
|
// toolStreamPreview caps one message's preview in traffic_messages, so a page
|
|
// of messages stays small in the agent's context.
|
|
toolStreamPreview = 1024
|
|
)
|
|
|
|
// streamState tracks one live stream. mu serializes the two WebSocket forwarding
|
|
// goroutines, which both fire WebSocketMessage on the same flow.
|
|
type streamState struct {
|
|
mu sync.Mutex
|
|
id string // exchange id; "" when the stream could not be recorded
|
|
kind string // "websocket" | "sse"
|
|
consumed int // library messages/events already handled
|
|
seq int64
|
|
stored int
|
|
// storedBytes counts what the kept messages occupy: the payload after
|
|
// truncation, plus the inline preview spill keeps for a spilled payload.
|
|
storedBytes int64
|
|
dropped bool
|
|
droppedMsgs int64
|
|
droppedBytes int64
|
|
// reqBytes and respBytes are the bytes observed in each direction, dropped
|
|
// messages included.
|
|
reqBytes, respBytes int64
|
|
}
|
|
|
|
// streamRow is one stream_messages row before it is written.
|
|
type streamRow struct {
|
|
ts int64 // unix seconds, like exchanges.ts; seq carries the order
|
|
direction string
|
|
kind int
|
|
event string
|
|
eventID string
|
|
retry int
|
|
size int
|
|
payload []byte // kept bytes (after maxStreamPayload truncation)
|
|
truncated bool
|
|
ctype string // content type handed to spill for the binary check
|
|
}
|
|
|
|
func (s *sink) WebSocketStart(f *mproxy.Flow) { s.startStream(f, "websocket") }
|
|
func (s *sink) SSEStart(f *mproxy.Flow) { s.startStream(f, "sse") }
|
|
|
|
func (s *sink) WebSocketMessage(f *mproxy.Flow) {
|
|
st := s.stream(f)
|
|
if st == nil || f.WebScoket == nil {
|
|
return
|
|
}
|
|
st.mu.Lock()
|
|
defer st.mu.Unlock()
|
|
s.t.persistStream(st, wsMessagesSince(f.WebScoket, st))
|
|
}
|
|
|
|
func (s *sink) SSEMessage(f *mproxy.Flow) {
|
|
st := s.stream(f)
|
|
if st == nil || f.SSE == nil {
|
|
return
|
|
}
|
|
st.mu.Lock()
|
|
defer st.mu.Unlock()
|
|
s.t.persistStream(st, sseEventsSince(f.SSE, st))
|
|
}
|
|
|
|
func (s *sink) WebSocketEnd(f *mproxy.Flow) {
|
|
v, ok := s.streams.LoadAndDelete(f)
|
|
if !ok {
|
|
return
|
|
}
|
|
st := v.(*streamState)
|
|
st.mu.Lock()
|
|
defer st.mu.Unlock()
|
|
if f.WebScoket != nil {
|
|
s.t.persistStream(st, wsMessagesSince(f.WebScoket, st))
|
|
}
|
|
s.t.finishStream(st)
|
|
}
|
|
|
|
func (s *sink) SSEEnd(f *mproxy.Flow) {
|
|
v, ok := s.streams.LoadAndDelete(f)
|
|
if !ok {
|
|
return
|
|
}
|
|
st := v.(*streamState)
|
|
st.mu.Lock()
|
|
defer st.mu.Unlock()
|
|
if f.SSE != nil {
|
|
s.t.persistStream(st, sseEventsSince(f.SSE, st))
|
|
}
|
|
s.t.finishStream(st)
|
|
}
|
|
|
|
func (s *sink) stream(f *mproxy.Flow) *streamState {
|
|
v, ok := s.streams.Load(f)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return v.(*streamState)
|
|
}
|
|
|
|
// startStream registers the flow before recording its exchange, so a message
|
|
// hook racing the start waits on st.mu instead of being dropped.
|
|
func (s *sink) startStream(f *mproxy.Flow, kind string) {
|
|
st := &streamState{kind: kind}
|
|
st.mu.Lock()
|
|
defer st.mu.Unlock()
|
|
s.streams.Store(f, st)
|
|
if f.Request == nil || f.Request.URL == nil || (kind == "sse" && f.Response == nil) {
|
|
return
|
|
}
|
|
rq, rs, lines, res := s.rewriteStream(f, kind)
|
|
id, err := s.t.recordStreamStart(rq, rs, kind, lines)
|
|
s.applyStreamResponse(f, res)
|
|
if err != nil {
|
|
if rewritten, ruleIDs := appliedRules(lines); rewritten {
|
|
var re *recordError
|
|
errors.As(err, &re)
|
|
log.Printf("[traffic] failed to record a rewritten %s stream of %s (%s; rules %v)", kind, f.Request.URL.Hostname(), re.step, ruleIDs)
|
|
return
|
|
}
|
|
log.Printf("[traffic] failed to record %s stream %s: %v", kind, f.Request.URL, err)
|
|
return
|
|
}
|
|
st.id = id
|
|
if kind == "sse" {
|
|
st.reqBytes = int64(len(rq.body))
|
|
}
|
|
}
|
|
|
|
// recordError names the step of a recording that failed. Its text is the step
|
|
// and the cause, as recordStreamStart always reported.
|
|
type recordError struct {
|
|
step string
|
|
err error
|
|
}
|
|
|
|
func (e *recordError) Error() string { return e.step + ": " + e.err.Error() }
|
|
func (e *recordError) Unwrap() error { return e.err }
|
|
|
|
// libMutex returns the unexported mu sync.Mutex that go-mitmproxy holds while it
|
|
// appends to WebSocketData.Messages / SSEData.Events. The library exposes no
|
|
// accessor, and the two WebSocket goroutines append concurrently, so reading the
|
|
// slice without that lock is a data race. Reaching a private field ties this to
|
|
// the pinned go-mitmproxy v1.9.2: TestLibMutexFound is the upgrade guard. Nil
|
|
// when the field is absent, and then nothing is read (see lockLib).
|
|
func libMutex(v any) *sync.Mutex {
|
|
rv := reflect.ValueOf(v)
|
|
if rv.Kind() != reflect.Pointer || rv.IsNil() {
|
|
return nil
|
|
}
|
|
f := rv.Elem().FieldByName("mu")
|
|
if !f.IsValid() || f.Type() != reflect.TypeOf(sync.Mutex{}) || !f.CanAddr() {
|
|
return nil
|
|
}
|
|
return (*sync.Mutex)(unsafe.Pointer(f.UnsafeAddr()))
|
|
}
|
|
|
|
var libMutexMissing sync.Once
|
|
|
|
// lockLib takes the library's lock on v. ok is false when the lock cannot be
|
|
// found; callers then read nothing rather than race the library.
|
|
func lockLib(v any) (unlock func(), ok bool) {
|
|
mu := libMutex(v)
|
|
if mu == nil {
|
|
libMutexMissing.Do(func() {
|
|
log.Printf("[traffic] go-mitmproxy stream data has no mu field; stream messages are not recorded")
|
|
})
|
|
return nil, false
|
|
}
|
|
mu.Lock()
|
|
return mu.Unlock, true
|
|
}
|
|
|
|
// wsMessagesSince returns every WebSocket message appended since the last call,
|
|
// in the library's order, so each is handled exactly once whichever goroutine's
|
|
// hook gets there first. The library slice itself is never modified.
|
|
func wsMessagesSince(ws *mproxy.WebSocketData, st *streamState) []streamRow {
|
|
unlock, ok := lockLib(ws)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
msgs := ws.Messages
|
|
if st.consumed > len(msgs) {
|
|
st.consumed = len(msgs)
|
|
}
|
|
fresh := append([]*mproxy.WebSocketMessage(nil), msgs[st.consumed:]...)
|
|
st.consumed = len(msgs)
|
|
unlock()
|
|
rows := make([]streamRow, 0, len(fresh))
|
|
for _, m := range fresh {
|
|
if m == nil {
|
|
continue
|
|
}
|
|
dir, ctype := "s2c", "text/plain; charset=utf-8"
|
|
if m.FromClient {
|
|
dir = "c2s"
|
|
st.reqBytes += int64(len(m.Content))
|
|
} else {
|
|
st.respBytes += int64(len(m.Content))
|
|
}
|
|
if m.Type == wsBinary {
|
|
ctype = "application/octet-stream"
|
|
}
|
|
rows = append(rows, streamRow{ts: m.Timestamp.Unix(), direction: dir, kind: m.Type, size: len(m.Content), payload: m.Content, ctype: ctype})
|
|
}
|
|
return rows
|
|
}
|
|
|
|
// sseEventsSince is wsMessagesSince for Server-Sent Events.
|
|
func sseEventsSince(sse *mproxy.SSEData, st *streamState) []streamRow {
|
|
unlock, ok := lockLib(sse)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
events := sse.Events
|
|
if st.consumed > len(events) {
|
|
st.consumed = len(events)
|
|
}
|
|
fresh := append([]*mproxy.SSEEvent(nil), events[st.consumed:]...)
|
|
st.consumed = len(events)
|
|
unlock()
|
|
rows := make([]streamRow, 0, len(fresh))
|
|
for _, e := range fresh {
|
|
if e == nil {
|
|
continue
|
|
}
|
|
st.respBytes += int64(len(e.Raw))
|
|
rows = append(rows, streamRow{ts: e.Time.Unix(), direction: "s2c", event: e.Event, eventID: e.ID,
|
|
retry: e.Retry, size: len(e.Data), payload: []byte(e.Data), ctype: "text/event-stream"})
|
|
}
|
|
return rows
|
|
}
|
|
|
|
// streamDisplayURL is the URL shown for a WebSocket exchange: http→ws and
|
|
// https→wss. A plain ws:// connection reaches the WebSocket hooks through its
|
|
// CONNECT tunnel, where only the target authority is known, not the path.
|
|
func streamDisplayURL(reqMethod string, reqURL *url.URL, kind string) (method, display, tmpl string) {
|
|
u := *reqURL
|
|
method = reqMethod
|
|
if kind == "websocket" && strings.EqualFold(method, "CONNECT") {
|
|
host := u.Host
|
|
if host == "" {
|
|
host = u.Opaque
|
|
}
|
|
return "GET", (&url.URL{Scheme: "ws", Host: host, Path: "/"}).String(), "/"
|
|
}
|
|
if kind == "websocket" {
|
|
switch strings.ToLower(u.Scheme) {
|
|
case "http", "":
|
|
u.Scheme = "ws"
|
|
case "https":
|
|
u.Scheme = "wss"
|
|
}
|
|
}
|
|
return method, u.String(), db.TemplatePath(reqURL.EscapedPath())
|
|
}
|
|
|
|
// recordStreamStart writes the stream's exchange: metadata, request head and
|
|
// body, the response head (synthetic 101 for WebSocket), the rewrite trace,
|
|
// and the header text in the full-text index. Messages are never indexed. rq
|
|
// is the request the client sent and rs (SSE only) the status and headers the
|
|
// target sent, whatever the rules changed.
|
|
func (t *Traffic) recordStreamStart(rq requestView, rs responseView, kind string, lines []rewrite.TraceLine) (string, error) {
|
|
now := time.Now()
|
|
id := t.newExchangeID(now)
|
|
method, display, tmpl, host, target := rq.method, "", "", "", ""
|
|
if rq.url != nil {
|
|
method, display, tmpl = streamDisplayURL(rq.method, rq.url, kind)
|
|
host = rq.url.Hostname()
|
|
target = rq.url.RequestURI()
|
|
if strings.EqualFold(rq.method, "CONNECT") {
|
|
target = rq.url.Host // authority-form, as on the wire
|
|
}
|
|
}
|
|
reqHead := requestHead(rq, target)
|
|
status, ct, respHead := 101, "websocket", "HTTP/1.1 101 Switching Protocols\n"
|
|
if kind == "sse" {
|
|
status = rs.status
|
|
ct = rs.header.Get("Content-Type")
|
|
respHead = fmt.Sprintf("HTTP %d\n%s", status, headerLines(rs.header))
|
|
}
|
|
|
|
t.wmu.Lock()
|
|
defer t.wmu.Unlock()
|
|
reqB := t.spill(rq.body, rq.header.Get("Content-Type"))
|
|
trace := t.spillTrace(lines)
|
|
tx, err := t.db.Begin()
|
|
if err != nil {
|
|
return "", &recordError{"beginning transaction", err}
|
|
}
|
|
defer tx.Rollback() //nolint:errcheck // no-op once committed
|
|
// An id can repeat (seconds plus a rolling counter); the trace of the
|
|
// exchange it replaces must not stay attached to this one.
|
|
if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id=?`, id); err != nil {
|
|
return "", &recordError{"clearing rewrite trace", err}
|
|
}
|
|
res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
|
VALUES(?,?,?,?,?,?,?,?,?,?,'')`, id, now.Unix(), host, method, tmpl, display, status, ct, len(rq.body), 0)
|
|
if err != nil {
|
|
return "", &recordError{"writing index", err}
|
|
}
|
|
rowid, err := res.LastInsertId()
|
|
if err != nil {
|
|
return "", &recordError{"fetching rowid", err}
|
|
}
|
|
if _, err := tx.Exec(`INSERT OR REPLACE INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob)
|
|
VALUES(?,?,?,?,?,NULL,NULL)`, id, reqHead, reqB.inline, nullIfEmpty(reqB.hash), respHead); err != nil {
|
|
return "", &recordError{"writing body", err}
|
|
}
|
|
if reqB.hash != "" {
|
|
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, reqB.hash, id); err != nil {
|
|
return "", &recordError{"registering blob reference", err}
|
|
}
|
|
}
|
|
if err := writeTrace(tx, id, lines, trace); err != nil {
|
|
return "", err
|
|
}
|
|
if t.fts {
|
|
idx := strings.Join([]string{display, reqHead, reqB.index, respHead}, "\n")
|
|
if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowid, idx); err != nil {
|
|
return "", &recordError{"writing full-text index", err}
|
|
}
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return "", &recordError{"committing", err}
|
|
}
|
|
return id, nil
|
|
}
|
|
|
|
// persistStream applies the per-message and per-exchange limits to rows and
|
|
// writes what is kept. The first message past a per-exchange limit stores one
|
|
// 'meta' row instead; nothing is stored after it. Caller holds st.mu. Errors are
|
|
// logged: a recording failure never interrupts the stream.
|
|
func (t *Traffic) persistStream(st *streamState, rows []streamRow) {
|
|
if st.id == "" || len(rows) == 0 {
|
|
return
|
|
}
|
|
keep := make([]streamRow, 0, len(rows))
|
|
addMeta := false
|
|
for _, r := range rows {
|
|
if len(r.payload) > maxStreamPayload {
|
|
r.payload = r.payload[:maxStreamPayload]
|
|
r.truncated = true
|
|
}
|
|
cost := streamStoredCost(r)
|
|
if st.dropped || st.stored >= maxStreamMessages || st.storedBytes+cost > maxStreamBytes {
|
|
if !st.dropped {
|
|
st.dropped = true
|
|
addMeta = true
|
|
}
|
|
st.droppedMsgs++
|
|
st.droppedBytes += int64(r.size)
|
|
continue
|
|
}
|
|
st.stored++
|
|
st.storedBytes += cost
|
|
keep = append(keep, r)
|
|
}
|
|
var meta []byte
|
|
if addMeta {
|
|
meta = streamMetaPayload(st)
|
|
}
|
|
if len(keep) == 0 && meta == nil {
|
|
return
|
|
}
|
|
|
|
t.wmu.Lock()
|
|
defer t.wmu.Unlock()
|
|
// Sequence numbers and blob writes come first, as in record(): blob writes are
|
|
// filesystem work and must not sit inside the SQLite transaction. Holding wmu
|
|
// keeps gcBlobs from collecting a blob before its reference is committed.
|
|
type preparedRow struct {
|
|
streamRow
|
|
seq int64
|
|
body storedBody
|
|
}
|
|
prepared := make([]preparedRow, 0, len(keep))
|
|
for _, r := range keep {
|
|
st.seq++
|
|
prepared = append(prepared, preparedRow{streamRow: r, seq: st.seq, body: t.spillAt(r.payload, r.ctype, maxStreamInline)})
|
|
}
|
|
var metaSeq int64
|
|
if meta != nil {
|
|
st.seq++
|
|
metaSeq = st.seq
|
|
}
|
|
|
|
tx, err := t.db.Begin()
|
|
if err != nil {
|
|
log.Printf("[traffic] failed to record stream messages of %s (beginning transaction): %v", st.id, err)
|
|
return
|
|
}
|
|
defer tx.Rollback() //nolint:errcheck // no-op once committed
|
|
for _, p := range prepared {
|
|
inline := p.body.inline
|
|
if inline == nil {
|
|
inline = []byte{}
|
|
}
|
|
if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,event,event_id,retry,size,payload,payload_blob,truncated)
|
|
VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`, st.id, p.seq, p.ts, p.direction, p.kind, p.event, p.eventID, p.retry, p.size,
|
|
inline, nullIfEmpty(p.body.hash), boolInt(p.truncated)); err != nil {
|
|
log.Printf("[traffic] failed to record stream message %d of %s: %v", p.seq, st.id, err)
|
|
return
|
|
}
|
|
if p.body.hash != "" {
|
|
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, p.body.hash, st.id); err != nil {
|
|
log.Printf("[traffic] failed to record stream message %d of %s (registering blob reference): %v", p.seq, st.id, err)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
if meta != nil {
|
|
if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload)
|
|
VALUES(?,?,?,'meta',0,?,?)`, st.id, metaSeq, time.Now().Unix(), len(meta), meta); err != nil {
|
|
log.Printf("[traffic] failed to record stream limit marker of %s: %v", st.id, err)
|
|
return
|
|
}
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
log.Printf("[traffic] failed to record stream messages of %s (committing): %v", st.id, err)
|
|
}
|
|
}
|
|
|
|
// streamStoredCost is what keeping r adds to the index and blob store: the
|
|
// payload, plus the inline preview spillAt keeps when the payload spills (a text
|
|
// prefix of at most blobPreview, or a one-line tag for binary).
|
|
func streamStoredCost(r streamRow) int64 {
|
|
n := int64(len(r.payload))
|
|
if len(r.payload) <= maxStreamInline {
|
|
return n
|
|
}
|
|
if isBinaryBody(r.ctype, r.payload) {
|
|
return n + int64(len(binaryTag(r.ctype, r.payload)))
|
|
}
|
|
return n + int64(len(truncateUTF8(r.payload, blobPreview)))
|
|
}
|
|
|
|
func streamMetaPayload(st *streamState) []byte {
|
|
b, _ := json.Marshal(map[string]int64{"dropped_messages": st.droppedMsgs, "dropped_bytes": st.droppedBytes})
|
|
return b
|
|
}
|
|
|
|
func boolInt(b bool) int {
|
|
if b {
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// finishStream records the bytes observed in each direction and the final
|
|
// counts of the limit marker. Caller holds st.mu.
|
|
func (t *Traffic) finishStream(st *streamState) {
|
|
if st.id == "" {
|
|
return
|
|
}
|
|
t.wmu.Lock()
|
|
defer t.wmu.Unlock()
|
|
if _, err := t.db.Exec(`UPDATE exchanges SET req_len=?, resp_len=? WHERE id=?`, st.reqBytes, st.respBytes, st.id); err != nil {
|
|
log.Printf("[traffic] failed to record stream sizes of %s: %v", st.id, err)
|
|
}
|
|
if st.dropped {
|
|
meta := streamMetaPayload(st)
|
|
if _, err := t.db.Exec(`UPDATE stream_messages SET payload=?, size=? WHERE exchange_id=? AND direction='meta'`, meta, len(meta), st.id); err != nil {
|
|
log.Printf("[traffic] failed to update stream limit marker of %s: %v", st.id, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// StreamMessage is one recorded WebSocket message or SSE event.
|
|
type StreamMessage struct {
|
|
Seq int64 `json:"seq"`
|
|
TS int64 `json:"ts"` // unix seconds, like an exchange's ts; Seq orders
|
|
Direction string `json:"direction"`
|
|
Kind int `json:"kind"`
|
|
// Type is "text" or "binary" (WebSocket), "event" (SSE), or "meta" (the
|
|
// limit marker).
|
|
Type string `json:"type"`
|
|
Event string `json:"event,omitempty"`
|
|
EventID string `json:"event_id,omitempty"`
|
|
Retry int `json:"retry,omitempty"`
|
|
Size int `json:"size"`
|
|
Payload string `json:"payload"`
|
|
// Encoding is "base64" for binary payloads, empty for UTF-8 text.
|
|
Encoding string `json:"encoding,omitempty"`
|
|
Blob string `json:"blob,omitempty"`
|
|
Truncated bool `json:"truncated,omitempty"`
|
|
}
|
|
|
|
// StreamMessages returns one page of an exchange's stream messages in order.
|
|
// limit is 1..200 (default 50) and offset >= 0. Every payload comes back as at
|
|
// most streamPreview (4 KiB); a longer one lives in the blob store and the row
|
|
// carries its hash for the rest (traffic_blob, GET /api/traffic/blob).
|
|
func (t *Traffic) StreamMessages(exchangeID string, offset, limit int) ([]StreamMessage, int, error) {
|
|
if limit <= 0 {
|
|
limit = 50
|
|
}
|
|
if limit > 200 {
|
|
limit = 200
|
|
}
|
|
if offset < 0 {
|
|
offset = 0
|
|
}
|
|
var total int
|
|
if err := t.db.QueryRow(`SELECT COUNT(*) FROM stream_messages WHERE exchange_id=?`, exchangeID).Scan(&total); err != nil {
|
|
return nil, 0, err
|
|
}
|
|
rows, err := t.db.Query(`SELECT seq,ts,direction,kind,event,event_id,retry,size,payload,payload_blob,truncated
|
|
FROM stream_messages WHERE exchange_id=? ORDER BY seq LIMIT ? OFFSET ?`, exchangeID, limit, offset)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
defer rows.Close()
|
|
out := []StreamMessage{}
|
|
for rows.Next() {
|
|
var m StreamMessage
|
|
var payload []byte
|
|
var blob sql.NullString
|
|
var truncated int
|
|
if err := rows.Scan(&m.Seq, &m.TS, &m.Direction, &m.Kind, &m.Event, &m.EventID, &m.Retry, &m.Size, &payload, &blob, &truncated); err != nil {
|
|
return nil, 0, err
|
|
}
|
|
m.Truncated = truncated != 0
|
|
m.Blob = blob.String
|
|
switch {
|
|
case m.Direction == "meta":
|
|
m.Type = "meta"
|
|
case m.Kind == wsBinary:
|
|
m.Type = "binary"
|
|
case m.Kind == wsText:
|
|
m.Type = "text"
|
|
default:
|
|
m.Type = "event"
|
|
}
|
|
if m.Type == "binary" {
|
|
if m.Blob != "" {
|
|
// A spilled binary payload keeps only a descriptive tag inline;
|
|
// the preview is the payload's own leading bytes.
|
|
payload, _, _ = t.BlobRange(m.Blob, 0, streamPreview)
|
|
}
|
|
m.Payload, m.Encoding = base64.StdEncoding.EncodeToString(clipBytes(payload, streamPreview)), "base64"
|
|
} else {
|
|
m.Payload = truncateUTF8(payload, streamPreview)
|
|
}
|
|
out = append(out, m)
|
|
}
|
|
return out, total, rows.Err()
|
|
}
|
|
|
|
// StreamInfo reports how many stream messages an exchange holds and whether it
|
|
// is a WebSocket or SSE exchange ("" for an ordinary exchange).
|
|
func (t *Traffic) StreamInfo(exchangeID string) (count int, kind string, err error) {
|
|
var ct sql.NullString
|
|
err = t.db.QueryRow(`SELECT content_type FROM exchanges WHERE id=?`, exchangeID).Scan(&ct)
|
|
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
|
return 0, "", err
|
|
}
|
|
switch c := strings.ToLower(ct.String); {
|
|
case c == "websocket":
|
|
kind = "websocket"
|
|
case strings.Contains(c, "text/event-stream"):
|
|
kind = "sse"
|
|
}
|
|
if err := t.db.QueryRow(`SELECT COUNT(*) FROM stream_messages WHERE exchange_id=?`, exchangeID).Scan(&count); err != nil {
|
|
return 0, "", err
|
|
}
|
|
return count, kind, nil
|
|
}
|
|
|
|
// streamToolPreview renders one message for traffic_messages: at most 1 KiB of
|
|
// text (hex for binary), plus the blob hash when the payload spilled.
|
|
func streamToolPreview(m StreamMessage) string {
|
|
if m.Encoding == "base64" {
|
|
raw, _ := base64.StdEncoding.DecodeString(m.Payload)
|
|
return "hex:" + fmt.Sprintf("%x", clipBytes(raw, toolStreamPreview/2))
|
|
}
|
|
p := m.Payload
|
|
if len(p) > toolStreamPreview {
|
|
p = truncateUTF8([]byte(p), toolStreamPreview)
|
|
}
|
|
if !utf8.ValidString(p) {
|
|
p = strings.ToValidUTF8(p, "�")
|
|
}
|
|
return p
|
|
}
|