feat(traffic): apply rewrite rules in the capture proxy (P3.3b1)

The sink runs the request phase in the Request hook and the response phase
at the start of the Response hook; at SSEStart only status and header
actions apply. WebSocketStart comes after the client is upgraded, so the
handshake is traced "not rewritten: stream" and recorded as sent.

Exchanges keep what the client sent and what the target answered. The
original of a rewritten request lives in a per-flow snapshot slot (entry or
tombstone, 1024 entries / 64 MiB, 4096 tombstones), reserved before the
request is changed, released on Response, RequestError, SSEStart, a 5 MiB
response, or by a sweep of finished flows (ClientDisconnected, 1 s
debounce, 1 min period). Under pressure an entry older than 15 minutes is
turned into a tombstone in place, only when that frees room for the new
reservation; its exchange is recorded "original request unavailable".

record(), recordStreamStart() and insertExchange() clear any trace under
the id before INSERT OR REPLACE; the trace (exchange_rewrites, created
with CREATE TABLE IF NOT EXISTS) is written in the exchange's transaction,
its spilled values registered in blob_refs, and purged with the exchange.
The full-text index covers the originals only. Logs of a rewritten flow
name the host, the step or error class and the rule ids, never the URL or
the error text. A rewritten request's transport error never switches its
host to passthrough. traffic_get shows the rewritten flag and rule ids.

Without a rule that applies, what is recorded is unchanged byte for byte.
This commit is contained in:
BaaRai
2026-11-17 22:23:12 +01:00
parent b3e1f5359f
commit fa07d69f1d
7 changed files with 2239 additions and 177 deletions
+5
View File
@@ -76,6 +76,11 @@ func (t *Traffic) insertExchange(id string, ts int64, host, method, urlTemplate,
}
defer tx.Rollback() //nolint:errcheck // no-op once committed
// An id can repeat (seconds plus a rolling counter); the trace of the
// exchange it replaces must not stay attached to this one.
if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id=?`, id); err != nil {
return fmt.Errorf("clearing rewrite trace: %w", err)
}
res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
VALUES(?,?,?,?,?,?,?,?,?,?,'')`,
id, ts, host, method, urlTemplate, rawURL, status, contentType, reqLen, respLen)
+5 -1
View File
@@ -40,11 +40,15 @@ func (s *sink) Requestheaders(f *mproxy.Flow) {
}
}
// ClientDisconnected forgets a closed connection's mark.
// ClientDisconnected forgets a closed connection's mark, and asks for a sweep
// of the request snapshots its flows may have left (rewrite_flow.go).
func (s *sink) ClientDisconnected(c *mproxy.ClientConn) {
if c != nil {
s.paced.Delete(c.Id)
}
if s.t != nil {
s.t.snaps.requestSweep()
}
}
// pace waits for the request's host under the current noise profile, bounded
+663
View File
@@ -0,0 +1,663 @@
package traffic
import (
"database/sql"
"io"
"log"
"net/http"
"net/url"
"sync"
"sync/atomic"
"time"
"github.com/Autumn-27/artex/traffic/rewrite"
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
)
// Applying the rewrite rules to the proxied flows.
//
// The request phase runs in the Request hook. A request a rule rewrote leaves
// the proxy changed, but its exchange records what the client sent: the
// original is kept in a per-flow request snapshot until the Response hook
// records the exchange. The response phase runs at the start of the Response
// hook (status and headers only at SSEStart), and the exchange records the
// response as the target sent it. The trace of both phases is written in the
// exchange's own transaction (exchange_rewrites).
//
// A request is rewritten only once its snapshot slot is reserved, and a slot
// stays until the flow's exchange is recorded or the flow is finished, so the
// Response hook never takes a rewritten request for the client's.
// Limits of the request snapshots.
const (
maxSnapEntries = 1024 // entries holding an original request
maxSnapBytes = 64 << 20 // what those entries hold
maxSnapTombs = 4096 // tombstones: slots whose data was released
// snapTTL deletes nothing: past it, the entry of a live flow may be turned
// into a tombstone when a new reservation finds no room.
snapTTL = 15 * time.Minute
sweepDebounce = time.Second // at most one sweep per second
sweepPeriod = time.Minute // a sweep at least this often while slots exist
// libraryStreamBody is go-mitmproxy's StreamLargeBodies default: a body this
// large reaches neither the Request nor the Response hook.
libraryStreamBody = 5 << 20
reasonSnapshotCapacity = "not rewritten: snapshot capacity"
reasonOriginalUnavailable = "original request unavailable"
)
// testRuleHook, when a test sets it, runs inside each rule that checks a body
// ("rule") and in the sink right after the engine ("request", "response"), so
// a test can make a rule, or the sink itself, panic.
var testRuleHook atomic.Pointer[func(stage string)]
func runTestHook(stage string) {
if h := testRuleHook.Load(); h != nil {
(*h)(stage)
}
}
// rulesIsBinary is the binary predicate handed to the engine: the recorder's
// own, so "not rewritten: binary" follows what the recorder treats as binary.
func rulesIsBinary(contentType string, body []byte) bool {
runTestHook("rule")
return isBinaryBody(contentType, body)
}
type flowKey [16]byte
func keyOf(f *mproxy.Flow) flowKey { return flowKey(f.Id) }
// reqSnap is one slot of the table: an entry, which holds the original of a
// request, or a tombstone, which holds only why it has none.
type reqSnap struct {
// done is the flow's Done() channel: closed, no hook will come for the
// slot any more. The flow itself is not referenced.
done <-chan struct{}
rules *rewrite.Set // the snapshot of rules the flow runs with, both phases
host string // host name: the only part of the request a log shows
tomb bool
// Entry.
created time.Time
cost int64
rewritten bool // the request phase changed the request
method string
url *url.URL
header http.Header
rawHost string
body []byte // the client's body, kept when a body action applied
keepBody bool
lines []rewrite.TraceLine // request-phase trace
// Tombstone.
reason string // reasonSnapshotCapacity or reasonOriginalUnavailable
ruleIDs []int64 // rules that had rewritten the request
}
// requestRewritten reports a flow whose request left the proxy changed.
func (s *reqSnap) requestRewritten() bool {
if s.tomb {
return s.reason == reasonOriginalUnavailable
}
return s.rewritten
}
func (s *reqSnap) appliedRuleIDs() []int64 {
if s.tomb {
return s.ruleIDs
}
_, ids := appliedRules(s.lines)
return ids
}
// snapTable holds the slots, keyed by flow id. Its mutex is a leaf lock.
type snapTable struct {
mu sync.Mutex
slots map[flowKey]*reqSnap
entries int
bytes int64
tombs int
lastSweep time.Time
lastFullLog time.Time
sweepWanted atomic.Bool
now func() time.Time
maxEntries int
maxBytes int64
maxTombs int
ttl time.Duration
sweeps, capacity, tombDrops, conversions, largeResponses atomic.Int64
}
func newSnapTable() *snapTable {
return &snapTable{slots: map[flowKey]*reqSnap{}, now: time.Now,
maxEntries: maxSnapEntries, maxBytes: maxSnapBytes, maxTombs: maxSnapTombs, ttl: snapTTL}
}
func doneClosed(ch <-chan struct{}) bool {
if ch == nil {
return false // a nil channel is never ready: the flow counts as live
}
select {
case <-ch:
return true
default:
return false
}
}
// reserve stores the entry of a rewritten request. On failure the request must
// leave unchanged: a capacity tombstone then stands for the flow, when there is
// room for one.
func (t *snapTable) reserve(k flowKey, e *reqSnap) bool {
if t == nil {
return false
}
t.mu.Lock()
defer t.mu.Unlock()
t.removeLocked(k)
if !t.fitsLocked(e.cost) {
// A failed reservation asks for an immediate sweep, under the same
// debounce; a sweep refused by the debounce fails the reservation.
now := t.now()
if now.Sub(t.lastSweep) < sweepDebounce {
t.refuseLocked(k, e)
return false
}
t.sweepLocked(now)
if !t.fitsLocked(e.cost) && !t.convertLocked(e.cost) {
t.refuseLocked(k, e)
return false
}
}
e.created = t.now()
t.slots[k] = e
t.entries++
t.bytes += e.cost
return true
}
func (t *snapTable) fitsLocked(cost int64) bool {
return t.entries < t.maxEntries && t.bytes+cost <= t.maxBytes
}
// convertLocked turns the oldest entry of a live flow older than the TTL into
// a tombstone, in place, so the flow is never without a slot. It does so only
// when releasing that entry lets a reservation of cost succeed, and when a
// tombstone place is free (after evicting tombstones of finished flows only).
// Otherwise nothing changes and it reports false.
func (t *snapTable) convertLocked(cost int64) bool {
now := t.now()
var victim *reqSnap
for _, s := range t.slots {
if s.tomb || doneClosed(s.done) || now.Sub(s.created) < t.ttl {
continue
}
if t.entries-1 >= t.maxEntries || t.bytes-s.cost+cost > t.maxBytes {
continue // released, it would still leave no room
}
if victim == nil || s.created.Before(victim.created) {
victim = s
}
}
if victim == nil {
return false
}
if t.tombs >= t.maxTombs {
t.evictFinishedTombsLocked()
}
if t.tombs >= t.maxTombs {
return false
}
ids := victim.appliedRuleIDs()
t.entries--
t.bytes -= victim.cost
t.tombs++
*victim = reqSnap{done: victim.done, rules: victim.rules, host: victim.host, tomb: true, reason: reasonOriginalUnavailable, ruleIDs: ids}
t.conversions.Add(1)
log.Printf("[traffic] rewrite snapshot of a request to %s (rules %v) released under pressure: its exchange will be recorded as original request unavailable", victim.host, ids)
return true
}
// refuseLocked is the capacity path: the request is not rewritten, and a
// tombstone records why, when a place is free.
func (t *snapTable) refuseLocked(k flowKey, e *reqSnap) {
t.capacity.Add(1)
if t.tombs >= t.maxTombs {
t.evictFinishedTombsLocked()
}
full := t.tombs >= t.maxTombs
if full {
t.tombDrops.Add(1)
} else {
t.slots[k] = &reqSnap{done: e.done, rules: e.rules, host: e.host, tomb: true, reason: reasonSnapshotCapacity}
t.tombs++
}
if now := t.now(); now.Sub(t.lastFullLog) >= time.Minute {
t.lastFullLog = now
log.Printf("[traffic] rewrite snapshot table full: requests are sent without rewriting (%d refused, %d without a trace so far)", t.capacity.Load(), t.tombDrops.Load())
}
}
func (t *snapTable) evictFinishedTombsLocked() {
for k, s := range t.slots {
if s.tomb && doneClosed(s.done) {
t.removeLocked(k)
}
}
}
// sweepLocked deletes every slot whose flow is finished. The slot of a live
// flow is never touched, whatever its age.
func (t *snapTable) sweepLocked(now time.Time) {
for k, s := range t.slots {
if doneClosed(s.done) {
t.removeLocked(k)
}
}
t.lastSweep = now
t.sweepWanted.Store(false)
t.sweeps.Add(1)
}
func (t *snapTable) removeLocked(k flowKey) {
s := t.slots[k]
if s == nil {
return
}
if s.tomb {
t.tombs--
} else {
t.entries--
t.bytes -= s.cost
}
delete(t.slots, k)
}
// take removes and returns the slot of a flow, nil when it has none.
func (t *snapTable) take(k flowKey) *reqSnap {
if t == nil {
return nil
}
t.mu.Lock()
defer t.mu.Unlock()
s := t.slots[k]
t.removeLocked(k)
return s
}
// requestSweep asks the sweeper for a pass; it runs at most once a second.
func (t *snapTable) requestSweep() {
if t != nil {
t.sweepWanted.Store(true)
}
}
// tick is one beat of the sweeper: a sweep when one was asked for and the last
// is a second old, or when the last is a minute old. Nothing on an empty table.
func (t *snapTable) tick() {
t.mu.Lock()
defer t.mu.Unlock()
if len(t.slots) == 0 {
t.sweepWanted.Store(false)
return
}
now := t.now()
since := now.Sub(t.lastSweep)
if (t.sweepWanted.Load() && since >= sweepDebounce) || since >= sweepPeriod {
t.sweepLocked(now)
}
}
// counts reports the entries and tombstones in the table.
func (t *snapTable) counts() (entries, tombs int) {
t.mu.Lock()
defer t.mu.Unlock()
return t.entries, t.tombs
}
// sweepSnapshots is the single sweeper goroutine, started by Open and stopped
// by Close.
func (t *Traffic) sweepSnapshots() {
tick := time.NewTicker(time.Second)
defer tick.Stop()
for {
select {
case <-t.closed:
return
case <-tick.C:
t.snaps.tick()
}
}
}
// RewriteStats are the proxy-side counters of the rewrite rules since start.
type RewriteStats struct {
// LargeResponses counts flows with a snapshot whose response reached the
// library's 5 MiB stream threshold: passed through, not recorded.
LargeResponses int64 `json:"large_responses"`
// CapacityRefusals counts requests sent without rewriting for lack of room.
CapacityRefusals int64 `json:"capacity_refusals"`
// TombstoneDrops counts those of them that got no capacity tombstone either.
TombstoneDrops int64 `json:"tombstone_drops"`
// Conversions counts entries released under pressure.
Conversions int64 `json:"conversions"`
Sweeps int64 `json:"sweeps"`
}
func (t *Traffic) RewriteStats() RewriteStats {
s := t.snaps
if s == nil {
return RewriteStats{}
}
return RewriteStats{LargeResponses: s.largeResponses.Load(), CapacityRefusals: s.capacity.Load(),
TombstoneDrops: s.tombDrops.Load(), Conversions: s.conversions.Load(), Sweeps: s.sweeps.Load()}
}
// newEntry builds the snapshot of a request before the rules' result is put on
// the flow: URL and headers cloned, the body referenced, and only when a body
// action applied (otherwise the flow's body is still the client's).
func newEntry(f *mproxy.Flow, set *rewrite.Set, res rewrite.Result) *reqSnap {
r := f.Request
u := *r.URL
e := &reqSnap{done: f.Done(), rules: set, host: r.URL.Hostname(), rewritten: res.Changed,
method: r.Method, url: &u, header: r.Header.Clone(), lines: res.Lines}
if raw := r.Raw(); raw != nil {
e.rawHost = raw.Host
}
for _, l := range res.Lines {
if l.Outcome == rewrite.OutcomeApplied && l.Field == "body" && len(r.Body) <= rewrite.MaxBodySize {
e.body, e.keepBody = r.Body, true
break
}
}
e.cost = int64(len(e.method) + len(u.String()) + len(e.rawHost) + len(e.body))
for k, vs := range e.header {
for _, v := range vs {
e.cost += int64(len(k) + len(v))
}
}
for _, l := range e.lines {
e.cost += int64(len(l.After) + len(l.RuleSnapshot))
}
return e
}
// originalRequest is the request an exchange records, with the request-phase
// trace: the flow's own request when no rule rewrote it, the entry's copy when
// one did, and only what Raw() still knows when the entry was released.
func originalRequest(f *mproxy.Flow, snap *reqSnap) (requestView, []rewrite.TraceLine) {
switch {
case snap == nil:
return flowRequestView(f.Request), nil
case !snap.tomb:
v := requestView{method: snap.method, url: snap.url, proto: f.Request.Proto, header: snap.header, rawHost: snap.rawHost, body: f.Request.Body}
if snap.keepBody {
v.body = snap.body
}
return v, snap.lines
case snap.reason == reasonSnapshotCapacity:
// The request was not rewritten: the flow's request is the client's.
return flowRequestView(f.Request), []rewrite.TraceLine{{Phase: rewrite.PhaseRequest, Outcome: rewrite.OutcomeSkipped, Reason: reasonSnapshotCapacity}}
default:
// Rewritten, original released: never record the rewritten request as
// the client's. Raw() is the client's request line and Host.
v := requestView{unavailable: true}
if raw := f.Request.Raw(); raw != nil {
v.method, v.url, v.proto, v.rawHost = raw.Method, raw.URL, raw.Proto, raw.Host
}
lines := []rewrite.TraceLine{{Phase: rewrite.PhaseRequest, Outcome: rewrite.OutcomeError, Reason: reasonOriginalUnavailable}}
for _, id := range snap.ruleIDs {
lines = append(lines, rewrite.TraceLine{Phase: rewrite.PhaseRequest, RuleID: id, Outcome: rewrite.OutcomeApplied})
}
return v, lines
}
}
func engineRequest(r *mproxy.Request) rewrite.Request {
return rewrite.Request{Method: r.Method, URL: r.URL, Header: r.Header, Body: r.Body}
}
// Request runs the request phase. The result goes on the flow only once the
// request's snapshot slot is reserved; without one the request leaves as the
// client sent it. Rules fail open: a panic leaves the request unchanged.
func (s *sink) Request(f *mproxy.Flow) {
if s.t == nil || f.Request == nil || f.Request.URL == nil {
return
}
set := s.t.rewriteRules.Load()
if set.Len(rewrite.PhaseRequest) == 0 {
return
}
host := f.Request.URL.Hostname()
defer func() {
if recover() != nil {
log.Printf("[traffic] rewrite rules stopped on a request to %s (class %s): it is sent as the client sent it", host, rewrite.ErrClassPanic)
}
}()
res := set.Apply(rewrite.PhaseRequest, rewrite.Input{Request: engineRequest(f.Request), IsBinary: rulesIsBinary})
runTestHook("request")
if len(res.Lines) == 0 {
return
}
if !s.t.snaps.reserve(keyOf(f), newEntry(f, set, res)) {
return
}
if res.Changed {
r := f.Request
r.Method, r.URL, r.Header, r.Body = res.Request.Method, res.Request.URL, res.Request.Header, res.Request.Body
}
}
// Response records the exchange and runs the response phase: the exchange
// keeps the client's request and the target's response, the client gets the
// rewritten response. Without a slot and without response rules this is the
// plain record.
func (s *sink) Response(f *mproxy.Flow) {
if f.Request == nil || f.Response == nil {
return
}
snap := s.t.snaps.take(keyOf(f))
set := s.t.rewriteRules.Load()
if snap != nil {
set = snap.rules
}
if snap == nil && set.Len(rewrite.PhaseResponse) == 0 {
s.t.record(f)
return
}
rq, lines := originalRequest(f, snap)
rs := flowResponseView(f.Response)
res := s.applyResponse(f, set, rewrite.Input{
Request: engineRequest(f.Request), // as sent to the target
Response: rewrite.Response{Status: rs.status, Header: rs.header, Body: rs.body},
IsBinary: rulesIsBinary,
})
s.t.recordViews(rq, rs, append(lines, res.Lines...))
if res.Changed {
f.Response.StatusCode, f.Response.Header, f.Response.Body = res.Response.Status, res.Response.Header, res.Response.Body
}
}
// applyResponse runs the response phase; a panic yields no change and no line.
func (s *sink) applyResponse(f *mproxy.Flow, set *rewrite.Set, in rewrite.Input) (res rewrite.Result) {
if set.Len(rewrite.PhaseResponse) == 0 {
return rewrite.Result{}
}
defer func() {
if recover() != nil {
res = rewrite.Result{}
log.Printf("[traffic] rewrite rules stopped on a response from %s (class %s): it is sent as the target sent it", f.Request.URL.Hostname(), rewrite.ErrClassPanic)
}
}()
r := set.Apply(rewrite.PhaseResponse, in)
runTestHook("response")
return r
}
// rewriteStream prepares the exchange of a stream at SSEStart or
// WebSocketStart: the client's request, the target's status and headers (SSE),
// and the trace. For SSE, the response phase runs in stream mode: status and
// header actions apply (res, put on the flow by applyStreamResponse), rules
// that read or write the response body are traced "not rewritten: stream".
// WebSocketStart runs once go-mitmproxy has already answered the client's
// upgrade, with no response on the flow: nothing of the handshake can change
// there, so every candidate rule is traced "not rewritten: stream".
func (s *sink) rewriteStream(f *mproxy.Flow, kind string) (rq requestView, rs responseView, lines []rewrite.TraceLine, res rewrite.Result) {
snap := s.t.snaps.take(keyOf(f))
set := s.t.rewriteRules.Load()
if snap != nil {
set = snap.rules
}
rq, lines = originalRequest(f, snap)
if kind == "sse" {
rs = flowResponseView(f.Response)
}
cur := engineRequest(f.Request)
if kind != "websocket" {
res = s.applyResponse(f, set, rewrite.Input{Request: cur, Response: rewrite.Response{Status: rs.status, Header: rs.header}, Stream: true, IsBinary: rulesIsBinary})
return rq, rs, append(lines, res.Lines...), res
}
if set.Len(rewrite.PhaseRequest)+set.Len(rewrite.PhaseResponse) == 0 {
return rq, rs, lines, rewrite.Result{}
}
// The engine never changes a request with Upgrade; a ws:// stream arrives
// on its CONNECT flow, which has none.
if cur.Header.Get("Upgrade") == "" {
cur.Header = cur.Header.Clone()
if cur.Header == nil {
cur.Header = http.Header{}
}
cur.Header.Set("Upgrade", "websocket")
}
if snap == nil {
lines = append(lines, s.applyRequestTrace(f, set, cur)...)
}
r := s.applyResponse(f, set, rewrite.Input{Request: cur, Response: rewrite.Response{Status: http.StatusSwitchingProtocols}, Stream: true, IsBinary: rulesIsBinary})
return rq, rs, append(lines, r.Lines...), rewrite.Result{}
}
// applyRequestTrace runs the request phase for its trace only.
func (s *sink) applyRequestTrace(f *mproxy.Flow, set *rewrite.Set, req rewrite.Request) (lines []rewrite.TraceLine) {
defer func() {
if recover() != nil {
lines = nil
log.Printf("[traffic] rewrite rules stopped on a request to %s (class %s)", f.Request.URL.Hostname(), rewrite.ErrClassPanic)
}
}()
return set.Apply(rewrite.PhaseRequest, rewrite.Input{Request: req, IsBinary: rulesIsBinary}).Lines
}
// applyStreamResponse puts the rewritten status and headers of an SSE
// response on the flow, before go-mitmproxy replies to the client.
func (s *sink) applyStreamResponse(f *mproxy.Flow, res rewrite.Result) {
if res.Changed && f.Response != nil {
f.Response.StatusCode, f.Response.Header = res.Response.Status, res.Response.Header
}
}
// RequestError fires when a request through an established MITM tunnel fails. If
// the failure looks proxy/protocol-caused (h2 quirks, HEAD-with-body, protocol
// errors) and not a plain target-unreachable error, the host is tunneled for
// passthroughTTL: this request has already failed, later requests in the window
// connect straight through, and the first one after the window tries
// interception again. A request a rule rewrote never flags its host: the rule,
// not the interception, may be at fault. Its log names the host, the error class
// and the rules, never the URL or the error text.
func (s *sink) RequestError(f *mproxy.Flow, err error) {
if f != nil {
if snap := s.t.snaps.take(keyOf(f)); snap != nil && snap.requestRewritten() {
ids := snap.appliedRuleIDs()
for _, id := range ids {
snap.rules.NoteError(id, rewrite.ErrClassTransport)
}
log.Printf("[traffic] a request to %s rewritten by rules %v failed (class %s); the host stays intercepted", snap.host, ids, rewrite.ErrClassTransport)
return
}
}
s.t.maybePassthrough(f, err)
}
// StreamResponseModifier passes every body through as received, nil included,
// without wrapping it. A response that reached the library's stream threshold
// is never recorded: the slot of its flow, if any, is released here.
func (s *sink) StreamResponseModifier(f *mproxy.Flow, in io.Reader) (out io.Reader) {
out = in
defer func() {
if recover() != nil {
out = in
}
}()
if s == nil || s.t == nil || f == nil || !f.Stream || f.SSE != nil {
return in
}
snap := s.t.snaps.take(keyOf(f))
if snap == nil {
return in
}
s.t.snaps.largeResponses.Add(1)
log.Printf("[traffic] the response to a request to %s with rewrite rules %v reached %d MiB: passed through, not recorded", snap.host, snap.appliedRuleIDs(), libraryStreamBody>>20)
return in
}
// spillTrace stores the values of the trace lines as record stores bodies.
// Called under wmu, before the transaction.
func (t *Traffic) spillTrace(lines []rewrite.TraceLine) []storedBody {
out := make([]storedBody, len(lines))
for i, l := range lines {
out[i] = t.spill(l.After, "")
}
return out
}
// writeTrace inserts an exchange's trace rows and the blob references of their
// spilled values, inside the exchange's transaction.
func writeTrace(tx *sql.Tx, id string, lines []rewrite.TraceLine, trace []storedBody) error {
for _, b := range trace {
if b.hash == "" {
continue
}
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, b.hash, id); err != nil {
return &recordError{"registering blob reference", err}
}
}
for i, l := range lines {
var ruleID, after any
if l.RuleID != 0 {
ruleID = l.RuleID
}
if trace[i].inline != nil {
after = trace[i].inline
}
if _, err := tx.Exec(`INSERT INTO exchange_rewrites(exchange_id,seq,phase,rule_id,rule_snapshot,action,field,outcome,reason,before_hash,after_hash,after_inline,after_blob)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`,
id, i, string(l.Phase), ruleID, nullIfEmpty(l.RuleSnapshot), nullIfEmpty(l.Action), nullIfEmpty(l.Field), l.Outcome, l.Reason,
nullIfEmpty(l.BeforeHash), nullIfEmpty(l.AfterHash), after, nullIfEmpty(trace[i].hash)); err != nil {
return &recordError{"writing rewrite trace", err}
}
}
return nil
}
// RewriteInfo reports whether rules rewrote an exchange (at least one applied
// trace line) and which rules did, by id.
func (t *Traffic) RewriteInfo(id string) (rewritten bool, ruleIDs []int64, err error) {
if err := t.db.QueryRow(`SELECT EXISTS(SELECT 1 FROM exchange_rewrites r WHERE r.exchange_id=? AND r.outcome='applied')`, id).Scan(&rewritten); err != nil || !rewritten {
return false, nil, err
}
rows, err := t.db.Query(`SELECT DISTINCT rule_id FROM exchange_rewrites WHERE exchange_id=? AND outcome='applied' AND rule_id IS NOT NULL ORDER BY rule_id`, id)
if err != nil {
return false, nil, err
}
defer rows.Close()
for rows.Next() {
var rid int64
if err := rows.Scan(&rid); err != nil {
return false, nil, err
}
ruleIDs = append(ruleIDs, rid)
}
return true, ruleIDs, rows.Err()
}
File diff suppressed because it is too large Load Diff
-102
View File
@@ -1,102 +0,0 @@
package traffic
import (
"io"
"net/http"
"net/http/httptest"
"slices"
"sort"
"strings"
"sync"
"testing"
"github.com/Autumn-27/artex/traffic/rewrite"
)
// The proxy stores the compiled rewrite rules and does not apply them. With a
// snapshot whose rules match every exchange, what the target receives, what the
// client receives and what is recorded are the same as with no rule at all.
func TestRewriteRulesStoredNotApplied(t *testing.T) {
var mu sync.Mutex
var seen []string
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
mu.Lock()
seen = append(seen, r.Method+" "+r.RequestURI+" X-Client="+r.Header.Get("X-Client")+" "+string(body))
mu.Unlock()
w.Header().Set("X-Target", "t")
w.WriteHeader(http.StatusTeapot)
io.WriteString(w, "target body")
}))
t.Cleanup(target.Close)
set := rewrite.Compile([]rewrite.Rule{
{ID: 1, Name: "request", Enabled: true, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{
{Type: rewrite.ActionSetMethod, Value: "PUT"},
{Type: rewrite.ActionSetPathQuery, Value: "/rewritten"},
{Type: rewrite.ActionSetHeader, Name: "X-Client", Value: "rewritten"},
{Type: rewrite.ActionSetBody, Value: "rewritten"}}},
{ID: 2, Name: "response", Enabled: true, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{
{Type: rewrite.ActionSetStatus, Value: "200"},
{Type: rewrite.ActionSetHeader, Name: "X-Rewritten", Value: "1"},
{Type: rewrite.ActionSetBody, Value: "rewritten"}}},
}, rewrite.Options{})
if set.Len(rewrite.PhaseRequest) != 1 || set.Len(rewrite.PhaseResponse) != 1 || len(set.LoadErrors()) != 0 {
t.Fatalf("the snapshot must hold both rules: %v", set.LoadErrors())
}
// canon makes a recorded message comparable: the recorder writes headers in
// map order, and the target dates its response.
canon := func(raw string) string {
head, body, _ := strings.Cut(raw, "\n\n")
lines := strings.Split(head, "\n")
sort.Strings(lines[1:])
lines = slices.DeleteFunc(lines, func(l string) bool { return strings.HasPrefix(l, "Date: ") })
return strings.Join(lines, "\n") + "\n\n" + body
}
// exchange sends one request through a fresh proxy holding set and returns
// what the client received and what the proxy recorded.
exchange := func(set *rewrite.Set) (client, recorded string) {
tr, _, proxyURL := startProxy(t)
if tr.RewriteRules() != nil {
t.Fatal("Open published rewrite rules")
}
tr.SetRewriteRules(set)
if tr.RewriteRules() != set {
t.Fatal("RewriteRules must return the snapshot last published")
}
req, err := http.NewRequest("POST", target.URL+"/path?q=1", strings.NewReader("client body"))
if err != nil {
t.Fatal(err)
}
req.Header.Set("X-Client", "c")
resp, err := proxyClient(t, tr, proxyURL).Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
client = resp.Status + " X-Target=" + resp.Header.Get("X-Target") + " X-Rewritten=" + resp.Header.Get("X-Rewritten") + " " + string(body)
reqText, respText, err := tr.Get(onlyExchangeID(t, tr))
if err != nil {
t.Fatal(err)
}
return client, canon(reqText) + "\n--\n" + canon(respText)
}
baseClient, baseRecorded := exchange(nil)
client, recorded := exchange(set)
if want := "418 I'm a teapot X-Target=t X-Rewritten= target body"; baseClient != want || client != want {
t.Fatalf("client received %q without rules and %q with rules, want %q", baseClient, client, want)
}
if recorded != baseRecorded {
t.Fatalf("the recorded exchange differs with rules stored:\n without: %s\n with: %s", baseRecorded, recorded)
}
if !strings.Contains(recorded, "client body") || !strings.Contains(recorded, "target body") || strings.Contains(recorded, "rewritten") {
t.Fatalf("recorded exchange: %s", recorded)
}
want := "POST /path?q=1 X-Client=c client body"
if len(seen) != 2 || seen[0] != want || seen[1] != want {
t.Fatalf("the target received %q, want %q twice", seen, want)
}
}
+60 -27
View File
@@ -16,6 +16,7 @@ import (
"unsafe"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/traffic/rewrite"
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
)
@@ -144,17 +145,35 @@ func (s *sink) startStream(f *mproxy.Flow, kind string) {
if f.Request == nil || f.Request.URL == nil || (kind == "sse" && f.Response == nil) {
return
}
id, err := s.t.recordStreamStart(f, kind)
rq, rs, lines, res := s.rewriteStream(f, kind)
id, err := s.t.recordStreamStart(rq, rs, kind, lines)
s.applyStreamResponse(f, res)
if err != nil {
if rewritten, ruleIDs := appliedRules(lines); rewritten {
var re *recordError
errors.As(err, &re)
log.Printf("[traffic] failed to record a rewritten %s stream of %s (%s; rules %v)", kind, f.Request.URL.Hostname(), re.step, ruleIDs)
return
}
log.Printf("[traffic] failed to record %s stream %s: %v", kind, f.Request.URL, err)
return
}
st.id = id
if kind == "sse" {
st.reqBytes = int64(len(f.Request.Body))
st.reqBytes = int64(len(rq.body))
}
}
// recordError names the step of a recording that failed. Its text is the step
// and the cause, as recordStreamStart always reported.
type recordError struct {
step string
err error
}
func (e *recordError) Error() string { return e.step + ": " + e.err.Error() }
func (e *recordError) Unwrap() error { return e.err }
// libMutex returns the unexported mu sync.Mutex that go-mitmproxy holds while it
// appends to WebSocketData.Messages / SSEData.Events. The library exposes no
// accessor, and the two WebSocket goroutines append concurrently, so reading the
@@ -252,9 +271,9 @@ func sseEventsSince(sse *mproxy.SSEData, st *streamState) []streamRow {
// streamDisplayURL is the URL shown for a WebSocket exchange: http→ws and
// https→wss. A plain ws:// connection reaches the WebSocket hooks through its
// CONNECT tunnel, where only the target authority is known, not the path.
func streamDisplayURL(f *mproxy.Flow, kind string) (method, display, tmpl string) {
u := *f.Request.URL
method = f.Request.Method
func streamDisplayURL(reqMethod string, reqURL *url.URL, kind string) (method, display, tmpl string) {
u := *reqURL
method = reqMethod
if kind == "websocket" && strings.EqualFold(method, "CONNECT") {
host := u.Host
if host == "" {
@@ -270,63 +289,77 @@ func streamDisplayURL(f *mproxy.Flow, kind string) (method, display, tmpl string
u.Scheme = "wss"
}
}
return method, u.String(), db.TemplatePath(f.Request.URL.EscapedPath())
return method, u.String(), db.TemplatePath(reqURL.EscapedPath())
}
// recordStreamStart writes the stream's exchange: metadata, request head and
// body, the response head (synthetic 101 for WebSocket), and the header text in
// the full-text index. Messages are never indexed.
func (t *Traffic) recordStreamStart(f *mproxy.Flow, kind string) (string, error) {
// body, the response head (synthetic 101 for WebSocket), the rewrite trace,
// and the header text in the full-text index. Messages are never indexed. rq
// is the request the client sent and rs (SSE only) the status and headers the
// target sent, whatever the rules changed.
func (t *Traffic) recordStreamStart(rq requestView, rs responseView, kind string, lines []rewrite.TraceLine) (string, error) {
now := time.Now()
id := t.newExchangeID(now)
method, display, tmpl := streamDisplayURL(f, kind)
host := f.Request.URL.Hostname()
target := f.Request.URL.RequestURI()
if strings.EqualFold(f.Request.Method, "CONNECT") {
target = f.Request.URL.Host // authority-form, as on the wire
method, display, tmpl, host, target := rq.method, "", "", "", ""
if rq.url != nil {
method, display, tmpl = streamDisplayURL(rq.method, rq.url, kind)
host = rq.url.Hostname()
target = rq.url.RequestURI()
if strings.EqualFold(rq.method, "CONNECT") {
target = rq.url.Host // authority-form, as on the wire
}
}
reqHead := fmt.Sprintf("%s %s %s\n%s", f.Request.Method, target, f.Request.Proto, requestHeaderLines(f.Request))
reqHead := requestHead(rq, target)
status, ct, respHead := 101, "websocket", "HTTP/1.1 101 Switching Protocols\n"
if kind == "sse" {
status = f.Response.StatusCode
ct = f.Response.Header.Get("Content-Type")
respHead = fmt.Sprintf("HTTP %d\n%s", status, headerLines(f.Response.Header))
status = rs.status
ct = rs.header.Get("Content-Type")
respHead = fmt.Sprintf("HTTP %d\n%s", status, headerLines(rs.header))
}
t.wmu.Lock()
defer t.wmu.Unlock()
reqB := t.spill(f.Request.Body, f.Request.Header.Get("Content-Type"))
reqB := t.spill(rq.body, rq.header.Get("Content-Type"))
trace := t.spillTrace(lines)
tx, err := t.db.Begin()
if err != nil {
return "", fmt.Errorf("beginning transaction: %w", err)
return "", &recordError{"beginning transaction", err}
}
defer tx.Rollback() //nolint:errcheck // no-op once committed
// An id can repeat (seconds plus a rolling counter); the trace of the
// exchange it replaces must not stay attached to this one.
if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id=?`, id); err != nil {
return "", &recordError{"clearing rewrite trace", err}
}
res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
VALUES(?,?,?,?,?,?,?,?,?,?,'')`, id, now.Unix(), host, method, tmpl, display, status, ct, len(f.Request.Body), 0)
VALUES(?,?,?,?,?,?,?,?,?,?,'')`, id, now.Unix(), host, method, tmpl, display, status, ct, len(rq.body), 0)
if err != nil {
return "", fmt.Errorf("writing index: %w", err)
return "", &recordError{"writing index", err}
}
rowid, err := res.LastInsertId()
if err != nil {
return "", fmt.Errorf("fetching rowid: %w", err)
return "", &recordError{"fetching rowid", err}
}
if _, err := tx.Exec(`INSERT OR REPLACE INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob)
VALUES(?,?,?,?,?,NULL,NULL)`, id, reqHead, reqB.inline, nullIfEmpty(reqB.hash), respHead); err != nil {
return "", fmt.Errorf("writing body: %w", err)
return "", &recordError{"writing body", err}
}
if reqB.hash != "" {
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, reqB.hash, id); err != nil {
return "", fmt.Errorf("registering blob reference: %w", err)
return "", &recordError{"registering blob reference", err}
}
}
if err := writeTrace(tx, id, lines, trace); err != nil {
return "", err
}
if t.fts {
idx := strings.Join([]string{display, reqHead, reqB.index, respHead}, "\n")
if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowid, idx); err != nil {
return "", fmt.Errorf("writing full-text index: %w", err)
return "", &recordError{"writing full-text index", err}
}
}
if err := tx.Commit(); err != nil {
return "", fmt.Errorf("committing: %w", err)
return "", &recordError{"committing", err}
}
return id, nil
}
+190 -47
View File
@@ -22,6 +22,7 @@ import (
"os"
"path/filepath"
"regexp"
"slices"
"sort"
"strconv"
"strings"
@@ -92,6 +93,26 @@ CREATE TABLE IF NOT EXISTS stream_messages (
PRIMARY KEY (exchange_id, seq)
);
CREATE INDEX IF NOT EXISTS idx_stream_messages_ex ON stream_messages(exchange_id, seq);
-- Rewrite-rule trace of an exchange, in order. The exchange row keeps what the
-- client sent and what the target answered; these rows say what the rules
-- changed on the way. An exchange is rewritten when it has an 'applied' row.
CREATE TABLE IF NOT EXISTS exchange_rewrites (
exchange_id TEXT NOT NULL,
seq INTEGER NOT NULL,
phase TEXT NOT NULL, -- 'request' | 'response'
rule_id INTEGER, -- NULL on a global line
rule_snapshot TEXT, -- JSON, no sensitive action value
action TEXT,
field TEXT, -- 'method' | 'url' | 'header:<Name>' | 'body' | 'status'
outcome TEXT NOT NULL, -- 'applied' | 'skipped' | 'error'
reason TEXT NOT NULL DEFAULT '',
before_hash TEXT,
after_hash TEXT,
after_inline BLOB,
after_blob TEXT,
PRIMARY KEY (exchange_id, seq)
);
`
// ftsSchema is applied separately from indexSchema: a driver build without FTS5
@@ -160,6 +181,10 @@ const (
// and the catalog migration cannot drift apart.
const TrafficSearchDescription = "Query the target traffic the recording proxy has captured (host is required; accepts a bare host, host:port, or full URL, optionally filtered further by URL substring or body keyword). When a port is specified, only that service's traffic is returned, avoiding cross-talk between different ports on the same IP. body_contains runs a full-text search over the captured request/response headers and bodies, supporting any substring including Chinese (at least 3 characters). Returns only an ultra-light index (id/method/url/status/resp_len) with no response content; once results are non-empty, use traffic_get to verify the request/response of each one, then hand the IDs that actually support the current vulnerability to bind_finding_traffic. Returns 3 by default, at most 10 per page; use page to paginate when there are more results."
// TrafficGetDescription is the traffic_get description, kept here for the same
// reason as TrafficSearchDescription.
const TrafficGetDescription = "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling. A WebSocket (status 101) or SSE (text/event-stream) record shows only its handshake here; read its messages with traffic_messages. When proxy rewrite rules changed the exchange, the result starts with a REWRITTEN line naming the rule ids, and shows the request as the client sent it and the response as the target sent it. traffic_search searches only these original values: it never finds a value a rewrite rule wrote."
// Traffic runs the recording proxy and owns the file tree + index.
type Traffic struct {
dir string
@@ -199,8 +224,14 @@ type Traffic struct {
// so no host escapes it. Hot-swappable at runtime via SetUpstreamProxy.
upstream atomic.Pointer[url.URL]
// rewriteRules is the compiled snapshot of the rewrite rules, replaced as a
// whole by SetRewriteRules. The proxy does not apply it yet.
// whole by SetRewriteRules and applied by the sink (rewrite_flow.go).
rewriteRules atomic.Pointer[rewrite.Set]
// snaps keeps the original of each request a rule rewrote until its
// exchange is recorded (rewrite_flow.go). Nil on the zero value.
snaps *snapTable
// sweeping tracks the snapshot sweeper, which runs until Close. It is kept
// apart from reaping, which only ever holds work that finishes on its own.
sweeping sync.WaitGroup
}
// Open initializes the traffic tree, blob store and SQLite index under dir.
@@ -226,7 +257,7 @@ func Open(dir, addr string) (*Traffic, error) {
if err != nil {
return nil, err
}
t := &Traffic{dir: dir, addr: addr, db: db, closed: make(chan struct{})}
t := &Traffic{dir: dir, addr: addr, db: db, closed: make(chan struct{}), snaps: newSnapTable()}
if err := t.initIndex(); err != nil {
db.Close()
return nil, err
@@ -258,6 +289,7 @@ func Open(dir, addr string) (*Traffic, error) {
})
p.AddAddon(&sink{t: t})
t.proxy = p
t.sweeping.Go(t.sweepSnapshots) // stops on t.closed; Close waits for it
return t, nil
}
@@ -358,8 +390,8 @@ func (t *Traffic) SetUpstreamProxy(raw string) error {
}
// SetRewriteRules publishes a compiled snapshot of the rewrite rules. The swap
// is atomic; nil clears the rules. The snapshot is stored only: the proxy does
// not apply rewrite rules yet, so capture is unchanged whatever it holds.
// is atomic; nil clears the rules. A flow already rewritten keeps the snapshot
// its request saw for its response as well.
func (t *Traffic) SetRewriteRules(s *rewrite.Set) { t.rewriteRules.Store(s) }
// RewriteRules returns the snapshot last published, nil when there is none.
@@ -406,6 +438,7 @@ func (t *Traffic) Close() error {
t.closeOnce.Do(func() { close(t.closed) })
}
t.reaping.Wait()
t.sweeping.Wait()
return t.db.Close()
}
@@ -430,21 +463,6 @@ type sink struct {
streams sync.Map // *mproxy.Flow of a live WebSocket/SSE stream -> *streamState (stream.go)
}
func (s *sink) Response(f *mproxy.Flow) {
if f.Request == nil || f.Response == nil {
return
}
s.t.record(f)
}
// RequestError fires when a request through an established MITM tunnel fails. If
// the failure looks proxy/protocol-caused (h2 quirks, HEAD-with-body, protocol
// errors) and not a plain target-unreachable error, the host is tunneled for
// passthroughTTL. This request has already failed. Later requests in the window
// connect straight through; the first request after the window tries
// interception again.
func (s *sink) RequestError(f *mproxy.Flow, err error) { s.t.maybePassthrough(f, err) }
// maybePassthrough marks a host to be tunneled transparently on the next
// connection, but only for errors the proxy itself caused — a target that is
// simply down/filtered would fail without us too, and must stay MITM'd+recorded.
@@ -502,52 +520,121 @@ func proxyCausedErr(err error) bool {
// full-text index. Nothing is written to a per-request directory — only bodies
// above maxInlineBody spill to the content-addressed blob store.
func (t *Traffic) record(f *mproxy.Flow) {
t.recordViews(flowRequestView(f.Request), flowResponseView(f.Response), nil)
}
// requestView is the request an exchange records: what the client sent, which
// is not the flow's request once a rule rewrote it.
type requestView struct {
method string
url *url.URL // nil when the original request is unavailable and Raw() is nil
proto string
header http.Header
// rawHost is the client's Host header (Raw().Host), "" when unknown.
rawHost string
body []byte
// unavailable marks a request rewritten by a rule whose original was not
// kept: only its method, URL and Host are known.
unavailable bool
}
// responseView is the response an exchange records: what the target answered.
type responseView struct {
status int
header http.Header
body []byte
}
func flowRequestView(r *mproxy.Request) requestView {
v := requestView{method: r.Method, url: r.URL, proto: r.Proto, header: r.Header, body: r.Body}
if raw := r.Raw(); raw != nil {
v.rawHost = raw.Host
}
return v
}
func flowResponseView(r *mproxy.Response) responseView {
return responseView{status: r.StatusCode, header: r.Header, body: r.Body}
}
// unavailableMarker stands for the headers and body of a request whose
// original was not kept (see rewrite_flow.go).
const unavailableMarker = "[original request unavailable]\n"
// recordViews writes one exchange from explicit views, with its rewrite trace,
// in one transaction: the exchange row, its bodies, blob references, the trace
// and the full-text index of the views only. A failure rolls everything back,
// so there is never a rewritten exchange without its trace. Without trace
// lines this is exactly what record always wrote.
func (t *Traffic) recordViews(rq requestView, rs responseView, lines []rewrite.TraceLine) {
// The write lock covers blob + index writes, so DeleteHost (and its blob GC)
// can run under the same lock without racing a concurrent record.
t.wmu.Lock()
defer t.wmu.Unlock()
host := f.Request.URL.Hostname()
method := f.Request.Method
tmpl := db.TemplatePath(f.Request.URL.EscapedPath())
var host, tmpl, url, uri string
if rq.url != nil {
host = rq.url.Hostname()
tmpl = db.TemplatePath(rq.url.EscapedPath())
url = rq.url.String()
uri = rq.url.RequestURI()
}
method := rq.method
now := time.Now()
id := t.newExchangeID(now)
ct := f.Response.Header.Get("Content-Type")
url := f.Request.URL.String()
reqHead := fmt.Sprintf("%s %s %s\n%s", method, f.Request.URL.RequestURI(), f.Request.Proto, requestHeaderLines(f.Request))
respHead := fmt.Sprintf("HTTP %d\n%s", f.Response.StatusCode, headerLines(f.Response.Header))
ct := rs.header.Get("Content-Type")
reqHead := requestHead(rq, uri)
respHead := fmt.Sprintf("HTTP %d\n%s", rs.status, headerLines(rs.header))
// Bodies are spilled before the transaction opens: blob writes are filesystem
// work and must not sit inside the SQLite write lock.
reqB := t.spill(f.Request.Body, f.Request.Header.Get("Content-Type"))
respB := t.spill(f.Response.Body, ct)
reqB := t.spill(rq.body, rq.header.Get("Content-Type"))
respB := t.spill(rs.body, ct)
trace := t.spillTrace(lines)
// A rewritten exchange is logged by host, step and rule ids only: its URL
// and the error text may carry a value a rule wrote.
rewritten, ruleIDs := appliedRules(lines)
fail := func(step string, err error) {
if rewritten {
log.Printf("[traffic] failed to record a rewritten exchange of %s (%s; rules %v)", host, step, ruleIDs)
return
}
log.Printf("[traffic] failed to record %s (%s): %v", url, step, err)
}
tx, err := t.db.Begin()
if err != nil {
log.Printf("[traffic] failed to record %s (beginning transaction): %v", url, err)
fail("beginning transaction", err)
return
}
defer tx.Rollback() //nolint:errcheck // no-op once committed
// An id can repeat (seconds plus a rolling counter); the trace of the
// exchange it replaces must not stay attached to this one.
if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id=?`, id); err != nil {
fail("clearing rewrite trace", err)
return
}
// path stays empty for db-resident exchanges; a non-empty path marks a legacy
// row whose bodies still live in the old on-disk tree (see Get).
res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
VALUES(?,?,?,?,?,?,?,?,?,?,'')`,
id, now.Unix(), host, method, tmpl, url, f.Response.StatusCode, ct,
len(f.Request.Body), len(f.Response.Body))
id, now.Unix(), host, method, tmpl, url, rs.status, ct,
len(rq.body), len(rs.body))
if err != nil {
log.Printf("[traffic] failed to record %s (writing index): %v", url, err)
fail("writing index", err)
return
}
rowid, err := res.LastInsertId()
if err != nil {
log.Printf("[traffic] failed to record %s (fetching rowid): %v", url, err)
fail("fetching rowid", err)
return
}
if _, err := tx.Exec(`INSERT OR REPLACE INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob)
VALUES(?,?,?,?,?,?,?)`,
id, reqHead, reqB.inline, nullIfEmpty(reqB.hash), respHead, respB.inline, nullIfEmpty(respB.hash)); err != nil {
log.Printf("[traffic] failed to record %s (writing body): %v", url, err)
fail("writing body", err)
return
}
@@ -556,26 +643,51 @@ VALUES(?,?,?,?,?,?,?)`,
continue
}
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, h, id); err != nil {
log.Printf("[traffic] failed to record %s (registering blob reference): %v", url, err)
fail("registering blob reference", err)
return
}
}
if err := writeTrace(tx, id, lines, trace); err != nil {
var re *recordError
errors.As(err, &re)
fail(re.step, re.err)
return
}
if t.fts {
// The index is fed from memory, so a body that spilled to _blobs is still
// fully searchable even though only its preview is stored inline.
// fully searchable even though only its preview is stored inline. Only the
// recorded views are indexed, never a value a rule wrote.
idx := strings.Join([]string{url, reqHead, reqB.index, respHead, respB.index}, "\n")
if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowid, idx); err != nil {
log.Printf("[traffic] failed to record %s (writing full-text index): %v", url, err)
fail("writing full-text index", err)
return
}
}
if err := tx.Commit(); err != nil {
log.Printf("[traffic] failed to record %s (committing): %v", url, err)
fail("committing", err)
}
}
// appliedRules reports whether a trace rewrote its exchange, with the ids of
// the rules that did, in order of first appearance.
func appliedRules(lines []rewrite.TraceLine) (bool, []int64) {
rewritten := false
var ids []int64
for _, l := range lines {
if l.Outcome != rewrite.OutcomeApplied {
continue
}
rewritten = true
if l.RuleID != 0 && !slices.Contains(ids, l.RuleID) {
ids = append(ids, l.RuleID)
}
}
return rewritten, ids
}
// newExchangeID names a recorded exchange: unix seconds plus a rolling sequence.
func (t *Traffic) newExchangeID(now time.Time) string {
n := t.seq.Add(1)
@@ -726,15 +838,35 @@ func headerLines(h map[string][]string) string {
return b.String()
}
// requestHeaderLines restores the HTTP Host header, which net/http stores on
// Request.Host rather than in Header. Keeping it in request.http makes the raw
// capture complete and directly replayable.
// requestHeaderLines is requestHeaderLinesOf for a flow's request.
func requestHeaderLines(req *mproxy.Request) string {
headers := req.Header.Clone()
v := flowRequestView(req)
return requestHeaderLinesOf(v.header, req.URL.Host, v.rawHost)
}
// requestHead is the head an exchange records for a request view: request
// line and headers, or only the Host when the original request is unavailable.
func requestHead(rq requestView, target string) string {
urlHost := ""
if rq.url != nil {
urlHost = rq.url.Host
}
if rq.unavailable {
return fmt.Sprintf("%s %s %s\n%s%s", rq.method, target, rq.proto, requestHeaderLinesOf(nil, urlHost, rq.rawHost), unavailableMarker)
}
return fmt.Sprintf("%s %s %s\n%s", rq.method, target, rq.proto, requestHeaderLinesOf(rq.header, urlHost, rq.rawHost))
}
// requestHeaderLinesOf restores the HTTP Host header, which net/http stores on
// Request.Host rather than in Header. Keeping it in request.http makes the raw
// capture complete and directly replayable. rawHost is the client's Host
// header (Raw().Host); urlHost stands in when it is empty.
func requestHeaderLinesOf(header http.Header, urlHost, rawHost string) string {
headers := header.Clone()
headers.Del("Host")
host := req.URL.Host
if raw := req.Raw(); raw != nil && strings.TrimSpace(raw.Host) != "" {
host = raw.Host
host := urlHost
if strings.TrimSpace(rawHost) != "" {
host = rawHost
}
var b strings.Builder
if host = strings.TrimSpace(host); host != "" {
@@ -1271,6 +1403,9 @@ func (t *Traffic) deleteWhere(tx *sql.Tx, where string, args ...any) (int64, err
if _, err := tx.Exec(`DELETE FROM stream_messages WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil {
return 0, fmt.Errorf("delete stream messages: %w", err)
}
if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil {
return 0, fmt.Errorf("delete rewrite trace: %w", err)
}
if _, err := tx.Exec(`DELETE FROM blob_refs WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil {
return 0, fmt.Errorf("delete blob references: %w", err)
}
@@ -2138,7 +2273,7 @@ func (t *Traffic) Tools() []actool.CoreTool {
get := actool.Build(actool.Spec{
Name: "traffic_get",
Description: "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling. A WebSocket (status 101) or SSE (text/event-stream) record shows only its handshake here; read its messages with traffic_messages.",
Description: TrafficGetDescription,
Schema: map[string]any{
"type": "object",
"properties": map[string]any{"id": map[string]any{"type": "string", "description": "the id returned by traffic_search"}},
@@ -2153,7 +2288,15 @@ func (t *Traffic) Tools() []actool.CoreTool {
if err != nil {
return actool.Errorf(err.Error()), nil
}
return actool.Text("=== REQUEST ===\n" + clip(req, 2500) + "\n\n=== RESPONSE ===\n" + clip(resp, 4000)), nil
rewritten, ruleIDs, err := t.RewriteInfo(a.ID)
if err != nil {
return actool.Errorf(err.Error()), nil
}
head := ""
if rewritten {
head = fmt.Sprintf("=== REWRITTEN === rewrite rules %v changed this exchange in the proxy. Shown below: the request as the client sent it and the response as the target sent it, not the rewritten values.\n\n", ruleIDs)
}
return actool.Text(head + "=== REQUEST ===\n" + clip(req, 2500) + "\n\n=== RESPONSE ===\n" + clip(resp, 4000)), nil
},
})