diff --git a/traffic/external.go b/traffic/external.go index 41c9dba..9e849dc 100644 --- a/traffic/external.go +++ b/traffic/external.go @@ -76,6 +76,11 @@ func (t *Traffic) insertExchange(id string, ts int64, host, method, urlTemplate, } defer tx.Rollback() //nolint:errcheck // no-op once committed + // An id can repeat (seconds plus a rolling counter); the trace of the + // exchange it replaces must not stay attached to this one. + if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id=?`, id); err != nil { + return fmt.Errorf("clearing rewrite trace: %w", err) + } res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path) VALUES(?,?,?,?,?,?,?,?,?,?,'')`, id, ts, host, method, urlTemplate, rawURL, status, contentType, reqLen, respLen) diff --git a/traffic/pace.go b/traffic/pace.go index 756c330..438a839 100644 --- a/traffic/pace.go +++ b/traffic/pace.go @@ -40,11 +40,15 @@ func (s *sink) Requestheaders(f *mproxy.Flow) { } } -// ClientDisconnected forgets a closed connection's mark. +// ClientDisconnected forgets a closed connection's mark, and asks for a sweep +// of the request snapshots its flows may have left (rewrite_flow.go). func (s *sink) ClientDisconnected(c *mproxy.ClientConn) { if c != nil { s.paced.Delete(c.Id) } + if s.t != nil { + s.t.snaps.requestSweep() + } } // pace waits for the request's host under the current noise profile, bounded diff --git a/traffic/rewrite_flow.go b/traffic/rewrite_flow.go new file mode 100644 index 0000000..b740093 --- /dev/null +++ b/traffic/rewrite_flow.go @@ -0,0 +1,663 @@ +package traffic + +import ( + "database/sql" + "io" + "log" + "net/http" + "net/url" + "sync" + "sync/atomic" + "time" + + "github.com/Autumn-27/artex/traffic/rewrite" + mproxy "github.com/lqqyt2423/go-mitmproxy/proxy" +) + +// Applying the rewrite rules to the proxied flows. +// +// The request phase runs in the Request hook. A request a rule rewrote leaves +// the proxy changed, but its exchange records what the client sent: the +// original is kept in a per-flow request snapshot until the Response hook +// records the exchange. The response phase runs at the start of the Response +// hook (status and headers only at SSEStart), and the exchange records the +// response as the target sent it. The trace of both phases is written in the +// exchange's own transaction (exchange_rewrites). +// +// A request is rewritten only once its snapshot slot is reserved, and a slot +// stays until the flow's exchange is recorded or the flow is finished, so the +// Response hook never takes a rewritten request for the client's. + +// Limits of the request snapshots. +const ( + maxSnapEntries = 1024 // entries holding an original request + maxSnapBytes = 64 << 20 // what those entries hold + maxSnapTombs = 4096 // tombstones: slots whose data was released + // snapTTL deletes nothing: past it, the entry of a live flow may be turned + // into a tombstone when a new reservation finds no room. + snapTTL = 15 * time.Minute + sweepDebounce = time.Second // at most one sweep per second + sweepPeriod = time.Minute // a sweep at least this often while slots exist + // libraryStreamBody is go-mitmproxy's StreamLargeBodies default: a body this + // large reaches neither the Request nor the Response hook. + libraryStreamBody = 5 << 20 + + reasonSnapshotCapacity = "not rewritten: snapshot capacity" + reasonOriginalUnavailable = "original request unavailable" +) + +// testRuleHook, when a test sets it, runs inside each rule that checks a body +// ("rule") and in the sink right after the engine ("request", "response"), so +// a test can make a rule, or the sink itself, panic. +var testRuleHook atomic.Pointer[func(stage string)] + +func runTestHook(stage string) { + if h := testRuleHook.Load(); h != nil { + (*h)(stage) + } +} + +// rulesIsBinary is the binary predicate handed to the engine: the recorder's +// own, so "not rewritten: binary" follows what the recorder treats as binary. +func rulesIsBinary(contentType string, body []byte) bool { + runTestHook("rule") + return isBinaryBody(contentType, body) +} + +type flowKey [16]byte + +func keyOf(f *mproxy.Flow) flowKey { return flowKey(f.Id) } + +// reqSnap is one slot of the table: an entry, which holds the original of a +// request, or a tombstone, which holds only why it has none. +type reqSnap struct { + // done is the flow's Done() channel: closed, no hook will come for the + // slot any more. The flow itself is not referenced. + done <-chan struct{} + rules *rewrite.Set // the snapshot of rules the flow runs with, both phases + host string // host name: the only part of the request a log shows + tomb bool + + // Entry. + created time.Time + cost int64 + rewritten bool // the request phase changed the request + method string + url *url.URL + header http.Header + rawHost string + body []byte // the client's body, kept when a body action applied + keepBody bool + lines []rewrite.TraceLine // request-phase trace + + // Tombstone. + reason string // reasonSnapshotCapacity or reasonOriginalUnavailable + ruleIDs []int64 // rules that had rewritten the request +} + +// requestRewritten reports a flow whose request left the proxy changed. +func (s *reqSnap) requestRewritten() bool { + if s.tomb { + return s.reason == reasonOriginalUnavailable + } + return s.rewritten +} + +func (s *reqSnap) appliedRuleIDs() []int64 { + if s.tomb { + return s.ruleIDs + } + _, ids := appliedRules(s.lines) + return ids +} + +// snapTable holds the slots, keyed by flow id. Its mutex is a leaf lock. +type snapTable struct { + mu sync.Mutex + slots map[flowKey]*reqSnap + entries int + bytes int64 + tombs int + lastSweep time.Time + lastFullLog time.Time + sweepWanted atomic.Bool + + now func() time.Time + maxEntries int + maxBytes int64 + maxTombs int + ttl time.Duration + + sweeps, capacity, tombDrops, conversions, largeResponses atomic.Int64 +} + +func newSnapTable() *snapTable { + return &snapTable{slots: map[flowKey]*reqSnap{}, now: time.Now, + maxEntries: maxSnapEntries, maxBytes: maxSnapBytes, maxTombs: maxSnapTombs, ttl: snapTTL} +} + +func doneClosed(ch <-chan struct{}) bool { + if ch == nil { + return false // a nil channel is never ready: the flow counts as live + } + select { + case <-ch: + return true + default: + return false + } +} + +// reserve stores the entry of a rewritten request. On failure the request must +// leave unchanged: a capacity tombstone then stands for the flow, when there is +// room for one. +func (t *snapTable) reserve(k flowKey, e *reqSnap) bool { + if t == nil { + return false + } + t.mu.Lock() + defer t.mu.Unlock() + t.removeLocked(k) + if !t.fitsLocked(e.cost) { + // A failed reservation asks for an immediate sweep, under the same + // debounce; a sweep refused by the debounce fails the reservation. + now := t.now() + if now.Sub(t.lastSweep) < sweepDebounce { + t.refuseLocked(k, e) + return false + } + t.sweepLocked(now) + if !t.fitsLocked(e.cost) && !t.convertLocked(e.cost) { + t.refuseLocked(k, e) + return false + } + } + e.created = t.now() + t.slots[k] = e + t.entries++ + t.bytes += e.cost + return true +} + +func (t *snapTable) fitsLocked(cost int64) bool { + return t.entries < t.maxEntries && t.bytes+cost <= t.maxBytes +} + +// convertLocked turns the oldest entry of a live flow older than the TTL into +// a tombstone, in place, so the flow is never without a slot. It does so only +// when releasing that entry lets a reservation of cost succeed, and when a +// tombstone place is free (after evicting tombstones of finished flows only). +// Otherwise nothing changes and it reports false. +func (t *snapTable) convertLocked(cost int64) bool { + now := t.now() + var victim *reqSnap + for _, s := range t.slots { + if s.tomb || doneClosed(s.done) || now.Sub(s.created) < t.ttl { + continue + } + if t.entries-1 >= t.maxEntries || t.bytes-s.cost+cost > t.maxBytes { + continue // released, it would still leave no room + } + if victim == nil || s.created.Before(victim.created) { + victim = s + } + } + if victim == nil { + return false + } + if t.tombs >= t.maxTombs { + t.evictFinishedTombsLocked() + } + if t.tombs >= t.maxTombs { + return false + } + ids := victim.appliedRuleIDs() + t.entries-- + t.bytes -= victim.cost + t.tombs++ + *victim = reqSnap{done: victim.done, rules: victim.rules, host: victim.host, tomb: true, reason: reasonOriginalUnavailable, ruleIDs: ids} + t.conversions.Add(1) + log.Printf("[traffic] rewrite snapshot of a request to %s (rules %v) released under pressure: its exchange will be recorded as original request unavailable", victim.host, ids) + return true +} + +// refuseLocked is the capacity path: the request is not rewritten, and a +// tombstone records why, when a place is free. +func (t *snapTable) refuseLocked(k flowKey, e *reqSnap) { + t.capacity.Add(1) + if t.tombs >= t.maxTombs { + t.evictFinishedTombsLocked() + } + full := t.tombs >= t.maxTombs + if full { + t.tombDrops.Add(1) + } else { + t.slots[k] = &reqSnap{done: e.done, rules: e.rules, host: e.host, tomb: true, reason: reasonSnapshotCapacity} + t.tombs++ + } + if now := t.now(); now.Sub(t.lastFullLog) >= time.Minute { + t.lastFullLog = now + log.Printf("[traffic] rewrite snapshot table full: requests are sent without rewriting (%d refused, %d without a trace so far)", t.capacity.Load(), t.tombDrops.Load()) + } +} + +func (t *snapTable) evictFinishedTombsLocked() { + for k, s := range t.slots { + if s.tomb && doneClosed(s.done) { + t.removeLocked(k) + } + } +} + +// sweepLocked deletes every slot whose flow is finished. The slot of a live +// flow is never touched, whatever its age. +func (t *snapTable) sweepLocked(now time.Time) { + for k, s := range t.slots { + if doneClosed(s.done) { + t.removeLocked(k) + } + } + t.lastSweep = now + t.sweepWanted.Store(false) + t.sweeps.Add(1) +} + +func (t *snapTable) removeLocked(k flowKey) { + s := t.slots[k] + if s == nil { + return + } + if s.tomb { + t.tombs-- + } else { + t.entries-- + t.bytes -= s.cost + } + delete(t.slots, k) +} + +// take removes and returns the slot of a flow, nil when it has none. +func (t *snapTable) take(k flowKey) *reqSnap { + if t == nil { + return nil + } + t.mu.Lock() + defer t.mu.Unlock() + s := t.slots[k] + t.removeLocked(k) + return s +} + +// requestSweep asks the sweeper for a pass; it runs at most once a second. +func (t *snapTable) requestSweep() { + if t != nil { + t.sweepWanted.Store(true) + } +} + +// tick is one beat of the sweeper: a sweep when one was asked for and the last +// is a second old, or when the last is a minute old. Nothing on an empty table. +func (t *snapTable) tick() { + t.mu.Lock() + defer t.mu.Unlock() + if len(t.slots) == 0 { + t.sweepWanted.Store(false) + return + } + now := t.now() + since := now.Sub(t.lastSweep) + if (t.sweepWanted.Load() && since >= sweepDebounce) || since >= sweepPeriod { + t.sweepLocked(now) + } +} + +// counts reports the entries and tombstones in the table. +func (t *snapTable) counts() (entries, tombs int) { + t.mu.Lock() + defer t.mu.Unlock() + return t.entries, t.tombs +} + +// sweepSnapshots is the single sweeper goroutine, started by Open and stopped +// by Close. +func (t *Traffic) sweepSnapshots() { + tick := time.NewTicker(time.Second) + defer tick.Stop() + for { + select { + case <-t.closed: + return + case <-tick.C: + t.snaps.tick() + } + } +} + +// RewriteStats are the proxy-side counters of the rewrite rules since start. +type RewriteStats struct { + // LargeResponses counts flows with a snapshot whose response reached the + // library's 5 MiB stream threshold: passed through, not recorded. + LargeResponses int64 `json:"large_responses"` + // CapacityRefusals counts requests sent without rewriting for lack of room. + CapacityRefusals int64 `json:"capacity_refusals"` + // TombstoneDrops counts those of them that got no capacity tombstone either. + TombstoneDrops int64 `json:"tombstone_drops"` + // Conversions counts entries released under pressure. + Conversions int64 `json:"conversions"` + Sweeps int64 `json:"sweeps"` +} + +func (t *Traffic) RewriteStats() RewriteStats { + s := t.snaps + if s == nil { + return RewriteStats{} + } + return RewriteStats{LargeResponses: s.largeResponses.Load(), CapacityRefusals: s.capacity.Load(), + TombstoneDrops: s.tombDrops.Load(), Conversions: s.conversions.Load(), Sweeps: s.sweeps.Load()} +} + +// newEntry builds the snapshot of a request before the rules' result is put on +// the flow: URL and headers cloned, the body referenced, and only when a body +// action applied (otherwise the flow's body is still the client's). +func newEntry(f *mproxy.Flow, set *rewrite.Set, res rewrite.Result) *reqSnap { + r := f.Request + u := *r.URL + e := &reqSnap{done: f.Done(), rules: set, host: r.URL.Hostname(), rewritten: res.Changed, + method: r.Method, url: &u, header: r.Header.Clone(), lines: res.Lines} + if raw := r.Raw(); raw != nil { + e.rawHost = raw.Host + } + for _, l := range res.Lines { + if l.Outcome == rewrite.OutcomeApplied && l.Field == "body" && len(r.Body) <= rewrite.MaxBodySize { + e.body, e.keepBody = r.Body, true + break + } + } + e.cost = int64(len(e.method) + len(u.String()) + len(e.rawHost) + len(e.body)) + for k, vs := range e.header { + for _, v := range vs { + e.cost += int64(len(k) + len(v)) + } + } + for _, l := range e.lines { + e.cost += int64(len(l.After) + len(l.RuleSnapshot)) + } + return e +} + +// originalRequest is the request an exchange records, with the request-phase +// trace: the flow's own request when no rule rewrote it, the entry's copy when +// one did, and only what Raw() still knows when the entry was released. +func originalRequest(f *mproxy.Flow, snap *reqSnap) (requestView, []rewrite.TraceLine) { + switch { + case snap == nil: + return flowRequestView(f.Request), nil + case !snap.tomb: + v := requestView{method: snap.method, url: snap.url, proto: f.Request.Proto, header: snap.header, rawHost: snap.rawHost, body: f.Request.Body} + if snap.keepBody { + v.body = snap.body + } + return v, snap.lines + case snap.reason == reasonSnapshotCapacity: + // The request was not rewritten: the flow's request is the client's. + return flowRequestView(f.Request), []rewrite.TraceLine{{Phase: rewrite.PhaseRequest, Outcome: rewrite.OutcomeSkipped, Reason: reasonSnapshotCapacity}} + default: + // Rewritten, original released: never record the rewritten request as + // the client's. Raw() is the client's request line and Host. + v := requestView{unavailable: true} + if raw := f.Request.Raw(); raw != nil { + v.method, v.url, v.proto, v.rawHost = raw.Method, raw.URL, raw.Proto, raw.Host + } + lines := []rewrite.TraceLine{{Phase: rewrite.PhaseRequest, Outcome: rewrite.OutcomeError, Reason: reasonOriginalUnavailable}} + for _, id := range snap.ruleIDs { + lines = append(lines, rewrite.TraceLine{Phase: rewrite.PhaseRequest, RuleID: id, Outcome: rewrite.OutcomeApplied}) + } + return v, lines + } +} + +func engineRequest(r *mproxy.Request) rewrite.Request { + return rewrite.Request{Method: r.Method, URL: r.URL, Header: r.Header, Body: r.Body} +} + +// Request runs the request phase. The result goes on the flow only once the +// request's snapshot slot is reserved; without one the request leaves as the +// client sent it. Rules fail open: a panic leaves the request unchanged. +func (s *sink) Request(f *mproxy.Flow) { + if s.t == nil || f.Request == nil || f.Request.URL == nil { + return + } + set := s.t.rewriteRules.Load() + if set.Len(rewrite.PhaseRequest) == 0 { + return + } + host := f.Request.URL.Hostname() + defer func() { + if recover() != nil { + log.Printf("[traffic] rewrite rules stopped on a request to %s (class %s): it is sent as the client sent it", host, rewrite.ErrClassPanic) + } + }() + res := set.Apply(rewrite.PhaseRequest, rewrite.Input{Request: engineRequest(f.Request), IsBinary: rulesIsBinary}) + runTestHook("request") + if len(res.Lines) == 0 { + return + } + if !s.t.snaps.reserve(keyOf(f), newEntry(f, set, res)) { + return + } + if res.Changed { + r := f.Request + r.Method, r.URL, r.Header, r.Body = res.Request.Method, res.Request.URL, res.Request.Header, res.Request.Body + } +} + +// Response records the exchange and runs the response phase: the exchange +// keeps the client's request and the target's response, the client gets the +// rewritten response. Without a slot and without response rules this is the +// plain record. +func (s *sink) Response(f *mproxy.Flow) { + if f.Request == nil || f.Response == nil { + return + } + snap := s.t.snaps.take(keyOf(f)) + set := s.t.rewriteRules.Load() + if snap != nil { + set = snap.rules + } + if snap == nil && set.Len(rewrite.PhaseResponse) == 0 { + s.t.record(f) + return + } + rq, lines := originalRequest(f, snap) + rs := flowResponseView(f.Response) + res := s.applyResponse(f, set, rewrite.Input{ + Request: engineRequest(f.Request), // as sent to the target + Response: rewrite.Response{Status: rs.status, Header: rs.header, Body: rs.body}, + IsBinary: rulesIsBinary, + }) + s.t.recordViews(rq, rs, append(lines, res.Lines...)) + if res.Changed { + f.Response.StatusCode, f.Response.Header, f.Response.Body = res.Response.Status, res.Response.Header, res.Response.Body + } +} + +// applyResponse runs the response phase; a panic yields no change and no line. +func (s *sink) applyResponse(f *mproxy.Flow, set *rewrite.Set, in rewrite.Input) (res rewrite.Result) { + if set.Len(rewrite.PhaseResponse) == 0 { + return rewrite.Result{} + } + defer func() { + if recover() != nil { + res = rewrite.Result{} + log.Printf("[traffic] rewrite rules stopped on a response from %s (class %s): it is sent as the target sent it", f.Request.URL.Hostname(), rewrite.ErrClassPanic) + } + }() + r := set.Apply(rewrite.PhaseResponse, in) + runTestHook("response") + return r +} + +// rewriteStream prepares the exchange of a stream at SSEStart or +// WebSocketStart: the client's request, the target's status and headers (SSE), +// and the trace. For SSE, the response phase runs in stream mode: status and +// header actions apply (res, put on the flow by applyStreamResponse), rules +// that read or write the response body are traced "not rewritten: stream". +// WebSocketStart runs once go-mitmproxy has already answered the client's +// upgrade, with no response on the flow: nothing of the handshake can change +// there, so every candidate rule is traced "not rewritten: stream". +func (s *sink) rewriteStream(f *mproxy.Flow, kind string) (rq requestView, rs responseView, lines []rewrite.TraceLine, res rewrite.Result) { + snap := s.t.snaps.take(keyOf(f)) + set := s.t.rewriteRules.Load() + if snap != nil { + set = snap.rules + } + rq, lines = originalRequest(f, snap) + if kind == "sse" { + rs = flowResponseView(f.Response) + } + cur := engineRequest(f.Request) + if kind != "websocket" { + res = s.applyResponse(f, set, rewrite.Input{Request: cur, Response: rewrite.Response{Status: rs.status, Header: rs.header}, Stream: true, IsBinary: rulesIsBinary}) + return rq, rs, append(lines, res.Lines...), res + } + if set.Len(rewrite.PhaseRequest)+set.Len(rewrite.PhaseResponse) == 0 { + return rq, rs, lines, rewrite.Result{} + } + // The engine never changes a request with Upgrade; a ws:// stream arrives + // on its CONNECT flow, which has none. + if cur.Header.Get("Upgrade") == "" { + cur.Header = cur.Header.Clone() + if cur.Header == nil { + cur.Header = http.Header{} + } + cur.Header.Set("Upgrade", "websocket") + } + if snap == nil { + lines = append(lines, s.applyRequestTrace(f, set, cur)...) + } + r := s.applyResponse(f, set, rewrite.Input{Request: cur, Response: rewrite.Response{Status: http.StatusSwitchingProtocols}, Stream: true, IsBinary: rulesIsBinary}) + return rq, rs, append(lines, r.Lines...), rewrite.Result{} +} + +// applyRequestTrace runs the request phase for its trace only. +func (s *sink) applyRequestTrace(f *mproxy.Flow, set *rewrite.Set, req rewrite.Request) (lines []rewrite.TraceLine) { + defer func() { + if recover() != nil { + lines = nil + log.Printf("[traffic] rewrite rules stopped on a request to %s (class %s)", f.Request.URL.Hostname(), rewrite.ErrClassPanic) + } + }() + return set.Apply(rewrite.PhaseRequest, rewrite.Input{Request: req, IsBinary: rulesIsBinary}).Lines +} + +// applyStreamResponse puts the rewritten status and headers of an SSE +// response on the flow, before go-mitmproxy replies to the client. +func (s *sink) applyStreamResponse(f *mproxy.Flow, res rewrite.Result) { + if res.Changed && f.Response != nil { + f.Response.StatusCode, f.Response.Header = res.Response.Status, res.Response.Header + } +} + +// RequestError fires when a request through an established MITM tunnel fails. If +// the failure looks proxy/protocol-caused (h2 quirks, HEAD-with-body, protocol +// errors) and not a plain target-unreachable error, the host is tunneled for +// passthroughTTL: this request has already failed, later requests in the window +// connect straight through, and the first one after the window tries +// interception again. A request a rule rewrote never flags its host: the rule, +// not the interception, may be at fault. Its log names the host, the error class +// and the rules, never the URL or the error text. +func (s *sink) RequestError(f *mproxy.Flow, err error) { + if f != nil { + if snap := s.t.snaps.take(keyOf(f)); snap != nil && snap.requestRewritten() { + ids := snap.appliedRuleIDs() + for _, id := range ids { + snap.rules.NoteError(id, rewrite.ErrClassTransport) + } + log.Printf("[traffic] a request to %s rewritten by rules %v failed (class %s); the host stays intercepted", snap.host, ids, rewrite.ErrClassTransport) + return + } + } + s.t.maybePassthrough(f, err) +} + +// StreamResponseModifier passes every body through as received, nil included, +// without wrapping it. A response that reached the library's stream threshold +// is never recorded: the slot of its flow, if any, is released here. +func (s *sink) StreamResponseModifier(f *mproxy.Flow, in io.Reader) (out io.Reader) { + out = in + defer func() { + if recover() != nil { + out = in + } + }() + if s == nil || s.t == nil || f == nil || !f.Stream || f.SSE != nil { + return in + } + snap := s.t.snaps.take(keyOf(f)) + if snap == nil { + return in + } + s.t.snaps.largeResponses.Add(1) + log.Printf("[traffic] the response to a request to %s with rewrite rules %v reached %d MiB: passed through, not recorded", snap.host, snap.appliedRuleIDs(), libraryStreamBody>>20) + return in +} + +// spillTrace stores the values of the trace lines as record stores bodies. +// Called under wmu, before the transaction. +func (t *Traffic) spillTrace(lines []rewrite.TraceLine) []storedBody { + out := make([]storedBody, len(lines)) + for i, l := range lines { + out[i] = t.spill(l.After, "") + } + return out +} + +// writeTrace inserts an exchange's trace rows and the blob references of their +// spilled values, inside the exchange's transaction. +func writeTrace(tx *sql.Tx, id string, lines []rewrite.TraceLine, trace []storedBody) error { + for _, b := range trace { + if b.hash == "" { + continue + } + if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, b.hash, id); err != nil { + return &recordError{"registering blob reference", err} + } + } + for i, l := range lines { + var ruleID, after any + if l.RuleID != 0 { + ruleID = l.RuleID + } + if trace[i].inline != nil { + after = trace[i].inline + } + if _, err := tx.Exec(`INSERT INTO exchange_rewrites(exchange_id,seq,phase,rule_id,rule_snapshot,action,field,outcome,reason,before_hash,after_hash,after_inline,after_blob) +VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`, + id, i, string(l.Phase), ruleID, nullIfEmpty(l.RuleSnapshot), nullIfEmpty(l.Action), nullIfEmpty(l.Field), l.Outcome, l.Reason, + nullIfEmpty(l.BeforeHash), nullIfEmpty(l.AfterHash), after, nullIfEmpty(trace[i].hash)); err != nil { + return &recordError{"writing rewrite trace", err} + } + } + return nil +} + +// RewriteInfo reports whether rules rewrote an exchange (at least one applied +// trace line) and which rules did, by id. +func (t *Traffic) RewriteInfo(id string) (rewritten bool, ruleIDs []int64, err error) { + if err := t.db.QueryRow(`SELECT EXISTS(SELECT 1 FROM exchange_rewrites r WHERE r.exchange_id=? AND r.outcome='applied')`, id).Scan(&rewritten); err != nil || !rewritten { + return false, nil, err + } + rows, err := t.db.Query(`SELECT DISTINCT rule_id FROM exchange_rewrites WHERE exchange_id=? AND outcome='applied' AND rule_id IS NOT NULL ORDER BY rule_id`, id) + if err != nil { + return false, nil, err + } + defer rows.Close() + for rows.Next() { + var rid int64 + if err := rows.Scan(&rid); err != nil { + return false, nil, err + } + ruleIDs = append(ruleIDs, rid) + } + return true, ruleIDs, rows.Err() +} diff --git a/traffic/rewrite_flow_test.go b/traffic/rewrite_flow_test.go new file mode 100644 index 0000000..a549d34 --- /dev/null +++ b/traffic/rewrite_flow_test.go @@ -0,0 +1,1316 @@ +package traffic + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "slices" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Autumn-27/artex/traffic/rewrite" + "github.com/gorilla/websocket" + mproxy "github.com/lqqyt2423/go-mitmproxy/proxy" +) + +// --- helpers --------------------------------------------------------------- + +// rules compiles enabled rules, failing the test on a load error. +func rules(t *testing.T, key []byte, errs *rewrite.ErrorLog, rs ...rewrite.Rule) *rewrite.Set { + t.Helper() + for i := range rs { + rs[i].Enabled = true + if rs[i].Name == "" { + rs[i].Name = fmt.Sprintf("rule %d", rs[i].ID) + } + } + set := rewrite.Compile(rs, rewrite.Options{TraceKey: key, Errors: errs}) + if len(set.LoadErrors()) != 0 { + t.Fatalf("load errors: %v", set.LoadErrors()) + } + return set +} + +type traceRow struct { + seq int + phase, outcome, reason string + ruleID sql.NullInt64 + snapshot, action, field sql.NullString + beforeHash, afterHash, blob sql.NullString + inline []byte +} + +func traceRows(t *testing.T, tr *Traffic, id string) []traceRow { + t.Helper() + rows, err := tr.DB().Query(`SELECT seq,phase,rule_id,rule_snapshot,action,field,outcome,reason,before_hash,after_hash,after_inline,after_blob +FROM exchange_rewrites WHERE exchange_id=? ORDER BY seq`, id) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var out []traceRow + for rows.Next() { + var r traceRow + if err := rows.Scan(&r.seq, &r.phase, &r.ruleID, &r.snapshot, &r.action, &r.field, &r.outcome, &r.reason, &r.beforeHash, &r.afterHash, &r.inline, &r.blob); err != nil { + t.Fatal(err) + } + out = append(out, r) + } + return out +} + +func countRows(t *testing.T, tr *Traffic, table string) int { + t.Helper() + var n int + if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM ` + table).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +func exchangeIDByPath(t *testing.T, tr *Traffic, suffix string) string { + t.Helper() + var id string + if err := tr.DB().QueryRow(`SELECT id FROM exchanges WHERE url LIKE ?`, "%"+suffix).Scan(&id); err != nil { + t.Fatalf("exchange %s: %v", suffix, err) + } + return id +} + +// logBuf captures the standard logger, where every ARTEX log goes. +type logBuf struct { + mu sync.Mutex + b bytes.Buffer +} + +func (l *logBuf) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.b.Write(p) +} + +func (l *logBuf) String() string { + l.mu.Lock() + defer l.mu.Unlock() + return l.b.String() +} + +func captureLog(t *testing.T) *logBuf { + b := &logBuf{} + prev := log.Writer() + log.SetOutput(b) + t.Cleanup(func() { log.SetOutput(prev) }) + return b +} + +func setTestHook(t *testing.T, h func(stage string)) { + testRuleHook.Store(&h) + t.Cleanup(func() { testRuleHook.Store(nil) }) +} + +// limits changes the table's bounds and clock for a test. +func (t *snapTable) limits(maxEntries int, maxBytes int64, maxTombs int, now func() time.Time) { + t.mu.Lock() + defer t.mu.Unlock() + t.maxEntries, t.maxBytes, t.maxTombs = maxEntries, maxBytes, maxTombs + if now != nil { + t.now = now + } +} + +func (t *snapTable) slot(k flowKey) *reqSnap { + t.mu.Lock() + defer t.mu.Unlock() + if s := t.slots[k]; s != nil { + c := *s + return &c + } + return nil +} + +func key(n int) flowKey { + var k flowKey + k[0], k[1] = byte(n), byte(n>>8) + return k +} + +// withID gives a synthetic flow a distinct id, as real flows have. +func withID(f *mproxy.Flow, n int) *mproxy.Flow { + f.Id[0], f.Id[1], f.Id[15] = byte(n), byte(n>>8), 0xAB + return f +} + +type received struct { + mu sync.Mutex + reqs []string +} + +func (r *received) add(s string) { r.mu.Lock(); r.reqs = append(r.reqs, s); r.mu.Unlock() } +func (r *received) all() []string { + r.mu.Lock() + defer r.mu.Unlock() + return slices.Clone(r.reqs) +} + +func sendThrough(t *testing.T, tr *Traffic, proxyURL, method, target, body string, header http.Header) (*http.Response, []byte) { + t.Helper() + req, err := http.NewRequest(method, target, strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + for k, vs := range header { + req.Header[k] = vs + } + resp, err := proxyClient(t, tr, proxyURL).Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + return resp, b +} + +// --- integration: rewritten exchange ----------------------------------------- + +// A request and a response rewritten by rules: the target gets the rewritten +// request, the client the rewritten response, and the exchange records what +// the client sent and what the target answered, with the trace of both phases. +// Lengths stay coherent after a body rewrite on both sides, and full-text +// search finds the originals only. +func TestRewriteAppliedRecordsOriginals(t *testing.T) { + var got received + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + got.add(fmt.Sprintf("%s %s X-Client=%s len=%d %s", r.Method, r.RequestURI, r.Header.Get("X-Client"), r.ContentLength, body)) + w.Header().Set("X-Target", "t") + w.Header().Set("Content-Length", "11") + w.WriteHeader(http.StatusTeapot) + io.WriteString(w, "target body") + })) + t.Cleanup(target.Close) + tr, _, proxyURL := startProxy(t) + tr.SetRewriteRules(rules(t, nil, nil, + rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{ + {Type: rewrite.ActionSetMethod, Value: "PUT"}, + {Type: rewrite.ActionSetPathQuery, Value: "/rewritten?r=1"}, + {Type: rewrite.ActionSetHeader, Name: "X-Client", Value: "rewritten"}, + {Type: rewrite.ActionSetBody, Value: "rewritten request body"}}}, + rewrite.Rule{ID: 2, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{ + {Type: rewrite.ActionSetStatus, Value: "200"}, + {Type: rewrite.ActionSetHeader, Name: "X-Rewritten", Value: "1"}, + {Type: rewrite.ActionReplaceBody, Regex: "target", Value: "REWRITTEN-RESPONSE"}}}, + )) + + resp, body := sendThrough(t, tr, proxyURL, "POST", target.URL+"/path?q=1", "client body", http.Header{"X-Client": {"c"}}) + + if want := []string{"PUT /rewritten?r=1 X-Client=rewritten len=22 rewritten request body"}; !slices.Equal(got.all(), want) { + t.Fatalf("target received %q, want %q", got.all(), want) + } + if resp.StatusCode != 200 || resp.Header.Get("X-Target") != "t" || resp.Header.Get("X-Rewritten") != "1" || string(body) != "REWRITTEN-RESPONSE body" { + t.Fatalf("client received %d %v %q", resp.StatusCode, resp.Header, body) + } + if resp.ContentLength != -1 && resp.ContentLength != int64(len(body)) { + t.Fatalf("client Content-Length %d for a %d-byte body", resp.ContentLength, len(body)) + } + + id := onlyExchangeID(t, tr) + var method, rawURL string + var status, reqLen, respLen int + if err := tr.DB().QueryRow(`SELECT method,url,status,req_len,resp_len FROM exchanges WHERE id=?`, id).Scan(&method, &rawURL, &status, &reqLen, &respLen); err != nil { + t.Fatal(err) + } + if method != "POST" || rawURL != target.URL+"/path?q=1" || status != 418 || reqLen != 11 || respLen != 11 { + t.Fatalf("exchange row %s %s %d %d %d", method, rawURL, status, reqLen, respLen) + } + reqText, respText, err := tr.Get(id) + if err != nil { + t.Fatal(err) + } + host := strings.TrimPrefix(target.URL, "http://") + for _, want := range []string{"POST /path?q=1 HTTP/1.1\n", "Host: " + host + "\n", "X-Client: c\n", "\n\nclient body"} { + if !strings.Contains(reqText, want) { + t.Fatalf("recorded request lacks %q:\n%s", want, reqText) + } + } + for _, want := range []string{"HTTP 418\n", "X-Target: t\n", "\n\ntarget body"} { + if !strings.Contains(respText, want) { + t.Fatalf("recorded response lacks %q:\n%s", want, respText) + } + } + if strings.Contains(strings.ToLower(reqText+respText), "rewritten") { + t.Fatalf("a rewritten value was recorded as original:\n%s\n%s", reqText, respText) + } + + lines := traceRows(t, tr, id) + var summary []string + for _, l := range lines { + summary = append(summary, fmt.Sprintf("%s/%d/%s/%s", l.phase, l.ruleID.Int64, l.field.String, l.outcome)) + } + want := []string{"request/1/method/applied", "request/1/url/applied", "request/1/header:X-Client/applied", "request/1/body/applied", + "response/2/header:X-Rewritten/applied", "response/2/body/applied", "response/2/status/applied"} + if !slices.Equal(summary, want) { + t.Fatalf("trace %q, want %q", summary, want) + } + for i, l := range lines { + first := i == 0 || l.ruleID != lines[i-1].ruleID + if l.snapshot.Valid != first { + t.Fatalf("line %d: rule_snapshot present=%v, want it on the first line of each rule only", i, l.snapshot.Valid) + } + } + if string(lines[3].inline) != "rewritten request body" || string(lines[5].inline) != "REWRITTEN-RESPONSE body" { + t.Fatalf("body lines keep %q and %q", lines[3].inline, lines[5].inline) + } + + if ok, ids, err := tr.RewriteInfo(id); err != nil || !ok || !slices.Equal(ids, []int64{1, 2}) { + t.Fatalf("RewriteInfo = %v %v %v", ok, ids, err) + } + out := toolText(t, tr, "traffic_get", `{"id":"`+id+`"}`) + if !strings.HasPrefix(out, "=== REWRITTEN === rewrite rules [1 2] ") || !strings.Contains(out, "client body") { + t.Fatalf("traffic_get output:\n%s", out) + } + + h := "127.0.0.1" + for term, found := range map[string]bool{"client body": true, "target body": true, "rewritten request": false, "REWRITTEN-RESPONSE": false, "/rewritten": false} { + rows, err := tr.query(h, "", term, 0, 10) + if err != nil { + t.Fatal(err) + } + if (len(rows) == 1) != found { + t.Fatalf("full-text search for %q found %d exchanges, want found=%v", term, len(rows), found) + } + } + if e, tb := tr.snaps.counts(); e != 0 || tb != 0 { + t.Fatalf("snapshot table holds %d entries, %d tombstones after Response", e, tb) + } +} + +// toolText runs one of the traffic agent tools and returns its text. +func toolText(t *testing.T, tr *Traffic, name, input string) string { + t.Helper() + for _, tool := range tr.Tools() { + if tool.Name() != name { + continue + } + res, err := tool.Call(context.Background(), json.RawMessage(input), nil) + if err != nil { + t.Fatal(err) + } + return res.Flatten() + } + t.Fatalf("no tool %s", name) + return "" +} + +// --- non-regression ---------------------------------------------------------- + +type recordedRow struct { + exchange, bodies string +} + +func dumpOnly(t *testing.T, tr *Traffic) recordedRow { + t.Helper() + var host, method, tmpl, u, ct, path, reqHead, respHead string + var status, reqLen, respLen int + var reqBody, respBody []byte + var reqBlob, respBlob sql.NullString + if err := tr.DB().QueryRow(`SELECT e.host,e.method,e.url_template,e.url,e.status,e.content_type,e.req_len,e.resp_len,e.path, +b.req_head,b.req_body,b.req_blob,b.resp_head,b.resp_body,b.resp_blob FROM exchanges e JOIN exchange_bodies b ON b.id=e.id`). + Scan(&host, &method, &tmpl, &u, &status, &ct, &reqLen, &respLen, &path, &reqHead, &reqBody, &reqBlob, &respHead, &respBody, &respBlob); err != nil { + t.Fatal(err) + } + return recordedRow{ + exchange: fmt.Sprintf("%q %q %q %q %d %q %d %d %q", host, method, tmpl, u, status, ct, reqLen, respLen, path), + bodies: fmt.Sprintf("%q %q %v %q %q %v", reqHead, reqBody, reqBlob, respHead, respBody, respBlob), + } +} + +// Without a rule that matches, the hooks record exactly what record() records, +// byte for byte, and write no trace row. +func TestRewriteNoRuleRecordsUnchanged(t *testing.T) { + flow := func(n int) *mproxy.Flow { + return withID(newFlow("api.example.com", "POST", "/v1/items/42?x=1", []byte(`{"a":1}`), []byte(`{"ok":true}`)), n) + } + plain, _ := openTraffic(t) + plain.record(flow(1)) + want := dumpOnly(t, plain) + + for name, set := range map[string]*rewrite.Set{ + "no snapshot": nil, + "no match": rules(t, nil, nil, + rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Matchers: rewrite.Matchers{Host: "other.example"}, Actions: []rewrite.Action{{Type: rewrite.ActionSetBody, Value: "x"}}}, + rewrite.Rule{ID: 2, Phase: rewrite.PhaseResponse, Matchers: rewrite.Matchers{Host: "*.other.example"}, Actions: []rewrite.Action{{Type: rewrite.ActionSetStatus, Value: "500"}}}), + } { + tr, _ := openTraffic(t) + tr.SetRewriteRules(set) + s := tr.proxy.Addons[0].(*sink) + f := flow(2) + s.Request(f) + s.Response(f) + if got := dumpOnly(t, tr); got != want { + t.Fatalf("%s: recorded\n%+v\nwant\n%+v", name, got, want) + } + if n := countRows(t, tr, "exchange_rewrites"); n != 0 { + t.Fatalf("%s: %d trace rows", name, n) + } + if f.Response.StatusCode != 200 || string(f.Request.Body) != `{"a":1}` { + t.Fatalf("%s: the flow changed", name) + } + } +} + +// Through the real proxy, a snapshot whose rules match nothing gives the same +// client response, target request and recorded exchange as no snapshot. +func TestRewriteNoMatchThroughProxy(t *testing.T) { + var got received + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + got.add(r.Method + " " + r.RequestURI + " " + string(body)) + w.Header().Set("Content-Type", "text/plain") + io.WriteString(w, "target body") + })) + t.Cleanup(target.Close) + canon := func(raw string) string { + head, body, _ := strings.Cut(raw, "\n\n") + lines := strings.Split(head, "\n") + sort.Strings(lines[1:]) + lines = slices.DeleteFunc(lines, func(l string) bool { return strings.HasPrefix(l, "Date: ") }) + return strings.Join(lines, "\n") + "\n\n" + body + } + exchange := func(set *rewrite.Set) string { + tr, _, proxyURL := startProxy(t) + tr.SetRewriteRules(set) + resp, body := sendThrough(t, tr, proxyURL, "POST", target.URL+"/p?q=1", "client body", nil) + reqText, respText, err := tr.Get(onlyExchangeID(t, tr)) + if err != nil { + t.Fatal(err) + } + if n := countRows(t, tr, "exchange_rewrites"); n != 0 { + t.Fatalf("%d trace rows", n) + } + return fmt.Sprintf("%d %s|%s|%s", resp.StatusCode, body, canon(reqText), canon(respText)) + } + base := exchange(nil) + other := exchange(rules(t, nil, nil, rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Matchers: rewrite.Matchers{Methods: []string{"DELETE"}}, + Actions: []rewrite.Action{{Type: rewrite.ActionSetBody, Value: "x"}}})) + if base != other { + t.Fatalf("without rules:\n%s\nwith non-matching rules:\n%s", base, other) + } + if all := got.all(); len(all) != 2 || all[0] != all[1] || all[0] != "POST /p?q=1 client body" { + t.Fatalf("target received %q", all) + } +} + +// --- atomicity and log redaction --------------------------------------------- + +// A trace that cannot be written rolls the whole exchange back: no rewritten +// exchange is ever left without its trace. The client is not affected, and the +// log names the host, the step and the rules, never the URL. +func TestRewriteTraceFailureRollsBack(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "target body") })) + t.Cleanup(target.Close) + tr, _, proxyURL := startProxy(t) + if _, err := tr.DB().Exec(`CREATE TRIGGER fail_trace BEFORE INSERT ON exchange_rewrites BEGIN SELECT RAISE(ABORT, 'forced'); END`); err != nil { + t.Fatal(err) + } + logs := captureLog(t) + tr.SetRewriteRules(rules(t, nil, nil, rewrite.Rule{ID: 7, Phase: rewrite.PhaseRequest, + Actions: []rewrite.Action{{Type: rewrite.ActionSetPathQuery, Value: "/secret-path?token=XYZ123"}}})) + resp, body := sendThrough(t, tr, proxyURL, "GET", target.URL+"/client-path?c=1", "", nil) + if resp.StatusCode != 200 || string(body) != "target body" { + t.Fatalf("client got %d %q", resp.StatusCode, body) + } + for _, table := range []string{"exchanges", "exchange_bodies", "exchange_rewrites", "blob_refs"} { + if n := countRows(t, tr, table); n != 0 { + t.Fatalf("%s holds %d rows after a failed trace insert", table, n) + } + } + out := logs.String() + if !strings.Contains(out, "failed to record a rewritten exchange of 127.0.0.1 (writing rewrite trace; rules [7])") { + t.Fatalf("log: %q", out) + } + for _, leak := range []string{"secret-path", "XYZ123", "client-path", "forced"} { + if strings.Contains(out, leak) { + t.Fatalf("log contains %q: %q", leak, out) + } + } +} + +// A transport error on a rewritten request clears its slot, keeps the host +// intercepted, and logs neither the URL a rule wrote nor the error text. +func TestRewriteRequestErrorRedactedAndCleared(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + dead := ln.Addr().String() + ln.Close() + tr, _, proxyURL := startProxy(t) + logs := captureLog(t) + tr.SetRewriteRules(rules(t, nil, nil, rewrite.Rule{ID: 3, Phase: rewrite.PhaseRequest, + Actions: []rewrite.Action{{Type: rewrite.ActionReplaceURL, Regex: "c=1", Value: "token=XYZ123"}}})) + resp, _ := sendThrough(t, tr, proxyURL, "GET", "http://"+dead+"/client-path?c=1", "", nil) + if resp.StatusCode != http.StatusBadGateway { + t.Fatalf("status %d", resp.StatusCode) + } + waitFor(t, "empty snapshot table", func() bool { e, tb := tr.snaps.counts(); return e == 0 && tb == 0 }) + out := logs.String() + if !strings.Contains(out, "a request to 127.0.0.1 rewritten by rules [3] failed (class transport_error)") { + t.Fatalf("log: %q", out) + } + for _, leak := range []string{"XYZ123", "client-path", dead, "refused"} { + if strings.Contains(out, leak) { + t.Fatalf("log contains %q: %q", leak, out) + } + } + if n := countRows(t, tr, "exchanges"); n != 0 { + t.Fatalf("%d exchanges", n) + } +} + +// RequestError never switches the host of a rewritten request to passthrough, +// records the rules' last error, and still does so for a request no rule changed. +func TestRewriteRequestErrorKeepsInterception(t *testing.T) { + tr, _ := openTraffic(t) + s := tr.proxy.Addons[0].(*sink) + errs := &rewrite.ErrorLog{} + tr.SetRewriteRules(rules(t, nil, errs, rewrite.Rule{ID: 9, Phase: rewrite.PhaseRequest, Matchers: rewrite.Matchers{Host: "rw.example"}, + Actions: []rewrite.Action{{Type: rewrite.ActionSetHeader, Name: "X-A", Value: "1"}}})) + protoErr := errors.New("http2: protocol error") + + f := withID(newFlow("rw.example", "GET", "/", nil, nil), 1) + s.Request(f) + if f.Request.Header.Get("X-A") != "1" { + t.Fatal("request not rewritten") + } + s.RequestError(f, protoErr) + if _, flagged := tr.pass.Load("rw.example"); flagged { + t.Fatal("a rewritten request flagged its host for passthrough") + } + if e, ok := errs.Get(9); !ok || e.Class != rewrite.ErrClassTransport { + t.Fatalf("last error %+v %v", e, ok) + } + g := withID(newFlow("plain.example", "GET", "/", nil, nil), 2) + s.Request(g) + s.RequestError(g, protoErr) + if _, flagged := tr.pass.Load("plain.example"); !flagged { + t.Fatal("an unrewritten request no longer flags its host") + } + if e, tb := tr.snaps.counts(); e != 0 || tb != 0 { + t.Fatalf("table holds %d entries, %d tombstones", e, tb) + } +} + +// --- snapshot table ---------------------------------------------------------- + +func liveEntry(cost int64) (*reqSnap, chan struct{}) { + ch := make(chan struct{}) + return &reqSnap{done: ch, cost: cost, host: "h.example", lines: []rewrite.TraceLine{{Phase: rewrite.PhaseRequest, RuleID: 4, Outcome: rewrite.OutcomeApplied}}, rewritten: true}, ch +} + +type fakeClock struct{ ns atomic.Int64 } + +func (c *fakeClock) now() time.Time { return time.Unix(0, c.ns.Load()) } +func (c *fakeClock) add(d time.Duration) { c.ns.Add(int64(d)) } + +func newFakeClock() *fakeClock { + c := &fakeClock{} + c.ns.Store(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).UnixNano()) + return c +} + +// The sweep a ClientDisconnected asks for removes the slots of finished flows +// only (Response and RequestError are covered by the proxy tests above). +func TestSnapshotClearedOnDisconnectSweep(t *testing.T) { + tr, _ := openTraffic(t) + s := tr.proxy.Addons[0].(*sink) + finished, done := liveEntry(10) + close(done) + live, _ := liveEntry(10) + if !tr.snaps.reserve(key(1), finished) || !tr.snaps.reserve(key(2), live) { + t.Fatal("reserve") + } + s.ClientDisconnected(&mproxy.ClientConn{}) + waitFor(t, "the sweep after a disconnect", func() bool { e, _ := tr.snaps.counts(); return e == 1 }) + if tr.snaps.slot(key(2)) == nil || tr.snaps.slot(key(1)) != nil { + t.Fatal("the sweep must remove the finished flow's slot and keep the live one") + } +} + +// One sweep per second at most, whatever the number of disconnects; a periodic +// pass without any; a live slot survives, even past the TTL. +func TestSnapshotSweepDebounce(t *testing.T) { + clk := newFakeClock() + tb := newSnapTable() + tb.limits(maxSnapEntries, maxSnapBytes, maxSnapTombs, clk.now) + tb.tick() + if tb.sweeps.Load() != 0 { + t.Fatal("an empty table was swept") + } + live, _ := liveEntry(10) + tb.reserve(key(1), live) + tb.tick() // first sweep: none before + start := tb.sweeps.Load() + for i := 0; i < 100; i++ { + tb.requestSweep() + tb.tick() + } + if tb.sweeps.Load() != start { + t.Fatalf("%d sweeps within the debounce", tb.sweeps.Load()-start) + } + clk.add(sweepDebounce) + tb.tick() + if tb.sweeps.Load() != start+1 { + t.Fatal("the requested sweep did not run after the debounce") + } + clk.add(30 * time.Second) + tb.tick() + if tb.sweeps.Load() != start+1 { + t.Fatal("swept without a request before the period") + } + clk.add(sweepPeriod + snapTTL) + tb.tick() + if tb.sweeps.Load() != start+2 { + t.Fatal("no periodic sweep") + } + if e, _ := tb.counts(); e != 1 { + t.Fatal("a live slot past the TTL was swept") + } +} + +// Close stops the sweeper goroutine. +func TestSnapshotSweeperStopsOnClose(t *testing.T) { + tr, err := Open(t.TempDir(), "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + go func() { _ = tr.Close(); close(done) }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Close did not return: the sweeper did not stop") + } +} + +// Past 1024 entries a reservation fails: the request is not rewritten and a +// capacity tombstone stands for the flow. +func TestSnapshotCapacityTombstone(t *testing.T) { + tb := newSnapTable() + for i := 0; i < maxSnapEntries; i++ { + e, _ := liveEntry(1) + if !tb.reserve(key(i), e) { + t.Fatalf("reservation %d failed", i) + } + } + e, _ := liveEntry(1) + if tb.reserve(key(5000), e) { + t.Fatal("reservation past 1024 entries succeeded") + } + if n, tombs := tb.counts(); n != maxSnapEntries || tombs != 1 { + t.Fatalf("%d entries, %d tombstones", n, tombs) + } + if s := tb.slot(key(5000)); s == nil || !s.tomb || s.reason != reasonSnapshotCapacity { + t.Fatalf("slot %+v", s) + } + if tb.capacity.Load() != 1 || tb.conversions.Load() != 0 { + t.Fatal("counters") + } + + // The 64 MiB budget bounds the entries too. + tb = newSnapTable() + for i := 0; i < 64; i++ { + e, _ := liveEntry(1 << 20) + if !tb.reserve(key(i), e) { + t.Fatalf("reservation %d failed", i) + } + } + if e, _ := liveEntry(1); tb.reserve(key(100), e) { + t.Fatal("reservation past 64 MiB succeeded") + } +} + +// Through the sink: a refused reservation leaves the request as the client +// sent it, and the exchange carries one global "snapshot capacity" line. +func TestSnapshotCapacityRecordsGlobalLine(t *testing.T) { + tr, _ := openTraffic(t) + s := tr.proxy.Addons[0].(*sink) + tr.snaps.limits(0, maxSnapBytes, maxSnapTombs, nil) + tr.SetRewriteRules(rules(t, nil, nil, rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{{Type: rewrite.ActionSetHeader, Name: "X-A", Value: "1"}}})) + f := withID(newFlow("api.example.com", "GET", "/", nil, []byte("ok")), 1) + s.Request(f) + if f.Request.Header.Get("X-A") != "" { + t.Fatal("the request was rewritten without a snapshot") + } + s.Response(f) + lines := traceRows(t, tr, onlyExchangeID(t, tr)) + if len(lines) != 1 || lines[0].ruleID.Valid || lines[0].outcome != "skipped" || lines[0].reason != reasonSnapshotCapacity { + t.Fatalf("trace %+v", lines) + } + if ok, _, _ := tr.RewriteInfo(onlyExchangeID(t, tr)); ok { + t.Fatal("an unrewritten exchange is flagged") + } +} + +// An entry of a live flow older than the TTL is turned into a tombstone in +// place when a reservation fails; the flow keeps its slot throughout. +func TestSnapshotConversionInPlace(t *testing.T) { + clk := newFakeClock() + tb := newSnapTable() + tb.limits(2, maxSnapBytes, maxSnapTombs, clk.now) + old, _ := liveEntry(10) + tb.reserve(key(1), old) + clk.add(time.Minute) + young, _ := liveEntry(10) + tb.reserve(key(2), young) + clk.add(snapTTL) + e, _ := liveEntry(10) + if !tb.reserve(key(3), e) { + t.Fatal("reservation failed although an old entry was convertible") + } + s := tb.slot(key(1)) + if s == nil || !s.tomb || s.reason != reasonOriginalUnavailable || !slices.Equal(s.ruleIDs, []int64{4}) || s.header != nil || s.lines != nil { + t.Fatalf("converted slot %+v", s) + } + if n, tombs := tb.counts(); n != 2 || tombs != 1 || tb.conversions.Load() != 1 { + t.Fatalf("%d entries %d tombstones %d conversions", n, tombs, tb.conversions.Load()) + } + if s := tb.slot(key(2)); s == nil || s.tomb { + t.Fatal("the younger entry was converted") + } +} + +// With every tombstone place held by a live flow, the conversion is refused: +// the old entry stays, the new reservation fails and, with no room for its +// tombstone either, is only counted. +func TestSnapshotConversionRefusedWithoutTombstoneRoom(t *testing.T) { + clk := newFakeClock() + tb := newSnapTable() + tb.limits(1, maxSnapBytes, 1, clk.now) + old, _ := liveEntry(10) + tb.reserve(key(1), old) + blocker, _ := liveEntry(10) + tb.reserve(key(2), blocker) // refused while old is young: the single capacity tombstone + clk.add(snapTTL + time.Minute) + e, _ := liveEntry(10) + if tb.reserve(key(3), e) { + t.Fatal("reservation succeeded") + } + if s := tb.slot(key(1)); s == nil || s.tomb { + t.Fatal("the old entry was converted without a tombstone place") + } + if tb.slot(key(3)) != nil || tb.tombDrops.Load() != 1 || tb.conversions.Load() != 0 { + t.Fatalf("drops %d conversions %d", tb.tombDrops.Load(), tb.conversions.Load()) + } + + // A tombstone of a finished flow is evicted to make room. + tb = newSnapTable() + tb.limits(1, maxSnapBytes, 1, clk.now) + old, _ = liveEntry(10) + tb.reserve(key(1), old) + gone, done := liveEntry(10) + tb.reserve(key(2), gone) // capacity tombstone of a flow that then ends + close(done) + clk.add(snapTTL + time.Minute) + tb.mu.Lock() // no sweep here: only the eviction can free the place + ok := tb.convertLocked(10) + tb.mu.Unlock() + if !ok { + t.Fatal("conversion refused although the only tombstone belongs to a finished flow") + } +} + +// The validated rule: an entry is converted only when releasing it lets the +// new reservation succeed. When the freed budget is too small, nothing is +// converted and the new request goes the capacity path. +func TestSnapshotConversionFreedBudgetInsufficient(t *testing.T) { + clk := newFakeClock() + tb := newSnapTable() + tb.limits(10, 1000, maxSnapTombs, clk.now) + old, _ := liveEntry(100) + tb.reserve(key(1), old) + clk.add(snapTTL + time.Minute) + young, _ := liveEntry(850) + tb.reserve(key(2), young) + e, _ := liveEntry(200) // 950 - 100 + 200 > 1000 + if tb.reserve(key(3), e) { + t.Fatal("reservation succeeded") + } + if s := tb.slot(key(1)); s == nil || s.tomb { + t.Fatal("the old entry was converted although its release frees too little") + } + if s := tb.slot(key(3)); s == nil || s.reason != reasonSnapshotCapacity || tb.conversions.Load() != 0 { + t.Fatalf("new slot %+v, conversions %d", s, tb.conversions.Load()) + } + // With enough freed, the same old entry is converted. + e, _ = liveEntry(140) // 950 - 100 + 140 <= 1000 + tb.mu.Lock() + tb.lastSweep = time.Time{} + tb.mu.Unlock() + if !tb.reserve(key(4), e) || tb.conversions.Load() != 1 { + t.Fatal("conversion refused although the release frees enough") + } +} + +// A failed reservation right after a sweep is refused by the debounce: no +// sweep, no conversion. +func TestSnapshotReservationDebounced(t *testing.T) { + clk := newFakeClock() + tb := newSnapTable() + tb.limits(1, maxSnapBytes, maxSnapTombs, clk.now) + old, _ := liveEntry(10) + tb.reserve(key(1), old) + clk.add(snapTTL + time.Minute) + tb.mu.Lock() + tb.lastSweep = clk.now() + tb.mu.Unlock() + e, _ := liveEntry(10) + if tb.reserve(key(2), e) || tb.conversions.Load() != 0 || tb.sweeps.Load() != 0 { + t.Fatal("a debounced reservation swept or converted") + } +} + +// End to end through the proxy: a live flow whose entry is converted under +// pressure is recorded "original request unavailable", with the client's +// method, URL and Host from Raw(), never the rewritten request. +func TestRewriteOriginalUnavailable(t *testing.T) { + arrived, release := make(chan struct{}), make(chan struct{}) + var got received + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got.add(r.URL.Path + " X-Rule=" + r.Header.Get("X-Rule")) + if r.URL.Path == "/slow" { + close(arrived) + <-release + } + io.WriteString(w, "ok") + })) + t.Cleanup(target.Close) + tr, _, proxyURL := startProxy(t) + clk := newFakeClock() + tr.snaps.limits(1, maxSnapBytes, maxSnapTombs, clk.now) + tr.SetRewriteRules(rules(t, nil, nil, rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{{Type: rewrite.ActionSetHeader, Name: "X-Rule", Value: "v"}}})) + + slowDone := make(chan string) + go func() { + resp, err := proxyClient(t, tr, proxyURL).Get(target.URL + "/slow?a=1") + if err != nil { + slowDone <- err.Error() + return + } + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + slowDone <- string(b) + }() + <-arrived + clk.add(snapTTL + time.Minute) + if _, body := sendThrough(t, tr, proxyURL, "GET", target.URL+"/fast", "", nil); string(body) != "ok" { + t.Fatalf("fast body %q", body) + } + close(release) + if b := <-slowDone; b != "ok" { + t.Fatalf("slow: %s", b) + } + if all := got.all(); !slices.Contains(all, "/slow X-Rule=v") || !slices.Contains(all, "/fast X-Rule=v") { + t.Fatalf("target received %q", all) + } + waitFor(t, "both exchanges", func() bool { return countRows(t, tr, "exchanges") == 2 }) + + id := exchangeIDByPath(t, tr, "/slow?a=1") + var method, u, host, reqHead string + var reqLen int + if err := tr.DB().QueryRow(`SELECT e.method,e.url,e.host,e.req_len,b.req_head FROM exchanges e JOIN exchange_bodies b ON b.id=e.id WHERE e.id=?`, id).Scan(&method, &u, &host, &reqLen, &reqHead); err != nil { + t.Fatal(err) + } + if method != "GET" || u != target.URL+"/slow?a=1" || host != "127.0.0.1" || reqLen != 0 || + reqHead != "GET /slow?a=1 HTTP/1.1\nHost: "+strings.TrimPrefix(target.URL, "http://")+"\n"+unavailableMarker { + t.Fatalf("exchange %s %s %s %d %q", method, u, host, reqLen, reqHead) + } + lines := traceRows(t, tr, id) + if len(lines) != 2 || lines[0].outcome != "error" || lines[0].reason != reasonOriginalUnavailable || lines[0].ruleID.Valid || + lines[1].outcome != "applied" || lines[1].ruleID.Int64 != 1 || lines[1].field.Valid { + t.Fatalf("trace %+v", lines) + } + if ok, ids, _ := tr.RewriteInfo(id); !ok || !slices.Equal(ids, []int64{1}) { + t.Fatal("RewriteInfo of the converted flow") + } + if fast := traceRows(t, tr, exchangeIDByPath(t, tr, "/fast")); len(fast) != 1 || fast[0].outcome != "applied" { + t.Fatalf("fast trace %+v", fast) + } + if tr.snaps.conversions.Load() != 1 { + t.Fatal("no conversion counted") + } +} + +// Without Raw(), the unavailable request records an empty method, URL and host. +func TestRewriteOriginalUnavailableWithoutRaw(t *testing.T) { + tr, _ := openTraffic(t) + s := tr.proxy.Addons[0].(*sink) + f := withID(newFlow("api.example.com", "PUT", "/rewritten", []byte("rewritten"), []byte("ok")), 1) + tr.snaps.mu.Lock() + tr.snaps.slots[keyOf(f)] = &reqSnap{tomb: true, reason: reasonOriginalUnavailable, ruleIDs: []int64{5}, host: "api.example.com"} + tr.snaps.tombs++ + tr.snaps.mu.Unlock() + s.Response(f) + var method, u, host, reqHead string + if err := tr.DB().QueryRow(`SELECT e.method,e.url,e.host,b.req_head FROM exchanges e JOIN exchange_bodies b ON b.id=e.id`).Scan(&method, &u, &host, &reqHead); err != nil { + t.Fatal(err) + } + if method != "" || u != "" || host != "" || strings.Contains(reqHead, "rewritten") || !strings.HasSuffix(reqHead, unavailableMarker) { + t.Fatalf("exchange %q %q %q %q", method, u, host, reqHead) + } +} + +// --- fail-open --------------------------------------------------------------- + +// A rule that panics is an error line; the flow goes on with the request and +// response untouched, and the exchange is recorded. +func TestRewriteRulePanicKeepsFlow(t *testing.T) { + var got received + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + got.add(string(b)) + io.WriteString(w, "target body") + })) + t.Cleanup(target.Close) + tr, _, proxyURL := startProxy(t) + tr.SetRewriteRules(rules(t, nil, nil, + rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{{Type: rewrite.ActionSetBody, Value: "x"}}}, + rewrite.Rule{ID: 2, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{{Type: rewrite.ActionSetBody, Value: "y"}}})) + setTestHook(t, func(stage string) { + if stage == "rule" { + panic("rule") + } + }) + resp, body := sendThrough(t, tr, proxyURL, "POST", target.URL+"/", "client body", nil) + if resp.StatusCode != 200 || string(body) != "target body" || !slices.Equal(got.all(), []string{"client body"}) { + t.Fatalf("client %d %q, target %q", resp.StatusCode, body, got.all()) + } + lines := traceRows(t, tr, onlyExchangeID(t, tr)) + if len(lines) != 2 || lines[0].outcome != "error" || lines[0].reason != rewrite.ErrClassPanic || lines[1].outcome != "error" { + t.Fatalf("trace %+v", lines) + } +} + +// A panic in the sink itself, after the engine, is recovered by the sink: the +// client still gets the target's response, not the library's empty 200. +func TestRewriteSinkRecoverKeepsFlow(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + io.WriteString(w, "target body") + })) + t.Cleanup(target.Close) + tr, _, proxyURL := startProxy(t) + tr.SetRewriteRules(rules(t, nil, nil, + rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{{Type: rewrite.ActionSetHeader, Name: "X-A", Value: "1"}}}, + rewrite.Rule{ID: 2, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{{Type: rewrite.ActionSetStatus, Value: "500"}}})) + logs := captureLog(t) + setTestHook(t, func(stage string) { + if stage == "request" || stage == "response" { + panic(stage) + } + }) + resp, body := sendThrough(t, tr, proxyURL, "GET", target.URL+"/p", "", nil) + if resp.StatusCode != http.StatusAccepted || string(body) != "target body" { + t.Fatalf("client %d %q", resp.StatusCode, body) + } + if n := countRows(t, tr, "exchanges"); n != 1 { + t.Fatalf("%d exchanges", n) + } + if out := logs.String(); !strings.Contains(out, "class panic") || strings.Contains(out, "/p") { + t.Fatalf("log %q", out) + } +} + +// --- 5 MiB passthrough ------------------------------------------------------- + +// StreamResponseModifier hands back the reader it got, nil included, and is +// nil-safe. +func TestStreamResponseModifierPassThrough(t *testing.T) { + tr, _ := openTraffic(t) + s := tr.proxy.Addons[0].(*sink) + f := withID(newFlow("a.example", "GET", "/", nil, nil), 1) + f.Stream = true + e, _ := liveEntry(1) + tr.snaps.reserve(keyOf(f), e) + if out := s.StreamResponseModifier(f, nil); out != nil { + t.Fatalf("nil reader became %v", out) + } + if n, _ := tr.snaps.counts(); n != 0 || tr.RewriteStats().LargeResponses != 1 { + t.Fatal("the slot of a streamed response was not released and counted") + } + r := strings.NewReader("x") + if out := s.StreamResponseModifier(f, r); out != io.Reader(r) { + t.Fatal("the reader was replaced") + } + if out := (&sink{}).StreamResponseModifier(nil, r); out != io.Reader(r) { + t.Fatal("nil flow / nil traffic") + } + if out := (*sink)(nil).StreamResponseModifier(f, r); out != io.Reader(r) { + t.Fatal("nil sink") + } +} + +// A rewritten request whose response reaches 5 MiB: the response reaches the +// client intact, nothing is recorded, the slot is released and counted. +func TestRewriteLargeResponsePassedThrough(t *testing.T) { + big := bytes.Repeat([]byte("0123456789abcdef"), (libraryStreamBody/16)+1024) + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-A") != "1" { + w.WriteHeader(400) + return + } + w.Write(big) + })) + t.Cleanup(target.Close) + tr, _, proxyURL := startProxy(t) + tr.SetRewriteRules(rules(t, nil, nil, rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{{Type: rewrite.ActionSetHeader, Name: "X-A", Value: "1"}}})) + resp, body := sendThrough(t, tr, proxyURL, "GET", target.URL+"/big", "", nil) + if resp.StatusCode != 200 || !bytes.Equal(body, big) { + t.Fatalf("status %d, %d bytes, intact=%v", resp.StatusCode, len(body), bytes.Equal(body, big)) + } + waitFor(t, "large response counted", func() bool { return tr.RewriteStats().LargeResponses == 1 }) + if n, tb := tr.snaps.counts(); n != 0 || tb != 0 || countRows(t, tr, "exchanges") != 0 { + t.Fatal("slot kept or exchange recorded") + } +} + +// --- purge, collisions, sensitive values, skip reasons ------------------------ + +// Deleting a host takes the trace with it, and gcBlobs frees the trace's blobs. +func TestRewriteDeleteHostTakesTrace(t *testing.T) { + tr, dir := openTraffic(t) + s := tr.proxy.Addons[0].(*sink) + large := strings.Repeat("L", maxInlineBody+1) + tr.SetRewriteRules(rules(t, nil, nil, rewrite.Rule{ID: 1, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{{Type: rewrite.ActionSetBody, Value: large}}})) + f := withID(newFlow("gone.example", "GET", "/", nil, []byte("small")), 1) + s.Request(f) + s.Response(f) + if string(f.Response.Body) != large { + t.Fatal("response not rewritten") + } + lines := traceRows(t, tr, onlyExchangeID(t, tr)) + if len(lines) != 1 || !lines[0].blob.Valid || len(lines[0].inline) == 0 { + t.Fatalf("trace %+v", lines) + } + blob := lines[0].blob.String + blobFile := dir + "/_blobs/sha256/" + blob[:2] + "/" + blob + ".bin" + if _, err := os.Stat(blobFile); err != nil { + t.Fatalf("trace blob: %v", err) + } + if n, err := tr.DeleteHost("gone.example"); err != nil || n != 1 { + t.Fatalf("DeleteHost = %d %v", n, err) + } + for _, table := range []string{"exchange_rewrites", "blob_refs", "exchanges"} { + if n := countRows(t, tr, table); n != 0 { + t.Fatalf("%s holds %d rows", table, n) + } + } + if _, err := os.Stat(blobFile); !os.IsNotExist(err) { + t.Fatalf("trace blob survived gcBlobs: %v", err) + } +} + +// An exchange id can repeat. Each of the three INSERT OR REPLACE paths clears +// the trace left under the id first, so the new exchange is never flagged. +func TestRewriteIDCollisionClearsTrace(t *testing.T) { + paths := map[string]func(tr *Traffic){ + "record": func(tr *Traffic) { tr.record(newFlow("c.example", "GET", "/", nil, nil)) }, + "RecordExternal": func(tr *Traffic) { + if err := tr.RecordExternal(ExternalExchange{Method: "GET", URL: "http://c.example/", Status: 200}); err != nil { + t.Fatal(err) + } + }, + "recordStreamStart": func(tr *Traffic) { + u, _ := url.Parse("https://c.example/ws") + f := &mproxy.Flow{Request: &mproxy.Request{Method: "GET", URL: u, Proto: "HTTP/1.1", Header: http.Header{}}, WebScoket: &mproxy.WebSocketData{}} + tr.proxy.Addons[0].(*sink).WebSocketStart(f) + }, + } + for name, write := range paths { + tr, _ := openTraffic(t) + var id string + for attempt := 0; ; attempt++ { + id = fmt.Sprintf("%d-%04d", time.Now().Unix(), (tr.seq.Load()+1)%10000) + if _, err := tr.DB().Exec(`INSERT INTO exchange_rewrites(exchange_id,seq,phase,rule_id,outcome) VALUES(?,0,'request',1,'applied')`, id); err != nil { + t.Fatal(err) + } + write(tr) + var got string + if err := tr.DB().QueryRow(`SELECT id FROM exchanges ORDER BY rowid DESC LIMIT 1`).Scan(&got); err != nil { + t.Fatal(err) + } + if got == id { + break + } + if attempt == 5 { + t.Fatalf("%s: could not land on the planted id", name) + } + if _, err := tr.DB().Exec(`DELETE FROM exchange_rewrites`); err != nil { + t.Fatal(err) + } + } + if ok, _, err := tr.RewriteInfo(id); err != nil || ok { + t.Fatalf("%s: the new exchange inherited the old trace", name) + } + } +} + +// A sensitive value is never in the trace: the line keeps rule id, field and +// a 128-bit HMAC under the trace sub-key, or no fingerprint without a key. +func TestRewriteSensitiveValueNotInTrace(t *testing.T) { + const secret = "Bearer s3cr3t-token-value" + master := []byte("instance key for the test") + m := hmac.New(sha256.New, master) + m.Write([]byte("artex-traffic-trace")) + sub := m.Sum(nil) + m = hmac.New(sha256.New, sub) + m.Write([]byte(secret)) + wantHash := hex.EncodeToString(m.Sum(nil)[:16]) + + for _, withKey := range []bool{true, false} { + tr, _ := openTraffic(t) + logs := captureLog(t) + s := tr.proxy.Addons[0].(*sink) + var k []byte + if withKey { + k = sub + } + tr.SetRewriteRules(rules(t, k, nil, rewrite.Rule{ID: 6, Phase: rewrite.PhaseRequest, + Actions: []rewrite.Action{{Type: rewrite.ActionSetHeader, Name: "Authorization", Value: secret, Sensitive: true}}})) + f := withID(newFlow("api.example.com", "GET", "/", nil, []byte("ok")), 1) + s.Request(f) + if f.Request.Header.Get("Authorization") != secret { + t.Fatal("the sensitive value did not reach the request") + } + s.Response(f) + id := onlyExchangeID(t, tr) + var dump string + if err := tr.DB().QueryRow(`SELECT group_concat(coalesce(rule_snapshot,'')||'|'||coalesce(after_inline,'')||'|'||coalesce(after_hash,'')||'|'||coalesce(before_hash,'')||'|'||reason, ';') FROM exchange_rewrites`).Scan(&dump); err != nil { + t.Fatal(err) + } + req, _, _ := tr.Get(id) + if strings.Contains(dump, "s3cr3t") || strings.Contains(req, "s3cr3t") || strings.Contains(logs.String(), "s3cr3t") { + t.Fatalf("secret leaked: trace %q request %q log %q", dump, req, logs.String()) + } + lines := traceRows(t, tr, id) + if len(lines) != 1 || lines[0].ruleID.Int64 != 6 || lines[0].field.String != "header:Authorization" || lines[0].inline != nil || lines[0].blob.Valid { + t.Fatalf("trace %+v", lines) + } + if withKey && lines[0].afterHash.String != wantHash { + t.Fatalf("fingerprint %q, want %q", lines[0].afterHash.String, wantHash) + } + if !withKey && lines[0].afterHash.Valid { + t.Fatal("a fingerprint without a key") + } + } +} + +// Body rules are skipped, with their reason, for a compressed, binary or too +// large body; rules without a body component still apply. +func TestRewriteBodySkipReasons(t *testing.T) { + tr, _ := openTraffic(t) + s := tr.proxy.Addons[0].(*sink) + tr.SetRewriteRules(rules(t, nil, nil, + rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{{Type: rewrite.ActionReplaceBody, Regex: "a", Value: "b"}}}, + rewrite.Rule{ID: 2, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{{Type: rewrite.ActionReplaceBody, Regex: "a", Value: "b"}}}, + rewrite.Rule{ID: 3, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{{Type: rewrite.ActionSetHeader, Name: "X-H", Value: "1"}}})) + cases := []struct { + name string + f *mproxy.Flow + want []string + }{ + {"content-encoding", withID(newFlow("e.example", "GET", "/ce", nil, []byte("aaa"), func(f *mproxy.Flow) { f.Response.Header.Set("Content-Encoding", "gzip") }), 1), + []string{"response/2/skipped/not rewritten: content-encoding", "response/3/applied/"}}, + {"binary", withID(newFlow("e.example", "GET", "/bin", nil, []byte("aaa"), withRespType("image/png")), 2), + []string{"response/2/skipped/not rewritten: binary", "response/3/applied/"}}, + {"too large", withID(newFlow("e.example", "POST", "/big", bytes.Repeat([]byte("a"), rewrite.MaxBodySize+1), []byte("x")), 3), + []string{"request/1/skipped/not rewritten: body too large", "response/3/applied/"}}, + } + for _, c := range cases { + s.Request(c.f) + s.Response(c.f) + var got []string + for _, l := range traceRows(t, tr, exchangeIDByPath(t, tr, c.f.Request.URL.Path)) { + got = append(got, fmt.Sprintf("%s/%d/%s/%s", l.phase, l.ruleID.Int64, l.outcome, l.reason)) + } + if !slices.Equal(got, c.want) { + t.Fatalf("%s: trace %q, want %q", c.name, got, c.want) + } + if c.f.Response.Header.Get("X-H") != "1" { + t.Fatalf("%s: the header rule did not apply", c.name) + } + } + if string(cases[0].f.Response.Body) != "aaa" || cases[0].f.Response.Header.Get("Content-Encoding") != "gzip" { + t.Fatal("a compressed body was rewritten") + } +} + +// --- streams ----------------------------------------------------------------- + +// SSE: the request is rewritten; at SSEStart the status and header actions +// reach the client, the body rule is traced "not rewritten: stream"; the +// exchange records the client's request and the target's status and headers, +// with the trace of both phases, and the events. +func TestRewriteSSEHeadersAndStatus(t *testing.T) { + var got received + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got.add(r.URL.RequestURI() + " X-Client=" + r.Header.Get("X-Client")) + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("X-Target", "t") + w.WriteHeader(200) + fl := w.(http.Flusher) + for _, ev := range []string{"id: 1\ndata: hello\n\n", "data: bye\n\n"} { + _, _ = io.WriteString(w, ev) + fl.Flush() + } + })) + t.Cleanup(srv.Close) + tr, _, proxyURL := startProxy(t) + tr.SetRewriteRules(rules(t, nil, nil, + rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{ + {Type: rewrite.ActionSetPathQuery, Value: "/rewritten-events?x=1"}, + {Type: rewrite.ActionSetHeader, Name: "X-Client", Value: "rewritten"}}}, + rewrite.Rule{ID: 2, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{ + {Type: rewrite.ActionSetStatus, Value: "203"}, + {Type: rewrite.ActionSetHeader, Name: "X-Rewritten", Value: "1"}}}, + rewrite.Rule{ID: 3, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{{Type: rewrite.ActionReplaceBody, Regex: "hello", Value: "HIJACKED"}}})) + + resp, body := sendThrough(t, tr, proxyURL, "GET", localhostURL(srv.URL)+"/events", "", http.Header{"X-Client": {"c"}}) + if resp.StatusCode != 203 || resp.Header.Get("X-Rewritten") != "1" || resp.Header.Get("X-Target") != "t" || + !strings.Contains(string(body), "data: hello") || strings.Contains(string(body), "HIJACKED") { + t.Fatalf("client got %d %v %q", resp.StatusCode, resp.Header, body) + } + if all := got.all(); !slices.Equal(all, []string{"/rewritten-events?x=1 X-Client=rewritten"}) { + t.Fatalf("target received %q", all) + } + var ex streamExchange + waitFor(t, "sse end", func() bool { + exs := streamExchanges(t, tr, "%event-stream%") + if len(exs) != 1 || exs[0].respLen == 0 { + return false + } + ex = exs[0] + return true + }) + if ex.status != 200 || ex.url != localhostURL(srv.URL)+"/events" { + t.Fatalf("exchange %+v", ex) + } + reqText, respText, err := tr.Get(ex.id) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(reqText, "GET /events HTTP/1.1\n") || !strings.Contains(reqText, "X-Client: c\n") || strings.Contains(reqText, "rewritten") || + !strings.HasPrefix(respText, "HTTP 200\n") || strings.Contains(respText, "X-Rewritten") { + t.Fatalf("recorded:\n%s\n%s", reqText, respText) + } + var summary []string + for _, l := range traceRows(t, tr, ex.id) { + summary = append(summary, fmt.Sprintf("%s/%d/%s/%s/%s", l.phase, l.ruleID.Int64, l.field.String, l.outcome, l.reason)) + } + want := []string{"request/1/url/applied/", "request/1/header:X-Client/applied/", + "response/2/header:X-Rewritten/applied/", "response/2/status/applied/", "response/3//skipped/not rewritten: stream"} + if !slices.Equal(summary, want) { + t.Fatalf("trace %q, want %q", summary, want) + } + if msgs := allMessages(t, tr, ex.id); len(msgs) != 2 || msgs[0].Payload != "hello" { + t.Fatalf("events %+v", msgs) + } + if n, tb := tr.snaps.counts(); n != 0 || tb != 0 { + t.Fatal("slot kept after SSEStart") + } +} + +// WebSocket: go-mitmproxy calls WebSocketStart after it has upgraded the +// client, with no response on the flow, so the handshake cannot be rewritten: +// every candidate rule is traced "not rewritten: stream", the handshake is +// recorded as the client sent it, and the frames pass unchanged. +func TestRewriteWebSocketTracedNotRewritten(t *testing.T) { + var gotRule atomic.Value + up := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }} + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotRule.Store(r.Header.Get("X-Rule")) + c, err := up.Upgrade(w, r, nil) + if err != nil { + return + } + defer c.Close() + for { + mt, msg, err := c.ReadMessage() + if err != nil { + return + } + if err := c.WriteMessage(mt, append([]byte("echo:"), msg...)); err != nil { + return + } + } + })) + t.Cleanup(srv.Close) + tr, _, proxyURL := startProxy(t) + tr.SetRewriteRules(rules(t, nil, nil, + rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{{Type: rewrite.ActionSetHeader, Name: "X-Rule", Value: "1"}}}, + rewrite.Rule{ID: 2, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{{Type: rewrite.ActionSetHeader, Name: "X-Resp", Value: "1"}}}, + rewrite.Rule{ID: 3, Phase: rewrite.PhaseResponse, Matchers: rewrite.Matchers{Host: "elsewhere.example"}, Actions: []rewrite.Action{{Type: rewrite.ActionSetHeader, Name: "X-Resp", Value: "2"}}})) + target := "wss://" + strings.TrimPrefix(localhostURL(srv.URL), "https://") + "/ws" + c, hs, err := wsDialer(t, tr, proxyURL).Dial(target, http.Header{"X-Client": {"c"}}) + if err != nil { + t.Fatal(err) + } + if hs.Header.Get("X-Resp") != "" { + t.Fatal("the handshake response was rewritten") + } + if err := c.WriteMessage(websocket.TextMessage, []byte("hi")); err != nil { + t.Fatal(err) + } + if _, msg, err := c.ReadMessage(); err != nil || string(msg) != "echo:hi" { + t.Fatalf("echo %q %v", msg, err) + } + closeWS(c) + if v, _ := gotRule.Load().(string); v != "" { + t.Fatal("the handshake request was rewritten") + } + var ex streamExchange + waitFor(t, "websocket end", func() bool { + exs := streamExchanges(t, tr, "websocket") + if len(exs) != 1 || exs[0].reqLen == 0 { + return false + } + ex = exs[0] + return true + }) + var summary []string + for _, l := range traceRows(t, tr, ex.id) { + summary = append(summary, fmt.Sprintf("%s/%d/%s/%s", l.phase, l.ruleID.Int64, l.outcome, l.reason)) + } + if want := []string{"request/1/skipped/not rewritten: stream", "response/2/skipped/not rewritten: stream"}; !slices.Equal(summary, want) { + t.Fatalf("trace %q, want %q", summary, want) + } + if ok, _, _ := tr.RewriteInfo(ex.id); ok { + t.Fatal("a websocket exchange is flagged rewritten") + } + reqText, _, _ := tr.Get(ex.id) + if !strings.Contains(reqText, "X-Client: c\n") || strings.Contains(reqText, "X-Rule") { + t.Fatalf("handshake recorded:\n%s", reqText) + } + if msgs := allMessages(t, tr, ex.id); len(msgs) != 2 || msgs[0].Payload != "hi" || msgs[1].Payload != "echo:hi" { + t.Fatalf("messages %+v", msgs) + } +} diff --git a/traffic/rewrite_rules_test.go b/traffic/rewrite_rules_test.go deleted file mode 100644 index 11ca1f4..0000000 --- a/traffic/rewrite_rules_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package traffic - -import ( - "io" - "net/http" - "net/http/httptest" - "slices" - "sort" - "strings" - "sync" - "testing" - - "github.com/Autumn-27/artex/traffic/rewrite" -) - -// The proxy stores the compiled rewrite rules and does not apply them. With a -// snapshot whose rules match every exchange, what the target receives, what the -// client receives and what is recorded are the same as with no rule at all. -func TestRewriteRulesStoredNotApplied(t *testing.T) { - var mu sync.Mutex - var seen []string - target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, _ := io.ReadAll(r.Body) - mu.Lock() - seen = append(seen, r.Method+" "+r.RequestURI+" X-Client="+r.Header.Get("X-Client")+" "+string(body)) - mu.Unlock() - w.Header().Set("X-Target", "t") - w.WriteHeader(http.StatusTeapot) - io.WriteString(w, "target body") - })) - t.Cleanup(target.Close) - - set := rewrite.Compile([]rewrite.Rule{ - {ID: 1, Name: "request", Enabled: true, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{ - {Type: rewrite.ActionSetMethod, Value: "PUT"}, - {Type: rewrite.ActionSetPathQuery, Value: "/rewritten"}, - {Type: rewrite.ActionSetHeader, Name: "X-Client", Value: "rewritten"}, - {Type: rewrite.ActionSetBody, Value: "rewritten"}}}, - {ID: 2, Name: "response", Enabled: true, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{ - {Type: rewrite.ActionSetStatus, Value: "200"}, - {Type: rewrite.ActionSetHeader, Name: "X-Rewritten", Value: "1"}, - {Type: rewrite.ActionSetBody, Value: "rewritten"}}}, - }, rewrite.Options{}) - if set.Len(rewrite.PhaseRequest) != 1 || set.Len(rewrite.PhaseResponse) != 1 || len(set.LoadErrors()) != 0 { - t.Fatalf("the snapshot must hold both rules: %v", set.LoadErrors()) - } - - // canon makes a recorded message comparable: the recorder writes headers in - // map order, and the target dates its response. - canon := func(raw string) string { - head, body, _ := strings.Cut(raw, "\n\n") - lines := strings.Split(head, "\n") - sort.Strings(lines[1:]) - lines = slices.DeleteFunc(lines, func(l string) bool { return strings.HasPrefix(l, "Date: ") }) - return strings.Join(lines, "\n") + "\n\n" + body - } - // exchange sends one request through a fresh proxy holding set and returns - // what the client received and what the proxy recorded. - exchange := func(set *rewrite.Set) (client, recorded string) { - tr, _, proxyURL := startProxy(t) - if tr.RewriteRules() != nil { - t.Fatal("Open published rewrite rules") - } - tr.SetRewriteRules(set) - if tr.RewriteRules() != set { - t.Fatal("RewriteRules must return the snapshot last published") - } - req, err := http.NewRequest("POST", target.URL+"/path?q=1", strings.NewReader("client body")) - if err != nil { - t.Fatal(err) - } - req.Header.Set("X-Client", "c") - resp, err := proxyClient(t, tr, proxyURL).Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - client = resp.Status + " X-Target=" + resp.Header.Get("X-Target") + " X-Rewritten=" + resp.Header.Get("X-Rewritten") + " " + string(body) - reqText, respText, err := tr.Get(onlyExchangeID(t, tr)) - if err != nil { - t.Fatal(err) - } - return client, canon(reqText) + "\n--\n" + canon(respText) - } - - baseClient, baseRecorded := exchange(nil) - client, recorded := exchange(set) - if want := "418 I'm a teapot X-Target=t X-Rewritten= target body"; baseClient != want || client != want { - t.Fatalf("client received %q without rules and %q with rules, want %q", baseClient, client, want) - } - if recorded != baseRecorded { - t.Fatalf("the recorded exchange differs with rules stored:\n without: %s\n with: %s", baseRecorded, recorded) - } - if !strings.Contains(recorded, "client body") || !strings.Contains(recorded, "target body") || strings.Contains(recorded, "rewritten") { - t.Fatalf("recorded exchange: %s", recorded) - } - want := "POST /path?q=1 X-Client=c client body" - if len(seen) != 2 || seen[0] != want || seen[1] != want { - t.Fatalf("the target received %q, want %q twice", seen, want) - } -} diff --git a/traffic/stream.go b/traffic/stream.go index c502ebc..d58c5a0 100644 --- a/traffic/stream.go +++ b/traffic/stream.go @@ -16,6 +16,7 @@ import ( "unsafe" "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/artex/traffic/rewrite" mproxy "github.com/lqqyt2423/go-mitmproxy/proxy" ) @@ -144,17 +145,35 @@ func (s *sink) startStream(f *mproxy.Flow, kind string) { if f.Request == nil || f.Request.URL == nil || (kind == "sse" && f.Response == nil) { return } - id, err := s.t.recordStreamStart(f, kind) + rq, rs, lines, res := s.rewriteStream(f, kind) + id, err := s.t.recordStreamStart(rq, rs, kind, lines) + s.applyStreamResponse(f, res) if err != nil { + if rewritten, ruleIDs := appliedRules(lines); rewritten { + var re *recordError + errors.As(err, &re) + log.Printf("[traffic] failed to record a rewritten %s stream of %s (%s; rules %v)", kind, f.Request.URL.Hostname(), re.step, ruleIDs) + return + } log.Printf("[traffic] failed to record %s stream %s: %v", kind, f.Request.URL, err) return } st.id = id if kind == "sse" { - st.reqBytes = int64(len(f.Request.Body)) + st.reqBytes = int64(len(rq.body)) } } +// recordError names the step of a recording that failed. Its text is the step +// and the cause, as recordStreamStart always reported. +type recordError struct { + step string + err error +} + +func (e *recordError) Error() string { return e.step + ": " + e.err.Error() } +func (e *recordError) Unwrap() error { return e.err } + // libMutex returns the unexported mu sync.Mutex that go-mitmproxy holds while it // appends to WebSocketData.Messages / SSEData.Events. The library exposes no // accessor, and the two WebSocket goroutines append concurrently, so reading the @@ -252,9 +271,9 @@ func sseEventsSince(sse *mproxy.SSEData, st *streamState) []streamRow { // streamDisplayURL is the URL shown for a WebSocket exchange: http→ws and // https→wss. A plain ws:// connection reaches the WebSocket hooks through its // CONNECT tunnel, where only the target authority is known, not the path. -func streamDisplayURL(f *mproxy.Flow, kind string) (method, display, tmpl string) { - u := *f.Request.URL - method = f.Request.Method +func streamDisplayURL(reqMethod string, reqURL *url.URL, kind string) (method, display, tmpl string) { + u := *reqURL + method = reqMethod if kind == "websocket" && strings.EqualFold(method, "CONNECT") { host := u.Host if host == "" { @@ -270,63 +289,77 @@ func streamDisplayURL(f *mproxy.Flow, kind string) (method, display, tmpl string u.Scheme = "wss" } } - return method, u.String(), db.TemplatePath(f.Request.URL.EscapedPath()) + return method, u.String(), db.TemplatePath(reqURL.EscapedPath()) } // recordStreamStart writes the stream's exchange: metadata, request head and -// body, the response head (synthetic 101 for WebSocket), and the header text in -// the full-text index. Messages are never indexed. -func (t *Traffic) recordStreamStart(f *mproxy.Flow, kind string) (string, error) { +// body, the response head (synthetic 101 for WebSocket), the rewrite trace, +// and the header text in the full-text index. Messages are never indexed. rq +// is the request the client sent and rs (SSE only) the status and headers the +// target sent, whatever the rules changed. +func (t *Traffic) recordStreamStart(rq requestView, rs responseView, kind string, lines []rewrite.TraceLine) (string, error) { now := time.Now() id := t.newExchangeID(now) - method, display, tmpl := streamDisplayURL(f, kind) - host := f.Request.URL.Hostname() - target := f.Request.URL.RequestURI() - if strings.EqualFold(f.Request.Method, "CONNECT") { - target = f.Request.URL.Host // authority-form, as on the wire + method, display, tmpl, host, target := rq.method, "", "", "", "" + if rq.url != nil { + method, display, tmpl = streamDisplayURL(rq.method, rq.url, kind) + host = rq.url.Hostname() + target = rq.url.RequestURI() + if strings.EqualFold(rq.method, "CONNECT") { + target = rq.url.Host // authority-form, as on the wire + } } - reqHead := fmt.Sprintf("%s %s %s\n%s", f.Request.Method, target, f.Request.Proto, requestHeaderLines(f.Request)) + reqHead := requestHead(rq, target) status, ct, respHead := 101, "websocket", "HTTP/1.1 101 Switching Protocols\n" if kind == "sse" { - status = f.Response.StatusCode - ct = f.Response.Header.Get("Content-Type") - respHead = fmt.Sprintf("HTTP %d\n%s", status, headerLines(f.Response.Header)) + status = rs.status + ct = rs.header.Get("Content-Type") + respHead = fmt.Sprintf("HTTP %d\n%s", status, headerLines(rs.header)) } t.wmu.Lock() defer t.wmu.Unlock() - reqB := t.spill(f.Request.Body, f.Request.Header.Get("Content-Type")) + reqB := t.spill(rq.body, rq.header.Get("Content-Type")) + trace := t.spillTrace(lines) tx, err := t.db.Begin() if err != nil { - return "", fmt.Errorf("beginning transaction: %w", err) + return "", &recordError{"beginning transaction", err} } defer tx.Rollback() //nolint:errcheck // no-op once committed + // An id can repeat (seconds plus a rolling counter); the trace of the + // exchange it replaces must not stay attached to this one. + if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id=?`, id); err != nil { + return "", &recordError{"clearing rewrite trace", err} + } res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path) -VALUES(?,?,?,?,?,?,?,?,?,?,'')`, id, now.Unix(), host, method, tmpl, display, status, ct, len(f.Request.Body), 0) +VALUES(?,?,?,?,?,?,?,?,?,?,'')`, id, now.Unix(), host, method, tmpl, display, status, ct, len(rq.body), 0) if err != nil { - return "", fmt.Errorf("writing index: %w", err) + return "", &recordError{"writing index", err} } rowid, err := res.LastInsertId() if err != nil { - return "", fmt.Errorf("fetching rowid: %w", err) + return "", &recordError{"fetching rowid", err} } if _, err := tx.Exec(`INSERT OR REPLACE INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob) VALUES(?,?,?,?,?,NULL,NULL)`, id, reqHead, reqB.inline, nullIfEmpty(reqB.hash), respHead); err != nil { - return "", fmt.Errorf("writing body: %w", err) + return "", &recordError{"writing body", err} } if reqB.hash != "" { if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, reqB.hash, id); err != nil { - return "", fmt.Errorf("registering blob reference: %w", err) + return "", &recordError{"registering blob reference", err} } } + if err := writeTrace(tx, id, lines, trace); err != nil { + return "", err + } if t.fts { idx := strings.Join([]string{display, reqHead, reqB.index, respHead}, "\n") if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowid, idx); err != nil { - return "", fmt.Errorf("writing full-text index: %w", err) + return "", &recordError{"writing full-text index", err} } } if err := tx.Commit(); err != nil { - return "", fmt.Errorf("committing: %w", err) + return "", &recordError{"committing", err} } return id, nil } diff --git a/traffic/traffic.go b/traffic/traffic.go index 17b1c05..3ace802 100644 --- a/traffic/traffic.go +++ b/traffic/traffic.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "sort" "strconv" "strings" @@ -92,6 +93,26 @@ CREATE TABLE IF NOT EXISTS stream_messages ( PRIMARY KEY (exchange_id, seq) ); CREATE INDEX IF NOT EXISTS idx_stream_messages_ex ON stream_messages(exchange_id, seq); + +-- Rewrite-rule trace of an exchange, in order. The exchange row keeps what the +-- client sent and what the target answered; these rows say what the rules +-- changed on the way. An exchange is rewritten when it has an 'applied' row. +CREATE TABLE IF NOT EXISTS exchange_rewrites ( + exchange_id TEXT NOT NULL, + seq INTEGER NOT NULL, + phase TEXT NOT NULL, -- 'request' | 'response' + rule_id INTEGER, -- NULL on a global line + rule_snapshot TEXT, -- JSON, no sensitive action value + action TEXT, + field TEXT, -- 'method' | 'url' | 'header:' | 'body' | 'status' + outcome TEXT NOT NULL, -- 'applied' | 'skipped' | 'error' + reason TEXT NOT NULL DEFAULT '', + before_hash TEXT, + after_hash TEXT, + after_inline BLOB, + after_blob TEXT, + PRIMARY KEY (exchange_id, seq) +); ` // ftsSchema is applied separately from indexSchema: a driver build without FTS5 @@ -160,6 +181,10 @@ const ( // and the catalog migration cannot drift apart. const TrafficSearchDescription = "Query the target traffic the recording proxy has captured (host is required; accepts a bare host, host:port, or full URL, optionally filtered further by URL substring or body keyword). When a port is specified, only that service's traffic is returned, avoiding cross-talk between different ports on the same IP. body_contains runs a full-text search over the captured request/response headers and bodies, supporting any substring including Chinese (at least 3 characters). Returns only an ultra-light index (id/method/url/status/resp_len) with no response content; once results are non-empty, use traffic_get to verify the request/response of each one, then hand the IDs that actually support the current vulnerability to bind_finding_traffic. Returns 3 by default, at most 10 per page; use page to paginate when there are more results." +// TrafficGetDescription is the traffic_get description, kept here for the same +// reason as TrafficSearchDescription. +const TrafficGetDescription = "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling. A WebSocket (status 101) or SSE (text/event-stream) record shows only its handshake here; read its messages with traffic_messages. When proxy rewrite rules changed the exchange, the result starts with a REWRITTEN line naming the rule ids, and shows the request as the client sent it and the response as the target sent it. traffic_search searches only these original values: it never finds a value a rewrite rule wrote." + // Traffic runs the recording proxy and owns the file tree + index. type Traffic struct { dir string @@ -199,8 +224,14 @@ type Traffic struct { // so no host escapes it. Hot-swappable at runtime via SetUpstreamProxy. upstream atomic.Pointer[url.URL] // rewriteRules is the compiled snapshot of the rewrite rules, replaced as a - // whole by SetRewriteRules. The proxy does not apply it yet. + // whole by SetRewriteRules and applied by the sink (rewrite_flow.go). rewriteRules atomic.Pointer[rewrite.Set] + // snaps keeps the original of each request a rule rewrote until its + // exchange is recorded (rewrite_flow.go). Nil on the zero value. + snaps *snapTable + // sweeping tracks the snapshot sweeper, which runs until Close. It is kept + // apart from reaping, which only ever holds work that finishes on its own. + sweeping sync.WaitGroup } // Open initializes the traffic tree, blob store and SQLite index under dir. @@ -226,7 +257,7 @@ func Open(dir, addr string) (*Traffic, error) { if err != nil { return nil, err } - t := &Traffic{dir: dir, addr: addr, db: db, closed: make(chan struct{})} + t := &Traffic{dir: dir, addr: addr, db: db, closed: make(chan struct{}), snaps: newSnapTable()} if err := t.initIndex(); err != nil { db.Close() return nil, err @@ -258,6 +289,7 @@ func Open(dir, addr string) (*Traffic, error) { }) p.AddAddon(&sink{t: t}) t.proxy = p + t.sweeping.Go(t.sweepSnapshots) // stops on t.closed; Close waits for it return t, nil } @@ -358,8 +390,8 @@ func (t *Traffic) SetUpstreamProxy(raw string) error { } // SetRewriteRules publishes a compiled snapshot of the rewrite rules. The swap -// is atomic; nil clears the rules. The snapshot is stored only: the proxy does -// not apply rewrite rules yet, so capture is unchanged whatever it holds. +// is atomic; nil clears the rules. A flow already rewritten keeps the snapshot +// its request saw for its response as well. func (t *Traffic) SetRewriteRules(s *rewrite.Set) { t.rewriteRules.Store(s) } // RewriteRules returns the snapshot last published, nil when there is none. @@ -406,6 +438,7 @@ func (t *Traffic) Close() error { t.closeOnce.Do(func() { close(t.closed) }) } t.reaping.Wait() + t.sweeping.Wait() return t.db.Close() } @@ -430,21 +463,6 @@ type sink struct { streams sync.Map // *mproxy.Flow of a live WebSocket/SSE stream -> *streamState (stream.go) } -func (s *sink) Response(f *mproxy.Flow) { - if f.Request == nil || f.Response == nil { - return - } - s.t.record(f) -} - -// RequestError fires when a request through an established MITM tunnel fails. If -// the failure looks proxy/protocol-caused (h2 quirks, HEAD-with-body, protocol -// errors) and not a plain target-unreachable error, the host is tunneled for -// passthroughTTL. This request has already failed. Later requests in the window -// connect straight through; the first request after the window tries -// interception again. -func (s *sink) RequestError(f *mproxy.Flow, err error) { s.t.maybePassthrough(f, err) } - // maybePassthrough marks a host to be tunneled transparently on the next // connection, but only for errors the proxy itself caused — a target that is // simply down/filtered would fail without us too, and must stay MITM'd+recorded. @@ -502,52 +520,121 @@ func proxyCausedErr(err error) bool { // full-text index. Nothing is written to a per-request directory — only bodies // above maxInlineBody spill to the content-addressed blob store. func (t *Traffic) record(f *mproxy.Flow) { + t.recordViews(flowRequestView(f.Request), flowResponseView(f.Response), nil) +} + +// requestView is the request an exchange records: what the client sent, which +// is not the flow's request once a rule rewrote it. +type requestView struct { + method string + url *url.URL // nil when the original request is unavailable and Raw() is nil + proto string + header http.Header + // rawHost is the client's Host header (Raw().Host), "" when unknown. + rawHost string + body []byte + // unavailable marks a request rewritten by a rule whose original was not + // kept: only its method, URL and Host are known. + unavailable bool +} + +// responseView is the response an exchange records: what the target answered. +type responseView struct { + status int + header http.Header + body []byte +} + +func flowRequestView(r *mproxy.Request) requestView { + v := requestView{method: r.Method, url: r.URL, proto: r.Proto, header: r.Header, body: r.Body} + if raw := r.Raw(); raw != nil { + v.rawHost = raw.Host + } + return v +} + +func flowResponseView(r *mproxy.Response) responseView { + return responseView{status: r.StatusCode, header: r.Header, body: r.Body} +} + +// unavailableMarker stands for the headers and body of a request whose +// original was not kept (see rewrite_flow.go). +const unavailableMarker = "[original request unavailable]\n" + +// recordViews writes one exchange from explicit views, with its rewrite trace, +// in one transaction: the exchange row, its bodies, blob references, the trace +// and the full-text index of the views only. A failure rolls everything back, +// so there is never a rewritten exchange without its trace. Without trace +// lines this is exactly what record always wrote. +func (t *Traffic) recordViews(rq requestView, rs responseView, lines []rewrite.TraceLine) { // The write lock covers blob + index writes, so DeleteHost (and its blob GC) // can run under the same lock without racing a concurrent record. t.wmu.Lock() defer t.wmu.Unlock() - host := f.Request.URL.Hostname() - method := f.Request.Method - tmpl := db.TemplatePath(f.Request.URL.EscapedPath()) + var host, tmpl, url, uri string + if rq.url != nil { + host = rq.url.Hostname() + tmpl = db.TemplatePath(rq.url.EscapedPath()) + url = rq.url.String() + uri = rq.url.RequestURI() + } + method := rq.method now := time.Now() id := t.newExchangeID(now) - ct := f.Response.Header.Get("Content-Type") - url := f.Request.URL.String() - reqHead := fmt.Sprintf("%s %s %s\n%s", method, f.Request.URL.RequestURI(), f.Request.Proto, requestHeaderLines(f.Request)) - respHead := fmt.Sprintf("HTTP %d\n%s", f.Response.StatusCode, headerLines(f.Response.Header)) + ct := rs.header.Get("Content-Type") + reqHead := requestHead(rq, uri) + respHead := fmt.Sprintf("HTTP %d\n%s", rs.status, headerLines(rs.header)) // Bodies are spilled before the transaction opens: blob writes are filesystem // work and must not sit inside the SQLite write lock. - reqB := t.spill(f.Request.Body, f.Request.Header.Get("Content-Type")) - respB := t.spill(f.Response.Body, ct) + reqB := t.spill(rq.body, rq.header.Get("Content-Type")) + respB := t.spill(rs.body, ct) + trace := t.spillTrace(lines) + + // A rewritten exchange is logged by host, step and rule ids only: its URL + // and the error text may carry a value a rule wrote. + rewritten, ruleIDs := appliedRules(lines) + fail := func(step string, err error) { + if rewritten { + log.Printf("[traffic] failed to record a rewritten exchange of %s (%s; rules %v)", host, step, ruleIDs) + return + } + log.Printf("[traffic] failed to record %s (%s): %v", url, step, err) + } tx, err := t.db.Begin() if err != nil { - log.Printf("[traffic] failed to record %s (beginning transaction): %v", url, err) + fail("beginning transaction", err) return } defer tx.Rollback() //nolint:errcheck // no-op once committed + // An id can repeat (seconds plus a rolling counter); the trace of the + // exchange it replaces must not stay attached to this one. + if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id=?`, id); err != nil { + fail("clearing rewrite trace", err) + return + } // path stays empty for db-resident exchanges; a non-empty path marks a legacy // row whose bodies still live in the old on-disk tree (see Get). res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path) VALUES(?,?,?,?,?,?,?,?,?,?,'')`, - id, now.Unix(), host, method, tmpl, url, f.Response.StatusCode, ct, - len(f.Request.Body), len(f.Response.Body)) + id, now.Unix(), host, method, tmpl, url, rs.status, ct, + len(rq.body), len(rs.body)) if err != nil { - log.Printf("[traffic] failed to record %s (writing index): %v", url, err) + fail("writing index", err) return } rowid, err := res.LastInsertId() if err != nil { - log.Printf("[traffic] failed to record %s (fetching rowid): %v", url, err) + fail("fetching rowid", err) return } if _, err := tx.Exec(`INSERT OR REPLACE INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob) VALUES(?,?,?,?,?,?,?)`, id, reqHead, reqB.inline, nullIfEmpty(reqB.hash), respHead, respB.inline, nullIfEmpty(respB.hash)); err != nil { - log.Printf("[traffic] failed to record %s (writing body): %v", url, err) + fail("writing body", err) return } @@ -556,26 +643,51 @@ VALUES(?,?,?,?,?,?,?)`, continue } if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, h, id); err != nil { - log.Printf("[traffic] failed to record %s (registering blob reference): %v", url, err) + fail("registering blob reference", err) return } } + if err := writeTrace(tx, id, lines, trace); err != nil { + var re *recordError + errors.As(err, &re) + fail(re.step, re.err) + return + } + if t.fts { // The index is fed from memory, so a body that spilled to _blobs is still - // fully searchable even though only its preview is stored inline. + // fully searchable even though only its preview is stored inline. Only the + // recorded views are indexed, never a value a rule wrote. idx := strings.Join([]string{url, reqHead, reqB.index, respHead, respB.index}, "\n") if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowid, idx); err != nil { - log.Printf("[traffic] failed to record %s (writing full-text index): %v", url, err) + fail("writing full-text index", err) return } } if err := tx.Commit(); err != nil { - log.Printf("[traffic] failed to record %s (committing): %v", url, err) + fail("committing", err) } } +// appliedRules reports whether a trace rewrote its exchange, with the ids of +// the rules that did, in order of first appearance. +func appliedRules(lines []rewrite.TraceLine) (bool, []int64) { + rewritten := false + var ids []int64 + for _, l := range lines { + if l.Outcome != rewrite.OutcomeApplied { + continue + } + rewritten = true + if l.RuleID != 0 && !slices.Contains(ids, l.RuleID) { + ids = append(ids, l.RuleID) + } + } + return rewritten, ids +} + // newExchangeID names a recorded exchange: unix seconds plus a rolling sequence. func (t *Traffic) newExchangeID(now time.Time) string { n := t.seq.Add(1) @@ -726,15 +838,35 @@ func headerLines(h map[string][]string) string { return b.String() } -// requestHeaderLines restores the HTTP Host header, which net/http stores on -// Request.Host rather than in Header. Keeping it in request.http makes the raw -// capture complete and directly replayable. +// requestHeaderLines is requestHeaderLinesOf for a flow's request. func requestHeaderLines(req *mproxy.Request) string { - headers := req.Header.Clone() + v := flowRequestView(req) + return requestHeaderLinesOf(v.header, req.URL.Host, v.rawHost) +} + +// requestHead is the head an exchange records for a request view: request +// line and headers, or only the Host when the original request is unavailable. +func requestHead(rq requestView, target string) string { + urlHost := "" + if rq.url != nil { + urlHost = rq.url.Host + } + if rq.unavailable { + return fmt.Sprintf("%s %s %s\n%s%s", rq.method, target, rq.proto, requestHeaderLinesOf(nil, urlHost, rq.rawHost), unavailableMarker) + } + return fmt.Sprintf("%s %s %s\n%s", rq.method, target, rq.proto, requestHeaderLinesOf(rq.header, urlHost, rq.rawHost)) +} + +// requestHeaderLinesOf restores the HTTP Host header, which net/http stores on +// Request.Host rather than in Header. Keeping it in request.http makes the raw +// capture complete and directly replayable. rawHost is the client's Host +// header (Raw().Host); urlHost stands in when it is empty. +func requestHeaderLinesOf(header http.Header, urlHost, rawHost string) string { + headers := header.Clone() headers.Del("Host") - host := req.URL.Host - if raw := req.Raw(); raw != nil && strings.TrimSpace(raw.Host) != "" { - host = raw.Host + host := urlHost + if strings.TrimSpace(rawHost) != "" { + host = rawHost } var b strings.Builder if host = strings.TrimSpace(host); host != "" { @@ -1271,6 +1403,9 @@ func (t *Traffic) deleteWhere(tx *sql.Tx, where string, args ...any) (int64, err if _, err := tx.Exec(`DELETE FROM stream_messages WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil { return 0, fmt.Errorf("delete stream messages: %w", err) } + if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil { + return 0, fmt.Errorf("delete rewrite trace: %w", err) + } if _, err := tx.Exec(`DELETE FROM blob_refs WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil { return 0, fmt.Errorf("delete blob references: %w", err) } @@ -2138,7 +2273,7 @@ func (t *Traffic) Tools() []actool.CoreTool { get := actool.Build(actool.Spec{ Name: "traffic_get", - Description: "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling. A WebSocket (status 101) or SSE (text/event-stream) record shows only its handshake here; read its messages with traffic_messages.", + Description: TrafficGetDescription, Schema: map[string]any{ "type": "object", "properties": map[string]any{"id": map[string]any{"type": "string", "description": "the id returned by traffic_search"}}, @@ -2153,7 +2288,15 @@ func (t *Traffic) Tools() []actool.CoreTool { if err != nil { return actool.Errorf(err.Error()), nil } - return actool.Text("=== REQUEST ===\n" + clip(req, 2500) + "\n\n=== RESPONSE ===\n" + clip(resp, 4000)), nil + rewritten, ruleIDs, err := t.RewriteInfo(a.ID) + if err != nil { + return actool.Errorf(err.Error()), nil + } + head := "" + if rewritten { + head = fmt.Sprintf("=== REWRITTEN === rewrite rules %v changed this exchange in the proxy. Shown below: the request as the client sent it and the response as the target sent it, not the rewritten values.\n\n", ruleIDs) + } + return actool.Text(head + "=== REQUEST ===\n" + clip(req, 2500) + "\n\n=== RESPONSE ===\n" + clip(resp, 4000)), nil }, })