mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-19 14:40:53 +08:00
feat(traffic): apply rewrite rules in the capture proxy (P3.3b1)
The sink runs the request phase in the Request hook and the response phase at the start of the Response hook; at SSEStart only status and header actions apply. WebSocketStart comes after the client is upgraded, so the handshake is traced "not rewritten: stream" and recorded as sent. Exchanges keep what the client sent and what the target answered. The original of a rewritten request lives in a per-flow snapshot slot (entry or tombstone, 1024 entries / 64 MiB, 4096 tombstones), reserved before the request is changed, released on Response, RequestError, SSEStart, a 5 MiB response, or by a sweep of finished flows (ClientDisconnected, 1 s debounce, 1 min period). Under pressure an entry older than 15 minutes is turned into a tombstone in place, only when that frees room for the new reservation; its exchange is recorded "original request unavailable". record(), recordStreamStart() and insertExchange() clear any trace under the id before INSERT OR REPLACE; the trace (exchange_rewrites, created with CREATE TABLE IF NOT EXISTS) is written in the exchange's transaction, its spilled values registered in blob_refs, and purged with the exchange. The full-text index covers the originals only. Logs of a rewritten flow name the host, the step or error class and the rule ids, never the URL or the error text. A rewritten request's transport error never switches its host to passthrough. traffic_get shows the rewritten flag and rule ids. Without a rule that applies, what is recorded is unchanged byte for byte.
This commit is contained in:
@@ -76,6 +76,11 @@ func (t *Traffic) insertExchange(id string, ts int64, host, method, urlTemplate,
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // no-op once committed
|
||||
|
||||
// An id can repeat (seconds plus a rolling counter); the trace of the
|
||||
// exchange it replaces must not stay attached to this one.
|
||||
if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id=?`, id); err != nil {
|
||||
return fmt.Errorf("clearing rewrite trace: %w", err)
|
||||
}
|
||||
res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,'')`,
|
||||
id, ts, host, method, urlTemplate, rawURL, status, contentType, reqLen, respLen)
|
||||
|
||||
+5
-1
@@ -40,11 +40,15 @@ func (s *sink) Requestheaders(f *mproxy.Flow) {
|
||||
}
|
||||
}
|
||||
|
||||
// ClientDisconnected forgets a closed connection's mark.
|
||||
// ClientDisconnected forgets a closed connection's mark, and asks for a sweep
|
||||
// of the request snapshots its flows may have left (rewrite_flow.go).
|
||||
func (s *sink) ClientDisconnected(c *mproxy.ClientConn) {
|
||||
if c != nil {
|
||||
s.paced.Delete(c.Id)
|
||||
}
|
||||
if s.t != nil {
|
||||
s.t.snaps.requestSweep()
|
||||
}
|
||||
}
|
||||
|
||||
// pace waits for the request's host under the current noise profile, bounded
|
||||
|
||||
@@ -0,0 +1,663 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/traffic/rewrite"
|
||||
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
|
||||
)
|
||||
|
||||
// Applying the rewrite rules to the proxied flows.
|
||||
//
|
||||
// The request phase runs in the Request hook. A request a rule rewrote leaves
|
||||
// the proxy changed, but its exchange records what the client sent: the
|
||||
// original is kept in a per-flow request snapshot until the Response hook
|
||||
// records the exchange. The response phase runs at the start of the Response
|
||||
// hook (status and headers only at SSEStart), and the exchange records the
|
||||
// response as the target sent it. The trace of both phases is written in the
|
||||
// exchange's own transaction (exchange_rewrites).
|
||||
//
|
||||
// A request is rewritten only once its snapshot slot is reserved, and a slot
|
||||
// stays until the flow's exchange is recorded or the flow is finished, so the
|
||||
// Response hook never takes a rewritten request for the client's.
|
||||
|
||||
// Limits of the request snapshots.
|
||||
const (
|
||||
maxSnapEntries = 1024 // entries holding an original request
|
||||
maxSnapBytes = 64 << 20 // what those entries hold
|
||||
maxSnapTombs = 4096 // tombstones: slots whose data was released
|
||||
// snapTTL deletes nothing: past it, the entry of a live flow may be turned
|
||||
// into a tombstone when a new reservation finds no room.
|
||||
snapTTL = 15 * time.Minute
|
||||
sweepDebounce = time.Second // at most one sweep per second
|
||||
sweepPeriod = time.Minute // a sweep at least this often while slots exist
|
||||
// libraryStreamBody is go-mitmproxy's StreamLargeBodies default: a body this
|
||||
// large reaches neither the Request nor the Response hook.
|
||||
libraryStreamBody = 5 << 20
|
||||
|
||||
reasonSnapshotCapacity = "not rewritten: snapshot capacity"
|
||||
reasonOriginalUnavailable = "original request unavailable"
|
||||
)
|
||||
|
||||
// testRuleHook, when a test sets it, runs inside each rule that checks a body
|
||||
// ("rule") and in the sink right after the engine ("request", "response"), so
|
||||
// a test can make a rule, or the sink itself, panic.
|
||||
var testRuleHook atomic.Pointer[func(stage string)]
|
||||
|
||||
func runTestHook(stage string) {
|
||||
if h := testRuleHook.Load(); h != nil {
|
||||
(*h)(stage)
|
||||
}
|
||||
}
|
||||
|
||||
// rulesIsBinary is the binary predicate handed to the engine: the recorder's
|
||||
// own, so "not rewritten: binary" follows what the recorder treats as binary.
|
||||
func rulesIsBinary(contentType string, body []byte) bool {
|
||||
runTestHook("rule")
|
||||
return isBinaryBody(contentType, body)
|
||||
}
|
||||
|
||||
type flowKey [16]byte
|
||||
|
||||
func keyOf(f *mproxy.Flow) flowKey { return flowKey(f.Id) }
|
||||
|
||||
// reqSnap is one slot of the table: an entry, which holds the original of a
|
||||
// request, or a tombstone, which holds only why it has none.
|
||||
type reqSnap struct {
|
||||
// done is the flow's Done() channel: closed, no hook will come for the
|
||||
// slot any more. The flow itself is not referenced.
|
||||
done <-chan struct{}
|
||||
rules *rewrite.Set // the snapshot of rules the flow runs with, both phases
|
||||
host string // host name: the only part of the request a log shows
|
||||
tomb bool
|
||||
|
||||
// Entry.
|
||||
created time.Time
|
||||
cost int64
|
||||
rewritten bool // the request phase changed the request
|
||||
method string
|
||||
url *url.URL
|
||||
header http.Header
|
||||
rawHost string
|
||||
body []byte // the client's body, kept when a body action applied
|
||||
keepBody bool
|
||||
lines []rewrite.TraceLine // request-phase trace
|
||||
|
||||
// Tombstone.
|
||||
reason string // reasonSnapshotCapacity or reasonOriginalUnavailable
|
||||
ruleIDs []int64 // rules that had rewritten the request
|
||||
}
|
||||
|
||||
// requestRewritten reports a flow whose request left the proxy changed.
|
||||
func (s *reqSnap) requestRewritten() bool {
|
||||
if s.tomb {
|
||||
return s.reason == reasonOriginalUnavailable
|
||||
}
|
||||
return s.rewritten
|
||||
}
|
||||
|
||||
func (s *reqSnap) appliedRuleIDs() []int64 {
|
||||
if s.tomb {
|
||||
return s.ruleIDs
|
||||
}
|
||||
_, ids := appliedRules(s.lines)
|
||||
return ids
|
||||
}
|
||||
|
||||
// snapTable holds the slots, keyed by flow id. Its mutex is a leaf lock.
|
||||
type snapTable struct {
|
||||
mu sync.Mutex
|
||||
slots map[flowKey]*reqSnap
|
||||
entries int
|
||||
bytes int64
|
||||
tombs int
|
||||
lastSweep time.Time
|
||||
lastFullLog time.Time
|
||||
sweepWanted atomic.Bool
|
||||
|
||||
now func() time.Time
|
||||
maxEntries int
|
||||
maxBytes int64
|
||||
maxTombs int
|
||||
ttl time.Duration
|
||||
|
||||
sweeps, capacity, tombDrops, conversions, largeResponses atomic.Int64
|
||||
}
|
||||
|
||||
func newSnapTable() *snapTable {
|
||||
return &snapTable{slots: map[flowKey]*reqSnap{}, now: time.Now,
|
||||
maxEntries: maxSnapEntries, maxBytes: maxSnapBytes, maxTombs: maxSnapTombs, ttl: snapTTL}
|
||||
}
|
||||
|
||||
func doneClosed(ch <-chan struct{}) bool {
|
||||
if ch == nil {
|
||||
return false // a nil channel is never ready: the flow counts as live
|
||||
}
|
||||
select {
|
||||
case <-ch:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// reserve stores the entry of a rewritten request. On failure the request must
|
||||
// leave unchanged: a capacity tombstone then stands for the flow, when there is
|
||||
// room for one.
|
||||
func (t *snapTable) reserve(k flowKey, e *reqSnap) bool {
|
||||
if t == nil {
|
||||
return false
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
t.removeLocked(k)
|
||||
if !t.fitsLocked(e.cost) {
|
||||
// A failed reservation asks for an immediate sweep, under the same
|
||||
// debounce; a sweep refused by the debounce fails the reservation.
|
||||
now := t.now()
|
||||
if now.Sub(t.lastSweep) < sweepDebounce {
|
||||
t.refuseLocked(k, e)
|
||||
return false
|
||||
}
|
||||
t.sweepLocked(now)
|
||||
if !t.fitsLocked(e.cost) && !t.convertLocked(e.cost) {
|
||||
t.refuseLocked(k, e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
e.created = t.now()
|
||||
t.slots[k] = e
|
||||
t.entries++
|
||||
t.bytes += e.cost
|
||||
return true
|
||||
}
|
||||
|
||||
func (t *snapTable) fitsLocked(cost int64) bool {
|
||||
return t.entries < t.maxEntries && t.bytes+cost <= t.maxBytes
|
||||
}
|
||||
|
||||
// convertLocked turns the oldest entry of a live flow older than the TTL into
|
||||
// a tombstone, in place, so the flow is never without a slot. It does so only
|
||||
// when releasing that entry lets a reservation of cost succeed, and when a
|
||||
// tombstone place is free (after evicting tombstones of finished flows only).
|
||||
// Otherwise nothing changes and it reports false.
|
||||
func (t *snapTable) convertLocked(cost int64) bool {
|
||||
now := t.now()
|
||||
var victim *reqSnap
|
||||
for _, s := range t.slots {
|
||||
if s.tomb || doneClosed(s.done) || now.Sub(s.created) < t.ttl {
|
||||
continue
|
||||
}
|
||||
if t.entries-1 >= t.maxEntries || t.bytes-s.cost+cost > t.maxBytes {
|
||||
continue // released, it would still leave no room
|
||||
}
|
||||
if victim == nil || s.created.Before(victim.created) {
|
||||
victim = s
|
||||
}
|
||||
}
|
||||
if victim == nil {
|
||||
return false
|
||||
}
|
||||
if t.tombs >= t.maxTombs {
|
||||
t.evictFinishedTombsLocked()
|
||||
}
|
||||
if t.tombs >= t.maxTombs {
|
||||
return false
|
||||
}
|
||||
ids := victim.appliedRuleIDs()
|
||||
t.entries--
|
||||
t.bytes -= victim.cost
|
||||
t.tombs++
|
||||
*victim = reqSnap{done: victim.done, rules: victim.rules, host: victim.host, tomb: true, reason: reasonOriginalUnavailable, ruleIDs: ids}
|
||||
t.conversions.Add(1)
|
||||
log.Printf("[traffic] rewrite snapshot of a request to %s (rules %v) released under pressure: its exchange will be recorded as original request unavailable", victim.host, ids)
|
||||
return true
|
||||
}
|
||||
|
||||
// refuseLocked is the capacity path: the request is not rewritten, and a
|
||||
// tombstone records why, when a place is free.
|
||||
func (t *snapTable) refuseLocked(k flowKey, e *reqSnap) {
|
||||
t.capacity.Add(1)
|
||||
if t.tombs >= t.maxTombs {
|
||||
t.evictFinishedTombsLocked()
|
||||
}
|
||||
full := t.tombs >= t.maxTombs
|
||||
if full {
|
||||
t.tombDrops.Add(1)
|
||||
} else {
|
||||
t.slots[k] = &reqSnap{done: e.done, rules: e.rules, host: e.host, tomb: true, reason: reasonSnapshotCapacity}
|
||||
t.tombs++
|
||||
}
|
||||
if now := t.now(); now.Sub(t.lastFullLog) >= time.Minute {
|
||||
t.lastFullLog = now
|
||||
log.Printf("[traffic] rewrite snapshot table full: requests are sent without rewriting (%d refused, %d without a trace so far)", t.capacity.Load(), t.tombDrops.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func (t *snapTable) evictFinishedTombsLocked() {
|
||||
for k, s := range t.slots {
|
||||
if s.tomb && doneClosed(s.done) {
|
||||
t.removeLocked(k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sweepLocked deletes every slot whose flow is finished. The slot of a live
|
||||
// flow is never touched, whatever its age.
|
||||
func (t *snapTable) sweepLocked(now time.Time) {
|
||||
for k, s := range t.slots {
|
||||
if doneClosed(s.done) {
|
||||
t.removeLocked(k)
|
||||
}
|
||||
}
|
||||
t.lastSweep = now
|
||||
t.sweepWanted.Store(false)
|
||||
t.sweeps.Add(1)
|
||||
}
|
||||
|
||||
func (t *snapTable) removeLocked(k flowKey) {
|
||||
s := t.slots[k]
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
if s.tomb {
|
||||
t.tombs--
|
||||
} else {
|
||||
t.entries--
|
||||
t.bytes -= s.cost
|
||||
}
|
||||
delete(t.slots, k)
|
||||
}
|
||||
|
||||
// take removes and returns the slot of a flow, nil when it has none.
|
||||
func (t *snapTable) take(k flowKey) *reqSnap {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
s := t.slots[k]
|
||||
t.removeLocked(k)
|
||||
return s
|
||||
}
|
||||
|
||||
// requestSweep asks the sweeper for a pass; it runs at most once a second.
|
||||
func (t *snapTable) requestSweep() {
|
||||
if t != nil {
|
||||
t.sweepWanted.Store(true)
|
||||
}
|
||||
}
|
||||
|
||||
// tick is one beat of the sweeper: a sweep when one was asked for and the last
|
||||
// is a second old, or when the last is a minute old. Nothing on an empty table.
|
||||
func (t *snapTable) tick() {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if len(t.slots) == 0 {
|
||||
t.sweepWanted.Store(false)
|
||||
return
|
||||
}
|
||||
now := t.now()
|
||||
since := now.Sub(t.lastSweep)
|
||||
if (t.sweepWanted.Load() && since >= sweepDebounce) || since >= sweepPeriod {
|
||||
t.sweepLocked(now)
|
||||
}
|
||||
}
|
||||
|
||||
// counts reports the entries and tombstones in the table.
|
||||
func (t *snapTable) counts() (entries, tombs int) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return t.entries, t.tombs
|
||||
}
|
||||
|
||||
// sweepSnapshots is the single sweeper goroutine, started by Open and stopped
|
||||
// by Close.
|
||||
func (t *Traffic) sweepSnapshots() {
|
||||
tick := time.NewTicker(time.Second)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-t.closed:
|
||||
return
|
||||
case <-tick.C:
|
||||
t.snaps.tick()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RewriteStats are the proxy-side counters of the rewrite rules since start.
|
||||
type RewriteStats struct {
|
||||
// LargeResponses counts flows with a snapshot whose response reached the
|
||||
// library's 5 MiB stream threshold: passed through, not recorded.
|
||||
LargeResponses int64 `json:"large_responses"`
|
||||
// CapacityRefusals counts requests sent without rewriting for lack of room.
|
||||
CapacityRefusals int64 `json:"capacity_refusals"`
|
||||
// TombstoneDrops counts those of them that got no capacity tombstone either.
|
||||
TombstoneDrops int64 `json:"tombstone_drops"`
|
||||
// Conversions counts entries released under pressure.
|
||||
Conversions int64 `json:"conversions"`
|
||||
Sweeps int64 `json:"sweeps"`
|
||||
}
|
||||
|
||||
func (t *Traffic) RewriteStats() RewriteStats {
|
||||
s := t.snaps
|
||||
if s == nil {
|
||||
return RewriteStats{}
|
||||
}
|
||||
return RewriteStats{LargeResponses: s.largeResponses.Load(), CapacityRefusals: s.capacity.Load(),
|
||||
TombstoneDrops: s.tombDrops.Load(), Conversions: s.conversions.Load(), Sweeps: s.sweeps.Load()}
|
||||
}
|
||||
|
||||
// newEntry builds the snapshot of a request before the rules' result is put on
|
||||
// the flow: URL and headers cloned, the body referenced, and only when a body
|
||||
// action applied (otherwise the flow's body is still the client's).
|
||||
func newEntry(f *mproxy.Flow, set *rewrite.Set, res rewrite.Result) *reqSnap {
|
||||
r := f.Request
|
||||
u := *r.URL
|
||||
e := &reqSnap{done: f.Done(), rules: set, host: r.URL.Hostname(), rewritten: res.Changed,
|
||||
method: r.Method, url: &u, header: r.Header.Clone(), lines: res.Lines}
|
||||
if raw := r.Raw(); raw != nil {
|
||||
e.rawHost = raw.Host
|
||||
}
|
||||
for _, l := range res.Lines {
|
||||
if l.Outcome == rewrite.OutcomeApplied && l.Field == "body" && len(r.Body) <= rewrite.MaxBodySize {
|
||||
e.body, e.keepBody = r.Body, true
|
||||
break
|
||||
}
|
||||
}
|
||||
e.cost = int64(len(e.method) + len(u.String()) + len(e.rawHost) + len(e.body))
|
||||
for k, vs := range e.header {
|
||||
for _, v := range vs {
|
||||
e.cost += int64(len(k) + len(v))
|
||||
}
|
||||
}
|
||||
for _, l := range e.lines {
|
||||
e.cost += int64(len(l.After) + len(l.RuleSnapshot))
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// originalRequest is the request an exchange records, with the request-phase
|
||||
// trace: the flow's own request when no rule rewrote it, the entry's copy when
|
||||
// one did, and only what Raw() still knows when the entry was released.
|
||||
func originalRequest(f *mproxy.Flow, snap *reqSnap) (requestView, []rewrite.TraceLine) {
|
||||
switch {
|
||||
case snap == nil:
|
||||
return flowRequestView(f.Request), nil
|
||||
case !snap.tomb:
|
||||
v := requestView{method: snap.method, url: snap.url, proto: f.Request.Proto, header: snap.header, rawHost: snap.rawHost, body: f.Request.Body}
|
||||
if snap.keepBody {
|
||||
v.body = snap.body
|
||||
}
|
||||
return v, snap.lines
|
||||
case snap.reason == reasonSnapshotCapacity:
|
||||
// The request was not rewritten: the flow's request is the client's.
|
||||
return flowRequestView(f.Request), []rewrite.TraceLine{{Phase: rewrite.PhaseRequest, Outcome: rewrite.OutcomeSkipped, Reason: reasonSnapshotCapacity}}
|
||||
default:
|
||||
// Rewritten, original released: never record the rewritten request as
|
||||
// the client's. Raw() is the client's request line and Host.
|
||||
v := requestView{unavailable: true}
|
||||
if raw := f.Request.Raw(); raw != nil {
|
||||
v.method, v.url, v.proto, v.rawHost = raw.Method, raw.URL, raw.Proto, raw.Host
|
||||
}
|
||||
lines := []rewrite.TraceLine{{Phase: rewrite.PhaseRequest, Outcome: rewrite.OutcomeError, Reason: reasonOriginalUnavailable}}
|
||||
for _, id := range snap.ruleIDs {
|
||||
lines = append(lines, rewrite.TraceLine{Phase: rewrite.PhaseRequest, RuleID: id, Outcome: rewrite.OutcomeApplied})
|
||||
}
|
||||
return v, lines
|
||||
}
|
||||
}
|
||||
|
||||
func engineRequest(r *mproxy.Request) rewrite.Request {
|
||||
return rewrite.Request{Method: r.Method, URL: r.URL, Header: r.Header, Body: r.Body}
|
||||
}
|
||||
|
||||
// Request runs the request phase. The result goes on the flow only once the
|
||||
// request's snapshot slot is reserved; without one the request leaves as the
|
||||
// client sent it. Rules fail open: a panic leaves the request unchanged.
|
||||
func (s *sink) Request(f *mproxy.Flow) {
|
||||
if s.t == nil || f.Request == nil || f.Request.URL == nil {
|
||||
return
|
||||
}
|
||||
set := s.t.rewriteRules.Load()
|
||||
if set.Len(rewrite.PhaseRequest) == 0 {
|
||||
return
|
||||
}
|
||||
host := f.Request.URL.Hostname()
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
log.Printf("[traffic] rewrite rules stopped on a request to %s (class %s): it is sent as the client sent it", host, rewrite.ErrClassPanic)
|
||||
}
|
||||
}()
|
||||
res := set.Apply(rewrite.PhaseRequest, rewrite.Input{Request: engineRequest(f.Request), IsBinary: rulesIsBinary})
|
||||
runTestHook("request")
|
||||
if len(res.Lines) == 0 {
|
||||
return
|
||||
}
|
||||
if !s.t.snaps.reserve(keyOf(f), newEntry(f, set, res)) {
|
||||
return
|
||||
}
|
||||
if res.Changed {
|
||||
r := f.Request
|
||||
r.Method, r.URL, r.Header, r.Body = res.Request.Method, res.Request.URL, res.Request.Header, res.Request.Body
|
||||
}
|
||||
}
|
||||
|
||||
// Response records the exchange and runs the response phase: the exchange
|
||||
// keeps the client's request and the target's response, the client gets the
|
||||
// rewritten response. Without a slot and without response rules this is the
|
||||
// plain record.
|
||||
func (s *sink) Response(f *mproxy.Flow) {
|
||||
if f.Request == nil || f.Response == nil {
|
||||
return
|
||||
}
|
||||
snap := s.t.snaps.take(keyOf(f))
|
||||
set := s.t.rewriteRules.Load()
|
||||
if snap != nil {
|
||||
set = snap.rules
|
||||
}
|
||||
if snap == nil && set.Len(rewrite.PhaseResponse) == 0 {
|
||||
s.t.record(f)
|
||||
return
|
||||
}
|
||||
rq, lines := originalRequest(f, snap)
|
||||
rs := flowResponseView(f.Response)
|
||||
res := s.applyResponse(f, set, rewrite.Input{
|
||||
Request: engineRequest(f.Request), // as sent to the target
|
||||
Response: rewrite.Response{Status: rs.status, Header: rs.header, Body: rs.body},
|
||||
IsBinary: rulesIsBinary,
|
||||
})
|
||||
s.t.recordViews(rq, rs, append(lines, res.Lines...))
|
||||
if res.Changed {
|
||||
f.Response.StatusCode, f.Response.Header, f.Response.Body = res.Response.Status, res.Response.Header, res.Response.Body
|
||||
}
|
||||
}
|
||||
|
||||
// applyResponse runs the response phase; a panic yields no change and no line.
|
||||
func (s *sink) applyResponse(f *mproxy.Flow, set *rewrite.Set, in rewrite.Input) (res rewrite.Result) {
|
||||
if set.Len(rewrite.PhaseResponse) == 0 {
|
||||
return rewrite.Result{}
|
||||
}
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
res = rewrite.Result{}
|
||||
log.Printf("[traffic] rewrite rules stopped on a response from %s (class %s): it is sent as the target sent it", f.Request.URL.Hostname(), rewrite.ErrClassPanic)
|
||||
}
|
||||
}()
|
||||
r := set.Apply(rewrite.PhaseResponse, in)
|
||||
runTestHook("response")
|
||||
return r
|
||||
}
|
||||
|
||||
// rewriteStream prepares the exchange of a stream at SSEStart or
|
||||
// WebSocketStart: the client's request, the target's status and headers (SSE),
|
||||
// and the trace. For SSE, the response phase runs in stream mode: status and
|
||||
// header actions apply (res, put on the flow by applyStreamResponse), rules
|
||||
// that read or write the response body are traced "not rewritten: stream".
|
||||
// WebSocketStart runs once go-mitmproxy has already answered the client's
|
||||
// upgrade, with no response on the flow: nothing of the handshake can change
|
||||
// there, so every candidate rule is traced "not rewritten: stream".
|
||||
func (s *sink) rewriteStream(f *mproxy.Flow, kind string) (rq requestView, rs responseView, lines []rewrite.TraceLine, res rewrite.Result) {
|
||||
snap := s.t.snaps.take(keyOf(f))
|
||||
set := s.t.rewriteRules.Load()
|
||||
if snap != nil {
|
||||
set = snap.rules
|
||||
}
|
||||
rq, lines = originalRequest(f, snap)
|
||||
if kind == "sse" {
|
||||
rs = flowResponseView(f.Response)
|
||||
}
|
||||
cur := engineRequest(f.Request)
|
||||
if kind != "websocket" {
|
||||
res = s.applyResponse(f, set, rewrite.Input{Request: cur, Response: rewrite.Response{Status: rs.status, Header: rs.header}, Stream: true, IsBinary: rulesIsBinary})
|
||||
return rq, rs, append(lines, res.Lines...), res
|
||||
}
|
||||
if set.Len(rewrite.PhaseRequest)+set.Len(rewrite.PhaseResponse) == 0 {
|
||||
return rq, rs, lines, rewrite.Result{}
|
||||
}
|
||||
// The engine never changes a request with Upgrade; a ws:// stream arrives
|
||||
// on its CONNECT flow, which has none.
|
||||
if cur.Header.Get("Upgrade") == "" {
|
||||
cur.Header = cur.Header.Clone()
|
||||
if cur.Header == nil {
|
||||
cur.Header = http.Header{}
|
||||
}
|
||||
cur.Header.Set("Upgrade", "websocket")
|
||||
}
|
||||
if snap == nil {
|
||||
lines = append(lines, s.applyRequestTrace(f, set, cur)...)
|
||||
}
|
||||
r := s.applyResponse(f, set, rewrite.Input{Request: cur, Response: rewrite.Response{Status: http.StatusSwitchingProtocols}, Stream: true, IsBinary: rulesIsBinary})
|
||||
return rq, rs, append(lines, r.Lines...), rewrite.Result{}
|
||||
}
|
||||
|
||||
// applyRequestTrace runs the request phase for its trace only.
|
||||
func (s *sink) applyRequestTrace(f *mproxy.Flow, set *rewrite.Set, req rewrite.Request) (lines []rewrite.TraceLine) {
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
lines = nil
|
||||
log.Printf("[traffic] rewrite rules stopped on a request to %s (class %s)", f.Request.URL.Hostname(), rewrite.ErrClassPanic)
|
||||
}
|
||||
}()
|
||||
return set.Apply(rewrite.PhaseRequest, rewrite.Input{Request: req, IsBinary: rulesIsBinary}).Lines
|
||||
}
|
||||
|
||||
// applyStreamResponse puts the rewritten status and headers of an SSE
|
||||
// response on the flow, before go-mitmproxy replies to the client.
|
||||
func (s *sink) applyStreamResponse(f *mproxy.Flow, res rewrite.Result) {
|
||||
if res.Changed && f.Response != nil {
|
||||
f.Response.StatusCode, f.Response.Header = res.Response.Status, res.Response.Header
|
||||
}
|
||||
}
|
||||
|
||||
// RequestError fires when a request through an established MITM tunnel fails. If
|
||||
// the failure looks proxy/protocol-caused (h2 quirks, HEAD-with-body, protocol
|
||||
// errors) and not a plain target-unreachable error, the host is tunneled for
|
||||
// passthroughTTL: this request has already failed, later requests in the window
|
||||
// connect straight through, and the first one after the window tries
|
||||
// interception again. A request a rule rewrote never flags its host: the rule,
|
||||
// not the interception, may be at fault. Its log names the host, the error class
|
||||
// and the rules, never the URL or the error text.
|
||||
func (s *sink) RequestError(f *mproxy.Flow, err error) {
|
||||
if f != nil {
|
||||
if snap := s.t.snaps.take(keyOf(f)); snap != nil && snap.requestRewritten() {
|
||||
ids := snap.appliedRuleIDs()
|
||||
for _, id := range ids {
|
||||
snap.rules.NoteError(id, rewrite.ErrClassTransport)
|
||||
}
|
||||
log.Printf("[traffic] a request to %s rewritten by rules %v failed (class %s); the host stays intercepted", snap.host, ids, rewrite.ErrClassTransport)
|
||||
return
|
||||
}
|
||||
}
|
||||
s.t.maybePassthrough(f, err)
|
||||
}
|
||||
|
||||
// StreamResponseModifier passes every body through as received, nil included,
|
||||
// without wrapping it. A response that reached the library's stream threshold
|
||||
// is never recorded: the slot of its flow, if any, is released here.
|
||||
func (s *sink) StreamResponseModifier(f *mproxy.Flow, in io.Reader) (out io.Reader) {
|
||||
out = in
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
out = in
|
||||
}
|
||||
}()
|
||||
if s == nil || s.t == nil || f == nil || !f.Stream || f.SSE != nil {
|
||||
return in
|
||||
}
|
||||
snap := s.t.snaps.take(keyOf(f))
|
||||
if snap == nil {
|
||||
return in
|
||||
}
|
||||
s.t.snaps.largeResponses.Add(1)
|
||||
log.Printf("[traffic] the response to a request to %s with rewrite rules %v reached %d MiB: passed through, not recorded", snap.host, snap.appliedRuleIDs(), libraryStreamBody>>20)
|
||||
return in
|
||||
}
|
||||
|
||||
// spillTrace stores the values of the trace lines as record stores bodies.
|
||||
// Called under wmu, before the transaction.
|
||||
func (t *Traffic) spillTrace(lines []rewrite.TraceLine) []storedBody {
|
||||
out := make([]storedBody, len(lines))
|
||||
for i, l := range lines {
|
||||
out[i] = t.spill(l.After, "")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// writeTrace inserts an exchange's trace rows and the blob references of their
|
||||
// spilled values, inside the exchange's transaction.
|
||||
func writeTrace(tx *sql.Tx, id string, lines []rewrite.TraceLine, trace []storedBody) error {
|
||||
for _, b := range trace {
|
||||
if b.hash == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, b.hash, id); err != nil {
|
||||
return &recordError{"registering blob reference", err}
|
||||
}
|
||||
}
|
||||
for i, l := range lines {
|
||||
var ruleID, after any
|
||||
if l.RuleID != 0 {
|
||||
ruleID = l.RuleID
|
||||
}
|
||||
if trace[i].inline != nil {
|
||||
after = trace[i].inline
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO exchange_rewrites(exchange_id,seq,phase,rule_id,rule_snapshot,action,field,outcome,reason,before_hash,after_hash,after_inline,after_blob)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
id, i, string(l.Phase), ruleID, nullIfEmpty(l.RuleSnapshot), nullIfEmpty(l.Action), nullIfEmpty(l.Field), l.Outcome, l.Reason,
|
||||
nullIfEmpty(l.BeforeHash), nullIfEmpty(l.AfterHash), after, nullIfEmpty(trace[i].hash)); err != nil {
|
||||
return &recordError{"writing rewrite trace", err}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RewriteInfo reports whether rules rewrote an exchange (at least one applied
|
||||
// trace line) and which rules did, by id.
|
||||
func (t *Traffic) RewriteInfo(id string) (rewritten bool, ruleIDs []int64, err error) {
|
||||
if err := t.db.QueryRow(`SELECT EXISTS(SELECT 1 FROM exchange_rewrites r WHERE r.exchange_id=? AND r.outcome='applied')`, id).Scan(&rewritten); err != nil || !rewritten {
|
||||
return false, nil, err
|
||||
}
|
||||
rows, err := t.db.Query(`SELECT DISTINCT rule_id FROM exchange_rewrites WHERE exchange_id=? AND outcome='applied' AND rule_id IS NOT NULL ORDER BY rule_id`, id)
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var rid int64
|
||||
if err := rows.Scan(&rid); err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
ruleIDs = append(ruleIDs, rid)
|
||||
}
|
||||
return true, ruleIDs, rows.Err()
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,102 +0,0 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/traffic/rewrite"
|
||||
)
|
||||
|
||||
// The proxy stores the compiled rewrite rules and does not apply them. With a
|
||||
// snapshot whose rules match every exchange, what the target receives, what the
|
||||
// client receives and what is recorded are the same as with no rule at all.
|
||||
func TestRewriteRulesStoredNotApplied(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var seen []string
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
mu.Lock()
|
||||
seen = append(seen, r.Method+" "+r.RequestURI+" X-Client="+r.Header.Get("X-Client")+" "+string(body))
|
||||
mu.Unlock()
|
||||
w.Header().Set("X-Target", "t")
|
||||
w.WriteHeader(http.StatusTeapot)
|
||||
io.WriteString(w, "target body")
|
||||
}))
|
||||
t.Cleanup(target.Close)
|
||||
|
||||
set := rewrite.Compile([]rewrite.Rule{
|
||||
{ID: 1, Name: "request", Enabled: true, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetMethod, Value: "PUT"},
|
||||
{Type: rewrite.ActionSetPathQuery, Value: "/rewritten"},
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Client", Value: "rewritten"},
|
||||
{Type: rewrite.ActionSetBody, Value: "rewritten"}}},
|
||||
{ID: 2, Name: "response", Enabled: true, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetStatus, Value: "200"},
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Rewritten", Value: "1"},
|
||||
{Type: rewrite.ActionSetBody, Value: "rewritten"}}},
|
||||
}, rewrite.Options{})
|
||||
if set.Len(rewrite.PhaseRequest) != 1 || set.Len(rewrite.PhaseResponse) != 1 || len(set.LoadErrors()) != 0 {
|
||||
t.Fatalf("the snapshot must hold both rules: %v", set.LoadErrors())
|
||||
}
|
||||
|
||||
// canon makes a recorded message comparable: the recorder writes headers in
|
||||
// map order, and the target dates its response.
|
||||
canon := func(raw string) string {
|
||||
head, body, _ := strings.Cut(raw, "\n\n")
|
||||
lines := strings.Split(head, "\n")
|
||||
sort.Strings(lines[1:])
|
||||
lines = slices.DeleteFunc(lines, func(l string) bool { return strings.HasPrefix(l, "Date: ") })
|
||||
return strings.Join(lines, "\n") + "\n\n" + body
|
||||
}
|
||||
// exchange sends one request through a fresh proxy holding set and returns
|
||||
// what the client received and what the proxy recorded.
|
||||
exchange := func(set *rewrite.Set) (client, recorded string) {
|
||||
tr, _, proxyURL := startProxy(t)
|
||||
if tr.RewriteRules() != nil {
|
||||
t.Fatal("Open published rewrite rules")
|
||||
}
|
||||
tr.SetRewriteRules(set)
|
||||
if tr.RewriteRules() != set {
|
||||
t.Fatal("RewriteRules must return the snapshot last published")
|
||||
}
|
||||
req, err := http.NewRequest("POST", target.URL+"/path?q=1", strings.NewReader("client body"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Set("X-Client", "c")
|
||||
resp, err := proxyClient(t, tr, proxyURL).Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
client = resp.Status + " X-Target=" + resp.Header.Get("X-Target") + " X-Rewritten=" + resp.Header.Get("X-Rewritten") + " " + string(body)
|
||||
reqText, respText, err := tr.Get(onlyExchangeID(t, tr))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return client, canon(reqText) + "\n--\n" + canon(respText)
|
||||
}
|
||||
|
||||
baseClient, baseRecorded := exchange(nil)
|
||||
client, recorded := exchange(set)
|
||||
if want := "418 I'm a teapot X-Target=t X-Rewritten= target body"; baseClient != want || client != want {
|
||||
t.Fatalf("client received %q without rules and %q with rules, want %q", baseClient, client, want)
|
||||
}
|
||||
if recorded != baseRecorded {
|
||||
t.Fatalf("the recorded exchange differs with rules stored:\n without: %s\n with: %s", baseRecorded, recorded)
|
||||
}
|
||||
if !strings.Contains(recorded, "client body") || !strings.Contains(recorded, "target body") || strings.Contains(recorded, "rewritten") {
|
||||
t.Fatalf("recorded exchange: %s", recorded)
|
||||
}
|
||||
want := "POST /path?q=1 X-Client=c client body"
|
||||
if len(seen) != 2 || seen[0] != want || seen[1] != want {
|
||||
t.Fatalf("the target received %q, want %q twice", seen, want)
|
||||
}
|
||||
}
|
||||
+60
-27
@@ -16,6 +16,7 @@ import (
|
||||
"unsafe"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/traffic/rewrite"
|
||||
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
|
||||
)
|
||||
|
||||
@@ -144,17 +145,35 @@ func (s *sink) startStream(f *mproxy.Flow, kind string) {
|
||||
if f.Request == nil || f.Request.URL == nil || (kind == "sse" && f.Response == nil) {
|
||||
return
|
||||
}
|
||||
id, err := s.t.recordStreamStart(f, kind)
|
||||
rq, rs, lines, res := s.rewriteStream(f, kind)
|
||||
id, err := s.t.recordStreamStart(rq, rs, kind, lines)
|
||||
s.applyStreamResponse(f, res)
|
||||
if err != nil {
|
||||
if rewritten, ruleIDs := appliedRules(lines); rewritten {
|
||||
var re *recordError
|
||||
errors.As(err, &re)
|
||||
log.Printf("[traffic] failed to record a rewritten %s stream of %s (%s; rules %v)", kind, f.Request.URL.Hostname(), re.step, ruleIDs)
|
||||
return
|
||||
}
|
||||
log.Printf("[traffic] failed to record %s stream %s: %v", kind, f.Request.URL, err)
|
||||
return
|
||||
}
|
||||
st.id = id
|
||||
if kind == "sse" {
|
||||
st.reqBytes = int64(len(f.Request.Body))
|
||||
st.reqBytes = int64(len(rq.body))
|
||||
}
|
||||
}
|
||||
|
||||
// recordError names the step of a recording that failed. Its text is the step
|
||||
// and the cause, as recordStreamStart always reported.
|
||||
type recordError struct {
|
||||
step string
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *recordError) Error() string { return e.step + ": " + e.err.Error() }
|
||||
func (e *recordError) Unwrap() error { return e.err }
|
||||
|
||||
// libMutex returns the unexported mu sync.Mutex that go-mitmproxy holds while it
|
||||
// appends to WebSocketData.Messages / SSEData.Events. The library exposes no
|
||||
// accessor, and the two WebSocket goroutines append concurrently, so reading the
|
||||
@@ -252,9 +271,9 @@ func sseEventsSince(sse *mproxy.SSEData, st *streamState) []streamRow {
|
||||
// streamDisplayURL is the URL shown for a WebSocket exchange: http→ws and
|
||||
// https→wss. A plain ws:// connection reaches the WebSocket hooks through its
|
||||
// CONNECT tunnel, where only the target authority is known, not the path.
|
||||
func streamDisplayURL(f *mproxy.Flow, kind string) (method, display, tmpl string) {
|
||||
u := *f.Request.URL
|
||||
method = f.Request.Method
|
||||
func streamDisplayURL(reqMethod string, reqURL *url.URL, kind string) (method, display, tmpl string) {
|
||||
u := *reqURL
|
||||
method = reqMethod
|
||||
if kind == "websocket" && strings.EqualFold(method, "CONNECT") {
|
||||
host := u.Host
|
||||
if host == "" {
|
||||
@@ -270,63 +289,77 @@ func streamDisplayURL(f *mproxy.Flow, kind string) (method, display, tmpl string
|
||||
u.Scheme = "wss"
|
||||
}
|
||||
}
|
||||
return method, u.String(), db.TemplatePath(f.Request.URL.EscapedPath())
|
||||
return method, u.String(), db.TemplatePath(reqURL.EscapedPath())
|
||||
}
|
||||
|
||||
// recordStreamStart writes the stream's exchange: metadata, request head and
|
||||
// body, the response head (synthetic 101 for WebSocket), and the header text in
|
||||
// the full-text index. Messages are never indexed.
|
||||
func (t *Traffic) recordStreamStart(f *mproxy.Flow, kind string) (string, error) {
|
||||
// body, the response head (synthetic 101 for WebSocket), the rewrite trace,
|
||||
// and the header text in the full-text index. Messages are never indexed. rq
|
||||
// is the request the client sent and rs (SSE only) the status and headers the
|
||||
// target sent, whatever the rules changed.
|
||||
func (t *Traffic) recordStreamStart(rq requestView, rs responseView, kind string, lines []rewrite.TraceLine) (string, error) {
|
||||
now := time.Now()
|
||||
id := t.newExchangeID(now)
|
||||
method, display, tmpl := streamDisplayURL(f, kind)
|
||||
host := f.Request.URL.Hostname()
|
||||
target := f.Request.URL.RequestURI()
|
||||
if strings.EqualFold(f.Request.Method, "CONNECT") {
|
||||
target = f.Request.URL.Host // authority-form, as on the wire
|
||||
method, display, tmpl, host, target := rq.method, "", "", "", ""
|
||||
if rq.url != nil {
|
||||
method, display, tmpl = streamDisplayURL(rq.method, rq.url, kind)
|
||||
host = rq.url.Hostname()
|
||||
target = rq.url.RequestURI()
|
||||
if strings.EqualFold(rq.method, "CONNECT") {
|
||||
target = rq.url.Host // authority-form, as on the wire
|
||||
}
|
||||
}
|
||||
reqHead := fmt.Sprintf("%s %s %s\n%s", f.Request.Method, target, f.Request.Proto, requestHeaderLines(f.Request))
|
||||
reqHead := requestHead(rq, target)
|
||||
status, ct, respHead := 101, "websocket", "HTTP/1.1 101 Switching Protocols\n"
|
||||
if kind == "sse" {
|
||||
status = f.Response.StatusCode
|
||||
ct = f.Response.Header.Get("Content-Type")
|
||||
respHead = fmt.Sprintf("HTTP %d\n%s", status, headerLines(f.Response.Header))
|
||||
status = rs.status
|
||||
ct = rs.header.Get("Content-Type")
|
||||
respHead = fmt.Sprintf("HTTP %d\n%s", status, headerLines(rs.header))
|
||||
}
|
||||
|
||||
t.wmu.Lock()
|
||||
defer t.wmu.Unlock()
|
||||
reqB := t.spill(f.Request.Body, f.Request.Header.Get("Content-Type"))
|
||||
reqB := t.spill(rq.body, rq.header.Get("Content-Type"))
|
||||
trace := t.spillTrace(lines)
|
||||
tx, err := t.db.Begin()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("beginning transaction: %w", err)
|
||||
return "", &recordError{"beginning transaction", err}
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // no-op once committed
|
||||
// An id can repeat (seconds plus a rolling counter); the trace of the
|
||||
// exchange it replaces must not stay attached to this one.
|
||||
if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id=?`, id); err != nil {
|
||||
return "", &recordError{"clearing rewrite trace", err}
|
||||
}
|
||||
res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,'')`, id, now.Unix(), host, method, tmpl, display, status, ct, len(f.Request.Body), 0)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,'')`, id, now.Unix(), host, method, tmpl, display, status, ct, len(rq.body), 0)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("writing index: %w", err)
|
||||
return "", &recordError{"writing index", err}
|
||||
}
|
||||
rowid, err := res.LastInsertId()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fetching rowid: %w", err)
|
||||
return "", &recordError{"fetching rowid", err}
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT OR REPLACE INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob)
|
||||
VALUES(?,?,?,?,?,NULL,NULL)`, id, reqHead, reqB.inline, nullIfEmpty(reqB.hash), respHead); err != nil {
|
||||
return "", fmt.Errorf("writing body: %w", err)
|
||||
return "", &recordError{"writing body", err}
|
||||
}
|
||||
if reqB.hash != "" {
|
||||
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, reqB.hash, id); err != nil {
|
||||
return "", fmt.Errorf("registering blob reference: %w", err)
|
||||
return "", &recordError{"registering blob reference", err}
|
||||
}
|
||||
}
|
||||
if err := writeTrace(tx, id, lines, trace); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if t.fts {
|
||||
idx := strings.Join([]string{display, reqHead, reqB.index, respHead}, "\n")
|
||||
if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowid, idx); err != nil {
|
||||
return "", fmt.Errorf("writing full-text index: %w", err)
|
||||
return "", &recordError{"writing full-text index", err}
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return "", fmt.Errorf("committing: %w", err)
|
||||
return "", &recordError{"committing", err}
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
+190
-47
@@ -22,6 +22,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -92,6 +93,26 @@ CREATE TABLE IF NOT EXISTS stream_messages (
|
||||
PRIMARY KEY (exchange_id, seq)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_stream_messages_ex ON stream_messages(exchange_id, seq);
|
||||
|
||||
-- Rewrite-rule trace of an exchange, in order. The exchange row keeps what the
|
||||
-- client sent and what the target answered; these rows say what the rules
|
||||
-- changed on the way. An exchange is rewritten when it has an 'applied' row.
|
||||
CREATE TABLE IF NOT EXISTS exchange_rewrites (
|
||||
exchange_id TEXT NOT NULL,
|
||||
seq INTEGER NOT NULL,
|
||||
phase TEXT NOT NULL, -- 'request' | 'response'
|
||||
rule_id INTEGER, -- NULL on a global line
|
||||
rule_snapshot TEXT, -- JSON, no sensitive action value
|
||||
action TEXT,
|
||||
field TEXT, -- 'method' | 'url' | 'header:<Name>' | 'body' | 'status'
|
||||
outcome TEXT NOT NULL, -- 'applied' | 'skipped' | 'error'
|
||||
reason TEXT NOT NULL DEFAULT '',
|
||||
before_hash TEXT,
|
||||
after_hash TEXT,
|
||||
after_inline BLOB,
|
||||
after_blob TEXT,
|
||||
PRIMARY KEY (exchange_id, seq)
|
||||
);
|
||||
`
|
||||
|
||||
// ftsSchema is applied separately from indexSchema: a driver build without FTS5
|
||||
@@ -160,6 +181,10 @@ const (
|
||||
// and the catalog migration cannot drift apart.
|
||||
const TrafficSearchDescription = "Query the target traffic the recording proxy has captured (host is required; accepts a bare host, host:port, or full URL, optionally filtered further by URL substring or body keyword). When a port is specified, only that service's traffic is returned, avoiding cross-talk between different ports on the same IP. body_contains runs a full-text search over the captured request/response headers and bodies, supporting any substring including Chinese (at least 3 characters). Returns only an ultra-light index (id/method/url/status/resp_len) with no response content; once results are non-empty, use traffic_get to verify the request/response of each one, then hand the IDs that actually support the current vulnerability to bind_finding_traffic. Returns 3 by default, at most 10 per page; use page to paginate when there are more results."
|
||||
|
||||
// TrafficGetDescription is the traffic_get description, kept here for the same
|
||||
// reason as TrafficSearchDescription.
|
||||
const TrafficGetDescription = "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling. A WebSocket (status 101) or SSE (text/event-stream) record shows only its handshake here; read its messages with traffic_messages. When proxy rewrite rules changed the exchange, the result starts with a REWRITTEN line naming the rule ids, and shows the request as the client sent it and the response as the target sent it. traffic_search searches only these original values: it never finds a value a rewrite rule wrote."
|
||||
|
||||
// Traffic runs the recording proxy and owns the file tree + index.
|
||||
type Traffic struct {
|
||||
dir string
|
||||
@@ -199,8 +224,14 @@ type Traffic struct {
|
||||
// so no host escapes it. Hot-swappable at runtime via SetUpstreamProxy.
|
||||
upstream atomic.Pointer[url.URL]
|
||||
// rewriteRules is the compiled snapshot of the rewrite rules, replaced as a
|
||||
// whole by SetRewriteRules. The proxy does not apply it yet.
|
||||
// whole by SetRewriteRules and applied by the sink (rewrite_flow.go).
|
||||
rewriteRules atomic.Pointer[rewrite.Set]
|
||||
// snaps keeps the original of each request a rule rewrote until its
|
||||
// exchange is recorded (rewrite_flow.go). Nil on the zero value.
|
||||
snaps *snapTable
|
||||
// sweeping tracks the snapshot sweeper, which runs until Close. It is kept
|
||||
// apart from reaping, which only ever holds work that finishes on its own.
|
||||
sweeping sync.WaitGroup
|
||||
}
|
||||
|
||||
// Open initializes the traffic tree, blob store and SQLite index under dir.
|
||||
@@ -226,7 +257,7 @@ func Open(dir, addr string) (*Traffic, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t := &Traffic{dir: dir, addr: addr, db: db, closed: make(chan struct{})}
|
||||
t := &Traffic{dir: dir, addr: addr, db: db, closed: make(chan struct{}), snaps: newSnapTable()}
|
||||
if err := t.initIndex(); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
@@ -258,6 +289,7 @@ func Open(dir, addr string) (*Traffic, error) {
|
||||
})
|
||||
p.AddAddon(&sink{t: t})
|
||||
t.proxy = p
|
||||
t.sweeping.Go(t.sweepSnapshots) // stops on t.closed; Close waits for it
|
||||
return t, nil
|
||||
}
|
||||
|
||||
@@ -358,8 +390,8 @@ func (t *Traffic) SetUpstreamProxy(raw string) error {
|
||||
}
|
||||
|
||||
// SetRewriteRules publishes a compiled snapshot of the rewrite rules. The swap
|
||||
// is atomic; nil clears the rules. The snapshot is stored only: the proxy does
|
||||
// not apply rewrite rules yet, so capture is unchanged whatever it holds.
|
||||
// is atomic; nil clears the rules. A flow already rewritten keeps the snapshot
|
||||
// its request saw for its response as well.
|
||||
func (t *Traffic) SetRewriteRules(s *rewrite.Set) { t.rewriteRules.Store(s) }
|
||||
|
||||
// RewriteRules returns the snapshot last published, nil when there is none.
|
||||
@@ -406,6 +438,7 @@ func (t *Traffic) Close() error {
|
||||
t.closeOnce.Do(func() { close(t.closed) })
|
||||
}
|
||||
t.reaping.Wait()
|
||||
t.sweeping.Wait()
|
||||
return t.db.Close()
|
||||
}
|
||||
|
||||
@@ -430,21 +463,6 @@ type sink struct {
|
||||
streams sync.Map // *mproxy.Flow of a live WebSocket/SSE stream -> *streamState (stream.go)
|
||||
}
|
||||
|
||||
func (s *sink) Response(f *mproxy.Flow) {
|
||||
if f.Request == nil || f.Response == nil {
|
||||
return
|
||||
}
|
||||
s.t.record(f)
|
||||
}
|
||||
|
||||
// RequestError fires when a request through an established MITM tunnel fails. If
|
||||
// the failure looks proxy/protocol-caused (h2 quirks, HEAD-with-body, protocol
|
||||
// errors) and not a plain target-unreachable error, the host is tunneled for
|
||||
// passthroughTTL. This request has already failed. Later requests in the window
|
||||
// connect straight through; the first request after the window tries
|
||||
// interception again.
|
||||
func (s *sink) RequestError(f *mproxy.Flow, err error) { s.t.maybePassthrough(f, err) }
|
||||
|
||||
// maybePassthrough marks a host to be tunneled transparently on the next
|
||||
// connection, but only for errors the proxy itself caused — a target that is
|
||||
// simply down/filtered would fail without us too, and must stay MITM'd+recorded.
|
||||
@@ -502,52 +520,121 @@ func proxyCausedErr(err error) bool {
|
||||
// full-text index. Nothing is written to a per-request directory — only bodies
|
||||
// above maxInlineBody spill to the content-addressed blob store.
|
||||
func (t *Traffic) record(f *mproxy.Flow) {
|
||||
t.recordViews(flowRequestView(f.Request), flowResponseView(f.Response), nil)
|
||||
}
|
||||
|
||||
// requestView is the request an exchange records: what the client sent, which
|
||||
// is not the flow's request once a rule rewrote it.
|
||||
type requestView struct {
|
||||
method string
|
||||
url *url.URL // nil when the original request is unavailable and Raw() is nil
|
||||
proto string
|
||||
header http.Header
|
||||
// rawHost is the client's Host header (Raw().Host), "" when unknown.
|
||||
rawHost string
|
||||
body []byte
|
||||
// unavailable marks a request rewritten by a rule whose original was not
|
||||
// kept: only its method, URL and Host are known.
|
||||
unavailable bool
|
||||
}
|
||||
|
||||
// responseView is the response an exchange records: what the target answered.
|
||||
type responseView struct {
|
||||
status int
|
||||
header http.Header
|
||||
body []byte
|
||||
}
|
||||
|
||||
func flowRequestView(r *mproxy.Request) requestView {
|
||||
v := requestView{method: r.Method, url: r.URL, proto: r.Proto, header: r.Header, body: r.Body}
|
||||
if raw := r.Raw(); raw != nil {
|
||||
v.rawHost = raw.Host
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func flowResponseView(r *mproxy.Response) responseView {
|
||||
return responseView{status: r.StatusCode, header: r.Header, body: r.Body}
|
||||
}
|
||||
|
||||
// unavailableMarker stands for the headers and body of a request whose
|
||||
// original was not kept (see rewrite_flow.go).
|
||||
const unavailableMarker = "[original request unavailable]\n"
|
||||
|
||||
// recordViews writes one exchange from explicit views, with its rewrite trace,
|
||||
// in one transaction: the exchange row, its bodies, blob references, the trace
|
||||
// and the full-text index of the views only. A failure rolls everything back,
|
||||
// so there is never a rewritten exchange without its trace. Without trace
|
||||
// lines this is exactly what record always wrote.
|
||||
func (t *Traffic) recordViews(rq requestView, rs responseView, lines []rewrite.TraceLine) {
|
||||
// The write lock covers blob + index writes, so DeleteHost (and its blob GC)
|
||||
// can run under the same lock without racing a concurrent record.
|
||||
t.wmu.Lock()
|
||||
defer t.wmu.Unlock()
|
||||
host := f.Request.URL.Hostname()
|
||||
method := f.Request.Method
|
||||
tmpl := db.TemplatePath(f.Request.URL.EscapedPath())
|
||||
var host, tmpl, url, uri string
|
||||
if rq.url != nil {
|
||||
host = rq.url.Hostname()
|
||||
tmpl = db.TemplatePath(rq.url.EscapedPath())
|
||||
url = rq.url.String()
|
||||
uri = rq.url.RequestURI()
|
||||
}
|
||||
method := rq.method
|
||||
now := time.Now()
|
||||
id := t.newExchangeID(now)
|
||||
|
||||
ct := f.Response.Header.Get("Content-Type")
|
||||
url := f.Request.URL.String()
|
||||
reqHead := fmt.Sprintf("%s %s %s\n%s", method, f.Request.URL.RequestURI(), f.Request.Proto, requestHeaderLines(f.Request))
|
||||
respHead := fmt.Sprintf("HTTP %d\n%s", f.Response.StatusCode, headerLines(f.Response.Header))
|
||||
ct := rs.header.Get("Content-Type")
|
||||
reqHead := requestHead(rq, uri)
|
||||
respHead := fmt.Sprintf("HTTP %d\n%s", rs.status, headerLines(rs.header))
|
||||
// Bodies are spilled before the transaction opens: blob writes are filesystem
|
||||
// work and must not sit inside the SQLite write lock.
|
||||
reqB := t.spill(f.Request.Body, f.Request.Header.Get("Content-Type"))
|
||||
respB := t.spill(f.Response.Body, ct)
|
||||
reqB := t.spill(rq.body, rq.header.Get("Content-Type"))
|
||||
respB := t.spill(rs.body, ct)
|
||||
trace := t.spillTrace(lines)
|
||||
|
||||
// A rewritten exchange is logged by host, step and rule ids only: its URL
|
||||
// and the error text may carry a value a rule wrote.
|
||||
rewritten, ruleIDs := appliedRules(lines)
|
||||
fail := func(step string, err error) {
|
||||
if rewritten {
|
||||
log.Printf("[traffic] failed to record a rewritten exchange of %s (%s; rules %v)", host, step, ruleIDs)
|
||||
return
|
||||
}
|
||||
log.Printf("[traffic] failed to record %s (%s): %v", url, step, err)
|
||||
}
|
||||
|
||||
tx, err := t.db.Begin()
|
||||
if err != nil {
|
||||
log.Printf("[traffic] failed to record %s (beginning transaction): %v", url, err)
|
||||
fail("beginning transaction", err)
|
||||
return
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // no-op once committed
|
||||
|
||||
// An id can repeat (seconds plus a rolling counter); the trace of the
|
||||
// exchange it replaces must not stay attached to this one.
|
||||
if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id=?`, id); err != nil {
|
||||
fail("clearing rewrite trace", err)
|
||||
return
|
||||
}
|
||||
// path stays empty for db-resident exchanges; a non-empty path marks a legacy
|
||||
// row whose bodies still live in the old on-disk tree (see Get).
|
||||
res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,'')`,
|
||||
id, now.Unix(), host, method, tmpl, url, f.Response.StatusCode, ct,
|
||||
len(f.Request.Body), len(f.Response.Body))
|
||||
id, now.Unix(), host, method, tmpl, url, rs.status, ct,
|
||||
len(rq.body), len(rs.body))
|
||||
if err != nil {
|
||||
log.Printf("[traffic] failed to record %s (writing index): %v", url, err)
|
||||
fail("writing index", err)
|
||||
return
|
||||
}
|
||||
rowid, err := res.LastInsertId()
|
||||
if err != nil {
|
||||
log.Printf("[traffic] failed to record %s (fetching rowid): %v", url, err)
|
||||
fail("fetching rowid", err)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(`INSERT OR REPLACE INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob)
|
||||
VALUES(?,?,?,?,?,?,?)`,
|
||||
id, reqHead, reqB.inline, nullIfEmpty(reqB.hash), respHead, respB.inline, nullIfEmpty(respB.hash)); err != nil {
|
||||
log.Printf("[traffic] failed to record %s (writing body): %v", url, err)
|
||||
fail("writing body", err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -556,26 +643,51 @@ VALUES(?,?,?,?,?,?,?)`,
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, h, id); err != nil {
|
||||
log.Printf("[traffic] failed to record %s (registering blob reference): %v", url, err)
|
||||
fail("registering blob reference", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := writeTrace(tx, id, lines, trace); err != nil {
|
||||
var re *recordError
|
||||
errors.As(err, &re)
|
||||
fail(re.step, re.err)
|
||||
return
|
||||
}
|
||||
|
||||
if t.fts {
|
||||
// The index is fed from memory, so a body that spilled to _blobs is still
|
||||
// fully searchable even though only its preview is stored inline.
|
||||
// fully searchable even though only its preview is stored inline. Only the
|
||||
// recorded views are indexed, never a value a rule wrote.
|
||||
idx := strings.Join([]string{url, reqHead, reqB.index, respHead, respB.index}, "\n")
|
||||
if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowid, idx); err != nil {
|
||||
log.Printf("[traffic] failed to record %s (writing full-text index): %v", url, err)
|
||||
fail("writing full-text index", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
log.Printf("[traffic] failed to record %s (committing): %v", url, err)
|
||||
fail("committing", err)
|
||||
}
|
||||
}
|
||||
|
||||
// appliedRules reports whether a trace rewrote its exchange, with the ids of
|
||||
// the rules that did, in order of first appearance.
|
||||
func appliedRules(lines []rewrite.TraceLine) (bool, []int64) {
|
||||
rewritten := false
|
||||
var ids []int64
|
||||
for _, l := range lines {
|
||||
if l.Outcome != rewrite.OutcomeApplied {
|
||||
continue
|
||||
}
|
||||
rewritten = true
|
||||
if l.RuleID != 0 && !slices.Contains(ids, l.RuleID) {
|
||||
ids = append(ids, l.RuleID)
|
||||
}
|
||||
}
|
||||
return rewritten, ids
|
||||
}
|
||||
|
||||
// newExchangeID names a recorded exchange: unix seconds plus a rolling sequence.
|
||||
func (t *Traffic) newExchangeID(now time.Time) string {
|
||||
n := t.seq.Add(1)
|
||||
@@ -726,15 +838,35 @@ func headerLines(h map[string][]string) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// requestHeaderLines restores the HTTP Host header, which net/http stores on
|
||||
// Request.Host rather than in Header. Keeping it in request.http makes the raw
|
||||
// capture complete and directly replayable.
|
||||
// requestHeaderLines is requestHeaderLinesOf for a flow's request.
|
||||
func requestHeaderLines(req *mproxy.Request) string {
|
||||
headers := req.Header.Clone()
|
||||
v := flowRequestView(req)
|
||||
return requestHeaderLinesOf(v.header, req.URL.Host, v.rawHost)
|
||||
}
|
||||
|
||||
// requestHead is the head an exchange records for a request view: request
|
||||
// line and headers, or only the Host when the original request is unavailable.
|
||||
func requestHead(rq requestView, target string) string {
|
||||
urlHost := ""
|
||||
if rq.url != nil {
|
||||
urlHost = rq.url.Host
|
||||
}
|
||||
if rq.unavailable {
|
||||
return fmt.Sprintf("%s %s %s\n%s%s", rq.method, target, rq.proto, requestHeaderLinesOf(nil, urlHost, rq.rawHost), unavailableMarker)
|
||||
}
|
||||
return fmt.Sprintf("%s %s %s\n%s", rq.method, target, rq.proto, requestHeaderLinesOf(rq.header, urlHost, rq.rawHost))
|
||||
}
|
||||
|
||||
// requestHeaderLinesOf restores the HTTP Host header, which net/http stores on
|
||||
// Request.Host rather than in Header. Keeping it in request.http makes the raw
|
||||
// capture complete and directly replayable. rawHost is the client's Host
|
||||
// header (Raw().Host); urlHost stands in when it is empty.
|
||||
func requestHeaderLinesOf(header http.Header, urlHost, rawHost string) string {
|
||||
headers := header.Clone()
|
||||
headers.Del("Host")
|
||||
host := req.URL.Host
|
||||
if raw := req.Raw(); raw != nil && strings.TrimSpace(raw.Host) != "" {
|
||||
host = raw.Host
|
||||
host := urlHost
|
||||
if strings.TrimSpace(rawHost) != "" {
|
||||
host = rawHost
|
||||
}
|
||||
var b strings.Builder
|
||||
if host = strings.TrimSpace(host); host != "" {
|
||||
@@ -1271,6 +1403,9 @@ func (t *Traffic) deleteWhere(tx *sql.Tx, where string, args ...any) (int64, err
|
||||
if _, err := tx.Exec(`DELETE FROM stream_messages WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil {
|
||||
return 0, fmt.Errorf("delete stream messages: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil {
|
||||
return 0, fmt.Errorf("delete rewrite trace: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM blob_refs WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil {
|
||||
return 0, fmt.Errorf("delete blob references: %w", err)
|
||||
}
|
||||
@@ -2138,7 +2273,7 @@ func (t *Traffic) Tools() []actool.CoreTool {
|
||||
|
||||
get := actool.Build(actool.Spec{
|
||||
Name: "traffic_get",
|
||||
Description: "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling. A WebSocket (status 101) or SSE (text/event-stream) record shows only its handshake here; read its messages with traffic_messages.",
|
||||
Description: TrafficGetDescription,
|
||||
Schema: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{"id": map[string]any{"type": "string", "description": "the id returned by traffic_search"}},
|
||||
@@ -2153,7 +2288,15 @@ func (t *Traffic) Tools() []actool.CoreTool {
|
||||
if err != nil {
|
||||
return actool.Errorf(err.Error()), nil
|
||||
}
|
||||
return actool.Text("=== REQUEST ===\n" + clip(req, 2500) + "\n\n=== RESPONSE ===\n" + clip(resp, 4000)), nil
|
||||
rewritten, ruleIDs, err := t.RewriteInfo(a.ID)
|
||||
if err != nil {
|
||||
return actool.Errorf(err.Error()), nil
|
||||
}
|
||||
head := ""
|
||||
if rewritten {
|
||||
head = fmt.Sprintf("=== REWRITTEN === rewrite rules %v changed this exchange in the proxy. Shown below: the request as the client sent it and the response as the target sent it, not the rewritten values.\n\n", ruleIDs)
|
||||
}
|
||||
return actool.Text(head + "=== REQUEST ===\n" + clip(req, 2500) + "\n\n=== RESPONSE ===\n" + clip(resp, 4000)), nil
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user