fix(traffic): replay reports recorded and headers, drops Connection-named headers, 413 for large bodies

The Host refusal now explains that go-mitmproxy rebuilds the upstream request from the URL and drops the Host override. ReplayResult gains Recorded (false for an https passthrough host) and the sorted response Headers, both exposed by the route. Headers named in Connection and Proxy-Authorization are dropped. A body over the limit returns ErrReplayTooLarge, mapped to 413.
This commit is contained in:
BaaRai
2026-11-16 16:17:55 +01:00
parent e770caa28e
commit 1263cdc185
4 changed files with 132 additions and 28 deletions
+7 -1
View File
@@ -3534,6 +3534,9 @@ func (s *Server) replayTraffic(w http.ResponseWriter, r *http.Request) {
defer cancel()
res, err := tr.Replay(ctx, traffic.ReplayRequest{Method: body.Method, URL: body.URL, Headers: body.Headers, Body: payload})
switch {
case errors.Is(err, traffic.ErrReplayTooLarge):
writeErr(w, 413, err.Error())
return
case errors.Is(err, traffic.ErrReplayInvalid):
writeErr(w, 400, err.Error())
return
@@ -3544,7 +3547,10 @@ func (s *Server) replayTraffic(w http.ResponseWriter, r *http.Request) {
writeErr(w, 502, err.Error())
return
}
writeJSON(w, 200, map[string]any{"status": res.Status, "raw": res.Raw, "duration_ms": res.DurationMS, "truncated": res.Truncated})
writeJSON(w, 200, map[string]any{
"status": res.Status, "headers": res.Headers, "raw": res.Raw, "duration_ms": res.DurationMS,
"truncated": res.Truncated, "recorded": res.Recorded,
})
}
// getTrafficBlob streams one oversized body by its sha256. Bodies past the inline
+15 -6
View File
@@ -99,7 +99,7 @@ func TestTrafficReplayRouteValidation(t *testing.T) {
"header CRLF": {`{"method":"GET","url":"http://example.com/","headers":[{"name":"X-A","value":"a\r\nb"}]}`, 400},
"too large": {`{"method":"POST","url":"http://example.com/","body":"` + strings.Repeat("a", 17<<20) + `"}`, 413},
"proxy down": {`{"method":"GET","url":"http://127.0.0.1:1/"}`, 502},
"body too big": {`{"method":"POST","url":"http://example.com/","body":"` + base64.StdEncoding.EncodeToString(make([]byte, 9<<20)) + `","body_base64":true}`, 400},
"body too big": {`{"method":"POST","url":"http://example.com/","body":"` + base64.StdEncoding.EncodeToString(make([]byte, 9<<20)) + `","body_base64":true}`, 413},
} {
if w := replay(c.body); w.Code != c.want {
t.Errorf("%s: %d %s, want %d", name, w.Code, strings.TrimSpace(w.Body.String()), c.want)
@@ -126,17 +126,26 @@ func TestTrafficReplayRouteEndToEnd(t *testing.T) {
t.Fatalf("replay = %d %s", w.Code, w.Body)
}
var res struct {
Status int `json:"status"`
Raw string `json:"raw"`
DurationMS int64 `json:"duration_ms"`
Truncated bool `json:"truncated"`
Status int `json:"status"`
Headers []traffic.ReplayHeader `json:"headers"`
Raw string `json:"raw"`
DurationMS int64 `json:"duration_ms"`
Truncated bool `json:"truncated"`
Recorded bool `json:"recorded"`
}
if err := json.Unmarshal(w.Body.Bytes(), &res); err != nil {
t.Fatal(err)
}
if res.Status != 200 || res.Truncated || !strings.Contains(res.Raw, "X-Seen: PUT edited") || !strings.HasSuffix(res.Raw, "got:payload") {
if res.Status != 200 || res.Truncated || !res.Recorded || !strings.Contains(res.Raw, "X-Seen: PUT edited") || !strings.HasSuffix(res.Raw, "got:payload") {
t.Fatalf("result %+v", res)
}
seen := false
for _, h := range res.Headers {
seen = seen || (h.Name == "X-Seen" && h.Value == "PUT edited")
}
if !seen {
t.Fatalf("headers %+v lack X-Seen", res.Headers)
}
deadline := time.Now().Add(5 * time.Second)
for {
var n int
+67 -17
View File
@@ -11,6 +11,7 @@ import (
"net/http"
"net/url"
"os"
"sort"
"strings"
"time"
)
@@ -32,6 +33,9 @@ const (
// ErrReplayInvalid marks a replay request rejected before anything is sent.
var ErrReplayInvalid = errors.New("invalid replay request")
// ErrReplayTooLarge marks a request body over maxReplayBody.
var ErrReplayTooLarge = errors.New("replay body too large")
// ReplayHeader is one request header line, kept as a list so duplicates and
// their order survive.
type ReplayHeader struct {
@@ -49,24 +53,41 @@ type ReplayRequest struct {
// ReplayResult is the response as received by the client side of the proxy.
type ReplayResult struct {
Status int
Status int
// Headers are the response headers, keys sorted, every value kept.
Headers []ReplayHeader
Raw string // status line, headers and (possibly truncated) body
DurationMS int64
Truncated bool // the body exceeded maxReplayResponse and was cut
// Recorded reports whether the proxy records this exchange: false for an
// https host the proxy tunnels without interception (passthrough).
Recorded bool
}
// replayDropped are the headers the client must not send as given: hop-by-hop
// fields and framing that net/http recomputes (Content-Length,
// Transfer-Encoding). Upgrade is rejected rather than dropped, see Replay.
// Transfer-Encoding). Headers named in Connection are dropped too. Upgrade is
// rejected rather than dropped, see Replay.
var replayDropped = map[string]bool{
"Content-Length": true,
"Transfer-Encoding": true,
"Connection": true,
"Proxy-Connection": true,
"Keep-Alive": true,
"Te": true,
"Trailer": true,
"Expect": true,
"Content-Length": true,
"Transfer-Encoding": true,
"Connection": true,
"Proxy-Connection": true,
"Proxy-Authorization": true,
"Keep-Alive": true,
"Te": true,
"Trailer": true,
"Expect": true,
}
// replayRecorded reports whether the proxy records a request to scheme://host:
// plain http is always intercepted, https is unless the host is in passthrough.
func (t *Traffic) replayRecorded(scheme, host string) bool {
if scheme != "https" {
return true
}
_, tunneled := t.pass.Load(hostOnly(host))
return !tunneled
}
func replayInvalid(format string, args ...any) error {
@@ -93,7 +114,7 @@ func validHeaderName(name string) bool {
// Replay sends req through the recording proxy and returns the response. The
// proxy's sink records the exchange; Replay never calls record() itself. A host
// the proxy tunnels without interception (passthrough, see maybePassthrough)
// still answers, but that exchange is not recorded.
// still answers, but that exchange is not recorded (Recorded is false).
//
// Redirects are not followed, no cookie jar is kept, and compression is not
// negotiated: the edited headers go out as given, without an Accept-Encoding
@@ -120,12 +141,25 @@ func (t *Traffic) Replay(ctx context.Context, req ReplayRequest) (ReplayResult,
return ReplayResult{}, replayInvalid("url must not carry user info; send an Authorization header instead")
}
if len(req.Body) > maxReplayBody {
return ReplayResult{}, replayInvalid("body is %d bytes, over the %d-byte limit", len(req.Body), maxReplayBody)
return ReplayResult{}, fmt.Errorf("%w: body is %d bytes, over the %d-byte limit", ErrReplayTooLarge, len(req.Body), maxReplayBody)
}
if len(req.Headers) > maxReplayHeaders {
return ReplayResult{}, replayInvalid("%d headers, over the limit of %d", len(req.Headers), maxReplayHeaders)
}
// Headers named in Connection are hop-by-hop too (RFC 9110 §7.6.1).
connectionNamed := map[string]bool{}
for _, h := range req.Headers {
if http.CanonicalHeaderKey(strings.TrimSpace(h.Name)) != "Connection" {
continue
}
for _, token := range strings.Split(h.Value, ",") {
if token = strings.TrimSpace(token); token != "" {
connectionNamed[http.CanonicalHeaderKey(token)] = true
}
}
}
header := http.Header{}
host := ""
for _, h := range req.Headers {
@@ -142,13 +176,18 @@ func (t *Traffic) Replay(ctx context.Context, req ReplayRequest) (ReplayResult,
// A protocol upgrade (WebSocket, h2c) cannot be replayed as one request.
return ReplayResult{}, replayInvalid("Upgrade requests cannot be replayed")
case canonical == "Host":
// The recording proxy routes a request by its Host, not by the URL, so
// a different Host would silently send the request to another server.
// A Host override cannot be replayed faithfully. go-mitmproxy rebuilds
// the upstream request from the URL (proxyReq from f.Request.URL, dialed
// at CanonicalAddr(req.URL)) and copies only Header, where net/http does
// not keep Host, so the override is lost: the recorded exchange would
// show a Host the target never received. For plain http the client also
// puts req.Host in the absolute-form request target, which sends the
// request to that host instead. Refusing avoids a misleading exchange.
host = strings.TrimSpace(h.Value)
if !strings.EqualFold(host, target.Host) {
return ReplayResult{}, replayInvalid("Host %q differs from the URL host %q; the proxy routes by Host, so edit the URL instead", host, target.Host)
return ReplayResult{}, replayInvalid("Host %q differs from the URL host %q; the proxy sends the URL host as Host, so edit the URL instead", host, target.Host)
}
case replayDropped[canonical]:
case replayDropped[canonical], connectionNamed[canonical]:
// recomputed by net/http, or hop-by-hop
default:
header.Add(canonical, h.Value)
@@ -190,7 +229,18 @@ func (t *Traffic) Replay(ctx context.Context, req ReplayRequest) (ReplayResult,
if err != nil {
return ReplayResult{}, fmt.Errorf("replay %s %s: reading response: %w", method, target.Redacted(), err)
}
res := ReplayResult{Status: resp.StatusCode}
res := ReplayResult{Status: resp.StatusCode, Recorded: t.replayRecorded(target.Scheme, target.Host)}
keys := make([]string, 0, len(resp.Header))
for k := range resp.Header {
keys = append(keys, k)
}
sort.Strings(keys)
res.Headers = make([]ReplayHeader, 0, len(keys))
for _, k := range keys {
for _, v := range resp.Header[k] {
res.Headers = append(res.Headers, ReplayHeader{Name: k, Value: v})
}
}
if len(data) > maxReplayResponse {
data = data[:maxReplayResponse]
res.Truncated = true
+43 -4
View File
@@ -24,6 +24,8 @@ type echoed struct {
ContentLength int64 `json:"content_length"`
Body string `json:"body"`
Host string `json:"host"`
XHop string `json:"x_hop"`
ProxyAuth string `json:"proxy_auth"`
}
func echoHandler(w http.ResponseWriter, r *http.Request) {
@@ -33,6 +35,7 @@ func echoHandler(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(echoed{
Method: r.Method, Path: r.URL.Path, Query: r.URL.RawQuery, Custom: r.Header.Get("X-Custom"),
Connection: r.Header.Get("Connection"), ContentLength: r.ContentLength, Body: string(b), Host: r.Host,
XHop: r.Header.Get("X-Hop"), ProxyAuth: r.Header.Get("Proxy-Authorization"),
})
}
@@ -81,8 +84,18 @@ func TestReplayThroughProxy(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if res.Status != 200 || res.Truncated || !strings.HasPrefix(res.Raw, "HTTP/1.1 200 OK\n") || !strings.Contains(res.Raw, "X-Echo: yes") {
t.Fatalf("result status=%d truncated=%v raw:\n%s", res.Status, res.Truncated, res.Raw)
if res.Status != 200 || res.Truncated || !res.Recorded || !strings.HasPrefix(res.Raw, "HTTP/1.1 200 OK\n") || !strings.Contains(res.Raw, "X-Echo: yes") {
t.Fatalf("result status=%d truncated=%v recorded=%v raw:\n%s", res.Status, res.Truncated, res.Recorded, res.Raw)
}
echoHeader := false
for i, h := range res.Headers {
if i > 0 && res.Headers[i-1].Name > h.Name {
t.Fatalf("headers not sorted: %+v", res.Headers)
}
echoHeader = echoHeader || (h.Name == "X-Echo" && h.Value == "yes")
}
if !echoHeader {
t.Fatalf("headers %+v lack X-Echo: yes", res.Headers)
}
got := replayBody(t, res)
if got.Method != "PATCH" || got.Path != "/edited/path" || got.Query != "q=1&r=two" || got.Custom != "edited" || got.Body != "edited body" {
@@ -101,7 +114,9 @@ func TestReplayDropsHopByHopHeaders(t *testing.T) {
Method: "POST",
URL: srv.URL + "/hop",
Headers: []ReplayHeader{
{Name: "Connection", Value: "close"},
{Name: "Connection", Value: "close, X-Hop"},
{Name: "X-Hop", Value: "v"},
{Name: "Proxy-Authorization", Value: "Basic dXNlcjpwYXNz"},
{Name: "Content-Length", Value: "999"},
{Name: "Keep-Alive", Value: "timeout=5"},
{Name: "Host", Value: strings.TrimPrefix(srv.URL, "http://")},
@@ -115,6 +130,9 @@ func TestReplayDropsHopByHopHeaders(t *testing.T) {
if got.Connection != "" || got.ContentLength != 3 || got.Body != "abc" {
t.Fatalf("target received connection=%q content_length=%d body=%q, want no Connection and a recomputed length", got.Connection, got.ContentLength, got.Body)
}
if got.XHop != "" || got.ProxyAuth != "" {
t.Fatalf("target received X-Hop=%q Proxy-Authorization=%q, want neither", got.XHop, got.ProxyAuth)
}
if got.Host != strings.TrimPrefix(srv.URL, "http://") {
t.Fatalf("target saw Host %q", got.Host)
}
@@ -211,7 +229,6 @@ func TestReplayRejectsInvalidRequests(t *testing.T) {
"empty method": {Method: " ", URL: "http://example.com/"},
"CONNECT": {Method: "CONNECT", URL: "http://example.com/"},
"method not token": {Method: "GE T", URL: "http://example.com/"},
"body too large": {Method: "POST", URL: "http://example.com/", Body: make([]byte, maxReplayBody+1)},
"too many headers": {Method: "GET", URL: "http://example.com/", Headers: many},
"CR in header": {Method: "GET", URL: "http://example.com/", Headers: []ReplayHeader{{Name: "X-A", Value: "a\r\nX-Injected: 1"}}},
"LF in header": {Method: "GET", URL: "http://example.com/", Headers: []ReplayHeader{{Name: "X-A", Value: "a\nb"}}},
@@ -223,4 +240,26 @@ func TestReplayRejectsInvalidRequests(t *testing.T) {
t.Errorf("%s: err = %v, want ErrReplayInvalid", name, err)
}
}
_, err = tr.Replay(t.Context(), ReplayRequest{Method: "POST", URL: "http://example.com/", Body: make([]byte, maxReplayBody+1)})
if !errors.Is(err, ErrReplayTooLarge) || errors.Is(err, ErrReplayInvalid) {
t.Errorf("body too large: err = %v, want ErrReplayTooLarge", err)
}
}
// Only an https host in passthrough is reported as not recorded.
func TestReplayRecordedHelper(t *testing.T) {
tr := &Traffic{}
tr.pass.Store("localhost", struct{}{})
for _, c := range []struct {
scheme, host string
want bool
}{
{"https", "localhost:443", false},
{"https", "example.com:443", true},
{"http", "localhost:80", true},
} {
if got := tr.replayRecorded(c.scheme, c.host); got != c.want {
t.Errorf("replayRecorded(%s, %s) = %v, want %v", c.scheme, c.host, got, c.want)
}
}
}