diff --git a/server/server.go b/server/server.go index a402a9f..245790a 100644 --- a/server/server.go +++ b/server/server.go @@ -3534,6 +3534,9 @@ func (s *Server) replayTraffic(w http.ResponseWriter, r *http.Request) { defer cancel() res, err := tr.Replay(ctx, traffic.ReplayRequest{Method: body.Method, URL: body.URL, Headers: body.Headers, Body: payload}) switch { + case errors.Is(err, traffic.ErrReplayTooLarge): + writeErr(w, 413, err.Error()) + return case errors.Is(err, traffic.ErrReplayInvalid): writeErr(w, 400, err.Error()) return @@ -3544,7 +3547,10 @@ func (s *Server) replayTraffic(w http.ResponseWriter, r *http.Request) { writeErr(w, 502, err.Error()) return } - writeJSON(w, 200, map[string]any{"status": res.Status, "raw": res.Raw, "duration_ms": res.DurationMS, "truncated": res.Truncated}) + writeJSON(w, 200, map[string]any{ + "status": res.Status, "headers": res.Headers, "raw": res.Raw, "duration_ms": res.DurationMS, + "truncated": res.Truncated, "recorded": res.Recorded, + }) } // getTrafficBlob streams one oversized body by its sha256. Bodies past the inline diff --git a/server/traffic_replay_test.go b/server/traffic_replay_test.go index ca50b44..43339b0 100644 --- a/server/traffic_replay_test.go +++ b/server/traffic_replay_test.go @@ -99,7 +99,7 @@ func TestTrafficReplayRouteValidation(t *testing.T) { "header CRLF": {`{"method":"GET","url":"http://example.com/","headers":[{"name":"X-A","value":"a\r\nb"}]}`, 400}, "too large": {`{"method":"POST","url":"http://example.com/","body":"` + strings.Repeat("a", 17<<20) + `"}`, 413}, "proxy down": {`{"method":"GET","url":"http://127.0.0.1:1/"}`, 502}, - "body too big": {`{"method":"POST","url":"http://example.com/","body":"` + base64.StdEncoding.EncodeToString(make([]byte, 9<<20)) + `","body_base64":true}`, 400}, + "body too big": {`{"method":"POST","url":"http://example.com/","body":"` + base64.StdEncoding.EncodeToString(make([]byte, 9<<20)) + `","body_base64":true}`, 413}, } { if w := replay(c.body); w.Code != c.want { t.Errorf("%s: %d %s, want %d", name, w.Code, strings.TrimSpace(w.Body.String()), c.want) @@ -126,17 +126,26 @@ func TestTrafficReplayRouteEndToEnd(t *testing.T) { t.Fatalf("replay = %d %s", w.Code, w.Body) } var res struct { - Status int `json:"status"` - Raw string `json:"raw"` - DurationMS int64 `json:"duration_ms"` - Truncated bool `json:"truncated"` + Status int `json:"status"` + Headers []traffic.ReplayHeader `json:"headers"` + Raw string `json:"raw"` + DurationMS int64 `json:"duration_ms"` + Truncated bool `json:"truncated"` + Recorded bool `json:"recorded"` } if err := json.Unmarshal(w.Body.Bytes(), &res); err != nil { t.Fatal(err) } - if res.Status != 200 || res.Truncated || !strings.Contains(res.Raw, "X-Seen: PUT edited") || !strings.HasSuffix(res.Raw, "got:payload") { + if res.Status != 200 || res.Truncated || !res.Recorded || !strings.Contains(res.Raw, "X-Seen: PUT edited") || !strings.HasSuffix(res.Raw, "got:payload") { t.Fatalf("result %+v", res) } + seen := false + for _, h := range res.Headers { + seen = seen || (h.Name == "X-Seen" && h.Value == "PUT edited") + } + if !seen { + t.Fatalf("headers %+v lack X-Seen", res.Headers) + } deadline := time.Now().Add(5 * time.Second) for { var n int diff --git a/traffic/replay.go b/traffic/replay.go index 7dd09e9..420f0aa 100644 --- a/traffic/replay.go +++ b/traffic/replay.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "os" + "sort" "strings" "time" ) @@ -32,6 +33,9 @@ const ( // ErrReplayInvalid marks a replay request rejected before anything is sent. var ErrReplayInvalid = errors.New("invalid replay request") +// ErrReplayTooLarge marks a request body over maxReplayBody. +var ErrReplayTooLarge = errors.New("replay body too large") + // ReplayHeader is one request header line, kept as a list so duplicates and // their order survive. type ReplayHeader struct { @@ -49,24 +53,41 @@ type ReplayRequest struct { // ReplayResult is the response as received by the client side of the proxy. type ReplayResult struct { - Status int + Status int + // Headers are the response headers, keys sorted, every value kept. + Headers []ReplayHeader Raw string // status line, headers and (possibly truncated) body DurationMS int64 Truncated bool // the body exceeded maxReplayResponse and was cut + // Recorded reports whether the proxy records this exchange: false for an + // https host the proxy tunnels without interception (passthrough). + Recorded bool } // replayDropped are the headers the client must not send as given: hop-by-hop // fields and framing that net/http recomputes (Content-Length, -// Transfer-Encoding). Upgrade is rejected rather than dropped, see Replay. +// Transfer-Encoding). Headers named in Connection are dropped too. Upgrade is +// rejected rather than dropped, see Replay. var replayDropped = map[string]bool{ - "Content-Length": true, - "Transfer-Encoding": true, - "Connection": true, - "Proxy-Connection": true, - "Keep-Alive": true, - "Te": true, - "Trailer": true, - "Expect": true, + "Content-Length": true, + "Transfer-Encoding": true, + "Connection": true, + "Proxy-Connection": true, + "Proxy-Authorization": true, + "Keep-Alive": true, + "Te": true, + "Trailer": true, + "Expect": true, +} + +// replayRecorded reports whether the proxy records a request to scheme://host: +// plain http is always intercepted, https is unless the host is in passthrough. +func (t *Traffic) replayRecorded(scheme, host string) bool { + if scheme != "https" { + return true + } + _, tunneled := t.pass.Load(hostOnly(host)) + return !tunneled } func replayInvalid(format string, args ...any) error { @@ -93,7 +114,7 @@ func validHeaderName(name string) bool { // Replay sends req through the recording proxy and returns the response. The // proxy's sink records the exchange; Replay never calls record() itself. A host // the proxy tunnels without interception (passthrough, see maybePassthrough) -// still answers, but that exchange is not recorded. +// still answers, but that exchange is not recorded (Recorded is false). // // Redirects are not followed, no cookie jar is kept, and compression is not // negotiated: the edited headers go out as given, without an Accept-Encoding @@ -120,12 +141,25 @@ func (t *Traffic) Replay(ctx context.Context, req ReplayRequest) (ReplayResult, return ReplayResult{}, replayInvalid("url must not carry user info; send an Authorization header instead") } if len(req.Body) > maxReplayBody { - return ReplayResult{}, replayInvalid("body is %d bytes, over the %d-byte limit", len(req.Body), maxReplayBody) + return ReplayResult{}, fmt.Errorf("%w: body is %d bytes, over the %d-byte limit", ErrReplayTooLarge, len(req.Body), maxReplayBody) } if len(req.Headers) > maxReplayHeaders { return ReplayResult{}, replayInvalid("%d headers, over the limit of %d", len(req.Headers), maxReplayHeaders) } + // Headers named in Connection are hop-by-hop too (RFC 9110 ยง7.6.1). + connectionNamed := map[string]bool{} + for _, h := range req.Headers { + if http.CanonicalHeaderKey(strings.TrimSpace(h.Name)) != "Connection" { + continue + } + for _, token := range strings.Split(h.Value, ",") { + if token = strings.TrimSpace(token); token != "" { + connectionNamed[http.CanonicalHeaderKey(token)] = true + } + } + } + header := http.Header{} host := "" for _, h := range req.Headers { @@ -142,13 +176,18 @@ func (t *Traffic) Replay(ctx context.Context, req ReplayRequest) (ReplayResult, // A protocol upgrade (WebSocket, h2c) cannot be replayed as one request. return ReplayResult{}, replayInvalid("Upgrade requests cannot be replayed") case canonical == "Host": - // The recording proxy routes a request by its Host, not by the URL, so - // a different Host would silently send the request to another server. + // A Host override cannot be replayed faithfully. go-mitmproxy rebuilds + // the upstream request from the URL (proxyReq from f.Request.URL, dialed + // at CanonicalAddr(req.URL)) and copies only Header, where net/http does + // not keep Host, so the override is lost: the recorded exchange would + // show a Host the target never received. For plain http the client also + // puts req.Host in the absolute-form request target, which sends the + // request to that host instead. Refusing avoids a misleading exchange. host = strings.TrimSpace(h.Value) if !strings.EqualFold(host, target.Host) { - return ReplayResult{}, replayInvalid("Host %q differs from the URL host %q; the proxy routes by Host, so edit the URL instead", host, target.Host) + return ReplayResult{}, replayInvalid("Host %q differs from the URL host %q; the proxy sends the URL host as Host, so edit the URL instead", host, target.Host) } - case replayDropped[canonical]: + case replayDropped[canonical], connectionNamed[canonical]: // recomputed by net/http, or hop-by-hop default: header.Add(canonical, h.Value) @@ -190,7 +229,18 @@ func (t *Traffic) Replay(ctx context.Context, req ReplayRequest) (ReplayResult, if err != nil { return ReplayResult{}, fmt.Errorf("replay %s %s: reading response: %w", method, target.Redacted(), err) } - res := ReplayResult{Status: resp.StatusCode} + res := ReplayResult{Status: resp.StatusCode, Recorded: t.replayRecorded(target.Scheme, target.Host)} + keys := make([]string, 0, len(resp.Header)) + for k := range resp.Header { + keys = append(keys, k) + } + sort.Strings(keys) + res.Headers = make([]ReplayHeader, 0, len(keys)) + for _, k := range keys { + for _, v := range resp.Header[k] { + res.Headers = append(res.Headers, ReplayHeader{Name: k, Value: v}) + } + } if len(data) > maxReplayResponse { data = data[:maxReplayResponse] res.Truncated = true diff --git a/traffic/replay_test.go b/traffic/replay_test.go index 01fa679..931569b 100644 --- a/traffic/replay_test.go +++ b/traffic/replay_test.go @@ -24,6 +24,8 @@ type echoed struct { ContentLength int64 `json:"content_length"` Body string `json:"body"` Host string `json:"host"` + XHop string `json:"x_hop"` + ProxyAuth string `json:"proxy_auth"` } func echoHandler(w http.ResponseWriter, r *http.Request) { @@ -33,6 +35,7 @@ func echoHandler(w http.ResponseWriter, r *http.Request) { _ = json.NewEncoder(w).Encode(echoed{ Method: r.Method, Path: r.URL.Path, Query: r.URL.RawQuery, Custom: r.Header.Get("X-Custom"), Connection: r.Header.Get("Connection"), ContentLength: r.ContentLength, Body: string(b), Host: r.Host, + XHop: r.Header.Get("X-Hop"), ProxyAuth: r.Header.Get("Proxy-Authorization"), }) } @@ -81,8 +84,18 @@ func TestReplayThroughProxy(t *testing.T) { if err != nil { t.Fatal(err) } - if res.Status != 200 || res.Truncated || !strings.HasPrefix(res.Raw, "HTTP/1.1 200 OK\n") || !strings.Contains(res.Raw, "X-Echo: yes") { - t.Fatalf("result status=%d truncated=%v raw:\n%s", res.Status, res.Truncated, res.Raw) + if res.Status != 200 || res.Truncated || !res.Recorded || !strings.HasPrefix(res.Raw, "HTTP/1.1 200 OK\n") || !strings.Contains(res.Raw, "X-Echo: yes") { + t.Fatalf("result status=%d truncated=%v recorded=%v raw:\n%s", res.Status, res.Truncated, res.Recorded, res.Raw) + } + echoHeader := false + for i, h := range res.Headers { + if i > 0 && res.Headers[i-1].Name > h.Name { + t.Fatalf("headers not sorted: %+v", res.Headers) + } + echoHeader = echoHeader || (h.Name == "X-Echo" && h.Value == "yes") + } + if !echoHeader { + t.Fatalf("headers %+v lack X-Echo: yes", res.Headers) } got := replayBody(t, res) if got.Method != "PATCH" || got.Path != "/edited/path" || got.Query != "q=1&r=two" || got.Custom != "edited" || got.Body != "edited body" { @@ -101,7 +114,9 @@ func TestReplayDropsHopByHopHeaders(t *testing.T) { Method: "POST", URL: srv.URL + "/hop", Headers: []ReplayHeader{ - {Name: "Connection", Value: "close"}, + {Name: "Connection", Value: "close, X-Hop"}, + {Name: "X-Hop", Value: "v"}, + {Name: "Proxy-Authorization", Value: "Basic dXNlcjpwYXNz"}, {Name: "Content-Length", Value: "999"}, {Name: "Keep-Alive", Value: "timeout=5"}, {Name: "Host", Value: strings.TrimPrefix(srv.URL, "http://")}, @@ -115,6 +130,9 @@ func TestReplayDropsHopByHopHeaders(t *testing.T) { if got.Connection != "" || got.ContentLength != 3 || got.Body != "abc" { t.Fatalf("target received connection=%q content_length=%d body=%q, want no Connection and a recomputed length", got.Connection, got.ContentLength, got.Body) } + if got.XHop != "" || got.ProxyAuth != "" { + t.Fatalf("target received X-Hop=%q Proxy-Authorization=%q, want neither", got.XHop, got.ProxyAuth) + } if got.Host != strings.TrimPrefix(srv.URL, "http://") { t.Fatalf("target saw Host %q", got.Host) } @@ -211,7 +229,6 @@ func TestReplayRejectsInvalidRequests(t *testing.T) { "empty method": {Method: " ", URL: "http://example.com/"}, "CONNECT": {Method: "CONNECT", URL: "http://example.com/"}, "method not token": {Method: "GE T", URL: "http://example.com/"}, - "body too large": {Method: "POST", URL: "http://example.com/", Body: make([]byte, maxReplayBody+1)}, "too many headers": {Method: "GET", URL: "http://example.com/", Headers: many}, "CR in header": {Method: "GET", URL: "http://example.com/", Headers: []ReplayHeader{{Name: "X-A", Value: "a\r\nX-Injected: 1"}}}, "LF in header": {Method: "GET", URL: "http://example.com/", Headers: []ReplayHeader{{Name: "X-A", Value: "a\nb"}}}, @@ -223,4 +240,26 @@ func TestReplayRejectsInvalidRequests(t *testing.T) { t.Errorf("%s: err = %v, want ErrReplayInvalid", name, err) } } + _, err = tr.Replay(t.Context(), ReplayRequest{Method: "POST", URL: "http://example.com/", Body: make([]byte, maxReplayBody+1)}) + if !errors.Is(err, ErrReplayTooLarge) || errors.Is(err, ErrReplayInvalid) { + t.Errorf("body too large: err = %v, want ErrReplayTooLarge", err) + } +} + +// Only an https host in passthrough is reported as not recorded. +func TestReplayRecordedHelper(t *testing.T) { + tr := &Traffic{} + tr.pass.Store("localhost", struct{}{}) + for _, c := range []struct { + scheme, host string + want bool + }{ + {"https", "localhost:443", false}, + {"https", "example.com:443", true}, + {"http", "localhost:80", true}, + } { + if got := tr.replayRecorded(c.scheme, c.host); got != c.want { + t.Errorf("replayRecorded(%s, %s) = %v, want %v", c.scheme, c.host, got, c.want) + } + } }