diff --git a/traffic/rewrite/compile.go b/traffic/rewrite/compile.go new file mode 100644 index 0000000..ed663f9 --- /dev/null +++ b/traffic/rewrite/compile.go @@ -0,0 +1,123 @@ +package rewrite + +import ( + "sort" + "sync" + "time" +) + +// Options configure a compiled Set. +type Options struct { + // TraceKey is the instance sub-key of the sensitive-value fingerprint. + // Empty: sensitive values get no fingerprint at all. + TraceKey []byte + // Errors receives the run-time rule errors. Pass the same log to every + // Compile so the entries survive a reload. Nil: errors are not kept. + Errors *ErrorLog +} + +// Set is an immutable snapshot of the enabled rules, compiled and in execution +// order: priority ascending, then id ascending, so the highest priority applies +// last. A nil *Set holds no rule. +type Set struct { + request, response []*compiledRule + loadErrors map[int64]string + traceKey []byte + errors *ErrorLog +} + +// Compile builds the snapshot of the enabled rules. An enabled rule that fails +// validation is left out and reported by LoadErrors. +func Compile(rules []Rule, opts Options) *Set { + s := &Set{loadErrors: map[int64]string{}, traceKey: opts.TraceKey, errors: opts.Errors} + enabled := make([]Rule, 0, len(rules)) + for _, r := range rules { + if r.Enabled { + enabled = append(enabled, r) + } + } + sort.SliceStable(enabled, func(i, j int) bool { + if enabled[i].Priority != enabled[j].Priority { + return enabled[i].Priority < enabled[j].Priority + } + return enabled[i].ID < enabled[j].ID + }) + for _, r := range enabled { + c, err := compileRule(r) + switch { + case err != nil: + s.loadErrors[r.ID] = err.Error() + case r.Phase == PhaseRequest: + s.request = append(s.request, c) + default: + s.response = append(s.response, c) + } + } + return s +} + +// Len is the number of rules the snapshot runs in a phase. +func (s *Set) Len(phase Phase) int { + if s == nil { + return 0 + } + return len(s.rules(phase)) +} + +func (s *Set) rules(phase Phase) []*compiledRule { + if phase == PhaseRequest { + return s.request + } + return s.response +} + +// LoadErrors maps the id of each enabled rule left out of the snapshot to the +// reason. The map must not be modified. +func (s *Set) LoadErrors() map[int64]string { + if s == nil { + return nil + } + return s.loadErrors +} + +// LastError is the most recent run-time error of a rule. It lives in memory +// only; the per-exchange trace keeps the history. +type LastError struct { + Class string `json:"class"` + At time.Time `json:"at"` + Count int64 `json:"count"` +} + +// ErrorLog keeps the last run-time error of each rule. The zero value is ready +// to use. +type ErrorLog struct { + mu sync.Mutex + m map[int64]LastError +} + +func (l *ErrorLog) record(id int64, class string) { + if l == nil { + return + } + l.mu.Lock() + if l.m == nil { + l.m = map[int64]LastError{} + } + l.m[id] = LastError{Class: class, At: time.Now(), Count: l.m[id].Count + 1} + l.mu.Unlock() +} + +// Get returns the last error of a rule, ok=false when it has none. +func (l *ErrorLog) Get(id int64) (LastError, bool) { + l.mu.Lock() + defer l.mu.Unlock() + e, ok := l.m[id] + return e, ok +} + +// Forget drops the entry of a rule that was edited or deleted. +func (l *ErrorLog) Forget(id int64) { + l.mu.Lock() + delete(l.m, id) + l.mu.Unlock() +} diff --git a/traffic/rewrite/engine.go b/traffic/rewrite/engine.go new file mode 100644 index 0000000..3643929 --- /dev/null +++ b/traffic/rewrite/engine.go @@ -0,0 +1,460 @@ +package rewrite + +import ( + "bytes" + "net/http" + "net/url" + "regexp" + "slices" + "strconv" + "strings" +) + +// Request is the request side of an exchange. +type Request struct { + Method string + URL *url.URL + Header http.Header + Body []byte +} + +// Response is the response side of an exchange. +type Response struct { + Status int + Header http.Header + Body []byte +} + +// Input is what one phase reads. The request phase rewrites Request. The +// response phase rewrites Response and reads Request, which must then be the +// request as it was sent to the target. +type Input struct { + Request Request + Response Response + // Stream marks a response whose body is not in memory (SSE): rules that + // read or rewrite the response body are skipped, the others apply. + Stream bool + // IsBinary reports a body that must not be rewritten. Nil: a body with a + // NUL byte in its first 512 bytes is binary. + IsBinary func(contentType string, body []byte) bool +} + +// Result is the outcome of one phase. When Changed is false, Request and +// Response are the input's. When it is true, every rewritten part is a new +// object: the engine never writes to the URL, the headers or the bodies it +// was given. +type Result struct { + Request Request + Response Response + Changed bool + Lines []TraceLine +} + +const ( + fieldMethod = "method" + fieldURL = "url" + fieldBody = "body" + fieldStatus = "status" +) + +// bodyDependentHeaders no longer describe a body once a rule rewrote it. +var bodyDependentHeaders = []string{"Content-Length", "Etag", "Content-Md5", "Content-Encoding"} + +type state struct { + req Request + resp Response +} + +type run struct { + set *Set + phase Phase + in *Input + st state + tainted map[string]bool // fields a sensitive action wrote + seen map[int64]bool // rules whose snapshot is already on a line + lines []TraceLine + changed bool +} + +// Apply runs the rules of one phase as a pipeline: each rule sees the output of +// the previous one. A rule is atomic: when one of its actions fails, or it +// panics, none of them applies and the message goes on unchanged by that rule. +func (s *Set) Apply(phase Phase, in Input) Result { + res := Result{Request: in.Request, Response: in.Response} + if s.Len(phase) == 0 { + return res + } + x := &run{set: s, phase: phase, in: &in, st: state{in.Request, in.Response}, tainted: map[string]bool{}, seen: map[int64]bool{}} + if x.st.req.URL == nil { + x.st.req.URL = &url.URL{} + } + // A protocol upgrade (WebSocket handshake) is never rewritten. + upgrade := len(in.Request.Header.Values("Upgrade")) > 0 + for _, r := range s.rules(phase) { + x.applyRule(r, upgrade) + } + // The trace keeps the body of the last body action of the phase only. + last := -1 + for i, l := range x.lines { + if l.Field == fieldBody && l.Outcome == OutcomeApplied { + last = i + } + } + for i := range x.lines { + if x.lines[i].Field == fieldBody && i != last { + x.lines[i].After = nil + } + } + res.Lines = x.lines + if x.changed { + res.Changed, res.Request, res.Response = true, x.st.req, x.st.resp + } + return res +} + +func (x *run) applyRule(r *compiledRule, upgrade bool) { + defer func() { + if recover() != nil { + x.fail(r, "", "", ErrClassPanic) + } + }() + if !r.matchHead(&x.st) { + return + } + if upgrade { + x.skip(r, ReasonStream) + return + } + // A rule that cannot read or rewrite a body it needs is skipped whole, + // its header, URL and status actions included. + if reason := x.ineligible(r); reason != "" { + x.skip(r, reason) + return + } + if !r.matchBody(&x.st) { + return + } + if r.writesConnectionHeader(*x.header(&x.st)) { + x.skip(r, ReasonConnectionHeader) + return + } + w := work{x: x, st: x.st} + for i := range r.actions { + a := &r.actions[i] + if class := w.exec(a); class != "" { + x.fail(r, a.typ, a.field(), class) + return + } + } + if len(w.lines) == 0 { + return + } + x.st, x.changed = w.st, true + for _, f := range w.taint { + x.tainted[f] = true + } + for _, l := range w.lines { + x.emit(r, l) + } +} + +func (x *run) skip(r *compiledRule, reason string) { + x.emit(r, TraceLine{Outcome: OutcomeSkipped, Reason: reason}) +} + +func (x *run) fail(r *compiledRule, action, field, class string) { + x.set.errors.record(r.id, class) + x.emit(r, TraceLine{Action: action, Field: field, Outcome: OutcomeError, Reason: class}) +} + +func (x *run) emit(r *compiledRule, l TraceLine) { + l.Phase, l.RuleID = x.phase, r.id + if !x.seen[r.id] { + x.seen[r.id] = true + l.RuleSnapshot = r.snapshot + } + x.lines = append(x.lines, l) +} + +// header and body point at the message the phase rewrites. +func (x *run) header(st *state) *http.Header { + if x.phase == PhaseRequest { + return &st.req.Header + } + return &st.resp.Header +} + +func (x *run) body(st *state) *[]byte { + if x.phase == PhaseRequest { + return &st.req.Body + } + return &st.resp.Body +} + +func (x *run) hash(b []byte, secret bool) string { + if secret { + return x.set.fingerprint(b) + } + return sha256Hex(b) +} + +func (x *run) ineligible(r *compiledRule) string { + if r.respBody { + if x.in.Stream { + return ReasonStream + } + if x.st.req.Method == http.MethodHead || !bodyAllowedForStatus(x.st.resp.Status) { + return ReasonNoBodyAllowed + } + if reason := x.bodyIneligible(x.st.resp.Header, x.st.resp.Body); reason != "" { + return reason + } + } + if r.reqBody { + return x.bodyIneligible(x.st.req.Header, x.st.req.Body) + } + return "" +} + +func (x *run) bodyIneligible(h http.Header, body []byte) string { + if len(body) > MaxBodySize { + return ReasonBodyTooLarge + } + if ce := strings.TrimSpace(h.Get("Content-Encoding")); ce != "" && !strings.EqualFold(ce, "identity") { + return ReasonContentEncoding + } + binary := x.in.IsBinary + if binary == nil { + binary = nulInHead + } + if binary(h.Get("Content-Type"), body) { + return ReasonBinary + } + return "" +} + +func nulInHead(_ string, body []byte) bool { + return bytes.IndexByte(body[:min(len(body), 512)], 0) >= 0 +} + +// matchHead evaluates the matchers that need no body, cheapest first. +func (r *compiledRule) matchHead(st *state) bool { + if r.methods != nil && !r.methods[strings.ToUpper(st.req.Method)] { + return false + } + if r.host != "" { + h := strings.ToLower(st.req.URL.Hostname()) + if r.hostGlob { + if len(h) == len(r.host) || !strings.HasSuffix(h, r.host) { + return false + } + } else if h != r.host { + return false + } + } + if r.status != nil && !slices.ContainsFunc(r.status, func(s statusRange) bool { return st.resp.Status >= s.lo && st.resp.Status < s.hi }) { + return false + } + if !matchHeaders(r.headers, st.req.Header) || !matchHeaders(r.respHeaders, st.resp.Header) { + return false + } + return r.urlRe == nil || r.urlRe.MatchString(st.req.URL.RequestURI()) +} + +func matchHeaders(ms []headerMatcher, h http.Header) bool { + for _, m := range ms { + values := h.Values(m.name) + if len(values) == 0 { + return false + } + if m.re != nil && !slices.ContainsFunc(values, m.re.MatchString) { + return false + } + } + return true +} + +func (r *compiledRule) matchBody(st *state) bool { + return (r.bodyRe == nil || r.bodyRe.Match(st.req.Body)) && (r.respBodyRe == nil || r.respBodyRe.Match(st.resp.Body)) +} + +// writesConnectionHeader reports a header action whose target the message +// lists in Connection: that header is hop-by-hop for this exchange. +func (r *compiledRule) writesConnectionHeader(h http.Header) bool { + for _, v := range h.Values("Connection") { + for _, token := range strings.Split(v, ",") { + name := http.CanonicalHeaderKey(strings.TrimSpace(token)) + if name != "" && slices.ContainsFunc(r.actions, func(a compiledAction) bool { return a.name == name }) { + return true + } + } + } + return false +} + +func (a *compiledAction) field() string { + switch actionClass(a.typ) { + case classTarget: + if a.typ == ActionSetMethod { + return fieldMethod + } + return fieldURL + case classDeleteHeader, classHeader: + return "header:" + a.name + case classBody: + return fieldBody + } + return fieldStatus +} + +// work is the copy one rule writes to; it replaces the run's state only when +// every action succeeded. +type work struct { + x *run + st state + ownHeader bool // the header of st is already this rule's copy + lines []TraceLine + taint []string +} + +// exec applies one action to the copy and returns an error class, or "". +// An action that changes nothing leaves no line. +func (w *work) exec(a *compiledAction) string { + switch a.typ { + case ActionSetMethod: + if old := w.st.req.Method; old != a.value { + w.note(a, []byte(old), true, []byte(a.value), true) + w.st.req.Method = a.value + } + case ActionSetPathQuery, ActionReplaceURL: + before := w.st.req.URL.RequestURI() + after := a.value + if a.typ == ActionReplaceURL { + out, ok := expandReplace(a.re, []byte(before), a.value) + if !ok { + return ErrClassExpansion + } + after = string(out) + } + if after == before { + return "" + } + pq, ok := parsePathQuery(after) // checked again here: the result depends on the request + if !ok { + return ErrClassInvalidURL + } + nu := *w.st.req.URL + nu.Path, nu.RawPath, nu.RawQuery, nu.ForceQuery = pq.Path, pq.RawPath, pq.RawQuery, pq.ForceQuery + w.note(a, []byte(before), true, []byte(nu.RequestURI()), true) + w.st.req.URL = &nu + case ActionDeleteHeader, ActionSetHeader, ActionReplaceHeader: + old := w.x.header(&w.st).Values(a.name) + var next []string + switch a.typ { + case ActionSetHeader: + next = []string{a.value} + case ActionReplaceHeader: + for _, v := range old { + out, ok := expandReplace(a.re, []byte(v), a.value) + if !ok { + return ErrClassExpansion + } + next = append(next, string(out)) + } + } + if slices.Equal(old, next) { + return "" + } + w.note(a, []byte(strings.Join(old, "\n")), len(old) > 0, []byte(strings.Join(next, "\n")), len(next) > 0) + if h := w.header(); len(next) == 0 { + h.Del(a.name) + } else { + h[a.name] = next + } + case ActionSetBody, ActionReplaceBody: + body := w.x.body(&w.st) + next := []byte(a.value) + if a.typ == ActionReplaceBody { + out, ok := expandReplace(a.re, *body, a.value) + if !ok { + return ErrClassExpansion + } + next = out + } + if bytes.Equal(*body, next) { + return "" + } + w.note(a, *body, true, next, true) + *body = next + h := w.header() + for _, name := range bodyDependentHeaders { + h.Del(name) + } + case ActionSetStatus: + if old := w.st.resp.Status; old != a.status { + w.note(a, []byte(strconv.Itoa(old)), true, []byte(a.value), true) + w.st.resp.Status = a.status + } + } + return "" +} + +// header returns the rule's own copy of the header it rewrites. +func (w *work) header() http.Header { + p := w.x.header(&w.st) + if !w.ownHeader { + if *p == nil { + *p = http.Header{} + } else { + *p = p.Clone() + } + w.ownHeader = true + } + return *p +} + +// note records an applied action. A sensitive action leaves neither its value +// nor a plain hash of it, and neither does any later action on the same field +// in the phase: the taint follows the field, not the value. +func (w *work) note(a *compiledAction, before []byte, hadBefore bool, after []byte, hasAfter bool) { + field := a.field() + tainted := w.x.tainted[field] || slices.Contains(w.taint, field) + secret := tainted || a.sensitive + l := TraceLine{Action: a.typ, Field: field, Outcome: OutcomeApplied} + if hadBefore { + l.BeforeHash = w.x.hash(before, tainted) + } + if hasAfter { + l.AfterHash = w.x.hash(after, secret) + if !secret { + l.After = after + } + } + if a.sensitive && !tainted { + w.taint = append(w.taint, field) + } + w.lines = append(w.lines, l) +} + +// expandReplace is regexp.ReplaceAll bounded by MaxExpansion. It returns src +// itself when nothing matches, and never writes to src. +func expandReplace(re *regexp.Regexp, src []byte, template string) ([]byte, bool) { + matches := re.FindAllSubmatchIndex(src, -1) + if len(matches) == 0 { + return src, true + } + var dst []byte + last := 0 + for _, m := range matches { + dst = append(dst, src[last:m[0]]...) + dst = re.Expand(dst, []byte(template), src, m) + last = m[1] + if len(dst) > MaxExpansion { + return nil, false + } + } + if dst = append(dst, src[last:]...); len(dst) > MaxExpansion { + return nil, false + } + return dst, true +} diff --git a/traffic/rewrite/engine_test.go b/traffic/rewrite/engine_test.go new file mode 100644 index 0000000..728b880 --- /dev/null +++ b/traffic/rewrite/engine_test.go @@ -0,0 +1,708 @@ +package rewrite + +import ( + "bytes" + "encoding/json" + "net/http" + "reflect" + "strings" + "testing" +) + +var testKey = []byte("0123456789abcdef0123456789abcdef") + +func response(status int, header http.Header, body string) Response { + if header == nil { + header = http.Header{} + } + return Response{Status: status, Header: header, Body: []byte(body)} +} + +// outcomes renders the lines as "rule:action:field:outcome:reason". +func outcomes(lines []TraceLine) []string { + out := make([]string, 0, len(lines)) + for _, l := range lines { + out = append(out, strings.Join([]string{string(rune('0' + l.RuleID)), l.Action, l.Field, l.Outcome, l.Reason}, ":")) + } + return out +} + +func wantLines(t *testing.T, got []TraceLine, want ...string) { + t.Helper() + if g := outcomes(got); !reflect.DeepEqual(g, append([]string{}, want...)) { + t.Fatalf("lines:\n got %q\n want %q", g, want) + } +} + +func TestMatchers(t *testing.T) { + mark := setHeader("X-Hit", "1") + req := request("POST", "https://api.example.com:8443/v1/users?id=7", http.Header{"X-Env": {"prod", "eu"}, "Accept": {"*/*"}}, `{"name":"bob"}`) + resp := response(404, http.Header{"Server": {"nginx/1.25"}}, "not found") + cases := []struct { + name string + phase Phase + m Matchers + want bool + }{ + {"no matcher", PhaseRequest, Matchers{}, true}, + {"method, any case", PhaseRequest, Matchers{Methods: []string{"get", "post"}}, true}, + {"method mismatch", PhaseRequest, Matchers{Methods: []string{"GET"}}, false}, + {"host exact, any case, port ignored", PhaseRequest, Matchers{Host: "API.example.com"}, true}, + {"host mismatch", PhaseRequest, Matchers{Host: "example.com"}, false}, + {"host suffix glob", PhaseRequest, Matchers{Host: "*.example.com"}, true}, + {"host suffix glob needs a dot boundary", PhaseRequest, Matchers{Host: "*.ample.com"}, false}, + {"url regex on path and query", PhaseRequest, Matchers{URLRegex: `^/v1/users\?id=\d+$`}, true}, + {"url regex does not see the host", PhaseRequest, Matchers{URLRegex: `example`}, false}, + {"header presence, any case", PhaseRequest, Matchers{Headers: []HeaderMatcher{{Name: "x-env"}}}, true}, + {"header absent", PhaseRequest, Matchers{Headers: []HeaderMatcher{{Name: "X-None"}}}, false}, + {"header regex on any value", PhaseRequest, Matchers{Headers: []HeaderMatcher{{Name: "X-Env", ValueRegex: "^eu$"}}}, true}, + {"header regex mismatch", PhaseRequest, Matchers{Headers: []HeaderMatcher{{Name: "X-Env", ValueRegex: "^us$"}}}, false}, + {"every header matcher must hold", PhaseRequest, Matchers{Headers: []HeaderMatcher{{Name: "X-Env"}, {Name: "X-None"}}}, false}, + {"body regex", PhaseRequest, Matchers{BodyRegex: `"name":"bob"`}, true}, + {"body regex mismatch", PhaseRequest, Matchers{BodyRegex: `alice`}, false}, + {"matchers are ANDed", PhaseRequest, Matchers{Methods: []string{"POST"}, Host: "other.example"}, false}, + {"status code", PhaseResponse, Matchers{Status: []string{"200", "404"}}, true}, + {"status class", PhaseResponse, Matchers{Status: []string{"4xx"}}, true}, + {"status mismatch", PhaseResponse, Matchers{Status: []string{"5xx", "200"}}, false}, + {"response header", PhaseResponse, Matchers{RespHeaders: []HeaderMatcher{{Name: "server", ValueRegex: "^nginx"}}}, true}, + {"response header mismatch", PhaseResponse, Matchers{RespHeaders: []HeaderMatcher{{Name: "Server", ValueRegex: "apache"}}}, false}, + {"response body regex", PhaseResponse, Matchers{RespBodyRegex: "found$"}, true}, + {"response body regex mismatch", PhaseResponse, Matchers{RespBodyRegex: "^ok"}, false}, + {"request matchers in the response phase", PhaseResponse, Matchers{Methods: []string{"POST"}, Host: "*.example.com", BodyRegex: "bob"}, true}, + {"request header in the response phase", PhaseResponse, Matchers{Headers: []HeaderMatcher{{Name: "X-None"}}}, false}, + } + for _, c := range cases { + s := compile(t, Options{}, rule(1, c.phase, 0, c.m, mark)) + res := s.Apply(c.phase, Input{Request: req, Response: resp}) + if res.Changed != c.want { + t.Errorf("%s: applied=%v, want %v", c.name, res.Changed, c.want) + } + } + apex := request("GET", "http://example.com/", nil, "") + if compile(t, Options{}, rule(1, PhaseRequest, 0, Matchers{Host: "*.example.com"}, mark)).Apply(PhaseRequest, Input{Request: apex}).Changed { + t.Error("*.example.com must not match example.com") + } +} + +func TestRequestActions(t *testing.T) { + req := request("GET", "http://h.example/old/path?q=1", http.Header{"X-Drop": {"a", "b"}, "Cookie": {"sid=abc", "theme=dark"}, "X-Keep": {"k"}}, "hello world") + s := compile(t, Options{}, rule(1, PhaseRequest, 0, Matchers{}, + Action{Type: ActionSetMethod, Value: "POST"}, + Action{Type: ActionSetPathQuery, Value: "/new/path?q=2"}, + Action{Type: ActionReplaceURL, Regex: `q=(\d)`, Value: "q=${1}0"}, + Action{Type: ActionDeleteHeader, Name: "x-drop"}, + setHeader("x-new", "v"), + Action{Type: ActionReplaceHeader, Name: "Cookie", Regex: `^sid=.*$`, Value: "sid=xyz"}, + Action{Type: ActionSetBody, Value: "hello there"}, + Action{Type: ActionReplaceBody, Regex: `(\w+) there`, Value: "$1, $1"})) + res := s.Apply(PhaseRequest, Input{Request: req}) + got := res.Request + if !res.Changed || got.Method != "POST" || got.URL.String() != "http://h.example/new/path?q=20" || string(got.Body) != "hello, hello" { + t.Fatalf("method=%s url=%s body=%q", got.Method, got.URL, got.Body) + } + wantHeader := http.Header{"Cookie": {"sid=xyz", "theme=dark"}, "X-Keep": {"k"}, "X-New": {"v"}} + if !reflect.DeepEqual(got.Header, wantHeader) { + t.Fatalf("header = %v, want %v", got.Header, wantHeader) + } + wantLines(t, res.Lines, + "1:set_method:method:applied:", "1:set_path_query:url:applied:", "1:replace_url:url:applied:", + "1:delete_header:header:X-Drop:applied:", "1:set_header:header:X-New:applied:", "1:replace_header:header:Cookie:applied:", + "1:set_body:body:applied:", "1:replace_body:body:applied:") + if l := res.Lines[2]; string(l.After) != "/new/path?q=20" || l.BeforeHash != sha256Hex([]byte("/new/path?q=2")) || l.AfterHash != sha256Hex(l.After) { + t.Fatalf("replace_url line = %+v", l) + } + if l := res.Lines[3]; l.After != nil || l.AfterHash != "" || l.BeforeHash != sha256Hex([]byte("a\nb")) { + t.Fatalf("delete_header line = %+v", l) + } +} + +func TestResponseActions(t *testing.T) { + req := request("GET", "http://h.example/", nil, "") + resp := response(403, http.Header{"Server": {"nginx"}, "Set-Cookie": {"a=1; Secure", "b=2; Secure"}}, "denied") + s := compile(t, Options{}, rule(1, PhaseResponse, 0, Matchers{}, + Action{Type: ActionSetStatus, Value: "200"}, + Action{Type: ActionReplaceBody, Regex: "denied", Value: "granted"}, + Action{Type: ActionReplaceHeader, Name: "Set-Cookie", Regex: "; Secure", Value: ""}, + setHeader("Server", "artex"), + Action{Type: ActionDeleteHeader, Name: "X-Absent"})) + res := s.Apply(PhaseResponse, Input{Request: req, Response: resp}) + got := res.Response + if got.Status != 200 || string(got.Body) != "granted" || !reflect.DeepEqual(got.Header, http.Header{"Server": {"artex"}, "Set-Cookie": {"a=1", "b=2"}}) { + t.Fatalf("status=%d body=%q header=%v", got.Status, got.Body, got.Header) + } + // Fixed order within a rule, whatever the order it lists its actions in; + // the delete of an absent header changed nothing and left no line. + wantLines(t, res.Lines, "1:replace_header:header:Set-Cookie:applied:", "1:set_header:header:Server:applied:", + "1:replace_body:body:applied:", "1:set_status:status:applied:") + if res.Request.URL != req.URL { + t.Fatal("the response phase must hand the request back untouched") + } +} + +// Within a rule: method and URL, delete_header, set_header and replace_header +// in their listed order, body, status. +func TestActionOrderWithinRule(t *testing.T) { + s := compile(t, Options{}, rule(1, PhaseRequest, 0, Matchers{}, + Action{Type: ActionSetBody, Value: "b"}, + Action{Type: ActionReplaceHeader, Name: "X-A", Regex: "$", Value: "2"}, + setHeader("X-A", "1"), + Action{Type: ActionDeleteHeader, Name: "X-A"}, + Action{Type: ActionReplaceURL, Regex: "^/a$", Value: "/b"}, + Action{Type: ActionSetMethod, Value: "PUT"})) + res := s.Apply(PhaseRequest, Input{Request: request("GET", "http://h.example/a", http.Header{"X-A": {"0"}}, "a")}) + // delete, then replace on the now absent header (no change, no line), then set. + wantLines(t, res.Lines, "1:replace_url:url:applied:", "1:set_method:method:applied:", "1:delete_header:header:X-A:applied:", + "1:set_header:header:X-A:applied:", "1:set_body:body:applied:") + if got := res.Request.Header["X-A"]; !reflect.DeepEqual(got, []string{"1"}) { + t.Fatalf("X-A = %v", got) + } +} + +func TestActionOrderKeepsListedOrderInClass(t *testing.T) { + s := compile(t, Options{}, rule(1, PhaseRequest, 0, Matchers{}, + setHeader("X-A", "1"), + Action{Type: ActionReplaceHeader, Name: "X-A", Regex: "$", Value: "2"})) + res := s.Apply(PhaseRequest, Input{Request: request("GET", "http://h.example/", nil, "")}) + if got := res.Request.Header.Get("X-A"); got != "12" { + t.Fatalf("X-A = %q, want set then replace", got) + } +} + +// Priority ascending then id ascending: the highest priority applies last and +// wins, and each rule sees the output of the previous one. +func TestPriorityOrderAndPipeline(t *testing.T) { + rules := []Rule{ + rule(5, PhaseRequest, 10, Matchers{}, setHeader("X-Who", "p10")), + rule(9, PhaseRequest, 0, Matchers{}, setHeader("X-Who", "p0-id9")), + rule(2, PhaseRequest, 0, Matchers{}, setHeader("X-Who", "p0-id2")), + // Matches only what rule 5 wrote. + rule(7, PhaseRequest, 20, Matchers{Headers: []HeaderMatcher{{Name: "X-Who", ValueRegex: "^p10$"}}}, setHeader("X-Seen", "yes")), + rule(8, PhaseRequest, -5, Matchers{Headers: []HeaderMatcher{{Name: "X-Who"}}}, setHeader("X-Early", "yes")), + } + res := compile(t, Options{}, rules...).Apply(PhaseRequest, Input{Request: request("GET", "http://h.example/", nil, "")}) + wantLines(t, res.Lines, "2:set_header:header:X-Who:applied:", "9:set_header:header:X-Who:applied:", + "5:set_header:header:X-Who:applied:", "7:set_header:header:X-Seen:applied:") + h := res.Request.Header + if h.Get("X-Who") != "p10" || h.Get("X-Seen") != "yes" || h.Get("X-Early") != "" { + t.Fatalf("header = %v", h) + } +} + +// A failing action cancels its whole rule; the others still apply. +func TestRuleAtomicFailOpen(t *testing.T) { + log := &ErrorLog{} + grow := strings.Repeat("x", 1024) + s := compile(t, Options{Errors: log}, + rule(1, PhaseRequest, 0, Matchers{}, setHeader("X-First", "1")), + rule(2, PhaseRequest, 1, Matchers{}, + Action{Type: ActionSetMethod, Value: "POST"}, + setHeader("X-Second", "2"), + Action{Type: ActionReplaceBody, Regex: "a", Value: grow}), + rule(3, PhaseRequest, 2, Matchers{}, setHeader("X-Third", "3"))) + req := request("GET", "http://h.example/", nil, strings.Repeat("a", 2048)) // 2048 * 1024 bytes > MaxExpansion + res := s.Apply(PhaseRequest, Input{Request: req}) + wantLines(t, res.Lines, "1:set_header:header:X-First:applied:", "2:replace_body:body:error:expansion_limit", "3:set_header:header:X-Third:applied:") + got := res.Request + if got.Method != "GET" || got.Header.Get("X-Second") != "" || !bytes.Equal(got.Body, req.Body) { + t.Fatalf("rule 2 left a trace on the request: method=%s header=%v", got.Method, got.Header) + } + if got.Header.Get("X-First") != "1" || got.Header.Get("X-Third") != "3" { + t.Fatalf("the other rules must apply: %v", got.Header) + } + if e, ok := log.Get(2); !ok || e.Class != ErrClassExpansion || e.Count != 1 { + t.Fatalf("last error of rule 2 = %+v %v", e, ok) + } + if _, ok := log.Get(1); ok { + t.Fatal("rule 1 has no error") + } + s.Apply(PhaseRequest, Input{Request: req}) + if e, _ := log.Get(2); e.Count != 2 { + t.Fatalf("count = %d, want 2", e.Count) + } +} + +// A rewritten URL is checked again at run time and can never leave the host. +func TestURLRewriteStaysOnHost(t *testing.T) { + req := request("GET", "https://user:pw@h.example:8443/a?x=1", nil, "") + for value, wantURL := range map[string]string{ + "http://evil.example/x": "", // not a path: the rule fails + "evil.example": "", // not a path + "/a\nb": "", // control character + "//evil.example/x": "https://user:pw@h.example:8443//evil.example/x", // a path, still on the host + "/b?y=2": "https://user:pw@h.example:8443/b?y=2", + } { + s := compile(t, Options{}, rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionReplaceURL, Regex: `^.*$`, Value: value})) + res := s.Apply(PhaseRequest, Input{Request: req}) + if wantURL == "" { + wantLines(t, res.Lines, "1:replace_url:url:error:invalid_url") + if res.Changed || res.Request.URL != req.URL { + t.Errorf("%q: the request must go on unchanged", value) + } + continue + } + u := res.Request.URL + if u.String() != wantURL || u.Scheme != "https" || u.Host != "h.example:8443" { + t.Errorf("%q: url = %s, want %s", value, u, wantURL) + } + } +} + +// A panic inside a rule cancels that rule only. +func TestPanicInRuleFailsOpen(t *testing.T) { + log := &ErrorLog{} + s := compile(t, Options{Errors: log}, + rule(1, PhaseRequest, 0, Matchers{}, setHeader("X-A", "1"), Action{Type: ActionSetBody, Value: "new"}), + rule(2, PhaseRequest, 1, Matchers{}, setHeader("X-B", "2"))) + req := request("POST", "http://h.example/", nil, "old") + res := s.Apply(PhaseRequest, Input{Request: req, IsBinary: func(string, []byte) bool { panic("boom") }}) + wantLines(t, res.Lines, "1:::error:panic", "2:set_header:header:X-B:applied:") + if string(res.Request.Body) != "old" || res.Request.Header.Get("X-A") != "" || res.Request.Header.Get("X-B") != "2" { + t.Fatalf("request after the panic: body=%q header=%v", res.Request.Body, res.Request.Header) + } + if e, ok := log.Get(1); !ok || e.Class != ErrClassPanic { + t.Fatalf("last error = %+v %v", e, ok) + } +} + +// The engine never writes to the objects it is given. +func TestInputsNotMutated(t *testing.T) { + newInput := func() Input { + return Input{ + Request: request("GET", "http://h.example/a?x=1", http.Header{"X-A": {"1", "2"}, "Content-Length": {"3"}}, "abc"), + Response: response(200, http.Header{"X-B": {"1"}, "Etag": {`"e"`}, "Content-Length": {"3"}}, "xyz"), + } + } + s := compile(t, Options{}, + rule(1, PhaseRequest, 0, Matchers{}, + Action{Type: ActionSetMethod, Value: "POST"}, + Action{Type: ActionSetPathQuery, Value: "/b"}, + Action{Type: ActionReplaceHeader, Name: "X-A", Regex: "1", Value: "9"}, + Action{Type: ActionDeleteHeader, Name: "X-A"}, + setHeader("X-N", "n"), + Action{Type: ActionReplaceBody, Regex: "b", Value: "B"}), + rule(2, PhaseRequest, 1, Matchers{}, Action{Type: ActionReplaceBody, Regex: "B", Value: "bb"}), + rule(3, PhaseResponse, 0, Matchers{}, + Action{Type: ActionSetStatus, Value: "500"}, + setHeader("X-B", "2"), + Action{Type: ActionReplaceBody, Regex: "y", Value: "Y"}), + // Fails after it wrote to its copy. + rule(4, PhaseResponse, 1, Matchers{}, setHeader("X-C", "c"), Action{Type: ActionReplaceBody, Regex: "x", Value: strings.Repeat("x", MaxValueSize)})) + for _, phase := range []Phase{PhaseRequest, PhaseResponse} { + in, pristine := newInput(), newInput() + res := s.Apply(phase, in) + if !res.Changed { + t.Fatalf("%s: nothing applied", phase) + } + if !reflect.DeepEqual(in, pristine) { + t.Fatalf("%s: the input was modified:\n got %+v\n want %+v", phase, in, pristine) + } + if phase == PhaseRequest { + if res.Request.URL == in.Request.URL || string(res.Request.Body) != "abbc" || res.Request.Header.Get("Content-Length") != "" { + t.Fatalf("request result: url=%s body=%q header=%v", res.Request.URL, res.Request.Body, res.Request.Header) + } + res.Request.Header.Set("X-After", "1") + res.Request.Body[0] = 'Z' + } else { + if res.Response.Status != 500 || string(res.Response.Body) != "xYz" || res.Response.Header.Get("X-C") != "" { + t.Fatalf("response result: %d %q %v", res.Response.Status, res.Response.Body, res.Response.Header) + } + res.Response.Header.Set("X-After", "1") + res.Response.Body[0] = 'Z' + } + if !reflect.DeepEqual(in, pristine) { + t.Fatalf("%s: the result shares memory with the input", phase) + } + } +} + +// After a body action the headers that described the old body are gone. +func TestBodyRewriteDropsBodyHeaders(t *testing.T) { + h := http.Header{"Content-Length": {"3"}, "Etag": {`"v1"`}, "Content-Md5": {"x"}, "Content-Encoding": {"identity"}, "Content-Type": {"text/plain"}} + s := compile(t, Options{}, + rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionSetBody, Value: "longer body"}), + rule(2, PhaseResponse, 0, Matchers{}, Action{Type: ActionReplaceBody, Regex: "a", Value: "bb"})) + want := http.Header{"Content-Type": {"text/plain"}} + if got := s.Apply(PhaseRequest, Input{Request: request("POST", "http://h.example/", h.Clone(), "abc")}).Request.Header; !reflect.DeepEqual(got, want) { + t.Fatalf("request header = %v", got) + } + in := Input{Request: request("GET", "http://h.example/", nil, ""), Response: response(200, h.Clone(), "abc")} + if got := s.Apply(PhaseResponse, in).Response.Header; !reflect.DeepEqual(got, want) { + t.Fatalf("response header = %v", got) + } + // A body action that changes nothing keeps them. + in.Response.Body = []byte("xyz") + if res := s.Apply(PhaseResponse, in); res.Changed || len(res.Lines) != 0 { + t.Fatalf("no match must leave no line: %v", outcomes(res.Lines)) + } +} + +// An action that changes nothing leaves no line, so the exchange is not +// flagged as rewritten. +func TestNoChangeNoLine(t *testing.T) { + s := compile(t, Options{}, rule(1, PhaseRequest, 0, Matchers{}, + Action{Type: ActionSetMethod, Value: "GET"}, + Action{Type: ActionSetPathQuery, Value: "/a?x=1"}, + Action{Type: ActionReplaceURL, Regex: "zzz", Value: "y"}, + Action{Type: ActionDeleteHeader, Name: "X-Absent"}, + setHeader("X-A", "1"), + Action{Type: ActionReplaceHeader, Name: "X-A", Regex: "zzz", Value: "y"}, + Action{Type: ActionSetBody, Value: "abc"}, + Action{Type: ActionReplaceBody, Regex: "zzz", Value: "y"})) + req := request("GET", "http://h.example/a?x=1", http.Header{"X-A": {"1"}}, "abc") + res := s.Apply(PhaseRequest, Input{Request: req}) + if res.Changed || len(res.Lines) != 0 || res.Request.URL != req.URL { + t.Fatalf("changed=%v lines=%v", res.Changed, outcomes(res.Lines)) + } +} + +// Limits: an oversized body skips the rules that need it; an oversized +// expansion fails the rule open. +func TestRunTimeLimits(t *testing.T) { + s := compile(t, Options{}, + rule(1, PhaseRequest, 0, Matchers{}, setHeader("X-A", "1"), Action{Type: ActionReplaceBody, Regex: "a", Value: "b"}), + rule(2, PhaseRequest, 1, Matchers{BodyRegex: "[ab]"}, setHeader("X-B", "1")), + rule(3, PhaseRequest, 2, Matchers{}, setHeader("X-C", "1"))) + atLimit := request("POST", "http://h.example/", nil, strings.Repeat("a", MaxBodySize)) + if res := s.Apply(PhaseRequest, Input{Request: atLimit}); len(res.Lines) != 4 || len(res.Request.Body) != MaxBodySize { + t.Fatalf("a body of exactly MaxBodySize is rewritten: %v", outcomes(res.Lines)) + } + over := request("POST", "http://h.example/", nil, strings.Repeat("a", MaxBodySize+1)) + res := s.Apply(PhaseRequest, Input{Request: over}) + wantLines(t, res.Lines, "1:::skipped:not rewritten: body too large", "2:::skipped:not rewritten: body too large", "3:set_header:header:X-C:applied:") + if res.Request.Header.Get("X-A") != "" || res.Request.Header.Get("X-B") != "" { + t.Fatalf("a skipped body rule applied a header: %v", res.Request.Header) + } + + grow := compile(t, Options{}, + rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionReplaceURL, Regex: "a", Value: strings.Repeat("b", MaxValueSize)}), + rule(2, PhaseRequest, 1, Matchers{}, Action{Type: ActionReplaceHeader, Name: "X-A", Regex: "a", Value: strings.Repeat("b", MaxValueSize)}), + rule(3, PhaseRequest, 2, Matchers{}, Action{Type: ActionReplaceBody, Regex: "a", Value: strings.Repeat("b", MaxValueSize)}), + rule(4, PhaseRequest, 3, Matchers{}, Action{Type: ActionReplaceBody, Regex: "^a", Value: strings.Repeat("b", MaxExpansion-1)})) + res = grow.Apply(PhaseRequest, Input{Request: request("POST", "http://h.example/aa", http.Header{"X-A": {"aa"}}, "aa")}) + wantLines(t, res.Lines, "1:replace_url:url:error:expansion_limit", "2:replace_header:header:X-A:error:expansion_limit", + "3:replace_body:body:error:expansion_limit", "4:replace_body:body:applied:") + if len(res.Request.Body) != MaxExpansion { + t.Fatalf("an expansion of exactly MaxExpansion is allowed, got %d bytes", len(res.Request.Body)) + } +} + +// A header the message lists in Connection is hop-by-hop: the rule is skipped. +func TestHeaderNamedInConnection(t *testing.T) { + s := compile(t, Options{}, + rule(1, PhaseRequest, 0, Matchers{}, setHeader("X-Other", "1"), Action{Type: ActionDeleteHeader, Name: "X-Hop"}), + rule(2, PhaseRequest, 1, Matchers{}, setHeader("X-Other", "2")), + rule(3, PhaseResponse, 0, Matchers{}, setHeader("X-Hop", "1")), + rule(4, PhaseResponse, 0, Matchers{}, setHeader("X-Resp-Hop", "1"))) + req := request("GET", "http://h.example/", http.Header{"Connection": {"keep-alive, x-hop"}, "X-Hop": {"v"}}, "") + res := s.Apply(PhaseRequest, Input{Request: req}) + wantLines(t, res.Lines, "1:::skipped:header named in Connection", "2:set_header:header:X-Other:applied:") + if res.Request.Header.Get("X-Hop") != "v" || res.Request.Header.Get("X-Other") != "2" { + t.Fatalf("header = %v", res.Request.Header) + } + // The response phase reads the Connection header of the response. + res = s.Apply(PhaseResponse, Input{Request: req, Response: response(200, http.Header{"Connection": {"X-Resp-Hop"}}, "")}) + wantLines(t, res.Lines, "3:set_header:header:X-Hop:applied:", "4:::skipped:header named in Connection") +} + +// In the response phase the request matchers run on the request that was sent +// to the target, that is, after the request phase. +func TestResponsePhaseSeesRewrittenRequest(t *testing.T) { + s := compile(t, Options{}, + rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionSetPathQuery, Value: "/internal"}, setHeader("X-Role", "admin")), + rule(2, PhaseResponse, 0, Matchers{URLRegex: "^/internal$", Headers: []HeaderMatcher{{Name: "X-Role", ValueRegex: "admin"}}}, setHeader("X-Matched", "1"))) + client := request("GET", "http://h.example/public", nil, "") + sent := s.Apply(PhaseRequest, Input{Request: client}).Request + resp := response(200, nil, "ok") + if res := s.Apply(PhaseResponse, Input{Request: sent, Response: resp}); res.Response.Header.Get("X-Matched") != "1" { + t.Fatal("the response rule must match the rewritten request") + } + if res := s.Apply(PhaseResponse, Input{Request: client, Response: resp}); res.Changed { + t.Fatal("the response rule must not match the client's request") + } +} + +// A rule that needs a body it cannot touch is skipped whole: its header, URL +// and status actions do not apply either. Rules without a body component do. +func TestBodyRuleNotEligibleSkippedWhole(t *testing.T) { + bodyRule := rule(1, PhaseResponse, 0, Matchers{}, + setHeader("X-Body-Rule", "1"), Action{Type: ActionSetStatus, Value: "200"}, Action{Type: ActionReplaceBody, Regex: "a", Value: "b"}) + matcherRule := rule(2, PhaseResponse, 1, Matchers{RespBodyRegex: "."}, setHeader("X-Matcher-Rule", "1")) + headerRule := rule(3, PhaseResponse, 2, Matchers{}, setHeader("X-Header-Rule", "1")) + s := compile(t, Options{}, bodyRule, matcherRule, headerRule) + get := request("GET", "http://h.example/", nil, "") + cases := []struct { + name string + req Request + resp Response + binary func(string, []byte) bool + reason string + }{ + {"NUL byte", get, response(500, nil, "a\x00a"), nil, ReasonBinary}, + {"binary content type", get, response(500, http.Header{"Content-Type": {"application/zip"}}, "aaa"), + func(ct string, _ []byte) bool { return ct == "application/zip" }, ReasonBinary}, + {"too large", get, response(500, nil, strings.Repeat("a", MaxBodySize+1)), nil, ReasonBodyTooLarge}, + {"content-encoding", get, response(500, http.Header{"Content-Encoding": {"gzip"}}, "aaa"), nil, ReasonContentEncoding}, + {"HEAD", request("HEAD", "http://h.example/", nil, ""), response(500, nil, ""), nil, ReasonNoBodyAllowed}, + {"204", get, response(204, nil, ""), nil, ReasonNoBodyAllowed}, + {"304", get, response(304, nil, ""), nil, ReasonNoBodyAllowed}, + {"101", get, response(101, nil, ""), nil, ReasonNoBodyAllowed}, + } + for _, c := range cases { + res := s.Apply(PhaseResponse, Input{Request: c.req, Response: c.resp, IsBinary: c.binary}) + wantLines(t, res.Lines, "1:::skipped:"+c.reason, "2:::skipped:"+c.reason, "3:set_header:header:X-Header-Rule:applied:") + h := res.Response.Header + if h.Get("X-Body-Rule") != "" || h.Get("X-Matcher-Rule") != "" || res.Response.Status != c.resp.Status || h.Get("X-Header-Rule") != "1" { + t.Errorf("%s: status=%d header=%v", c.name, res.Response.Status, h) + } + } + // Content-Encoding: identity is not an encoding. + res := s.Apply(PhaseResponse, Input{Request: get, Response: response(500, http.Header{"Content-Encoding": {"identity"}}, "aaa")}) + if string(res.Response.Body) != "bbb" || res.Response.Status != 200 { + t.Fatalf("identity body not rewritten: %d %q", res.Response.Status, res.Response.Body) + } + + // Request side: the request body of a request-phase rule, and the request + // body a response-phase rule reads through body_regex. + reqSet := compile(t, Options{}, + rule(1, PhaseRequest, 0, Matchers{}, setHeader("X-A", "1"), Action{Type: ActionSetBody, Value: "new"}), + rule(2, PhaseResponse, 0, Matchers{BodyRegex: "a"}, setHeader("X-B", "1"))) + gz := request("POST", "http://h.example/", http.Header{"Content-Encoding": {"br"}}, "aaa") + wantLines(t, reqSet.Apply(PhaseRequest, Input{Request: gz}).Lines, "1:::skipped:not rewritten: content-encoding") + wantLines(t, reqSet.Apply(PhaseResponse, Input{Request: gz, Response: response(200, nil, "")}).Lines, "2:::skipped:not rewritten: content-encoding") + // A skipped line needs a candidate rule: no line when the other matchers fail. + miss := compile(t, Options{}, rule(1, PhaseRequest, 0, Matchers{Methods: []string{"PUT"}}, Action{Type: ActionSetBody, Value: "new"})) + if res := miss.Apply(PhaseRequest, Input{Request: gz}); len(res.Lines) != 0 { + t.Fatalf("a rule that does not match left a line: %v", outcomes(res.Lines)) + } +} + +// Stream mode (SSE): the response body is out of reach, headers and status are not. +func TestStreamMode(t *testing.T) { + s := compile(t, Options{}, + rule(1, PhaseResponse, 0, Matchers{}, setHeader("X-A", "1"), Action{Type: ActionSetStatus, Value: "202"}), + rule(2, PhaseResponse, 1, Matchers{}, setHeader("X-B", "1"), Action{Type: ActionReplaceBody, Regex: "a", Value: "b"}), + rule(3, PhaseResponse, 2, Matchers{RespBodyRegex: "a"}, setHeader("X-C", "1")), + // Reads the request body only: it is in memory, so the rule applies. + rule(4, PhaseResponse, 3, Matchers{BodyRegex: "ping"}, setHeader("X-D", "1")), + rule(5, PhaseResponse, 4, Matchers{Status: []string{"500"}}, Action{Type: ActionSetBody, Value: "x"})) + in := Input{Request: request("POST", "http://h.example/events", nil, "ping"), + Response: response(200, http.Header{"Content-Type": {"text/event-stream"}}, ""), Stream: true} + res := s.Apply(PhaseResponse, in) + wantLines(t, res.Lines, "1:set_header:header:X-A:applied:", "1:set_status:status:applied:", + "2:::skipped:not rewritten: stream", "3:::skipped:not rewritten: stream", "4:set_header:header:X-D:applied:") + h := res.Response.Header + if res.Response.Status != 202 || h.Get("X-A") != "1" || h.Get("X-B") != "" || h.Get("X-C") != "" || h.Get("X-D") != "1" { + t.Fatalf("status=%d header=%v", res.Response.Status, h) + } +} + +// A request that asks for a protocol upgrade (WebSocket) is never rewritten. +func TestUpgradeRequestSkipsEveryRule(t *testing.T) { + s := compile(t, Options{}, + rule(1, PhaseRequest, 0, Matchers{}, setHeader("X-A", "1")), + rule(2, PhaseRequest, 1, Matchers{BodyRegex: "a"}, Action{Type: ActionSetBody, Value: "x"}), + rule(3, PhaseRequest, 2, Matchers{Methods: []string{"POST"}}, setHeader("X-C", "1"))) + req := request("GET", "http://h.example/ws", http.Header{"Connection": {"keep-alive, Upgrade"}, "Upgrade": {"websocket"}}, "") + res := s.Apply(PhaseRequest, Input{Request: req}) + wantLines(t, res.Lines, "1:::skipped:not rewritten: stream", "2:::skipped:not rewritten: stream") + if res.Changed { + t.Fatal("an upgrade request was rewritten") + } +} + +func TestSensitiveValueNeverTraced(t *testing.T) { + const secret = "tok-SECRET-0001" + leak := func(t *testing.T, res Result) { + t.Helper() + plain := sha256Hex([]byte(secret)) + for _, l := range res.Lines { + blob, _ := json.Marshal(l) + if strings.Contains(string(blob)+string(l.After), secret) || l.AfterHash == plain || l.BeforeHash == plain { + t.Fatalf("the sensitive value, or a plain hash of it, is in line %+v", l) + } + } + } + s := compile(t, Options{TraceKey: testKey}, + rule(1, PhaseRequest, 0, Matchers{}, + Action{Type: ActionSetHeader, Name: "Authorization", Value: secret, Sensitive: true}, + Action{Type: ActionReplaceURL, Regex: "token=[^&]*", Value: "token=" + secret, Sensitive: true}, + Action{Type: ActionSetBody, Value: secret, Sensitive: true}, + setHeader("X-Plain", "visible")), + // Not sensitive, but writes fields rule 1 tainted: no value, no plain hash. + rule(2, PhaseRequest, 1, Matchers{}, + Action{Type: ActionReplaceHeader, Name: "Authorization", Regex: "^", Value: "Bearer "}, + Action{Type: ActionReplaceBody, Regex: "$", Value: "!"}, + Action{Type: ActionReplaceHeader, Name: "X-Plain", Regex: "$", Value: "!"})) + res := s.Apply(PhaseRequest, Input{Request: request("GET", "http://h.example/a?token=old", http.Header{"Authorization": {"old"}}, "old body")}) + leak(t, res) + wantLines(t, res.Lines, "1:replace_url:url:applied:", "1:set_header:header:Authorization:applied:", "1:set_header:header:X-Plain:applied:", + "1:set_body:body:applied:", "2:replace_header:header:Authorization:applied:", "2:replace_header:header:X-Plain:applied:", "2:replace_body:body:applied:") + if got := res.Request.Header.Get("Authorization"); got != "Bearer "+secret { + t.Fatalf("the request itself carries the value: Authorization=%q", got) + } + byField := map[string][]TraceLine{} + for _, l := range res.Lines { + byField[l.Field] = append(byField[l.Field], l) + } + // The sensitive action: the original value is hashed in clear (it is in + // the exchange), the new one is fingerprinted with the key. + auth := byField["header:Authorization"] + if auth[0].BeforeHash != sha256Hex([]byte("old")) || auth[0].AfterHash != s.fingerprint([]byte(secret)) || auth[0].After != nil { + t.Fatalf("sensitive line = %+v", auth[0]) + } + // Taint by field: the later, non-sensitive action on the same field. + if auth[1].After != nil || auth[1].BeforeHash != s.fingerprint([]byte(secret)) || auth[1].AfterHash != s.fingerprint([]byte("Bearer "+secret)) { + t.Fatalf("tainted line = %+v", auth[1]) + } + if body := byField["body"]; body[0].After != nil || body[1].After != nil { + t.Fatalf("a tainted body is in the trace: %+v", body) + } + // A field no sensitive action wrote is traced in full. + if plain := byField["header:X-Plain"]; string(plain[0].After) != "visible" || string(plain[1].After) != "visible!" || plain[1].AfterHash != sha256Hex([]byte("visible!")) { + t.Fatalf("untainted lines = %+v", plain) + } + // The rule snapshot carries the matchers and the action, not the value. + if snap := res.Lines[0].RuleSnapshot; strings.Contains(snap, secret) || !strings.Contains(snap, `"sensitive":true`) || !strings.Contains(snap, `"name":"Authorization"`) { + t.Fatalf("rule snapshot = %s", snap) + } + + // A failing sensitive action reports its class only. + log := &ErrorLog{} + bad := compile(t, Options{TraceKey: testKey, Errors: log}, rule(1, PhaseRequest, 0, Matchers{}, + Action{Type: ActionReplaceURL, Regex: "^.*$", Value: secret, Sensitive: true})) + res = bad.Apply(PhaseRequest, Input{Request: request("GET", "http://h.example/a", nil, "")}) + leak(t, res) + wantLines(t, res.Lines, "1:replace_url:url:error:invalid_url") + if e, _ := log.Get(1); e.Class != ErrClassInvalidURL { + t.Fatalf("last error = %+v", e) + } +} + +// The taint follows the field, not the value: it does not reach another field +// that carries the same value, nor the other phase. This test pins that limit. +func TestSensitiveTaintLimit(t *testing.T) { + const secret = "tok-SECRET-0002" + s := compile(t, Options{TraceKey: testKey}, + rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionSetHeader, Name: "X-Token", Value: secret, Sensitive: true}), + rule(2, PhaseRequest, 1, Matchers{}, setHeader("X-Copy", secret)), + rule(3, PhaseResponse, 0, Matchers{}, setHeader("X-Token", secret))) + res := s.Apply(PhaseRequest, Input{Request: request("GET", "http://h.example/", nil, "")}) + if res.Lines[0].After != nil || string(res.Lines[1].After) != secret { + t.Fatalf("request lines = %+v", res.Lines) + } + res = s.Apply(PhaseResponse, Input{Request: res.Request, Response: response(200, nil, "")}) + if string(res.Lines[0].After) != secret { + t.Fatalf("the taint crossed to the response phase: %+v", res.Lines[0]) + } +} + +func TestRuleSnapshot(t *testing.T) { + long := strings.Repeat("v", snapshotValueMax+1) + short := strings.Repeat("w", snapshotValueMax) + s := compile(t, Options{}, + rule(1, PhaseRequest, 3, Matchers{Host: "h.example", Headers: []HeaderMatcher{{Name: "X-A", ValueRegex: "^matcher-in-clear$"}}}, + setHeader("X-Long", long), setHeader("X-Short", short), Action{Type: ActionSetMethod, Value: "POST"}), + rule(2, PhaseRequest, 4, Matchers{}, setHeader("X-B", "1"))) + res := s.Apply(PhaseRequest, Input{Request: request("GET", "http://h.example/", http.Header{"X-A": {"matcher-in-clear"}}, "")}) + if len(res.Lines) != 4 { + t.Fatalf("lines = %v", outcomes(res.Lines)) + } + // Once per rule, on its first line. + for i, want := range []bool{true, false, false, true} { + if (res.Lines[i].RuleSnapshot != "") != want { + t.Fatalf("line %d: snapshot present=%v, want %v", i, !want, want) + } + } + var snap struct { + ID int64 + Name string + Phase string + Priority int + Matchers Matchers + Actions []struct { + Type, Name string + Value *string + ValueLen int `json:"value_len"` + ValueSHA256 string `json:"value_sha256"` + } + } + if err := json.Unmarshal([]byte(res.Lines[0].RuleSnapshot), &snap); err != nil { + t.Fatal(err) + } + if snap.ID != 1 || snap.Name != "r1" || snap.Phase != "request" || snap.Priority != 3 || snap.Matchers.Headers[0].ValueRegex != "^matcher-in-clear$" { + t.Fatalf("snapshot = %+v", snap) + } + // A value over 4 KiB is summarized by its length and hash; 4 KiB exactly is kept. + if a := snap.Actions[0]; a.Value != nil || a.ValueLen != len(long) || a.ValueSHA256 != sha256Hex([]byte(long)) { + t.Fatalf("long action = %+v", a) + } + if a := snap.Actions[1]; a.Value == nil || *a.Value != short || a.ValueLen != 0 { + t.Fatal("a value of exactly 4 KiB must be kept") + } + if strings.Contains(res.Lines[0].RuleSnapshot, long) { + t.Fatal("the long value is in the snapshot") + } + // A skipped or failed rule carries its snapshot too. + skip := compile(t, Options{}, rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionSetBody, Value: "x"})) + res = skip.Apply(PhaseRequest, Input{Request: request("POST", "http://h.example/", nil, "\x00")}) + if len(res.Lines) != 1 || res.Lines[0].RuleSnapshot == "" { + t.Fatalf("skipped line = %+v", res.Lines) + } +} + +// The trace keeps the body of the last body action of the phase, and only it. +func TestBodyTraceOnLastBodyActionOnly(t *testing.T) { + s := compile(t, Options{}, + rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionSetBody, Value: "one"}, Action{Type: ActionReplaceBody, Regex: "one", Value: "two"}, setHeader("X-A", "1")), + rule(2, PhaseRequest, 1, Matchers{}, Action{Type: ActionReplaceBody, Regex: "two", Value: "three"}), + rule(3, PhaseRequest, 2, Matchers{}, setHeader("X-B", "1"))) + res := s.Apply(PhaseRequest, Input{Request: request("POST", "http://h.example/", nil, "zero")}) + wantLines(t, res.Lines, "1:set_header:header:X-A:applied:", "1:set_body:body:applied:", "1:replace_body:body:applied:", + "2:replace_body:body:applied:", "3:set_header:header:X-B:applied:") + for i, want := range []string{"1", "", "", "three", "1"} { + if got := string(res.Lines[i].After); got != want { + t.Errorf("line %d: after = %q, want %q", i, got, want) + } + } + // The intermediate steps keep their hashes. + if l := res.Lines[2]; l.BeforeHash != sha256Hex([]byte("one")) || l.AfterHash != sha256Hex([]byte("two")) { + t.Fatalf("intermediate body line = %+v", l) + } + // When the last body action is sensitive, no body is kept at all. + sens := compile(t, Options{}, + rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionSetBody, Value: "one"}), + rule(2, PhaseRequest, 1, Matchers{}, Action{Type: ActionSetBody, Value: "two", Sensitive: true})) + for _, l := range sens.Apply(PhaseRequest, Input{Request: request("POST", "http://h.example/", nil, "zero")}).Lines { + if l.After != nil { + t.Fatalf("a body is kept although the last body action is sensitive: %+v", l) + } + } +} + +// The fingerprint of a sensitive value is keyed, 128 bits long, and absent +// without a key: there is no plain SHA-256 fallback. +func TestSensitiveFingerprint(t *testing.T) { + const secret = "tok-SECRET-0003" + rules := []Rule{rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionSetHeader, Name: "X-Token", Value: secret, Sensitive: true})} + apply := func(key []byte) TraceLine { + res := Compile(rules, Options{TraceKey: key}).Apply(PhaseRequest, Input{Request: request("GET", "http://h.example/", http.Header{"X-Token": {"old"}}, "")}) + if len(res.Lines) != 1 { + t.Fatalf("lines = %v", outcomes(res.Lines)) + } + return res.Lines[0] + } + keyed := apply(testKey) + plain := sha256Hex([]byte(secret)) + if len(keyed.AfterHash) != 32 || keyed.AfterHash == plain[:32] || strings.Contains(plain, keyed.AfterHash) { + t.Fatalf("fingerprint %q: want 128 bits, unrelated to the plain SHA-256", keyed.AfterHash) + } + if again := apply(testKey); again.AfterHash != keyed.AfterHash { + t.Fatal("the fingerprint must be stable under one key") + } + if other := apply([]byte("another-key-another-key-another-k")); other.AfterHash == keyed.AfterHash || other.AfterHash == "" { + t.Fatal("the fingerprint must depend on the key") + } + if none := apply(nil); none.AfterHash != "" || none.BeforeHash != sha256Hex([]byte("old")) || none.After != nil { + t.Fatalf("without a key the fingerprint must be absent: %+v", none) + } +} diff --git a/traffic/rewrite/rule.go b/traffic/rewrite/rule.go new file mode 100644 index 0000000..9e954ec --- /dev/null +++ b/traffic/rewrite/rule.go @@ -0,0 +1,418 @@ +// Package rewrite is the engine of the capture proxy's rewrite rules: the rule +// model, its validation, an immutable compiled snapshot, and the transformation +// of one request or one response. It depends on neither the proxy, the traffic +// index nor the database. +// +// Rules fail open and are not a security control: a rule that errors leaves the +// message as it was. +package rewrite + +import ( + "errors" + "fmt" + "net" + "net/http" + "net/url" + "regexp" + "sort" + "strconv" + "strings" +) + +// Limits. The save-time ones make Validate fail; the run-time ones skip the +// rule or fail it open. +const ( + MaxBodySize = 1 << 20 // run time: a larger body skips every rule that reads or rewrites it + MaxPatternSize = 1 << 10 // save time: a regex + MaxValueSize = 1 << 20 // save time: a set value or a replacement template + MaxExpansion = 1 << 20 // run time: the result of a replace action + + snapshotValueMax = 4 << 10 // rule_snapshot: a longer action value is summarized +) + +// Phase is where a rule runs. +type Phase string + +const ( + PhaseRequest Phase = "request" + PhaseResponse Phase = "response" +) + +// Action types. +const ( + ActionSetMethod = "set_method" // request + ActionSetPathQuery = "set_path_query" // request + ActionReplaceURL = "replace_url" // request; path and query only + ActionDeleteHeader = "delete_header" + ActionSetHeader = "set_header" + ActionReplaceHeader = "replace_header" + ActionSetBody = "set_body" + ActionReplaceBody = "replace_body" + ActionSetStatus = "set_status" // response +) + +// HeaderMatcher matches a header by name. An empty ValueRegex tests presence; +// otherwise one value of the header must match. +type HeaderMatcher struct { + Name string `json:"name"` + ValueRegex string `json:"value_regex,omitempty"` +} + +// Matchers are combined with AND; an absent matcher is true. In the response +// phase the request matchers see the request as it was sent to the target. +type Matchers struct { + Methods []string `json:"methods,omitempty"` // case-insensitive + Host string `json:"host,omitempty"` // host name without port: exact, or "*.suffix" + URLRegex string `json:"url_regex,omitempty"` // on URL.RequestURI() + Headers []HeaderMatcher `json:"headers,omitempty"` // request headers + BodyRegex string `json:"body_regex,omitempty"` // request body + Status []string `json:"status,omitempty"` // response: "404" or "4xx", any of + RespHeaders []HeaderMatcher `json:"resp_headers,omitempty"` // response + RespBodyRegex string `json:"resp_body_regex,omitempty"` // response +} + +// Action is one step of a rule. Value is the new value (set_*) or the +// replacement template (replace_*, regexp.Expand syntax: $1, ${name}). +// Sensitive keeps Value out of the API, the trace and the rule snapshot. +type Action struct { + Type string `json:"type"` + Name string `json:"name,omitempty"` // header actions + Regex string `json:"regex,omitempty"` // replace_* actions + Value string `json:"value,omitempty"` + Sensitive bool `json:"sensitive,omitempty"` +} + +// Rule is one rewrite rule. Within a rule the actions run in a fixed order: +// method and URL, delete_header, set_header and replace_header, body, status. +type Rule struct { + ID int64 `json:"id"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + Phase Phase `json:"phase"` + Priority int `json:"priority"` + Matchers Matchers `json:"matchers"` + Actions []Action `json:"actions"` + Note string `json:"note"` +} + +// Validate reports the first reason a rule cannot be saved. The message never +// contains an action value. +func Validate(r Rule) error { + _, err := compileRule(r) + return err +} + +type headerMatcher struct { + name string // canonical + re *regexp.Regexp +} + +type statusRange struct{ lo, hi int } // lo <= status < hi + +type compiledAction struct { + typ string + name string // canonical header name + re *regexp.Regexp + value string + status int + sensitive bool +} + +type compiledRule struct { + id int64 + phase Phase + snapshot string // rule_snapshot JSON + + methods map[string]bool // upper case + host string // lower case; with hostGlob, the suffix including its leading dot + hostGlob bool + urlRe *regexp.Regexp + headers []headerMatcher + bodyRe *regexp.Regexp + status []statusRange + respHeaders []headerMatcher + respBodyRe *regexp.Regexp + + actions []compiledAction // in execution order + // reqBody and respBody report that the rule reads or rewrites that body. + reqBody, respBody bool +} + +func compileRule(r Rule) (*compiledRule, error) { + if strings.TrimSpace(r.Name) == "" { + return nil, errors.New("name cannot be empty") + } + if r.Phase != PhaseRequest && r.Phase != PhaseResponse { + return nil, errors.New("phase must be request or response") + } + c := &compiledRule{id: r.ID, phase: r.Phase} + if err := c.compileMatchers(r.Matchers); err != nil { + return nil, err + } + if len(r.Actions) == 0 { + return nil, errors.New("a rule needs at least one action") + } + bodyAction := false + for i, a := range r.Actions { + ca, err := compileAction(r.Phase, a) + if err != nil { + return nil, fmt.Errorf("action %d (%s): %w", i+1, a.Type, err) + } + bodyAction = bodyAction || actionClass(ca.typ) == classBody + c.actions = append(c.actions, ca) + } + sort.SliceStable(c.actions, func(i, j int) bool { return actionClass(c.actions[i].typ) < actionClass(c.actions[j].typ) }) + if r.Phase == PhaseRequest { + c.reqBody = c.bodyRe != nil || bodyAction + } else { + c.reqBody = c.bodyRe != nil + c.respBody = c.respBodyRe != nil || bodyAction + } + c.snapshot = ruleSnapshot(r) + return c, nil +} + +// Execution classes, in order. +const ( + classTarget = iota // method and URL + classDeleteHeader + classHeader + classBody + classStatus +) + +func actionClass(typ string) int { + switch typ { + case ActionSetMethod, ActionSetPathQuery, ActionReplaceURL: + return classTarget + case ActionDeleteHeader: + return classDeleteHeader + case ActionSetHeader, ActionReplaceHeader: + return classHeader + case ActionSetBody, ActionReplaceBody: + return classBody + } + return classStatus +} + +func (c *compiledRule) compileMatchers(m Matchers) (err error) { + if c.phase == PhaseRequest && (len(m.Status) > 0 || len(m.RespHeaders) > 0 || m.RespBodyRegex != "") { + return errors.New("status, resp_headers and resp_body_regex are response-phase matchers") + } + if len(m.Methods) > 0 { + c.methods = map[string]bool{} + for _, method := range m.Methods { + if !validToken(method) { + return errors.New("methods: invalid method") + } + c.methods[strings.ToUpper(method)] = true + } + } + if m.Host != "" { + if c.host, c.hostGlob, err = compileHost(m.Host); err != nil { + return err + } + } + if c.urlRe, err = compilePattern("url_regex", m.URLRegex); err != nil { + return err + } + if c.headers, err = compileHeaderMatchers("headers", m.Headers); err != nil { + return err + } + if c.bodyRe, err = compilePattern("body_regex", m.BodyRegex); err != nil { + return err + } + for _, s := range m.Status { + sr, ok := parseStatus(s) + if !ok { + return fmt.Errorf("status: %q is neither a code (404) nor a class (4xx)", s) + } + c.status = append(c.status, sr) + } + if c.respHeaders, err = compileHeaderMatchers("resp_headers", m.RespHeaders); err != nil { + return err + } + c.respBodyRe, err = compilePattern("resp_body_regex", m.RespBodyRegex) + return err +} + +// compileHost accepts an exact host name or a suffix glob "*.example.com", +// which does not match example.com itself. No other wildcard, no port. +func compileHost(h string) (host string, glob bool, err error) { + h = strings.ToLower(strings.TrimSpace(h)) + if strings.HasPrefix(h, "*.") { + h, glob = h[1:], true // keep the leading dot + } + bare := strings.TrimPrefix(h, ".") + if bare == "" || strings.ContainsAny(h, "*/ \t") { + return "", false, errors.New("host: use an exact host name or a suffix glob such as *.example.com") + } + if _, _, perr := net.SplitHostPort(h); perr == nil { + return "", false, errors.New("host: a host name without port is expected") + } + return h, glob, nil +} + +func compilePattern(field, expr string) (*regexp.Regexp, error) { + if expr == "" { + return nil, nil + } + if len(expr) > MaxPatternSize { + return nil, fmt.Errorf("%s: pattern longer than %d bytes", field, MaxPatternSize) + } + re, err := regexp.Compile(expr) + if err != nil { + return nil, fmt.Errorf("%s: %w", field, err) + } + return re, nil +} + +func compileHeaderMatchers(field string, in []HeaderMatcher) ([]headerMatcher, error) { + var out []headerMatcher + for _, h := range in { + if !validToken(h.Name) { + return nil, fmt.Errorf("%s: invalid header name", field) + } + re, err := compilePattern(field, h.ValueRegex) + if err != nil { + return nil, err + } + out = append(out, headerMatcher{http.CanonicalHeaderKey(h.Name), re}) + } + return out, nil +} + +// parseStatus reads a code ("404") or a class ("4xx"), the syntax of the +// traffic list's status filter. +func parseStatus(s string) (statusRange, bool) { + s = strings.ToLower(strings.TrimSpace(s)) + if len(s) != 3 || s[0] < '1' || s[0] > '5' { + return statusRange{}, false + } + if s[1:] == "xx" { + lo := int(s[0]-'0') * 100 + return statusRange{lo, lo + 100}, true + } + n, err := strconv.Atoi(s) + if err != nil { + return statusRange{}, false + } + return statusRange{n, n + 1}, true +} + +func compileAction(phase Phase, a Action) (compiledAction, error) { + ca := compiledAction{typ: a.Type, value: a.Value, sensitive: a.Sensitive} + if len(a.Value) > MaxValueSize { + return ca, fmt.Errorf("value longer than %d bytes", MaxValueSize) + } + var err error + switch a.Type { + case ActionSetMethod, ActionSetPathQuery, ActionReplaceURL: + if phase != PhaseRequest { + return ca, errors.New("request-phase action") + } + case ActionSetStatus: + if phase != PhaseResponse { + return ca, errors.New("response-phase action") + } + } + switch a.Type { + case ActionSetMethod: + if !validToken(a.Value) || strings.EqualFold(a.Value, http.MethodConnect) { + return ca, errors.New("value must be an HTTP method other than CONNECT") + } + case ActionSetPathQuery: + if _, ok := parsePathQuery(a.Value); !ok { + return ca, errors.New("value must start with / and be a valid request URI") + } + case ActionReplaceURL, ActionReplaceBody: + ca.re, err = requiredPattern(a.Regex) + case ActionDeleteHeader: + ca.name, err = headerTarget(a.Name) + case ActionSetHeader: + if ca.name, err = headerTarget(a.Name); err == nil { + err = headerValue(a.Value) + } + case ActionReplaceHeader: + if ca.name, err = headerTarget(a.Name); err == nil { + if ca.re, err = requiredPattern(a.Regex); err == nil { + err = headerValue(a.Value) + } + } + case ActionSetBody: + case ActionSetStatus: + // 1xx, 204 and 304 carry no body (net/http bodyAllowedForStatus). + n, serr := strconv.Atoi(a.Value) + if serr != nil || n < 200 || n > 599 || !bodyAllowedForStatus(n) { + return ca, errors.New("value must be a status from 200 to 599, except 204 and 304") + } + ca.status = n + default: + return ca, errors.New("unknown action type") + } + return ca, err +} + +func requiredPattern(expr string) (*regexp.Regexp, error) { + if expr == "" { + return nil, errors.New("regex cannot be empty") + } + return compilePattern("regex", expr) +} + +// headerTarget refuses the headers a rule must not write: the framing and +// destination ones net/http owns, and the hop-by-hop ones. +func headerTarget(name string) (string, error) { + if !validToken(name) { + return "", errors.New("invalid header name") + } + c := http.CanonicalHeaderKey(name) + switch c { + case "Host", "Content-Length", "Transfer-Encoding", "Connection", "Keep-Alive", "Te", "Trailer", "Expect", "Upgrade": + return "", fmt.Errorf("header %s cannot be rewritten", c) + } + if strings.HasPrefix(c, "Proxy-") { + return "", fmt.Errorf("header %s cannot be rewritten", c) + } + return c, nil +} + +func headerValue(v string) error { + if strings.ContainsAny(v, "\r\n\x00") { + return errors.New("value contains a line break or a NUL byte") + } + return nil +} + +// validToken reports whether s is an RFC 9110 token (a header name, a method). +func validToken(s string) bool { + if s == "" { + return false + } + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9': + case strings.IndexByte("!#$%&'*+-.^_`|~", c) >= 0: + default: + return false + } + } + return true +} + +// parsePathQuery parses a path and query. Only those two parts of a URL are +// ever rewritten, so a rule cannot change the scheme, the host or the port. +func parsePathQuery(s string) (*url.URL, bool) { + if !strings.HasPrefix(s, "/") { + return nil, false + } + u, err := url.ParseRequestURI(s) + return u, err == nil +} + +// bodyAllowedForStatus is net/http's unexported rule: 1xx, 204 and 304 +// responses carry no body. +func bodyAllowedForStatus(status int) bool { + return !(status >= 100 && status <= 199) && status != 204 && status != 304 +} diff --git a/traffic/rewrite/rule_test.go b/traffic/rewrite/rule_test.go new file mode 100644 index 0000000..aafb6f4 --- /dev/null +++ b/traffic/rewrite/rule_test.go @@ -0,0 +1,210 @@ +package rewrite + +import ( + "fmt" + "net/http" + "net/url" + "strings" + "testing" +) + +// rule builds an enabled rule named after its id. +func rule(id int64, phase Phase, priority int, m Matchers, actions ...Action) Rule { + return Rule{ID: id, Name: fmt.Sprintf("r%d", id), Enabled: true, Phase: phase, Priority: priority, Matchers: m, Actions: actions} +} + +// compile fails the test when a rule is left out of the snapshot. +func compile(t *testing.T, opts Options, rules ...Rule) *Set { + t.Helper() + s := Compile(rules, opts) + if len(s.LoadErrors()) > 0 { + t.Fatalf("load errors: %v", s.LoadErrors()) + } + return s +} + +func request(method, rawURL string, header http.Header, body string) Request { + u, err := url.Parse(rawURL) + if err != nil { + panic(err) + } + if header == nil { + header = http.Header{} + } + return Request{Method: method, URL: u, Header: header, Body: []byte(body)} +} + +func setHeader(name, value string) Action { + return Action{Type: ActionSetHeader, Name: name, Value: value} +} + +func TestValidateAcceptsEveryAction(t *testing.T) { + req := rule(1, PhaseRequest, 0, Matchers{Methods: []string{"get"}, Host: "*.example.com", URLRegex: `^/api/`, + Headers: []HeaderMatcher{{Name: "x-a"}, {Name: "X-B", ValueRegex: "^v"}}, BodyRegex: "a"}, + Action{Type: ActionSetMethod, Value: "POST"}, + Action{Type: ActionSetPathQuery, Value: "/x?y=1"}, + Action{Type: ActionReplaceURL, Regex: "a", Value: "b"}, + Action{Type: ActionDeleteHeader, Name: "X-Old"}, + setHeader("X-New", "v"), + Action{Type: ActionReplaceHeader, Name: "Cookie", Regex: "a", Value: "b", Sensitive: true}, + Action{Type: ActionSetBody, Value: "body"}, + Action{Type: ActionReplaceBody, Regex: "a", Value: "$0$0"}) + resp := rule(2, PhaseResponse, 0, Matchers{Status: []string{"404", "5xx"}, RespHeaders: []HeaderMatcher{{Name: "Server"}}, RespBodyRegex: "a", BodyRegex: "b"}, + Action{Type: ActionDeleteHeader, Name: "Server"}, + setHeader("X-New", "v"), + Action{Type: ActionReplaceHeader, Name: "Set-Cookie", Regex: "a", Value: "b"}, + Action{Type: ActionSetBody, Value: ""}, + Action{Type: ActionReplaceBody, Regex: "a", Value: "b"}, + Action{Type: ActionSetStatus, Value: "200"}) + for _, r := range []Rule{req, resp} { + if err := Validate(r); err != nil { + t.Errorf("rule %d: %v", r.ID, err) + } + } +} + +// Every refusal of the save-time validation. +func TestValidateForbidden(t *testing.T) { + long := strings.Repeat("a", MaxPatternSize+1) + big := strings.Repeat("a", MaxValueSize+1) + ok := setHeader("X-A", "v") + cases := map[string]Rule{ + "empty name": {Phase: PhaseRequest, Actions: []Action{ok}}, + "bad phase": {Name: "r", Phase: "both", Actions: []Action{ok}}, + "no action": rule(1, PhaseRequest, 0, Matchers{}), + "unknown action": rule(1, PhaseRequest, 0, Matchers{}, Action{Type: "set_host", Value: "evil.example"}), + "set_method CONNECT": rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionSetMethod, Value: "CONNECT"}), + "set_method connect": rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionSetMethod, Value: "connect"}), + "set_method not token": rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionSetMethod, Value: "GET /"}), + "set_method response": rule(1, PhaseResponse, 0, Matchers{}, Action{Type: ActionSetMethod, Value: "POST"}), + "path absolute URL": rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionSetPathQuery, Value: "http://evil.example/x"}), + "path relative": rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionSetPathQuery, Value: "evil.example/x"}), + "path control char": rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionSetPathQuery, Value: "/a\r\nHost: evil"}), + "path response": rule(1, PhaseResponse, 0, Matchers{}, Action{Type: ActionSetPathQuery, Value: "/x"}), + "replace_url response": rule(1, PhaseResponse, 0, Matchers{}, Action{Type: ActionReplaceURL, Regex: "a", Value: "b"}), + "replace no regex": rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionReplaceBody, Value: "b"}), + "replace bad regex": rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionReplaceBody, Regex: "(", Value: "b"}), + "action regex 1 KiB": rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionReplaceBody, Regex: long, Value: "b"}), + "value 1 MiB": rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionSetBody, Value: big}), + "replacement 1 MiB": rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionReplaceBody, Regex: "a", Value: big}), + "header name": rule(1, PhaseRequest, 0, Matchers{}, setHeader("X A", "v")), + "header name empty": rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionDeleteHeader}), + "header value CR": rule(1, PhaseRequest, 0, Matchers{}, setHeader("X-A", "a\rb")), + "header value LF": rule(1, PhaseRequest, 0, Matchers{}, setHeader("X-A", "a\nb")), + "header value NUL": rule(1, PhaseRequest, 0, Matchers{}, setHeader("X-A", "a\x00b")), + "replacement LF": rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionReplaceHeader, Name: "X-A", Regex: "a", Value: "b\nc"}), + "set_status request": rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionSetStatus, Value: "200"}), + "set_status text": rule(1, PhaseResponse, 0, Matchers{}, Action{Type: ActionSetStatus, Value: "ok"}), + "matcher regex 1 KiB": rule(1, PhaseRequest, 0, Matchers{URLRegex: long}, ok), + "matcher bad regex": rule(1, PhaseRequest, 0, Matchers{BodyRegex: "("}, ok), + "matcher header name": rule(1, PhaseRequest, 0, Matchers{Headers: []HeaderMatcher{{Name: "X A"}}}, ok), + "matcher header regex": rule(1, PhaseRequest, 0, Matchers{Headers: []HeaderMatcher{{Name: "X-A", ValueRegex: "("}}}, ok), + "matcher method": rule(1, PhaseRequest, 0, Matchers{Methods: []string{"GET POST"}}, ok), + "status in request": rule(1, PhaseRequest, 0, Matchers{Status: []string{"200"}}, ok), + "resp header request": rule(1, PhaseRequest, 0, Matchers{RespHeaders: []HeaderMatcher{{Name: "Server"}}}, ok), + "resp body request": rule(1, PhaseRequest, 0, Matchers{RespBodyRegex: "a"}, ok), + "status syntax": rule(1, PhaseResponse, 0, Matchers{Status: []string{"6xx"}}, ok), + "status syntax word": rule(1, PhaseResponse, 0, Matchers{Status: []string{"ok"}}, ok), + } + for _, name := range []string{"Host", "content-length", "Transfer-Encoding", "Connection", "Proxy-Authorization", "proxy-anything", + "Keep-Alive", "TE", "Trailer", "Expect", "Upgrade"} { + cases["set_header "+name] = rule(1, PhaseRequest, 0, Matchers{}, setHeader(name, "v")) + cases["delete_header "+name] = rule(1, PhaseResponse, 0, Matchers{}, Action{Type: ActionDeleteHeader, Name: name}) + cases["replace_header "+name] = rule(1, PhaseRequest, 0, Matchers{}, Action{Type: ActionReplaceHeader, Name: name, Regex: "a", Value: "b"}) + } + for _, status := range []string{"199", "204", "304", "600"} { + cases["set_status "+status] = rule(1, PhaseResponse, 0, Matchers{}, Action{Type: ActionSetStatus, Value: status}) + } + for name, r := range cases { + if err := Validate(r); err == nil { + t.Errorf("%s: accepted", name) + } + } +} + +func TestValidateHostGlob(t *testing.T) { + ok := setHeader("X-A", "v") + for host, want := range map[string]bool{ + "example.com": true, + "*.a.b": true, + "*.example.com": true, + "::1": true, + "a.*.b": false, + "*": false, + "*.": false, + "*.*.example.com": false, + "*example.com": false, + "example.com:8080": false, + "example.com/path": false, + } { + err := Validate(rule(1, PhaseRequest, 0, Matchers{Host: host}, ok)) + if (err == nil) != want { + t.Errorf("host %q: accepted=%v, want %v (%v)", host, err == nil, want, err) + } + } +} + +// A validation message names the action, never its value. +func TestValidateErrorOmitsValue(t *testing.T) { + const secret = "s3cr3t-value" + for _, a := range []Action{ + {Type: ActionSetMethod, Value: secret + " x", Sensitive: true}, + {Type: ActionSetPathQuery, Value: secret, Sensitive: true}, + {Type: ActionSetHeader, Name: "X-A", Value: secret + "\n", Sensitive: true}, + {Type: ActionSetStatus, Value: secret, Sensitive: true}, + {Type: "nope", Value: secret, Sensitive: true}, + } { + phase := PhaseRequest + if a.Type == ActionSetStatus { + phase = PhaseResponse + } + err := Validate(rule(1, phase, 0, Matchers{}, a)) + if err == nil || strings.Contains(err.Error(), secret) { + t.Errorf("%s: error %v", a.Type, err) + } + } +} + +func TestCompileLoadError(t *testing.T) { + valid := rule(1, PhaseRequest, 0, Matchers{}, setHeader("X-A", "v")) + invalid := rule(2, PhaseRequest, 0, Matchers{}, setHeader("Host", "evil.example")) + disabledInvalid := rule(3, PhaseRequest, 0, Matchers{}, setHeader("Host", "evil.example")) + disabledInvalid.Enabled = false + disabled := rule(4, PhaseResponse, 0, Matchers{}, setHeader("X-A", "v")) + disabled.Enabled = false + s := Compile([]Rule{valid, invalid, disabledInvalid, disabled}, Options{}) + if s.Len(PhaseRequest) != 1 || s.Len(PhaseResponse) != 0 { + t.Fatalf("active rules: request=%d response=%d, want 1 and 0", s.Len(PhaseRequest), s.Len(PhaseResponse)) + } + errs := s.LoadErrors() + if len(errs) != 1 || !strings.Contains(errs[2], "Host") { + t.Fatalf("load errors = %v, want one for rule 2 naming Host", errs) + } + res := s.Apply(PhaseRequest, Input{Request: request("GET", "http://h.example/", nil, "")}) + if got := res.Request.Header.Get("X-A"); got != "v" || res.Request.Header.Get("Host") != "" { + t.Fatalf("the valid rule must apply and the invalid one must not: X-A=%q", got) + } + var none *Set + if none.Len(PhaseRequest) != 0 || none.LoadErrors() != nil || none.Apply(PhaseRequest, Input{}).Changed { + t.Fatal("a nil Set must hold no rule") + } +} + +func TestErrorLog(t *testing.T) { + l := &ErrorLog{} + if _, ok := l.Get(7); ok { + t.Fatal("empty log returned an entry") + } + l.record(7, ErrClassExpansion) + l.record(7, ErrClassInvalidURL) + e, ok := l.Get(7) + if !ok || e.Class != ErrClassInvalidURL || e.Count != 2 || e.At.IsZero() { + t.Fatalf("entry = %+v %v", e, ok) + } + l.Forget(7) + if _, ok := l.Get(7); ok { + t.Fatal("Forget kept the entry") + } + var none *ErrorLog + none.record(1, ErrClassPanic) // a Set compiled without a log +} diff --git a/traffic/rewrite/trace.go b/traffic/rewrite/trace.go new file mode 100644 index 0000000..506a85e --- /dev/null +++ b/traffic/rewrite/trace.go @@ -0,0 +1,106 @@ +package rewrite + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" +) + +// Outcomes of a trace line. +const ( + OutcomeApplied = "applied" + OutcomeSkipped = "skipped" + OutcomeError = "error" +) + +// Reasons of a skipped line. +const ( + ReasonStream = "not rewritten: stream" + ReasonBinary = "not rewritten: binary" + ReasonBodyTooLarge = "not rewritten: body too large" + ReasonContentEncoding = "not rewritten: content-encoding" + ReasonNoBodyAllowed = "not rewritten: no body allowed" + ReasonConnectionHeader = "header named in Connection" +) + +// Error classes: the only thing the engine reports about a failed rule. They +// are the Reason of an error line and the Class of a LastError. +const ( + ErrClassInvalidURL = "invalid_url" // a rewritten path and query is not a valid request URI + ErrClassExpansion = "expansion_limit" // a replace result exceeds MaxExpansion + ErrClassPanic = "panic" +) + +// TraceLine is one row of an exchange's rewrite trace, in order. An exchange is +// rewritten when it has at least one applied line. +type TraceLine struct { + Phase Phase + RuleID int64 // 0 on a global line + // RuleSnapshot is the rule as JSON, on its first line in the exchange only: + // matchers in clear, sensitive action values left out, longer values summarized. + RuleSnapshot string + Action string + Field string // "method", "url", "header:", "body" or "status" + Outcome string + Reason string + // BeforeHash and AfterHash are SHA-256 in hex. Once a sensitive action wrote + // the field they are the keyed fingerprint instead, or empty without a key. + BeforeHash string + AfterHash string + // After is the new value. It is nil for a sensitive action, for a field a + // sensitive action wrote earlier in the phase, and for every body line but + // the last one of the phase. + After []byte +} + +func sha256Hex(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} + +// fingerprint is HMAC-SHA256 truncated to 128 bits. It is not comparable +// between instances, and there is no unkeyed fallback. +func (s *Set) fingerprint(b []byte) string { + if len(s.traceKey) == 0 { + return "" + } + m := hmac.New(sha256.New, s.traceKey) + m.Write(b) + return hex.EncodeToString(m.Sum(nil)[:16]) +} + +type snapshotAction struct { + Type string `json:"type"` + Name string `json:"name,omitempty"` + Regex string `json:"regex,omitempty"` + Value *string `json:"value,omitempty"` + Sensitive bool `json:"sensitive,omitempty"` + ValueLen int `json:"value_len,omitempty"` + ValueSHA256 string `json:"value_sha256,omitempty"` +} + +func ruleSnapshot(r Rule) string { + actions := make([]snapshotAction, 0, len(r.Actions)) + for _, a := range r.Actions { + sa := snapshotAction{Type: a.Type, Name: a.Name, Regex: a.Regex, Sensitive: a.Sensitive} + switch { + case a.Sensitive: + case len(a.Value) > snapshotValueMax: + sa.ValueLen, sa.ValueSHA256 = len(a.Value), sha256Hex([]byte(a.Value)) + default: + v := a.Value + sa.Value = &v + } + actions = append(actions, sa) + } + out, _ := json.Marshal(struct { + ID int64 `json:"id"` + Name string `json:"name"` + Phase Phase `json:"phase"` + Priority int `json:"priority"` + Matchers Matchers `json:"matchers"` + Actions []snapshotAction `json:"actions"` + }{r.ID, r.Name, r.Phase, r.Priority, r.Matchers, actions}) + return string(out) +}