From 7a9b4fdf762f4a7f89cde12f7c30b2dbb15bafcd Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Mon, 16 Nov 2026 23:15:20 +0100 Subject: [PATCH] feat(server): rewrite-rule API and compiled snapshot for the proxy GET, POST, PUT and DELETE /api/traffic/rewrite-rules and POST /api/traffic/rewrite-rules/{id}/toggle manage the rules behind requireAuth. A rule is validated on save (400), created disabled, and validated again before the toggle enables it. The value of a sensitive action is never returned; an update sent without it keeps the stored value when the stored rule has a sensitive action of the same type and header name at the same position. The manager compiles the enabled rules after traffic.Open and after every change, and publishes the snapshot with Traffic.SetRewriteRules, an atomic swap. An enabled rule the validation refuses is left out and reported as load_error. The proxy stores the snapshot and does not apply it: a test sends the same request with and without matching rules and compares what the target receives, what the client receives and what is recorded. --- server/manager.go | 8 + server/server.go | 7 + server/traffic_rewrite.go | 321 ++++++++++++++++++++++++++ server/traffic_rewrite_test.go | 401 +++++++++++++++++++++++++++++++++ traffic/rewrite_rules_test.go | 102 +++++++++ traffic/traffic.go | 12 + 6 files changed, 851 insertions(+) create mode 100644 server/traffic_rewrite.go create mode 100644 server/traffic_rewrite_test.go create mode 100644 traffic/rewrite_rules_test.go diff --git a/server/manager.go b/server/manager.go index 116524f..71b9c0f 100644 --- a/server/manager.go +++ b/server/manager.go @@ -24,6 +24,7 @@ import ( "github.com/Autumn-27/artex/guard" "github.com/Autumn-27/artex/intercept" "github.com/Autumn-27/artex/traffic" + "github.com/Autumn-27/artex/traffic/rewrite" actool "github.com/Autumn-27/norma/tool" ) @@ -260,6 +261,12 @@ type Manager struct { enrich *enrich.Engine // engine-side asset auto-completion (DNS/HTTP) interceptor *intercept.Interceptor // user-configured tool-call interception rules + // Capture-proxy rewrite rules: rewriteMu serializes reloads, rewriteSet is + // the snapshot last compiled, rewriteErrors outlives the reloads. + rewriteMu sync.Mutex + rewriteSet atomic.Pointer[rewrite.Set] + rewriteErrors rewrite.ErrorLog + companyMu sync.Mutex // serializes task/company-scope commits with live handle registration // taskStateMu preserves commit order between PostgreSQL lifecycle writes and // their in-memory mirrors. lifecycleMu makes snapshots race-free, but without @@ -449,6 +456,7 @@ func NewManager(dir, proxyAddr string) (*Manager, error) { } if tr != nil { m.traffic = tr + m.reloadRewriteRules() // in place before the proxy accepts a connection go func() { log.Printf("[traffic] recording proxy on %s (set HTTP_PROXY=%s + trust _ca CA)", proxyAddr, tr.ProxyAddr()) if err := tr.Start(); err != nil { diff --git a/server/server.go b/server/server.go index 6c32c87..c73c4f1 100644 --- a/server/server.go +++ b/server/server.go @@ -953,6 +953,13 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("DELETE /api/intercept/rules/{id}", s.interceptDeleteRule) mux.HandleFunc("POST /api/intercept/rules/{id}/toggle", s.interceptToggleRule) + // capture-proxy rewrite rule management (stored and compiled; not applied by the proxy yet) + mux.HandleFunc("GET /api/traffic/rewrite-rules", s.rewriteListRules) + mux.HandleFunc("POST /api/traffic/rewrite-rules", s.rewriteCreateRule) + mux.HandleFunc("PUT /api/traffic/rewrite-rules/{id}", s.rewriteUpdateRule) + mux.HandleFunc("DELETE /api/traffic/rewrite-rules/{id}", s.rewriteDeleteRule) + mux.HandleFunc("POST /api/traffic/rewrite-rules/{id}/toggle", s.rewriteToggleRule) + // asset interception rule management (global blocklist: domains/IPs/URLs/CIDRs) mux.HandleFunc("GET /api/asset-intercept/rules", s.assetInterceptListRules) mux.HandleFunc("POST /api/asset-intercept/rules", s.assetInterceptCreateRule) diff --git a/server/traffic_rewrite.go b/server/traffic_rewrite.go new file mode 100644 index 0000000..9ad8e8c --- /dev/null +++ b/server/traffic_rewrite.go @@ -0,0 +1,321 @@ +package server + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "log" + "net/http" + "time" + + "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/artex/traffic/rewrite" +) + +// --- capture-proxy rewrite rules --- +// +// Global rules, admin only, never an agent tool. This layer stores, validates +// and compiles them and hands the compiled snapshot to the proxy; the proxy +// does not apply rewrite rules yet. + +// rewriteActionDTO is an action on the wire. The value of a sensitive action is +// never returned: a read leaves Value out and sets ValueSet. On update, a +// sensitive action sent without a value keeps the stored one, provided the +// stored rule has a sensitive action of the same type and header name at the +// same position. +type rewriteActionDTO struct { + Type string `json:"type"` + Name string `json:"name,omitempty"` + Regex string `json:"regex,omitempty"` + Value *string `json:"value,omitempty"` + Sensitive bool `json:"sensitive,omitempty"` + ValueSet bool `json:"value_set,omitempty"` +} + +type rewriteRuleReq struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + Phase rewrite.Phase `json:"phase"` + Priority int `json:"priority"` + Matchers rewrite.Matchers `json:"matchers"` + Actions []rewriteActionDTO `json:"actions"` + Note string `json:"note"` +} + +type rewriteRuleDTO struct { + ID int64 `json:"id"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + Phase string `json:"phase"` + Priority int `json:"priority"` + Matchers rewrite.Matchers `json:"matchers"` + Actions []rewriteActionDTO `json:"actions"` + Note string `json:"note"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + // LoadError is why an enabled rule is left out of the running snapshot. + LoadError string `json:"load_error,omitempty"` + // LastError is the rule's most recent run-time error, kept in memory only. + LastError *rewrite.LastError `json:"last_error,omitempty"` +} + +// errRewriteInvalid marks a rule the validation refuses (HTTP 400). +var errRewriteInvalid = errors.New("invalid rewrite rule") + +const rewriteUndecodable = "the stored rule cannot be decoded" + +// decodeRewriteRule turns a stored row into the engine's rule. +func decodeRewriteRule(row db.TrafficRewriteRule) (rewrite.Rule, error) { + r := rewrite.Rule{ID: row.ID, Name: row.Name, Enabled: row.Enabled, Phase: rewrite.Phase(row.Phase), Priority: row.Priority, Note: row.Note} + if err := json.Unmarshal(row.Matchers, &r.Matchers); err != nil { + return r, err + } + return r, json.Unmarshal(row.Actions, &r.Actions) +} + +func encodeRewriteRule(r rewrite.Rule) db.TrafficRewriteRule { + matchers, _ := json.Marshal(r.Matchers) + actions, _ := json.Marshal(r.Actions) + return db.TrafficRewriteRule{Name: r.Name, Enabled: r.Enabled, Phase: string(r.Phase), Priority: r.Priority, Matchers: matchers, Actions: actions, Note: r.Note} +} + +// rule builds the rule a request describes. stored is the rule being updated, +// nil on create: it supplies the sensitive values the request left out. +func (req rewriteRuleReq) rule(stored *rewrite.Rule) (rewrite.Rule, error) { + r := rewrite.Rule{Name: req.Name, Enabled: req.Enabled, Phase: req.Phase, Priority: req.Priority, Matchers: req.Matchers, Note: req.Note} + for i, a := range req.Actions { + out := rewrite.Action{Type: a.Type, Name: a.Name, Regex: a.Regex, Sensitive: a.Sensitive} + switch { + case a.Value != nil: + out.Value = *a.Value + case a.Sensitive && a.Type != rewrite.ActionDeleteHeader: + if stored == nil || i >= len(stored.Actions) { + return r, fmt.Errorf("action %d (%s): a sensitive action needs a value", i+1, a.Type) + } + old := stored.Actions[i] + if !old.Sensitive || old.Type != a.Type || http.CanonicalHeaderKey(old.Name) != http.CanonicalHeaderKey(a.Name) { + return r, fmt.Errorf("action %d (%s): no stored value to keep, send the value", i+1, a.Type) + } + out.Value = old.Value + } + r.Actions = append(r.Actions, out) + } + return r, nil +} + +// rewriteRuleDTO renders a stored row without the value of any sensitive action. +func (m *Manager) rewriteRuleDTO(row db.TrafficRewriteRule) rewriteRuleDTO { + out := rewriteRuleDTO{ID: row.ID, Name: row.Name, Enabled: row.Enabled, Phase: row.Phase, Priority: row.Priority, + Actions: []rewriteActionDTO{}, Note: row.Note, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt} + r, err := decodeRewriteRule(row) + if err != nil { + out.LoadError = rewriteUndecodable + return out + } + out.Matchers = r.Matchers + for _, a := range r.Actions { + d := rewriteActionDTO{Type: a.Type, Name: a.Name, Regex: a.Regex, Sensitive: a.Sensitive} + if a.Sensitive { + d.ValueSet = a.Type != rewrite.ActionDeleteHeader + } else { + v := a.Value + d.Value = &v + } + out.Actions = append(out.Actions, d) + } + out.LoadError = m.rewriteSet.Load().LoadErrors()[row.ID] + if e, ok := m.rewriteErrors.Get(row.ID); ok { + out.LastError = &e + } + return out +} + +// reloadRewriteRules compiles the stored rules and publishes the snapshot to +// the proxy. It runs at startup once the traffic store is open, and after every +// rule change. On a read failure the previous snapshot stays. +func (m *Manager) reloadRewriteRules() { + m.rewriteMu.Lock() + defer m.rewriteMu.Unlock() + rows, err := m.pg.ListTrafficRewriteRules() + if err != nil { + log.Printf("[traffic] rewrite rules not reloaded: %v", err) + return + } + rules := make([]rewrite.Rule, 0, len(rows)) + for _, row := range rows { + r, err := decodeRewriteRule(row) + if err != nil { + log.Printf("[traffic] rewrite rule %d not loaded: %s", row.ID, rewriteUndecodable) + continue + } + rules = append(rules, r) + } + set := rewrite.Compile(rules, rewrite.Options{TraceKey: db.TrafficTraceKey(), Errors: &m.rewriteErrors}) + for id, reason := range set.LoadErrors() { + log.Printf("[traffic] rewrite rule %d not loaded: %s", id, reason) + } + m.rewriteSet.Store(set) + if m.traffic != nil { + m.traffic.SetRewriteRules(set) + } +} + +// rewriteWriteErr answers a failed rule write. +func rewriteWriteErr(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, sql.ErrNoRows): + writeErr(w, 404, "rewrite rule not found") + case errors.Is(err, errRewriteInvalid), errors.Is(err, db.ErrTrafficRewriteRuleLimit), errors.Is(err, db.ErrTrafficRewriteEnabledLimit): + writeErr(w, 400, err.Error()) + default: + writeErr(w, 500, err.Error()) + } +} + +func rewriteInvalid(err error) error { return fmt.Errorf("%w: %v", errRewriteInvalid, err) } + +func (s *Server) rewriteListRules(w http.ResponseWriter, r *http.Request) { + pg := s.pg(w) + if pg == nil { + return + } + rows, err := pg.ListTrafficRewriteRules() + if err != nil { + writeErr(w, 500, err.Error()) + return + } + rules := make([]rewriteRuleDTO, 0, len(rows)) + for _, row := range rows { + rules = append(rules, s.m.rewriteRuleDTO(row)) + } + writeJSON(w, 200, map[string]any{"rules": rules}) +} + +// rewriteCreateRule creates a rule disabled, whatever the request says: a rule +// is enabled by the toggle or by an update. +func (s *Server) rewriteCreateRule(w http.ResponseWriter, r *http.Request) { + pg := s.pg(w) + if pg == nil { + return + } + var req rewriteRuleReq + if err := decode(r, &req); err != nil { + writeErr(w, 400, err.Error()) + return + } + rule, err := req.rule(nil) + if err == nil { + rule.Enabled = false + err = rewrite.Validate(rule) + } + if err != nil { + rewriteWriteErr(w, rewriteInvalid(err)) + return + } + row, err := pg.CreateTrafficRewriteRule(encodeRewriteRule(rule)) + if err != nil { + rewriteWriteErr(w, err) + return + } + s.m.reloadRewriteRules() + writeJSON(w, 200, s.m.rewriteRuleDTO(row)) +} + +func (s *Server) rewriteUpdateRule(w http.ResponseWriter, r *http.Request) { + pg := s.pg(w) + if pg == nil { + return + } + id, ok := pathInt(r, "id") + if !ok { + writeErr(w, 400, "bad rule id") + return + } + var req rewriteRuleReq + if err := decode(r, &req); err != nil { + writeErr(w, 400, err.Error()) + return + } + row, err := pg.UpdateTrafficRewriteRule(id, func(stored db.TrafficRewriteRule) (db.TrafficRewriteRule, error) { + var keep *rewrite.Rule + if old, err := decodeRewriteRule(stored); err == nil { + keep = &old + } + rule, err := req.rule(keep) + if err == nil { + err = rewrite.Validate(rule) + } + if err != nil { + return stored, rewriteInvalid(err) + } + return encodeRewriteRule(rule), nil + }) + if err != nil { + rewriteWriteErr(w, err) + return + } + s.m.rewriteErrors.Forget(id) + s.m.reloadRewriteRules() + writeJSON(w, 200, s.m.rewriteRuleDTO(row)) +} + +func (s *Server) rewriteDeleteRule(w http.ResponseWriter, r *http.Request) { + pg := s.pg(w) + if pg == nil { + return + } + id, ok := pathInt(r, "id") + if !ok { + writeErr(w, 400, "bad rule id") + return + } + if err := pg.DeleteTrafficRewriteRule(id); err != nil { + rewriteWriteErr(w, err) + return + } + s.m.rewriteErrors.Forget(id) + s.m.reloadRewriteRules() + writeJSON(w, 200, map[string]any{"deleted": id}) +} + +func (s *Server) rewriteToggleRule(w http.ResponseWriter, r *http.Request) { + pg := s.pg(w) + if pg == nil { + return + } + id, ok := pathInt(r, "id") + if !ok { + writeErr(w, 400, "bad rule id") + return + } + var req struct { + Enabled bool `json:"enabled"` + } + if err := decode(r, &req); err != nil { + writeErr(w, 400, err.Error()) + return + } + if req.Enabled { + // A rule saved under older validation rules is checked again before it runs. + stored, err := pg.GetTrafficRewriteRule(id) + if err != nil { + rewriteWriteErr(w, err) + return + } + rule, err := decodeRewriteRule(stored) + if err == nil { + err = rewrite.Validate(rule) + } + if err != nil { + rewriteWriteErr(w, rewriteInvalid(err)) + return + } + } + if err := pg.ToggleTrafficRewriteRule(id, req.Enabled); err != nil { + rewriteWriteErr(w, err) + return + } + s.m.reloadRewriteRules() + writeJSON(w, 200, map[string]any{"ok": true, "enabled": req.Enabled}) +} diff --git a/server/traffic_rewrite_test.go b/server/traffic_rewrite_test.go new file mode 100644 index 0000000..4b473ea --- /dev/null +++ b/server/traffic_rewrite_test.go @@ -0,0 +1,401 @@ +package server + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "strings" + "testing" + + "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/artex/traffic" + "github.com/Autumn-27/artex/traffic/rewrite" +) + +const rewriteAPI = "/api/traffic/rewrite-rules" + +// cleanRewriteRules removes the rules of these tests, now and at the end, and +// skips when the table holds other rules: the limit test counts rows. +func cleanRewriteRules(t *testing.T, pg *db.DB) { + t.Helper() + clean := func() { pg.Exec(`DELETE FROM traffic_rewrite_rules WHERE name LIKE 't-rwapi-%'`) } + clean() + t.Cleanup(clean) + var others int + if err := pg.QueryRow(`SELECT count(*) FROM traffic_rewrite_rules`).Scan(&others); err != nil { + t.Fatal(err) + } + if others > 0 { + t.Skipf("traffic_rewrite_rules holds %d rule(s) that are not test fixtures — skipping", others) + } +} + +// rewriteServer is a server with an open traffic store whose proxy does not +// listen. request sends an authenticated call when auth is true. +func rewriteServer(t *testing.T) (*Server, func(auth bool, method, path, body string) *httptest.ResponseRecorder) { + t.Helper() + skipWithoutDSN(t) + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { m.Close() }) + cleanRewriteRules(t, m.pg) + if m.traffic, err = traffic.Open(filepath.Join(m.dir, "traffic"), ":0"); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir()) + s.archiveWG.Wait() + <-s.side.done + token, err := signJWT(s.jwtKey) + if err != nil { + t.Fatal(err) + } + return s, func(auth bool, method, path, body string) *httptest.ResponseRecorder { + t.Helper() + r := httptest.NewRequest(method, path, strings.NewReader(body)) + if auth { + r.Header.Set("Authorization", "Bearer "+token) + } + r.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + s.Handler().ServeHTTP(w, r) + return w + } +} + +type rewriteRuleJSON struct { + ID int64 + Name string + Enabled bool + Phase string + Priority int + Matchers rewrite.Matchers + Actions []struct { + Type, Name, Regex string + Value *string + Sensitive bool + ValueSet bool `json:"value_set"` + } + Note string + LoadError string `json:"load_error"` +} + +func decodeRewriteRuleJSON(t *testing.T, resp *httptest.ResponseRecorder) rewriteRuleJSON { + t.Helper() + var r rewriteRuleJSON + if resp.Code != 200 { + t.Fatalf("HTTP %d %s", resp.Code, resp.Body.String()) + } + if err := json.Unmarshal(resp.Body.Bytes(), &r); err != nil { + t.Fatal(err) + } + return r +} + +func listRewriteRules(t *testing.T, request func(bool, string, string, string) *httptest.ResponseRecorder) (rules []rewriteRuleJSON, raw string) { + t.Helper() + resp := request(true, "GET", rewriteAPI, "") + var out struct{ Rules []rewriteRuleJSON } + if resp.Code != 200 || json.Unmarshal(resp.Body.Bytes(), &out) != nil || out.Rules == nil { + t.Fatalf("list: HTTP %d %s", resp.Code, resp.Body.String()) + } + return out.Rules, resp.Body.String() +} + +// sentToTarget applies the request phase of the proxy's snapshot to a request: +// what the stored rules would do, read from the snapshot itself. +func sentToTarget(s *Server) rewrite.Request { + u, _ := url.Parse("http://api.example/v1") + return s.m.traffic.RewriteRules().Apply(rewrite.PhaseRequest, rewrite.Input{Request: rewrite.Request{Method: "GET", URL: u, Header: http.Header{}}}).Request +} + +func TestRewriteRulesRequireAuth(t *testing.T) { + _, request := rewriteServer(t) + for _, c := range [][2]string{{"GET", rewriteAPI}, {"POST", rewriteAPI}, {"PUT", rewriteAPI + "/1"}, {"DELETE", rewriteAPI + "/1"}, {"POST", rewriteAPI + "/1/toggle"}} { + if resp := request(false, c[0], c[1], `{}`); resp.Code != 401 { + t.Errorf("%s %s without a token: HTTP %d, want 401", c[0], c[1], resp.Code) + } + } + if rules, _ := listRewriteRules(t, request); len(rules) != 0 { + t.Fatalf("an unauthenticated call created a rule: %+v", rules) + } +} + +func TestRewriteRulesCRUD(t *testing.T) { + s, request := rewriteServer(t) + pg := s.m.pg + const secret, secret2 = "Bearer rwapi-plain-0001", "Bearer rwapi-plain-0002" + body := func(name, auth string) string { + return `{"name":"` + name + `","enabled":true,"phase":"request","priority":3,"note":"n", +"matchers":{"host":"*.example","methods":["GET"]}, +"actions":[{"type":"set_header","name":"Authorization",` + auth + `"sensitive":true},{"type":"set_header","name":"X-Plain","value":"visible"}]}` + } + if s.m.traffic.RewriteRules() != nil { + t.Fatal("a snapshot exists before any rule change on this fixture") + } + + created := request(true, "POST", rewriteAPI, body("t-rwapi-crud", `"value":"`+secret+`",`)) + r := decodeRewriteRuleJSON(t, created) + if r.ID == 0 || r.Name != "t-rwapi-crud" || r.Phase != "request" || r.Priority != 3 || r.Note != "n" || r.Matchers.Host != "*.example" { + t.Fatalf("created rule = %+v", r) + } + if r.Enabled { + t.Fatal("a rule must be created disabled, whatever the request says") + } + if a := r.Actions; len(a) != 2 || a[0].Value != nil || !a[0].ValueSet || !a[0].Sensitive || a[1].Value == nil || *a[1].Value != "visible" || a[1].ValueSet { + t.Fatalf("actions = %+v", a) + } + _, listed := listRewriteRules(t, request) + if strings.Contains(created.Body.String()+listed, "rwapi-plain") { + t.Fatal("the sensitive value was returned") + } + path := fmt.Sprintf("%s/%d", rewriteAPI, r.ID) + + // Both columns are sealed at rest. + var matchers, actions string + if err := pg.QueryRow(`SELECT matchers::text, actions::text FROM traffic_rewrite_rules WHERE id=$1`, r.ID).Scan(&matchers, &actions); err != nil { + t.Fatal(err) + } + for _, col := range []string{matchers, actions} { + if !strings.Contains(col, `"__artex_enc"`) || strings.Contains(col, "rwapi-plain") || strings.Contains(col, "example") { + t.Fatalf("column not sealed at rest: %s", col) + } + } + + // Every change republishes the snapshot. A disabled rule is not in it. + set := s.m.traffic.RewriteRules() + if set == nil || set.Len(rewrite.PhaseRequest) != 0 { + t.Fatalf("after create: the snapshot must exist and hold no rule") + } + if resp := request(true, "POST", path+"/toggle", `{"enabled":true}`); resp.Code != 200 { + t.Fatalf("toggle: HTTP %d %s", resp.Code, resp.Body.String()) + } + if s.m.traffic.RewriteRules() == set || s.m.traffic.RewriteRules().Len(rewrite.PhaseRequest) != 1 { + t.Fatal("after toggle: the snapshot must be replaced and hold the rule") + } + if got := sentToTarget(s).Header.Get("Authorization"); got != secret { + t.Fatalf("the snapshot holds Authorization=%q, want the stored value", got) + } + + // Update without the sensitive value: the stored one is kept. + updated := request(true, "PUT", path, body("t-rwapi-crud2", "")) + if r2 := decodeRewriteRuleJSON(t, updated); r2.Name != "t-rwapi-crud2" || !r2.Enabled || !r2.Actions[0].ValueSet || r2.Actions[0].Value != nil { + t.Fatalf("updated rule = %+v", r2) + } + if strings.Contains(updated.Body.String(), "rwapi-plain") { + t.Fatal("the sensitive value was returned by the update") + } + if got := sentToTarget(s).Header.Get("Authorization"); got != secret { + t.Fatalf("after an update without value: Authorization=%q, want the stored value kept", got) + } + // Update with a value: replaced. An explicit empty value is a value. + decodeRewriteRuleJSON(t, request(true, "PUT", path, body("t-rwapi-crud2", `"value":"`+secret2+`",`))) + if got := sentToTarget(s).Header.Get("Authorization"); got != secret2 { + t.Fatalf("after an update with a value: Authorization=%q", got) + } + decodeRewriteRuleJSON(t, request(true, "PUT", path, body("t-rwapi-crud2", `"value":"",`))) + if h := sentToTarget(s).Header; h.Get("Authorization") != "" || len(h.Values("Authorization")) != 1 { + t.Fatalf("after an update with an empty value: %v", h) + } + decodeRewriteRuleJSON(t, request(true, "PUT", path, body("t-rwapi-crud2", `"value":"`+secret2+`",`))) + + // A kept value never moves to another action, and never comes back in clear. + for name, bad := range map[string]string{ + "other type": `[{"type":"set_body","sensitive":true}]`, + "other header": `[{"type":"set_header","name":"X-Other","sensitive":true}]`, + "other index": `[{"type":"set_header","name":"X-Plain","value":"v"},{"type":"set_header","name":"Authorization","sensitive":true}]`, + } { + resp := request(true, "PUT", path, `{"name":"t-rwapi-crud2","phase":"request","actions":`+bad+`}`) + if resp.Code != 400 || strings.Contains(resp.Body.String(), "rwapi-plain") { + t.Fatalf("%s: HTTP %d %s, want 400", name, resp.Code, resp.Body.String()) + } + } + unmasked := request(true, "PUT", path, `{"name":"t-rwapi-crud2","enabled":true,"phase":"request","actions":[{"type":"set_header","name":"Authorization"}]}`) + if r3 := decodeRewriteRuleJSON(t, unmasked); r3.Actions[0].Value == nil || *r3.Actions[0].Value != "" || strings.Contains(unmasked.Body.String(), "rwapi-plain") { + t.Fatalf("dropping the sensitive flag must not reveal the stored value: %s", unmasked.Body.String()) + } + if stored, err := pg.GetTrafficRewriteRule(r.ID); err != nil || strings.Contains(string(stored.Actions), "rwapi-plain") { + t.Fatalf("the stored value must be gone once the action is saved in clear: %s (%v)", stored.Actions, err) + } + + // Unknown rule: 404. Bad id: 400. + for _, c := range [][3]string{{"PUT", rewriteAPI + "/999999999", body("t-rwapi-x", `"value":"v",`)}, + {"POST", rewriteAPI + "/999999999/toggle", `{"enabled":true}`}, {"POST", rewriteAPI + "/999999999/toggle", `{"enabled":false}`}, + {"DELETE", rewriteAPI + "/999999999", ""}} { + if resp := request(true, c[0], c[1], c[2]); resp.Code != 404 { + t.Errorf("%s %s: HTTP %d, want 404", c[0], c[1], resp.Code) + } + } + if resp := request(true, "DELETE", rewriteAPI+"/abc", ""); resp.Code != 400 { + t.Errorf("bad id: HTTP %d, want 400", resp.Code) + } + + if resp := request(true, "DELETE", path, ""); resp.Code != 200 { + t.Fatalf("delete: HTTP %d %s", resp.Code, resp.Body.String()) + } + if rules, _ := listRewriteRules(t, request); len(rules) != 0 || s.m.traffic.RewriteRules().Len(rewrite.PhaseRequest) != 0 { + t.Fatalf("after delete: %d rule(s) listed, %d in the snapshot", len(rules), s.m.traffic.RewriteRules().Len(rewrite.PhaseRequest)) + } +} + +// The validation refuses a rule with HTTP 400 on create and on update, and +// stores nothing. +func TestRewriteRulesValidation400(t *testing.T) { + s, request := rewriteServer(t) + valid := `{"name":"t-rwapi-valid","phase":"request","actions":[{"type":"set_header","name":"X-A","value":"v"}]}` + r := decodeRewriteRuleJSON(t, request(true, "POST", rewriteAPI, valid)) + path := fmt.Sprintf("%s/%d", rewriteAPI, r.ID) + rule := func(phase, matchers, actions string) string { + return `{"name":"t-rwapi-bad","phase":"` + phase + `","matchers":` + matchers + `,"actions":` + actions + `}` + } + cases := map[string]string{ + "malformed JSON": `{"name":`, + "empty name": `{"phase":"request","actions":[{"type":"set_header","name":"X-A","value":"v"}]}`, + "bad phase": rule("both", `{}`, `[{"type":"set_header","name":"X-A","value":"v"}]`), + "no action": rule("request", `{}`, `[]`), + "unknown action": rule("request", `{}`, `[{"type":"set_host","value":"evil.example"}]`), + "Host header": rule("request", `{}`, `[{"type":"set_header","name":"Host","value":"evil.example"}]`), + "Content-Length": rule("response", `{}`, `[{"type":"delete_header","name":"content-length"}]`), + "Proxy- header": rule("request", `{}`, `[{"type":"set_header","name":"Proxy-Authorization","value":"v"}]`), + "hop-by-hop header": rule("request", `{}`, `[{"type":"set_header","name":"Connection","value":"close"}]`), + "header line break": rule("request", `{}`, `[{"type":"set_header","name":"X-A","value":"a\r\nX-B: b"}]`), + "header name": rule("request", `{}`, `[{"type":"set_header","name":"X A","value":"v"}]`), + "absolute URL": rule("request", `{}`, `[{"type":"set_path_query","value":"http://evil.example/x"}]`), + "method CONNECT": rule("request", `{}`, `[{"type":"set_method","value":"CONNECT"}]`), + "status 204": rule("response", `{}`, `[{"type":"set_status","value":"204"}]`), + "status 600": rule("response", `{}`, `[{"type":"set_status","value":"600"}]`), + "status in request": rule("request", `{}`, `[{"type":"set_status","value":"200"}]`), + "host glob": rule("request", `{"host":"a.*.b"}`, `[{"type":"set_header","name":"X-A","value":"v"}]`), + "bad regex": rule("request", `{"url_regex":"("}`, `[{"type":"set_header","name":"X-A","value":"v"}]`), + "regex over 1 KiB": rule("request", `{"url_regex":"`+strings.Repeat("a", rewrite.MaxPatternSize+1)+`"}`, `[{"type":"set_header","name":"X-A","value":"v"}]`), + "value over 1 MiB": rule("request", `{}`, `[{"type":"set_body","value":"`+strings.Repeat("a", rewrite.MaxValueSize+1)+`"}]`), + "sensitive, no value": rule("request", `{}`, `[{"type":"set_body","sensitive":true}]`), + "response matcher": rule("request", `{"status":["200"]}`, `[{"type":"set_header","name":"X-A","value":"v"}]`), + "replace without expr": rule("request", `{}`, `[{"type":"replace_body","value":"v"}]`), + } + for name, body := range cases { + for _, c := range [][2]string{{"POST", rewriteAPI}, {"PUT", path}} { + if resp := request(true, c[0], c[1], body); resp.Code != 400 { + t.Errorf("%s: %s answered HTTP %d, want 400", name, c[0], resp.Code) + } + } + } + rules, _ := listRewriteRules(t, request) + if len(rules) != 1 || rules[0].Name != "t-rwapi-valid" || rules[0].Actions[0].Name != "X-A" { + t.Fatalf("a refused request changed the table: %+v", rules) + } + if resp := request(true, "POST", path+"/toggle", `{"enabled":`); resp.Code != 400 { + t.Errorf("toggle with malformed JSON: HTTP %d, want 400", resp.Code) + } + + // A stored rule the validation no longer accepts: reported by load_error, + // kept out of the snapshot, and refused by the toggle. + old, err := s.m.pg.CreateTrafficRewriteRule(db.TrafficRewriteRule{Name: "t-rwapi-old", Enabled: true, Phase: "request", + Matchers: json.RawMessage(`{}`), Actions: json.RawMessage(`[{"type":"set_header","name":"Host","value":"evil.example"}]`)}) + if err != nil { + t.Fatal(err) + } + s.m.reloadRewriteRules() + if set := s.m.traffic.RewriteRules(); set.Len(rewrite.PhaseRequest) != 0 || !strings.Contains(set.LoadErrors()[old.ID], "Host") { + t.Fatalf("snapshot: %d rule(s), load errors %v", set.Len(rewrite.PhaseRequest), set.LoadErrors()) + } + rules, _ = listRewriteRules(t, request) + for _, r := range rules { + if (r.ID == old.ID) != (r.LoadError != "") { + t.Errorf("rule %s: load_error=%q", r.Name, r.LoadError) + } + } + oldPath := fmt.Sprintf("%s/%d/toggle", rewriteAPI, old.ID) + if resp := request(true, "POST", oldPath, `{"enabled":true}`); resp.Code != 400 { + t.Errorf("enabling an invalid stored rule: HTTP %d, want 400", resp.Code) + } + if resp := request(true, "POST", oldPath, `{"enabled":false}`); resp.Code != 200 { + t.Errorf("disabling an invalid stored rule: HTTP %d, want 200", resp.Code) + } +} + +// 100 rules at most, 50 of them enabled. +func TestRewriteRulesLimits(t *testing.T) { + s, request := rewriteServer(t) + body := func(i int) string { + return fmt.Sprintf(`{"name":"t-rwapi-limit-%03d","phase":"response","actions":[{"type":"set_header","name":"X-A","value":"v"}]}`, i) + } + var ids []int64 + for i := 0; i < db.MaxTrafficRewriteRules; i++ { + ids = append(ids, decodeRewriteRuleJSON(t, request(true, "POST", rewriteAPI, body(i))).ID) + } + if resp := request(true, "POST", rewriteAPI, body(100)); resp.Code != 400 || !strings.Contains(resp.Body.String(), "100") { + t.Fatalf("101st rule: HTTP %d %s, want 400", resp.Code, resp.Body.String()) + } + toggle := func(id int64) *httptest.ResponseRecorder { + return request(true, "POST", fmt.Sprintf("%s/%d/toggle", rewriteAPI, id), `{"enabled":true}`) + } + for _, id := range ids[:db.MaxEnabledTrafficRewriteRules] { + if resp := toggle(id); resp.Code != 200 { + t.Fatalf("toggle: HTTP %d %s", resp.Code, resp.Body.String()) + } + } + over := ids[db.MaxEnabledTrafficRewriteRules] + if resp := toggle(over); resp.Code != 400 || !strings.Contains(resp.Body.String(), "50") { + t.Fatalf("51st enabled rule by toggle: HTTP %d %s, want 400", resp.Code, resp.Body.String()) + } + update := `{"name":"t-rwapi-limit-upd","enabled":true,"phase":"response","actions":[{"type":"set_header","name":"X-A","value":"v"}]}` + if resp := request(true, "PUT", fmt.Sprintf("%s/%d", rewriteAPI, over), update); resp.Code != 400 { + t.Fatalf("51st enabled rule by update: HTTP %d %s, want 400", resp.Code, resp.Body.String()) + } + if n := s.m.traffic.RewriteRules().Len(rewrite.PhaseResponse); n != db.MaxEnabledTrafficRewriteRules { + t.Fatalf("the snapshot holds %d rules, want %d", n, db.MaxEnabledTrafficRewriteRules) + } + rules, _ := listRewriteRules(t, request) + if len(rules) != db.MaxTrafficRewriteRules { + t.Fatalf("%d rules listed, want %d", len(rules), db.MaxTrafficRewriteRules) + } +} + +// NewManager loads the rules once the traffic store is open, and refuses to +// start under a key other than the one that sealed them. +func TestRewriteRulesLoadedAtStartup(t *testing.T) { + _, request := rewriteServer(t) + r := decodeRewriteRuleJSON(t, request(true, "POST", rewriteAPI, + `{"name":"t-rwapi-startup","phase":"request","actions":[{"type":"set_header","name":"X-Startup","value":"1"}]}`)) + if resp := request(true, "POST", fmt.Sprintf("%s/%d/toggle", rewriteAPI, r.ID), `{"enabled":true}`); resp.Code != 200 { + t.Fatalf("toggle: HTTP %d", resp.Code) + } + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + proxyAddr := ln.Addr().String() + ln.Close() + m, err := NewManager(t.TempDir(), proxyAddr) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { m.Close() }) + if m.traffic == nil { + t.Fatal("NewManager did not open the traffic store") + } + set := m.traffic.RewriteRules() + if set == nil || set.Len(rewrite.PhaseRequest) != 1 || set.Len(rewrite.PhaseResponse) != 0 || len(set.LoadErrors()) != 0 { + t.Fatalf("snapshot at startup = %+v", set) + } + + // Not the key that sealed the rule. + t.Setenv("ARTEX_SECRET_KEY", base64.StdEncoding.EncodeToString([]byte(strings.Repeat("f", 32)))) + if m2, err := NewManager(t.TempDir(), ""); err == nil { + m2.Close() + t.Fatal("NewManager started under a key that cannot open the stored rules") + } else if !strings.Contains(err.Error(), "secrets at rest") || !strings.Contains(err.Error(), "secret key check") { + t.Fatalf("startup error = %v", err) + } +} diff --git a/traffic/rewrite_rules_test.go b/traffic/rewrite_rules_test.go new file mode 100644 index 0000000..11ca1f4 --- /dev/null +++ b/traffic/rewrite_rules_test.go @@ -0,0 +1,102 @@ +package traffic + +import ( + "io" + "net/http" + "net/http/httptest" + "slices" + "sort" + "strings" + "sync" + "testing" + + "github.com/Autumn-27/artex/traffic/rewrite" +) + +// The proxy stores the compiled rewrite rules and does not apply them. With a +// snapshot whose rules match every exchange, what the target receives, what the +// client receives and what is recorded are the same as with no rule at all. +func TestRewriteRulesStoredNotApplied(t *testing.T) { + var mu sync.Mutex + var seen []string + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + mu.Lock() + seen = append(seen, r.Method+" "+r.RequestURI+" X-Client="+r.Header.Get("X-Client")+" "+string(body)) + mu.Unlock() + w.Header().Set("X-Target", "t") + w.WriteHeader(http.StatusTeapot) + io.WriteString(w, "target body") + })) + t.Cleanup(target.Close) + + set := rewrite.Compile([]rewrite.Rule{ + {ID: 1, Name: "request", Enabled: true, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{ + {Type: rewrite.ActionSetMethod, Value: "PUT"}, + {Type: rewrite.ActionSetPathQuery, Value: "/rewritten"}, + {Type: rewrite.ActionSetHeader, Name: "X-Client", Value: "rewritten"}, + {Type: rewrite.ActionSetBody, Value: "rewritten"}}}, + {ID: 2, Name: "response", Enabled: true, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{ + {Type: rewrite.ActionSetStatus, Value: "200"}, + {Type: rewrite.ActionSetHeader, Name: "X-Rewritten", Value: "1"}, + {Type: rewrite.ActionSetBody, Value: "rewritten"}}}, + }, rewrite.Options{}) + if set.Len(rewrite.PhaseRequest) != 1 || set.Len(rewrite.PhaseResponse) != 1 || len(set.LoadErrors()) != 0 { + t.Fatalf("the snapshot must hold both rules: %v", set.LoadErrors()) + } + + // canon makes a recorded message comparable: the recorder writes headers in + // map order, and the target dates its response. + canon := func(raw string) string { + head, body, _ := strings.Cut(raw, "\n\n") + lines := strings.Split(head, "\n") + sort.Strings(lines[1:]) + lines = slices.DeleteFunc(lines, func(l string) bool { return strings.HasPrefix(l, "Date: ") }) + return strings.Join(lines, "\n") + "\n\n" + body + } + // exchange sends one request through a fresh proxy holding set and returns + // what the client received and what the proxy recorded. + exchange := func(set *rewrite.Set) (client, recorded string) { + tr, _, proxyURL := startProxy(t) + if tr.RewriteRules() != nil { + t.Fatal("Open published rewrite rules") + } + tr.SetRewriteRules(set) + if tr.RewriteRules() != set { + t.Fatal("RewriteRules must return the snapshot last published") + } + req, err := http.NewRequest("POST", target.URL+"/path?q=1", strings.NewReader("client body")) + if err != nil { + t.Fatal(err) + } + req.Header.Set("X-Client", "c") + resp, err := proxyClient(t, tr, proxyURL).Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + client = resp.Status + " X-Target=" + resp.Header.Get("X-Target") + " X-Rewritten=" + resp.Header.Get("X-Rewritten") + " " + string(body) + reqText, respText, err := tr.Get(onlyExchangeID(t, tr)) + if err != nil { + t.Fatal(err) + } + return client, canon(reqText) + "\n--\n" + canon(respText) + } + + baseClient, baseRecorded := exchange(nil) + client, recorded := exchange(set) + if want := "418 I'm a teapot X-Target=t X-Rewritten= target body"; baseClient != want || client != want { + t.Fatalf("client received %q without rules and %q with rules, want %q", baseClient, client, want) + } + if recorded != baseRecorded { + t.Fatalf("the recorded exchange differs with rules stored:\n without: %s\n with: %s", baseRecorded, recorded) + } + if !strings.Contains(recorded, "client body") || !strings.Contains(recorded, "target body") || strings.Contains(recorded, "rewritten") { + t.Fatalf("recorded exchange: %s", recorded) + } + want := "POST /path?q=1 X-Client=c client body" + if len(seen) != 2 || seen[0] != want || seen[1] != want { + t.Fatalf("the target received %q, want %q twice", seen, want) + } +} diff --git a/traffic/traffic.go b/traffic/traffic.go index b6fe0f9..fd10097 100644 --- a/traffic/traffic.go +++ b/traffic/traffic.go @@ -31,6 +31,7 @@ import ( "unicode/utf8" "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/artex/traffic/rewrite" "github.com/Autumn-27/norma/permission" actool "github.com/Autumn-27/norma/tool" mproxy "github.com/lqqyt2423/go-mitmproxy/proxy" @@ -196,6 +197,9 @@ type Traffic struct { // transparent-passthrough paths honor it (go-mitmproxy's getUpstreamConn), // so no host escapes it. Hot-swappable at runtime via SetUpstreamProxy. upstream atomic.Pointer[url.URL] + // rewriteRules is the compiled snapshot of the rewrite rules, replaced as a + // whole by SetRewriteRules. The proxy does not apply it yet. + rewriteRules atomic.Pointer[rewrite.Set] } // Open initializes the traffic tree, blob store and SQLite index under dir. @@ -353,6 +357,14 @@ func (t *Traffic) SetUpstreamProxy(raw string) error { return nil } +// SetRewriteRules publishes a compiled snapshot of the rewrite rules. The swap +// is atomic; nil clears the rules. The snapshot is stored only: the proxy does +// not apply rewrite rules yet, so capture is unchanged whatever it holds. +func (t *Traffic) SetRewriteRules(s *rewrite.Set) { t.rewriteRules.Store(s) } + +// RewriteRules returns the snapshot last published, nil when there is none. +func (t *Traffic) RewriteRules() *rewrite.Set { return t.rewriteRules.Load() } + // ValidateProxyURL parses and checks a proxy URL (http/https/socks5, optional // user:pass), returning the parsed URL. Exposed so callers can validate a global // proxy before persisting it even when the traffic proxy itself is disabled.