diff --git a/server/manager.go b/server/manager.go index 116524f..71b9c0f 100644 --- a/server/manager.go +++ b/server/manager.go @@ -24,6 +24,7 @@ import ( "github.com/Autumn-27/artex/guard" "github.com/Autumn-27/artex/intercept" "github.com/Autumn-27/artex/traffic" + "github.com/Autumn-27/artex/traffic/rewrite" actool "github.com/Autumn-27/norma/tool" ) @@ -260,6 +261,12 @@ type Manager struct { enrich *enrich.Engine // engine-side asset auto-completion (DNS/HTTP) interceptor *intercept.Interceptor // user-configured tool-call interception rules + // Capture-proxy rewrite rules: rewriteMu serializes reloads, rewriteSet is + // the snapshot last compiled, rewriteErrors outlives the reloads. + rewriteMu sync.Mutex + rewriteSet atomic.Pointer[rewrite.Set] + rewriteErrors rewrite.ErrorLog + companyMu sync.Mutex // serializes task/company-scope commits with live handle registration // taskStateMu preserves commit order between PostgreSQL lifecycle writes and // their in-memory mirrors. lifecycleMu makes snapshots race-free, but without @@ -449,6 +456,7 @@ func NewManager(dir, proxyAddr string) (*Manager, error) { } if tr != nil { m.traffic = tr + m.reloadRewriteRules() // in place before the proxy accepts a connection go func() { log.Printf("[traffic] recording proxy on %s (set HTTP_PROXY=%s + trust _ca CA)", proxyAddr, tr.ProxyAddr()) if err := tr.Start(); err != nil { diff --git a/server/server.go b/server/server.go index 6c32c87..c73c4f1 100644 --- a/server/server.go +++ b/server/server.go @@ -953,6 +953,13 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("DELETE /api/intercept/rules/{id}", s.interceptDeleteRule) mux.HandleFunc("POST /api/intercept/rules/{id}/toggle", s.interceptToggleRule) + // capture-proxy rewrite rule management (stored and compiled; not applied by the proxy yet) + mux.HandleFunc("GET /api/traffic/rewrite-rules", s.rewriteListRules) + mux.HandleFunc("POST /api/traffic/rewrite-rules", s.rewriteCreateRule) + mux.HandleFunc("PUT /api/traffic/rewrite-rules/{id}", s.rewriteUpdateRule) + mux.HandleFunc("DELETE /api/traffic/rewrite-rules/{id}", s.rewriteDeleteRule) + mux.HandleFunc("POST /api/traffic/rewrite-rules/{id}/toggle", s.rewriteToggleRule) + // asset interception rule management (global blocklist: domains/IPs/URLs/CIDRs) mux.HandleFunc("GET /api/asset-intercept/rules", s.assetInterceptListRules) mux.HandleFunc("POST /api/asset-intercept/rules", s.assetInterceptCreateRule) diff --git a/server/traffic_rewrite.go b/server/traffic_rewrite.go new file mode 100644 index 0000000..9ad8e8c --- /dev/null +++ b/server/traffic_rewrite.go @@ -0,0 +1,321 @@ +package server + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "log" + "net/http" + "time" + + "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/artex/traffic/rewrite" +) + +// --- capture-proxy rewrite rules --- +// +// Global rules, admin only, never an agent tool. This layer stores, validates +// and compiles them and hands the compiled snapshot to the proxy; the proxy +// does not apply rewrite rules yet. + +// rewriteActionDTO is an action on the wire. The value of a sensitive action is +// never returned: a read leaves Value out and sets ValueSet. On update, a +// sensitive action sent without a value keeps the stored one, provided the +// stored rule has a sensitive action of the same type and header name at the +// same position. +type rewriteActionDTO struct { + Type string `json:"type"` + Name string `json:"name,omitempty"` + Regex string `json:"regex,omitempty"` + Value *string `json:"value,omitempty"` + Sensitive bool `json:"sensitive,omitempty"` + ValueSet bool `json:"value_set,omitempty"` +} + +type rewriteRuleReq struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + Phase rewrite.Phase `json:"phase"` + Priority int `json:"priority"` + Matchers rewrite.Matchers `json:"matchers"` + Actions []rewriteActionDTO `json:"actions"` + Note string `json:"note"` +} + +type rewriteRuleDTO struct { + ID int64 `json:"id"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + Phase string `json:"phase"` + Priority int `json:"priority"` + Matchers rewrite.Matchers `json:"matchers"` + Actions []rewriteActionDTO `json:"actions"` + Note string `json:"note"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + // LoadError is why an enabled rule is left out of the running snapshot. + LoadError string `json:"load_error,omitempty"` + // LastError is the rule's most recent run-time error, kept in memory only. + LastError *rewrite.LastError `json:"last_error,omitempty"` +} + +// errRewriteInvalid marks a rule the validation refuses (HTTP 400). +var errRewriteInvalid = errors.New("invalid rewrite rule") + +const rewriteUndecodable = "the stored rule cannot be decoded" + +// decodeRewriteRule turns a stored row into the engine's rule. +func decodeRewriteRule(row db.TrafficRewriteRule) (rewrite.Rule, error) { + r := rewrite.Rule{ID: row.ID, Name: row.Name, Enabled: row.Enabled, Phase: rewrite.Phase(row.Phase), Priority: row.Priority, Note: row.Note} + if err := json.Unmarshal(row.Matchers, &r.Matchers); err != nil { + return r, err + } + return r, json.Unmarshal(row.Actions, &r.Actions) +} + +func encodeRewriteRule(r rewrite.Rule) db.TrafficRewriteRule { + matchers, _ := json.Marshal(r.Matchers) + actions, _ := json.Marshal(r.Actions) + return db.TrafficRewriteRule{Name: r.Name, Enabled: r.Enabled, Phase: string(r.Phase), Priority: r.Priority, Matchers: matchers, Actions: actions, Note: r.Note} +} + +// rule builds the rule a request describes. stored is the rule being updated, +// nil on create: it supplies the sensitive values the request left out. +func (req rewriteRuleReq) rule(stored *rewrite.Rule) (rewrite.Rule, error) { + r := rewrite.Rule{Name: req.Name, Enabled: req.Enabled, Phase: req.Phase, Priority: req.Priority, Matchers: req.Matchers, Note: req.Note} + for i, a := range req.Actions { + out := rewrite.Action{Type: a.Type, Name: a.Name, Regex: a.Regex, Sensitive: a.Sensitive} + switch { + case a.Value != nil: + out.Value = *a.Value + case a.Sensitive && a.Type != rewrite.ActionDeleteHeader: + if stored == nil || i >= len(stored.Actions) { + return r, fmt.Errorf("action %d (%s): a sensitive action needs a value", i+1, a.Type) + } + old := stored.Actions[i] + if !old.Sensitive || old.Type != a.Type || http.CanonicalHeaderKey(old.Name) != http.CanonicalHeaderKey(a.Name) { + return r, fmt.Errorf("action %d (%s): no stored value to keep, send the value", i+1, a.Type) + } + out.Value = old.Value + } + r.Actions = append(r.Actions, out) + } + return r, nil +} + +// rewriteRuleDTO renders a stored row without the value of any sensitive action. +func (m *Manager) rewriteRuleDTO(row db.TrafficRewriteRule) rewriteRuleDTO { + out := rewriteRuleDTO{ID: row.ID, Name: row.Name, Enabled: row.Enabled, Phase: row.Phase, Priority: row.Priority, + Actions: []rewriteActionDTO{}, Note: row.Note, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt} + r, err := decodeRewriteRule(row) + if err != nil { + out.LoadError = rewriteUndecodable + return out + } + out.Matchers = r.Matchers + for _, a := range r.Actions { + d := rewriteActionDTO{Type: a.Type, Name: a.Name, Regex: a.Regex, Sensitive: a.Sensitive} + if a.Sensitive { + d.ValueSet = a.Type != rewrite.ActionDeleteHeader + } else { + v := a.Value + d.Value = &v + } + out.Actions = append(out.Actions, d) + } + out.LoadError = m.rewriteSet.Load().LoadErrors()[row.ID] + if e, ok := m.rewriteErrors.Get(row.ID); ok { + out.LastError = &e + } + return out +} + +// reloadRewriteRules compiles the stored rules and publishes the snapshot to +// the proxy. It runs at startup once the traffic store is open, and after every +// rule change. On a read failure the previous snapshot stays. +func (m *Manager) reloadRewriteRules() { + m.rewriteMu.Lock() + defer m.rewriteMu.Unlock() + rows, err := m.pg.ListTrafficRewriteRules() + if err != nil { + log.Printf("[traffic] rewrite rules not reloaded: %v", err) + return + } + rules := make([]rewrite.Rule, 0, len(rows)) + for _, row := range rows { + r, err := decodeRewriteRule(row) + if err != nil { + log.Printf("[traffic] rewrite rule %d not loaded: %s", row.ID, rewriteUndecodable) + continue + } + rules = append(rules, r) + } + set := rewrite.Compile(rules, rewrite.Options{TraceKey: db.TrafficTraceKey(), Errors: &m.rewriteErrors}) + for id, reason := range set.LoadErrors() { + log.Printf("[traffic] rewrite rule %d not loaded: %s", id, reason) + } + m.rewriteSet.Store(set) + if m.traffic != nil { + m.traffic.SetRewriteRules(set) + } +} + +// rewriteWriteErr answers a failed rule write. +func rewriteWriteErr(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, sql.ErrNoRows): + writeErr(w, 404, "rewrite rule not found") + case errors.Is(err, errRewriteInvalid), errors.Is(err, db.ErrTrafficRewriteRuleLimit), errors.Is(err, db.ErrTrafficRewriteEnabledLimit): + writeErr(w, 400, err.Error()) + default: + writeErr(w, 500, err.Error()) + } +} + +func rewriteInvalid(err error) error { return fmt.Errorf("%w: %v", errRewriteInvalid, err) } + +func (s *Server) rewriteListRules(w http.ResponseWriter, r *http.Request) { + pg := s.pg(w) + if pg == nil { + return + } + rows, err := pg.ListTrafficRewriteRules() + if err != nil { + writeErr(w, 500, err.Error()) + return + } + rules := make([]rewriteRuleDTO, 0, len(rows)) + for _, row := range rows { + rules = append(rules, s.m.rewriteRuleDTO(row)) + } + writeJSON(w, 200, map[string]any{"rules": rules}) +} + +// rewriteCreateRule creates a rule disabled, whatever the request says: a rule +// is enabled by the toggle or by an update. +func (s *Server) rewriteCreateRule(w http.ResponseWriter, r *http.Request) { + pg := s.pg(w) + if pg == nil { + return + } + var req rewriteRuleReq + if err := decode(r, &req); err != nil { + writeErr(w, 400, err.Error()) + return + } + rule, err := req.rule(nil) + if err == nil { + rule.Enabled = false + err = rewrite.Validate(rule) + } + if err != nil { + rewriteWriteErr(w, rewriteInvalid(err)) + return + } + row, err := pg.CreateTrafficRewriteRule(encodeRewriteRule(rule)) + if err != nil { + rewriteWriteErr(w, err) + return + } + s.m.reloadRewriteRules() + writeJSON(w, 200, s.m.rewriteRuleDTO(row)) +} + +func (s *Server) rewriteUpdateRule(w http.ResponseWriter, r *http.Request) { + pg := s.pg(w) + if pg == nil { + return + } + id, ok := pathInt(r, "id") + if !ok { + writeErr(w, 400, "bad rule id") + return + } + var req rewriteRuleReq + if err := decode(r, &req); err != nil { + writeErr(w, 400, err.Error()) + return + } + row, err := pg.UpdateTrafficRewriteRule(id, func(stored db.TrafficRewriteRule) (db.TrafficRewriteRule, error) { + var keep *rewrite.Rule + if old, err := decodeRewriteRule(stored); err == nil { + keep = &old + } + rule, err := req.rule(keep) + if err == nil { + err = rewrite.Validate(rule) + } + if err != nil { + return stored, rewriteInvalid(err) + } + return encodeRewriteRule(rule), nil + }) + if err != nil { + rewriteWriteErr(w, err) + return + } + s.m.rewriteErrors.Forget(id) + s.m.reloadRewriteRules() + writeJSON(w, 200, s.m.rewriteRuleDTO(row)) +} + +func (s *Server) rewriteDeleteRule(w http.ResponseWriter, r *http.Request) { + pg := s.pg(w) + if pg == nil { + return + } + id, ok := pathInt(r, "id") + if !ok { + writeErr(w, 400, "bad rule id") + return + } + if err := pg.DeleteTrafficRewriteRule(id); err != nil { + rewriteWriteErr(w, err) + return + } + s.m.rewriteErrors.Forget(id) + s.m.reloadRewriteRules() + writeJSON(w, 200, map[string]any{"deleted": id}) +} + +func (s *Server) rewriteToggleRule(w http.ResponseWriter, r *http.Request) { + pg := s.pg(w) + if pg == nil { + return + } + id, ok := pathInt(r, "id") + if !ok { + writeErr(w, 400, "bad rule id") + return + } + var req struct { + Enabled bool `json:"enabled"` + } + if err := decode(r, &req); err != nil { + writeErr(w, 400, err.Error()) + return + } + if req.Enabled { + // A rule saved under older validation rules is checked again before it runs. + stored, err := pg.GetTrafficRewriteRule(id) + if err != nil { + rewriteWriteErr(w, err) + return + } + rule, err := decodeRewriteRule(stored) + if err == nil { + err = rewrite.Validate(rule) + } + if err != nil { + rewriteWriteErr(w, rewriteInvalid(err)) + return + } + } + if err := pg.ToggleTrafficRewriteRule(id, req.Enabled); err != nil { + rewriteWriteErr(w, err) + return + } + s.m.reloadRewriteRules() + writeJSON(w, 200, map[string]any{"ok": true, "enabled": req.Enabled}) +} diff --git a/server/traffic_rewrite_test.go b/server/traffic_rewrite_test.go new file mode 100644 index 0000000..4b473ea --- /dev/null +++ b/server/traffic_rewrite_test.go @@ -0,0 +1,401 @@ +package server + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "strings" + "testing" + + "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/artex/traffic" + "github.com/Autumn-27/artex/traffic/rewrite" +) + +const rewriteAPI = "/api/traffic/rewrite-rules" + +// cleanRewriteRules removes the rules of these tests, now and at the end, and +// skips when the table holds other rules: the limit test counts rows. +func cleanRewriteRules(t *testing.T, pg *db.DB) { + t.Helper() + clean := func() { pg.Exec(`DELETE FROM traffic_rewrite_rules WHERE name LIKE 't-rwapi-%'`) } + clean() + t.Cleanup(clean) + var others int + if err := pg.QueryRow(`SELECT count(*) FROM traffic_rewrite_rules`).Scan(&others); err != nil { + t.Fatal(err) + } + if others > 0 { + t.Skipf("traffic_rewrite_rules holds %d rule(s) that are not test fixtures — skipping", others) + } +} + +// rewriteServer is a server with an open traffic store whose proxy does not +// listen. request sends an authenticated call when auth is true. +func rewriteServer(t *testing.T) (*Server, func(auth bool, method, path, body string) *httptest.ResponseRecorder) { + t.Helper() + skipWithoutDSN(t) + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { m.Close() }) + cleanRewriteRules(t, m.pg) + if m.traffic, err = traffic.Open(filepath.Join(m.dir, "traffic"), ":0"); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir()) + s.archiveWG.Wait() + <-s.side.done + token, err := signJWT(s.jwtKey) + if err != nil { + t.Fatal(err) + } + return s, func(auth bool, method, path, body string) *httptest.ResponseRecorder { + t.Helper() + r := httptest.NewRequest(method, path, strings.NewReader(body)) + if auth { + r.Header.Set("Authorization", "Bearer "+token) + } + r.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + s.Handler().ServeHTTP(w, r) + return w + } +} + +type rewriteRuleJSON struct { + ID int64 + Name string + Enabled bool + Phase string + Priority int + Matchers rewrite.Matchers + Actions []struct { + Type, Name, Regex string + Value *string + Sensitive bool + ValueSet bool `json:"value_set"` + } + Note string + LoadError string `json:"load_error"` +} + +func decodeRewriteRuleJSON(t *testing.T, resp *httptest.ResponseRecorder) rewriteRuleJSON { + t.Helper() + var r rewriteRuleJSON + if resp.Code != 200 { + t.Fatalf("HTTP %d %s", resp.Code, resp.Body.String()) + } + if err := json.Unmarshal(resp.Body.Bytes(), &r); err != nil { + t.Fatal(err) + } + return r +} + +func listRewriteRules(t *testing.T, request func(bool, string, string, string) *httptest.ResponseRecorder) (rules []rewriteRuleJSON, raw string) { + t.Helper() + resp := request(true, "GET", rewriteAPI, "") + var out struct{ Rules []rewriteRuleJSON } + if resp.Code != 200 || json.Unmarshal(resp.Body.Bytes(), &out) != nil || out.Rules == nil { + t.Fatalf("list: HTTP %d %s", resp.Code, resp.Body.String()) + } + return out.Rules, resp.Body.String() +} + +// sentToTarget applies the request phase of the proxy's snapshot to a request: +// what the stored rules would do, read from the snapshot itself. +func sentToTarget(s *Server) rewrite.Request { + u, _ := url.Parse("http://api.example/v1") + return s.m.traffic.RewriteRules().Apply(rewrite.PhaseRequest, rewrite.Input{Request: rewrite.Request{Method: "GET", URL: u, Header: http.Header{}}}).Request +} + +func TestRewriteRulesRequireAuth(t *testing.T) { + _, request := rewriteServer(t) + for _, c := range [][2]string{{"GET", rewriteAPI}, {"POST", rewriteAPI}, {"PUT", rewriteAPI + "/1"}, {"DELETE", rewriteAPI + "/1"}, {"POST", rewriteAPI + "/1/toggle"}} { + if resp := request(false, c[0], c[1], `{}`); resp.Code != 401 { + t.Errorf("%s %s without a token: HTTP %d, want 401", c[0], c[1], resp.Code) + } + } + if rules, _ := listRewriteRules(t, request); len(rules) != 0 { + t.Fatalf("an unauthenticated call created a rule: %+v", rules) + } +} + +func TestRewriteRulesCRUD(t *testing.T) { + s, request := rewriteServer(t) + pg := s.m.pg + const secret, secret2 = "Bearer rwapi-plain-0001", "Bearer rwapi-plain-0002" + body := func(name, auth string) string { + return `{"name":"` + name + `","enabled":true,"phase":"request","priority":3,"note":"n", +"matchers":{"host":"*.example","methods":["GET"]}, +"actions":[{"type":"set_header","name":"Authorization",` + auth + `"sensitive":true},{"type":"set_header","name":"X-Plain","value":"visible"}]}` + } + if s.m.traffic.RewriteRules() != nil { + t.Fatal("a snapshot exists before any rule change on this fixture") + } + + created := request(true, "POST", rewriteAPI, body("t-rwapi-crud", `"value":"`+secret+`",`)) + r := decodeRewriteRuleJSON(t, created) + if r.ID == 0 || r.Name != "t-rwapi-crud" || r.Phase != "request" || r.Priority != 3 || r.Note != "n" || r.Matchers.Host != "*.example" { + t.Fatalf("created rule = %+v", r) + } + if r.Enabled { + t.Fatal("a rule must be created disabled, whatever the request says") + } + if a := r.Actions; len(a) != 2 || a[0].Value != nil || !a[0].ValueSet || !a[0].Sensitive || a[1].Value == nil || *a[1].Value != "visible" || a[1].ValueSet { + t.Fatalf("actions = %+v", a) + } + _, listed := listRewriteRules(t, request) + if strings.Contains(created.Body.String()+listed, "rwapi-plain") { + t.Fatal("the sensitive value was returned") + } + path := fmt.Sprintf("%s/%d", rewriteAPI, r.ID) + + // Both columns are sealed at rest. + var matchers, actions string + if err := pg.QueryRow(`SELECT matchers::text, actions::text FROM traffic_rewrite_rules WHERE id=$1`, r.ID).Scan(&matchers, &actions); err != nil { + t.Fatal(err) + } + for _, col := range []string{matchers, actions} { + if !strings.Contains(col, `"__artex_enc"`) || strings.Contains(col, "rwapi-plain") || strings.Contains(col, "example") { + t.Fatalf("column not sealed at rest: %s", col) + } + } + + // Every change republishes the snapshot. A disabled rule is not in it. + set := s.m.traffic.RewriteRules() + if set == nil || set.Len(rewrite.PhaseRequest) != 0 { + t.Fatalf("after create: the snapshot must exist and hold no rule") + } + if resp := request(true, "POST", path+"/toggle", `{"enabled":true}`); resp.Code != 200 { + t.Fatalf("toggle: HTTP %d %s", resp.Code, resp.Body.String()) + } + if s.m.traffic.RewriteRules() == set || s.m.traffic.RewriteRules().Len(rewrite.PhaseRequest) != 1 { + t.Fatal("after toggle: the snapshot must be replaced and hold the rule") + } + if got := sentToTarget(s).Header.Get("Authorization"); got != secret { + t.Fatalf("the snapshot holds Authorization=%q, want the stored value", got) + } + + // Update without the sensitive value: the stored one is kept. + updated := request(true, "PUT", path, body("t-rwapi-crud2", "")) + if r2 := decodeRewriteRuleJSON(t, updated); r2.Name != "t-rwapi-crud2" || !r2.Enabled || !r2.Actions[0].ValueSet || r2.Actions[0].Value != nil { + t.Fatalf("updated rule = %+v", r2) + } + if strings.Contains(updated.Body.String(), "rwapi-plain") { + t.Fatal("the sensitive value was returned by the update") + } + if got := sentToTarget(s).Header.Get("Authorization"); got != secret { + t.Fatalf("after an update without value: Authorization=%q, want the stored value kept", got) + } + // Update with a value: replaced. An explicit empty value is a value. + decodeRewriteRuleJSON(t, request(true, "PUT", path, body("t-rwapi-crud2", `"value":"`+secret2+`",`))) + if got := sentToTarget(s).Header.Get("Authorization"); got != secret2 { + t.Fatalf("after an update with a value: Authorization=%q", got) + } + decodeRewriteRuleJSON(t, request(true, "PUT", path, body("t-rwapi-crud2", `"value":"",`))) + if h := sentToTarget(s).Header; h.Get("Authorization") != "" || len(h.Values("Authorization")) != 1 { + t.Fatalf("after an update with an empty value: %v", h) + } + decodeRewriteRuleJSON(t, request(true, "PUT", path, body("t-rwapi-crud2", `"value":"`+secret2+`",`))) + + // A kept value never moves to another action, and never comes back in clear. + for name, bad := range map[string]string{ + "other type": `[{"type":"set_body","sensitive":true}]`, + "other header": `[{"type":"set_header","name":"X-Other","sensitive":true}]`, + "other index": `[{"type":"set_header","name":"X-Plain","value":"v"},{"type":"set_header","name":"Authorization","sensitive":true}]`, + } { + resp := request(true, "PUT", path, `{"name":"t-rwapi-crud2","phase":"request","actions":`+bad+`}`) + if resp.Code != 400 || strings.Contains(resp.Body.String(), "rwapi-plain") { + t.Fatalf("%s: HTTP %d %s, want 400", name, resp.Code, resp.Body.String()) + } + } + unmasked := request(true, "PUT", path, `{"name":"t-rwapi-crud2","enabled":true,"phase":"request","actions":[{"type":"set_header","name":"Authorization"}]}`) + if r3 := decodeRewriteRuleJSON(t, unmasked); r3.Actions[0].Value == nil || *r3.Actions[0].Value != "" || strings.Contains(unmasked.Body.String(), "rwapi-plain") { + t.Fatalf("dropping the sensitive flag must not reveal the stored value: %s", unmasked.Body.String()) + } + if stored, err := pg.GetTrafficRewriteRule(r.ID); err != nil || strings.Contains(string(stored.Actions), "rwapi-plain") { + t.Fatalf("the stored value must be gone once the action is saved in clear: %s (%v)", stored.Actions, err) + } + + // Unknown rule: 404. Bad id: 400. + for _, c := range [][3]string{{"PUT", rewriteAPI + "/999999999", body("t-rwapi-x", `"value":"v",`)}, + {"POST", rewriteAPI + "/999999999/toggle", `{"enabled":true}`}, {"POST", rewriteAPI + "/999999999/toggle", `{"enabled":false}`}, + {"DELETE", rewriteAPI + "/999999999", ""}} { + if resp := request(true, c[0], c[1], c[2]); resp.Code != 404 { + t.Errorf("%s %s: HTTP %d, want 404", c[0], c[1], resp.Code) + } + } + if resp := request(true, "DELETE", rewriteAPI+"/abc", ""); resp.Code != 400 { + t.Errorf("bad id: HTTP %d, want 400", resp.Code) + } + + if resp := request(true, "DELETE", path, ""); resp.Code != 200 { + t.Fatalf("delete: HTTP %d %s", resp.Code, resp.Body.String()) + } + if rules, _ := listRewriteRules(t, request); len(rules) != 0 || s.m.traffic.RewriteRules().Len(rewrite.PhaseRequest) != 0 { + t.Fatalf("after delete: %d rule(s) listed, %d in the snapshot", len(rules), s.m.traffic.RewriteRules().Len(rewrite.PhaseRequest)) + } +} + +// The validation refuses a rule with HTTP 400 on create and on update, and +// stores nothing. +func TestRewriteRulesValidation400(t *testing.T) { + s, request := rewriteServer(t) + valid := `{"name":"t-rwapi-valid","phase":"request","actions":[{"type":"set_header","name":"X-A","value":"v"}]}` + r := decodeRewriteRuleJSON(t, request(true, "POST", rewriteAPI, valid)) + path := fmt.Sprintf("%s/%d", rewriteAPI, r.ID) + rule := func(phase, matchers, actions string) string { + return `{"name":"t-rwapi-bad","phase":"` + phase + `","matchers":` + matchers + `,"actions":` + actions + `}` + } + cases := map[string]string{ + "malformed JSON": `{"name":`, + "empty name": `{"phase":"request","actions":[{"type":"set_header","name":"X-A","value":"v"}]}`, + "bad phase": rule("both", `{}`, `[{"type":"set_header","name":"X-A","value":"v"}]`), + "no action": rule("request", `{}`, `[]`), + "unknown action": rule("request", `{}`, `[{"type":"set_host","value":"evil.example"}]`), + "Host header": rule("request", `{}`, `[{"type":"set_header","name":"Host","value":"evil.example"}]`), + "Content-Length": rule("response", `{}`, `[{"type":"delete_header","name":"content-length"}]`), + "Proxy- header": rule("request", `{}`, `[{"type":"set_header","name":"Proxy-Authorization","value":"v"}]`), + "hop-by-hop header": rule("request", `{}`, `[{"type":"set_header","name":"Connection","value":"close"}]`), + "header line break": rule("request", `{}`, `[{"type":"set_header","name":"X-A","value":"a\r\nX-B: b"}]`), + "header name": rule("request", `{}`, `[{"type":"set_header","name":"X A","value":"v"}]`), + "absolute URL": rule("request", `{}`, `[{"type":"set_path_query","value":"http://evil.example/x"}]`), + "method CONNECT": rule("request", `{}`, `[{"type":"set_method","value":"CONNECT"}]`), + "status 204": rule("response", `{}`, `[{"type":"set_status","value":"204"}]`), + "status 600": rule("response", `{}`, `[{"type":"set_status","value":"600"}]`), + "status in request": rule("request", `{}`, `[{"type":"set_status","value":"200"}]`), + "host glob": rule("request", `{"host":"a.*.b"}`, `[{"type":"set_header","name":"X-A","value":"v"}]`), + "bad regex": rule("request", `{"url_regex":"("}`, `[{"type":"set_header","name":"X-A","value":"v"}]`), + "regex over 1 KiB": rule("request", `{"url_regex":"`+strings.Repeat("a", rewrite.MaxPatternSize+1)+`"}`, `[{"type":"set_header","name":"X-A","value":"v"}]`), + "value over 1 MiB": rule("request", `{}`, `[{"type":"set_body","value":"`+strings.Repeat("a", rewrite.MaxValueSize+1)+`"}]`), + "sensitive, no value": rule("request", `{}`, `[{"type":"set_body","sensitive":true}]`), + "response matcher": rule("request", `{"status":["200"]}`, `[{"type":"set_header","name":"X-A","value":"v"}]`), + "replace without expr": rule("request", `{}`, `[{"type":"replace_body","value":"v"}]`), + } + for name, body := range cases { + for _, c := range [][2]string{{"POST", rewriteAPI}, {"PUT", path}} { + if resp := request(true, c[0], c[1], body); resp.Code != 400 { + t.Errorf("%s: %s answered HTTP %d, want 400", name, c[0], resp.Code) + } + } + } + rules, _ := listRewriteRules(t, request) + if len(rules) != 1 || rules[0].Name != "t-rwapi-valid" || rules[0].Actions[0].Name != "X-A" { + t.Fatalf("a refused request changed the table: %+v", rules) + } + if resp := request(true, "POST", path+"/toggle", `{"enabled":`); resp.Code != 400 { + t.Errorf("toggle with malformed JSON: HTTP %d, want 400", resp.Code) + } + + // A stored rule the validation no longer accepts: reported by load_error, + // kept out of the snapshot, and refused by the toggle. + old, err := s.m.pg.CreateTrafficRewriteRule(db.TrafficRewriteRule{Name: "t-rwapi-old", Enabled: true, Phase: "request", + Matchers: json.RawMessage(`{}`), Actions: json.RawMessage(`[{"type":"set_header","name":"Host","value":"evil.example"}]`)}) + if err != nil { + t.Fatal(err) + } + s.m.reloadRewriteRules() + if set := s.m.traffic.RewriteRules(); set.Len(rewrite.PhaseRequest) != 0 || !strings.Contains(set.LoadErrors()[old.ID], "Host") { + t.Fatalf("snapshot: %d rule(s), load errors %v", set.Len(rewrite.PhaseRequest), set.LoadErrors()) + } + rules, _ = listRewriteRules(t, request) + for _, r := range rules { + if (r.ID == old.ID) != (r.LoadError != "") { + t.Errorf("rule %s: load_error=%q", r.Name, r.LoadError) + } + } + oldPath := fmt.Sprintf("%s/%d/toggle", rewriteAPI, old.ID) + if resp := request(true, "POST", oldPath, `{"enabled":true}`); resp.Code != 400 { + t.Errorf("enabling an invalid stored rule: HTTP %d, want 400", resp.Code) + } + if resp := request(true, "POST", oldPath, `{"enabled":false}`); resp.Code != 200 { + t.Errorf("disabling an invalid stored rule: HTTP %d, want 200", resp.Code) + } +} + +// 100 rules at most, 50 of them enabled. +func TestRewriteRulesLimits(t *testing.T) { + s, request := rewriteServer(t) + body := func(i int) string { + return fmt.Sprintf(`{"name":"t-rwapi-limit-%03d","phase":"response","actions":[{"type":"set_header","name":"X-A","value":"v"}]}`, i) + } + var ids []int64 + for i := 0; i < db.MaxTrafficRewriteRules; i++ { + ids = append(ids, decodeRewriteRuleJSON(t, request(true, "POST", rewriteAPI, body(i))).ID) + } + if resp := request(true, "POST", rewriteAPI, body(100)); resp.Code != 400 || !strings.Contains(resp.Body.String(), "100") { + t.Fatalf("101st rule: HTTP %d %s, want 400", resp.Code, resp.Body.String()) + } + toggle := func(id int64) *httptest.ResponseRecorder { + return request(true, "POST", fmt.Sprintf("%s/%d/toggle", rewriteAPI, id), `{"enabled":true}`) + } + for _, id := range ids[:db.MaxEnabledTrafficRewriteRules] { + if resp := toggle(id); resp.Code != 200 { + t.Fatalf("toggle: HTTP %d %s", resp.Code, resp.Body.String()) + } + } + over := ids[db.MaxEnabledTrafficRewriteRules] + if resp := toggle(over); resp.Code != 400 || !strings.Contains(resp.Body.String(), "50") { + t.Fatalf("51st enabled rule by toggle: HTTP %d %s, want 400", resp.Code, resp.Body.String()) + } + update := `{"name":"t-rwapi-limit-upd","enabled":true,"phase":"response","actions":[{"type":"set_header","name":"X-A","value":"v"}]}` + if resp := request(true, "PUT", fmt.Sprintf("%s/%d", rewriteAPI, over), update); resp.Code != 400 { + t.Fatalf("51st enabled rule by update: HTTP %d %s, want 400", resp.Code, resp.Body.String()) + } + if n := s.m.traffic.RewriteRules().Len(rewrite.PhaseResponse); n != db.MaxEnabledTrafficRewriteRules { + t.Fatalf("the snapshot holds %d rules, want %d", n, db.MaxEnabledTrafficRewriteRules) + } + rules, _ := listRewriteRules(t, request) + if len(rules) != db.MaxTrafficRewriteRules { + t.Fatalf("%d rules listed, want %d", len(rules), db.MaxTrafficRewriteRules) + } +} + +// NewManager loads the rules once the traffic store is open, and refuses to +// start under a key other than the one that sealed them. +func TestRewriteRulesLoadedAtStartup(t *testing.T) { + _, request := rewriteServer(t) + r := decodeRewriteRuleJSON(t, request(true, "POST", rewriteAPI, + `{"name":"t-rwapi-startup","phase":"request","actions":[{"type":"set_header","name":"X-Startup","value":"1"}]}`)) + if resp := request(true, "POST", fmt.Sprintf("%s/%d/toggle", rewriteAPI, r.ID), `{"enabled":true}`); resp.Code != 200 { + t.Fatalf("toggle: HTTP %d", resp.Code) + } + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + proxyAddr := ln.Addr().String() + ln.Close() + m, err := NewManager(t.TempDir(), proxyAddr) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { m.Close() }) + if m.traffic == nil { + t.Fatal("NewManager did not open the traffic store") + } + set := m.traffic.RewriteRules() + if set == nil || set.Len(rewrite.PhaseRequest) != 1 || set.Len(rewrite.PhaseResponse) != 0 || len(set.LoadErrors()) != 0 { + t.Fatalf("snapshot at startup = %+v", set) + } + + // Not the key that sealed the rule. + t.Setenv("ARTEX_SECRET_KEY", base64.StdEncoding.EncodeToString([]byte(strings.Repeat("f", 32)))) + if m2, err := NewManager(t.TempDir(), ""); err == nil { + m2.Close() + t.Fatal("NewManager started under a key that cannot open the stored rules") + } else if !strings.Contains(err.Error(), "secrets at rest") || !strings.Contains(err.Error(), "secret key check") { + t.Fatalf("startup error = %v", err) + } +} diff --git a/traffic/rewrite_rules_test.go b/traffic/rewrite_rules_test.go new file mode 100644 index 0000000..11ca1f4 --- /dev/null +++ b/traffic/rewrite_rules_test.go @@ -0,0 +1,102 @@ +package traffic + +import ( + "io" + "net/http" + "net/http/httptest" + "slices" + "sort" + "strings" + "sync" + "testing" + + "github.com/Autumn-27/artex/traffic/rewrite" +) + +// The proxy stores the compiled rewrite rules and does not apply them. With a +// snapshot whose rules match every exchange, what the target receives, what the +// client receives and what is recorded are the same as with no rule at all. +func TestRewriteRulesStoredNotApplied(t *testing.T) { + var mu sync.Mutex + var seen []string + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + mu.Lock() + seen = append(seen, r.Method+" "+r.RequestURI+" X-Client="+r.Header.Get("X-Client")+" "+string(body)) + mu.Unlock() + w.Header().Set("X-Target", "t") + w.WriteHeader(http.StatusTeapot) + io.WriteString(w, "target body") + })) + t.Cleanup(target.Close) + + set := rewrite.Compile([]rewrite.Rule{ + {ID: 1, Name: "request", Enabled: true, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{ + {Type: rewrite.ActionSetMethod, Value: "PUT"}, + {Type: rewrite.ActionSetPathQuery, Value: "/rewritten"}, + {Type: rewrite.ActionSetHeader, Name: "X-Client", Value: "rewritten"}, + {Type: rewrite.ActionSetBody, Value: "rewritten"}}}, + {ID: 2, Name: "response", Enabled: true, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{ + {Type: rewrite.ActionSetStatus, Value: "200"}, + {Type: rewrite.ActionSetHeader, Name: "X-Rewritten", Value: "1"}, + {Type: rewrite.ActionSetBody, Value: "rewritten"}}}, + }, rewrite.Options{}) + if set.Len(rewrite.PhaseRequest) != 1 || set.Len(rewrite.PhaseResponse) != 1 || len(set.LoadErrors()) != 0 { + t.Fatalf("the snapshot must hold both rules: %v", set.LoadErrors()) + } + + // canon makes a recorded message comparable: the recorder writes headers in + // map order, and the target dates its response. + canon := func(raw string) string { + head, body, _ := strings.Cut(raw, "\n\n") + lines := strings.Split(head, "\n") + sort.Strings(lines[1:]) + lines = slices.DeleteFunc(lines, func(l string) bool { return strings.HasPrefix(l, "Date: ") }) + return strings.Join(lines, "\n") + "\n\n" + body + } + // exchange sends one request through a fresh proxy holding set and returns + // what the client received and what the proxy recorded. + exchange := func(set *rewrite.Set) (client, recorded string) { + tr, _, proxyURL := startProxy(t) + if tr.RewriteRules() != nil { + t.Fatal("Open published rewrite rules") + } + tr.SetRewriteRules(set) + if tr.RewriteRules() != set { + t.Fatal("RewriteRules must return the snapshot last published") + } + req, err := http.NewRequest("POST", target.URL+"/path?q=1", strings.NewReader("client body")) + if err != nil { + t.Fatal(err) + } + req.Header.Set("X-Client", "c") + resp, err := proxyClient(t, tr, proxyURL).Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + client = resp.Status + " X-Target=" + resp.Header.Get("X-Target") + " X-Rewritten=" + resp.Header.Get("X-Rewritten") + " " + string(body) + reqText, respText, err := tr.Get(onlyExchangeID(t, tr)) + if err != nil { + t.Fatal(err) + } + return client, canon(reqText) + "\n--\n" + canon(respText) + } + + baseClient, baseRecorded := exchange(nil) + client, recorded := exchange(set) + if want := "418 I'm a teapot X-Target=t X-Rewritten= target body"; baseClient != want || client != want { + t.Fatalf("client received %q without rules and %q with rules, want %q", baseClient, client, want) + } + if recorded != baseRecorded { + t.Fatalf("the recorded exchange differs with rules stored:\n without: %s\n with: %s", baseRecorded, recorded) + } + if !strings.Contains(recorded, "client body") || !strings.Contains(recorded, "target body") || strings.Contains(recorded, "rewritten") { + t.Fatalf("recorded exchange: %s", recorded) + } + want := "POST /path?q=1 X-Client=c client body" + if len(seen) != 2 || seen[0] != want || seen[1] != want { + t.Fatalf("the target received %q, want %q twice", seen, want) + } +} diff --git a/traffic/traffic.go b/traffic/traffic.go index b6fe0f9..fd10097 100644 --- a/traffic/traffic.go +++ b/traffic/traffic.go @@ -31,6 +31,7 @@ import ( "unicode/utf8" "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/artex/traffic/rewrite" "github.com/Autumn-27/norma/permission" actool "github.com/Autumn-27/norma/tool" mproxy "github.com/lqqyt2423/go-mitmproxy/proxy" @@ -196,6 +197,9 @@ type Traffic struct { // transparent-passthrough paths honor it (go-mitmproxy's getUpstreamConn), // so no host escapes it. Hot-swappable at runtime via SetUpstreamProxy. upstream atomic.Pointer[url.URL] + // rewriteRules is the compiled snapshot of the rewrite rules, replaced as a + // whole by SetRewriteRules. The proxy does not apply it yet. + rewriteRules atomic.Pointer[rewrite.Set] } // Open initializes the traffic tree, blob store and SQLite index under dir. @@ -353,6 +357,14 @@ func (t *Traffic) SetUpstreamProxy(raw string) error { return nil } +// SetRewriteRules publishes a compiled snapshot of the rewrite rules. The swap +// is atomic; nil clears the rules. The snapshot is stored only: the proxy does +// not apply rewrite rules yet, so capture is unchanged whatever it holds. +func (t *Traffic) SetRewriteRules(s *rewrite.Set) { t.rewriteRules.Store(s) } + +// RewriteRules returns the snapshot last published, nil when there is none. +func (t *Traffic) RewriteRules() *rewrite.Set { return t.rewriteRules.Load() } + // ValidateProxyURL parses and checks a proxy URL (http/https/socks5, optional // user:pass), returning the parsed URL. Exposed so callers can validate a global // proxy before persisting it even when the traffic proxy itself is disabled.