mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-18 06:00:54 +08:00
Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7c3d50d533 | ||
|
|
b57bafe74a | ||
|
|
fa07d69f1d | ||
|
|
b3e1f5359f | ||
|
|
8ae53f74cf | ||
|
|
03ac1c7d47 | ||
|
|
287cbb685c | ||
|
|
a1c6103cbf | ||
|
|
56d11bb51c | ||
|
|
c73fec7471 | ||
|
|
0fb4ed145d | ||
|
|
c0ce7a9494 | ||
|
|
758bc416dd | ||
|
|
0234025d28 | ||
|
|
08eb4ec244 |
+2
-1
@@ -23,7 +23,8 @@ All notable changes to this project are documented in this file. The format is b
|
||||
- **`ask_operator` target warning**: a request whose question names a host or URL outside the goal targets is tagged as containing target-derived text (#197).
|
||||
- **Opt-in update banner**: `ARTEX_UPDATE_CHECK` makes the backend check the latest GitHub release at most once per 24 hours and the UI show a banner; off by default (#202).
|
||||
- **Release provenance**: release archives carry GitHub build provenance attestations and a CycloneDX SBOM (`gh attestation verify <zip> --repo BaaRai/ARTEX`); release notes are the matching `CHANGELOG.md` section (#219, #201).
|
||||
- **Capture-proxy rewrite rules, storage and API**: `GET`, `POST`, `PUT` and `DELETE /api/traffic/rewrite-rules` and `POST /api/traffic/rewrite-rules/{id}/toggle` manage rules that match a request or a response (method, host, URL, headers, body, status) and describe a rewrite (method, path and query, headers, body, status). Rules are validated on save, limited to 100 of which 50 enabled, created disabled, and stored in the new table `traffic_rewrite_rules` (migration `0005_traffic_rewrite_rules.sql`) with `matchers` and `actions` encrypted at rest. The value of an action marked `sensitive` is never returned, and an update sent without it keeps the stored one. The server compiles the enabled rules at startup and after every change and hands the snapshot to the proxy. **The proxy does not apply the rules yet: capture is unchanged.** See `docs/traffic-rewrite-rules.md`.
|
||||
- **Capture-proxy rewrite rules, storage and API**: `GET`, `POST`, `PUT` and `DELETE /api/traffic/rewrite-rules` and `POST /api/traffic/rewrite-rules/{id}/toggle` manage rules that match a request or a response (method, host, URL, headers, body, status) and describe a rewrite (method, path and query, headers, body, status). Rules are validated on save, limited to 100 of which 50 enabled, created disabled, and stored in the new table `traffic_rewrite_rules` (migration `0005_traffic_rewrite_rules.sql`) with `matchers` and `actions` encrypted at rest. The value of an action marked `sensitive` is never returned, and an update sent without it keeps the stored one. The server compiles the enabled rules at startup and after every change and hands the snapshot to the proxy. The proxy applies them from the next entry. See `docs/traffic-rewrite-rules.md`.
|
||||
- **Capture-proxy rewrite rules applied**: the capture proxy applies enabled rules to each HTTP request before it is sent to the target and to each response before it reaches the client. Each action that ran is recorded in the new table `exchange_rewrites` in the same transaction as the exchange, `traffic_get` shows a `REWRITTEN` line, and `traffic_search` never finds a value a rule wrote. SSE status and headers are rewritten at the start of the stream. **WebSocket handshakes are not rewritten**: the capture library runs its hook after the upgrade, so the status and headers cannot be changed, and frames pass unchanged. **Responses of 5 MiB or more** are forwarded unchanged and not recorded in full, because the library does not call the response hook for them. Snapshots of original requests are capped at 1024 entries and 64 MiB. Replays go through the proxy, so the rules apply to them. Archives, a per-replay rule toggle, the web UI and the proof flag come later.
|
||||
|
||||
### Changed
|
||||
|
||||
|
||||
+18
-3
@@ -127,9 +127,24 @@ func run() int {
|
||||
<-ctx.Done()
|
||||
|
||||
log.Println("shutting down...")
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = httpSrv.Shutdown(shutdownCtx)
|
||||
// Close the listener before draining. Otherwise a request accepted during
|
||||
// the drain can start a detached run, and mgr.Close runs while that run
|
||||
// is still writing.
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
httpErr := httpSrv.Shutdown(shutdownCtx)
|
||||
cancel()
|
||||
if httpErr != nil {
|
||||
log.Printf("http shutdown: %v", httpErr)
|
||||
}
|
||||
drainCtx, drainCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
drainErr := srv.Wait(drainCtx)
|
||||
drainCancel()
|
||||
if drainErr != nil {
|
||||
log.Printf("task drain: %v", drainErr)
|
||||
}
|
||||
if httpErr != nil || drainErr != nil {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
)
|
||||
|
||||
// activityOutboxMaxAttempts is how many times a row may fail AppendActivity
|
||||
// before it is dropped. One stuck row must not sit at the head of the queue
|
||||
// and keep every later row from being replayed.
|
||||
const activityOutboxMaxAttempts = 5
|
||||
|
||||
// activityOutboxPayload is Activity plus Detail. Activity.Detail is omitted
|
||||
// from ordinary JSON (the list payload stays small); the outbox has to keep it
|
||||
// so a replayed row is the same step that failed to store.
|
||||
type activityOutboxPayload struct {
|
||||
Activity
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
func marshalActivity(a Activity) ([]byte, error) {
|
||||
return json.Marshal(activityOutboxPayload{Activity: a, Detail: a.Detail})
|
||||
}
|
||||
|
||||
func unmarshalActivity(raw []byte) (Activity, error) {
|
||||
var payload activityOutboxPayload
|
||||
if err := json.Unmarshal(raw, &payload); err != nil {
|
||||
return Activity{}, err
|
||||
}
|
||||
a := payload.Activity
|
||||
a.Detail = payload.Detail
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// EnqueueActivity keeps one activity that AppendActivity could not store.
|
||||
func (s *ExplorationStore) EnqueueActivity(a Activity, cause string) error {
|
||||
if s == nil || s.db == nil {
|
||||
return fmt.Errorf("activity outbox: no store")
|
||||
}
|
||||
raw, err := marshalActivity(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.Exec(`INSERT INTO activity_outbox (exploration_id, payload, last_error) VALUES ($1, $2::jsonb, $3)`, s.expID, string(raw), cause)
|
||||
return err
|
||||
}
|
||||
|
||||
// FlushActivityOutbox inserts queued activities in order and deletes each row
|
||||
// that lands. The insert and the delete commit together, so a crash cannot
|
||||
// leave the activity stored and the outbox row still queued. A row that still
|
||||
// cannot be inserted stays, with its attempt count increased, and the flush
|
||||
// continues with the next row. After activityOutboxMaxAttempts the row is
|
||||
// dropped and logged. A payload that cannot be decoded is discarded.
|
||||
func (s *ExplorationStore) FlushActivityOutbox() (int, error) {
|
||||
if s == nil || s.db == nil {
|
||||
return 0, fmt.Errorf("activity outbox: no store")
|
||||
}
|
||||
rows, err := s.db.Query(`SELECT id, payload, attempts FROM activity_outbox WHERE exploration_id=$1 ORDER BY id`, s.expID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
type item struct {
|
||||
id int64
|
||||
payload []byte
|
||||
attempts int
|
||||
}
|
||||
var items []item
|
||||
for rows.Next() {
|
||||
var it item
|
||||
if err := rows.Scan(&it.id, &it.payload, &it.attempts); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
items = append(items, it)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
stored := 0
|
||||
for _, it := range items {
|
||||
a, err := unmarshalActivity(it.payload)
|
||||
if err != nil {
|
||||
if _, delErr := s.db.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); delErr != nil {
|
||||
return stored, delErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
return stored, err
|
||||
}
|
||||
if _, err := s.appendActivity(tx, a); err != nil {
|
||||
_ = tx.Rollback()
|
||||
next := it.attempts + 1
|
||||
if next >= activityOutboxMaxAttempts {
|
||||
log.Printf("[activity] outbox %d for exploration %d dropped after %d attempts: %v", it.id, s.expID, next, err)
|
||||
if _, delErr := s.db.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); delErr != nil {
|
||||
return stored, delErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, updErr := s.db.Exec(`UPDATE activity_outbox SET attempts=$2, last_error=$3 WHERE id=$1`, it.id, next, err.Error()); updErr != nil {
|
||||
return stored, updErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM activity_outbox WHERE id=$1`, it.id); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return stored, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return stored, err
|
||||
}
|
||||
stored++
|
||||
}
|
||||
return stored, nil
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package db
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestActivityOutboxKeepsDetail(t *testing.T) {
|
||||
in := Activity{Worker: "worker", Kind: "tool_result", Summary: "done", Detail: "full body", IsError: true}
|
||||
raw, err := marshalActivity(in)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := unmarshalActivity(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Detail != in.Detail || out.Summary != in.Summary || out.Worker != in.Worker || out.Kind != in.Kind || !out.IsError {
|
||||
t.Fatalf("round trip = %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlushActivityOutboxReplaysOnce(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
defer d.Close()
|
||||
expID, err := d.CreateExploration("outbox", "replay")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = d.Exec(`DELETE FROM activity_outbox WHERE exploration_id=$1`, expID)
|
||||
_, _ = d.Exec(`DELETE FROM explorations WHERE id=$1`, expID)
|
||||
})
|
||||
es := d.Exploration(expID)
|
||||
if err := es.EnqueueActivity(Activity{Worker: "worker", Kind: "text", Summary: "kept", Detail: "body"}, "insert failed"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, err := es.FlushActivityOutbox()
|
||||
if err != nil || n != 1 {
|
||||
t.Fatalf("flush n=%d err=%v", n, err)
|
||||
}
|
||||
var activities, queued int
|
||||
if err := d.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='kept'`, expID).Scan(&activities); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, expID).Scan(&queued); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if activities != 1 || queued != 0 {
|
||||
t.Fatalf("activities=%d outbox=%d, want 1 and 0", activities, queued)
|
||||
}
|
||||
if n, err = es.FlushActivityOutbox(); err != nil || n != 0 {
|
||||
t.Fatalf("second flush n=%d err=%v", n, err)
|
||||
}
|
||||
if err := d.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='kept'`, expID).Scan(&activities); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if activities != 1 {
|
||||
t.Fatalf("activities=%d after a second flush, want 1", activities)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlushActivityOutboxDropsStuckRow(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
defer d.Close()
|
||||
const missing = int64(9_000_000_000_001)
|
||||
t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM activity_outbox WHERE exploration_id=$1`, missing) })
|
||||
es := d.Exploration(missing)
|
||||
if err := es.EnqueueActivity(Activity{Worker: "worker", Kind: "text", Summary: "stuck"}, "no parent"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for attempt := 1; attempt < activityOutboxMaxAttempts; attempt++ {
|
||||
n, err := es.FlushActivityOutbox()
|
||||
if err != nil || n != 0 {
|
||||
t.Fatalf("attempt %d: n=%d err=%v", attempt, n, err)
|
||||
}
|
||||
var queued int
|
||||
if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, missing).Scan(&queued); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if queued != 1 {
|
||||
t.Fatalf("attempt %d: outbox rows=%d, want the stuck row kept", attempt, queued)
|
||||
}
|
||||
}
|
||||
n, err := es.FlushActivityOutbox()
|
||||
if err != nil || n != 0 {
|
||||
t.Fatalf("final flush n=%d err=%v", n, err)
|
||||
}
|
||||
var queued int
|
||||
if err := d.QueryRow(`SELECT count(*) FROM activity_outbox WHERE exploration_id=$1`, missing).Scan(&queued); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if queued != 0 {
|
||||
t.Fatalf("outbox rows=%d, want the stuck row dropped", queued)
|
||||
}
|
||||
}
|
||||
+15
-2
@@ -354,7 +354,10 @@ WHERE exploration_id=$1 AND kind='intent' AND state='running'
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
@@ -1379,14 +1382,24 @@ func (s *ExplorationStore) Stats() (map[string]int, error) {
|
||||
|
||||
// --- activity (poll by global id cursor) ---
|
||||
|
||||
// queryRower is the subset of *sql.DB and *sql.Tx that AppendActivity needs,
|
||||
// so a replay can insert the activity and delete the outbox row together.
|
||||
type queryRower interface {
|
||||
QueryRow(query string, args ...any) *sql.Row
|
||||
}
|
||||
|
||||
// AppendActivity records one worker step and returns its id.
|
||||
func (s *ExplorationStore) AppendActivity(a Activity) (int64, error) {
|
||||
return s.appendActivity(s.db, a)
|
||||
}
|
||||
|
||||
func (s *ExplorationStore) appendActivity(q queryRower, a Activity) (int64, error) {
|
||||
metadata := a.Metadata
|
||||
if len(metadata) == 0 {
|
||||
metadata = json.RawMessage(`{}`)
|
||||
}
|
||||
var id int64
|
||||
err := s.db.QueryRow(`
|
||||
err := q.QueryRow(`
|
||||
INSERT INTO activity(exploration_id, node_id, worker, kind, tool, tool_use_id, is_error, summary, detail, metadata, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, main_seg)
|
||||
VALUES ($1,$2,NULLIF($3,''),NULLIF($4,''),NULLIF($5,''),NULLIF($6,''),$7,NULLIF($8,''),NULLIF($9,''),$10,$11,$12,$13,$14,$15)
|
||||
RETURNING id`, s.expID, a.NodeID, utf8Clean(a.Worker), utf8Clean(a.Kind), utf8Clean(a.Tool), utf8Clean(a.ToolUseID), a.IsError, utf8Clean(a.Summary), utf8Clean(a.Detail),
|
||||
|
||||
+105
-5
@@ -93,10 +93,15 @@ func lessonExtractCompanyID(companyIDs []int64) int64 {
|
||||
}
|
||||
|
||||
// ExtractLessonsForTask derives candidate lessons from one task's findings and
|
||||
// retests. It is idempotent: the evidence primary key means a second run adds
|
||||
// no support. A task with more than one company is skipped so a generic
|
||||
// (NULL company_id) lesson cannot leak across companies. Returns the number of
|
||||
// distinct lessons touched.
|
||||
// retests. It is idempotent for the same disposition: the evidence primary key
|
||||
// means a second run adds no support. A later run after a reversed disposition
|
||||
// retracts that finding's evidence from the opposite-kind lesson, including
|
||||
// every retest of the finding, and retires the lesson when nothing else
|
||||
// supports it. Confirming again restores the evidence; a retired lesson is
|
||||
// un-retired only when it has no remaining evidence. Pending and in_progress
|
||||
// produce nothing and retract nothing. A task with more than one company is
|
||||
// skipped so a generic (NULL company_id) lesson cannot leak across companies.
|
||||
// Returns the number of distinct lessons written.
|
||||
func (d *DB) ExtractLessonsForTask(ctx context.Context, taskID int64) (int, error) {
|
||||
categoryID, companyIDs, err := taskLessonScope(ctx, d, taskID)
|
||||
if err != nil {
|
||||
@@ -175,11 +180,20 @@ func (d *DB) ExtractLessonsForTask(ctx context.Context, taskID int64) (int, erro
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if err := d.retractLessonEvidence(ctx, "finding", findingID, "negative"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
case "false_positive", "duplicate", "ignored", "risk_accepted":
|
||||
if err := add("negative", "Do not re-report without new mechanics: "+label+" (prior "+status+")",
|
||||
"finding", findingID, sourceDetail); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := d.retractLessonEvidence(ctx, "finding", findingID, "positive"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := d.retractFindingRetestEvidence(ctx, findingID, "positive"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if retestID > 0 && (verdict == "fixed" || verdict == "inconclusive") {
|
||||
if err := add("caution", "Recheck before reporting "+cue+": a prior retest was "+verdict,
|
||||
@@ -196,11 +210,97 @@ func (d *DB) upsertLesson(ctx context.Context, key, kind, cue, action string, ca
|
||||
err := d.QueryRowContext(ctx, `
|
||||
INSERT INTO rsi_lessons(dedupe_key,kind,cue,action,category_id,company_id)
|
||||
VALUES($1,$2,$3,$4,NULLIF($5,0),NULLIF($6,0))
|
||||
ON CONFLICT (dedupe_key) DO UPDATE SET updated_at=now()
|
||||
ON CONFLICT (dedupe_key) DO UPDATE SET
|
||||
updated_at=now(),
|
||||
status=CASE
|
||||
WHEN rsi_lessons.status='retired'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id
|
||||
)
|
||||
THEN 'candidate'
|
||||
ELSE rsi_lessons.status
|
||||
END
|
||||
RETURNING id`, key, kind, cue, action, categoryID, companyID).Scan(&id)
|
||||
return id, err
|
||||
}
|
||||
|
||||
// retractLessonEvidence removes this source from lessons of fromKind and
|
||||
// retires any of those lessons that now have zero evidence. Shared lessons
|
||||
// that still have other evidence stay active.
|
||||
func (d *DB) retractLessonEvidence(ctx context.Context, sourceKind string, sourceID int64, fromKind string) error {
|
||||
rows, err := d.QueryContext(ctx, `
|
||||
DELETE FROM rsi_lesson_evidence e
|
||||
USING rsi_lessons l
|
||||
WHERE e.lesson_id=l.id
|
||||
AND l.kind=$1
|
||||
AND e.source_kind=$2
|
||||
AND e.source_id=$3
|
||||
RETURNING e.lesson_id`, fromKind, sourceKind, sourceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ids, err := collectLessonIDs(rows)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return d.retireLessonsIfEmpty(ctx, ids)
|
||||
}
|
||||
|
||||
// retractFindingRetestEvidence removes every retest of this finding from
|
||||
// lessons of fromKind, not only the latest row.
|
||||
func (d *DB) retractFindingRetestEvidence(ctx context.Context, findingID int64, fromKind string) error {
|
||||
rows, err := d.QueryContext(ctx, `
|
||||
DELETE FROM rsi_lesson_evidence e
|
||||
USING rsi_lessons l, finding_retests r
|
||||
WHERE e.lesson_id=l.id
|
||||
AND l.kind=$1
|
||||
AND e.source_kind='retest'
|
||||
AND e.source_id=r.id
|
||||
AND r.finding_id=$2
|
||||
RETURNING e.lesson_id`, fromKind, findingID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ids, err := collectLessonIDs(rows)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return d.retireLessonsIfEmpty(ctx, ids)
|
||||
}
|
||||
|
||||
func collectLessonIDs(rows *sql.Rows) ([]int64, error) {
|
||||
defer rows.Close()
|
||||
seen := map[int64]bool{}
|
||||
var ids []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !seen[id] {
|
||||
seen[id] = true
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (d *DB) retireLessonsIfEmpty(ctx context.Context, ids []int64) error {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
_, err := d.ExecContext(ctx, `
|
||||
UPDATE rsi_lessons l
|
||||
SET status='retired', updated_at=now()
|
||||
WHERE l.id=ANY($1::bigint[])
|
||||
AND l.status<>'retired'
|
||||
AND NOT EXISTS (SELECT 1 FROM rsi_lesson_evidence e WHERE e.lesson_id=l.id)`, ids)
|
||||
return err
|
||||
}
|
||||
|
||||
// TaskLessons selects lessons visible to one task: generic lessons plus those
|
||||
// scoped to the task's company and category. Scope matches rank first, then
|
||||
// positive before negative before caution, then support.
|
||||
|
||||
@@ -453,3 +453,459 @@ func insertLessonFinding(t *testing.T, d *DB, taskID int64, status string) int64
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, id) })
|
||||
return id
|
||||
}
|
||||
|
||||
func TestExtractLessonsRetractsReversedDisposition(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
task, err := d.CreateTask(fmt.Sprintf("lesson retract %d", stamp), "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "task", d.DeleteTask(task.ID)) })
|
||||
|
||||
cue := fmt.Sprintf("xss-retract-%d", stamp)
|
||||
label := fmt.Sprintf("reflected in search %d", stamp)
|
||||
var fid int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS',$3,'high','confirmed')
|
||||
RETURNING id`, task.ID, cue, label).Scan(&fid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
n, err := d.ExtractLessonsForTask(t.Context(), task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("first extract=%d, want 1", n)
|
||||
}
|
||||
first, err := d.TaskLessons(t.Context(), task.ID, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var pos *Lesson
|
||||
for i := range first {
|
||||
if first[i].Kind == "positive" && strings.Contains(first[i].Action, label) {
|
||||
pos = &first[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if pos == nil {
|
||||
t.Fatalf("no positive lesson: %+v", first)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, pos.ID) })
|
||||
posID := pos.ID
|
||||
|
||||
if _, err := d.SetFindingStatus(fid, FindingFalsePositive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var posStatus string
|
||||
var posSupport int
|
||||
if err := d.QueryRow(`SELECT status,
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id)
|
||||
FROM rsi_lessons WHERE id=$1`, posID).Scan(&posStatus, &posSupport); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if posStatus != "retired" || posSupport != 0 {
|
||||
t.Fatalf("positive after FP: status=%s support=%d", posStatus, posSupport)
|
||||
}
|
||||
|
||||
afterFP, err := d.TaskLessons(t.Context(), task.ID, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var neg *Lesson
|
||||
for i := range afterFP {
|
||||
if afterFP[i].ID == posID {
|
||||
t.Fatal("retired positive still selected")
|
||||
}
|
||||
if afterFP[i].Kind == "negative" && strings.Contains(afterFP[i].Action, label) {
|
||||
neg = &afterFP[i]
|
||||
}
|
||||
}
|
||||
if neg == nil {
|
||||
t.Fatalf("no negative lesson after FP: %+v", afterFP)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, neg.ID) })
|
||||
|
||||
if _, err := d.SetFindingStatus(fid, FindingConfirmed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := d.QueryRow(`SELECT status,
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id)
|
||||
FROM rsi_lessons WHERE id=$1`, posID).Scan(&posStatus, &posSupport); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if posStatus != "candidate" || posSupport != 1 {
|
||||
t.Fatalf("positive after re-confirm: status=%s support=%d", posStatus, posSupport)
|
||||
}
|
||||
|
||||
var negStatus string
|
||||
var negSupport int
|
||||
if err := d.QueryRow(`SELECT status,
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id)
|
||||
FROM rsi_lessons WHERE id=$1`, neg.ID).Scan(&negStatus, &negSupport); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if negStatus != "retired" || negSupport != 0 {
|
||||
t.Fatalf("negative after re-confirm: status=%s support=%d", negStatus, negSupport)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLessonsRetireOnlyWhenLastEvidence(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
task, err := d.CreateTask(fmt.Sprintf("lesson shared retract %d", stamp), "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "task", d.DeleteTask(task.ID)) })
|
||||
|
||||
cue := fmt.Sprintf("xss-shared-%d", stamp)
|
||||
label := fmt.Sprintf("reflected in search %d", stamp)
|
||||
var aID, bID int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS',$3,'high','confirmed')
|
||||
RETURNING id`, task.ID, cue, label).Scan(&aID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS',$3,'high','confirmed')
|
||||
RETURNING id`, task.ID, cue, label).Scan(&bID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lessons, err := d.TaskLessons(t.Context(), task.ID, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var pos *Lesson
|
||||
for i := range lessons {
|
||||
if lessons[i].Kind == "positive" && strings.Contains(lessons[i].Action, label) {
|
||||
pos = &lessons[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if pos == nil {
|
||||
t.Fatalf("no shared positive lesson: %+v", lessons)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, pos.ID) })
|
||||
if pos.Support != 2 {
|
||||
t.Fatalf("shared support=%d, want 2", pos.Support)
|
||||
}
|
||||
|
||||
if _, err := d.SetFindingStatus(aID, FindingFalsePositive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var status string
|
||||
var support int
|
||||
if err := d.QueryRow(`SELECT status,
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id)
|
||||
FROM rsi_lessons WHERE id=$1`, pos.ID).Scan(&status, &support); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status == "retired" || support != 1 {
|
||||
t.Fatalf("shared positive should stay active with support 1, got status=%s support=%d", status, support)
|
||||
}
|
||||
|
||||
after, err := d.TaskLessons(t.Context(), task.ID, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var sawNeg bool
|
||||
for i := range after {
|
||||
if after[i].ID == pos.ID && after[i].Kind == "positive" {
|
||||
if after[i].Support != 1 {
|
||||
t.Fatalf("selected support=%d, want 1", after[i].Support)
|
||||
}
|
||||
}
|
||||
if after[i].Kind == "negative" && strings.Contains(after[i].Action, label) {
|
||||
sawNeg = true
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, after[i].ID) })
|
||||
}
|
||||
}
|
||||
if !sawNeg {
|
||||
t.Fatalf("expected a negative lesson for the flipped finding: %+v", after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLessonsRetractsAllRetests(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
task, err := d.CreateTask(fmt.Sprintf("lesson all retests %d", stamp), "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "task", d.DeleteTask(task.ID)) })
|
||||
|
||||
cue := fmt.Sprintf("xss-all-retest-%d", stamp)
|
||||
label := fmt.Sprintf("reflected in search %d", stamp)
|
||||
var fid int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS',$3,'high','confirmed')
|
||||
RETURNING id`, task.ID, cue, label).Scan(&fid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
|
||||
r1 := insertLessonRetest(t, d, fid, "reproduced")
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reproduced := findLessonByAction(t, d, task.ID, "positive", "Reproduced", label)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, reproduced.ID) })
|
||||
confirmed := findLessonByAction(t, d, task.ID, "positive", "Confirmed", label)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, confirmed.ID) })
|
||||
if positiveRetestEvidence(t, d, r1) != 1 {
|
||||
t.Fatal("first retest was not extracted")
|
||||
}
|
||||
|
||||
r2 := insertLessonRetest(t, d, fid, "fixed")
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
caution := findLessonByAction(t, d, task.ID, "caution", "Recheck", cue)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, caution.ID) })
|
||||
if positiveRetestEvidence(t, d, r1) != 1 {
|
||||
t.Fatal("older reproduced retest lost its evidence after a later fixed retest")
|
||||
}
|
||||
|
||||
if _, err := d.SetFindingStatus(fid, FindingFalsePositive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := positiveRetestEvidence(t, d, r1); n != 0 {
|
||||
t.Fatalf("older retest still has %d positive evidence rows after FP", n)
|
||||
}
|
||||
if n := positiveRetestEvidence(t, d, r2); n != 0 {
|
||||
t.Fatalf("latest retest still has %d positive evidence rows after FP", n)
|
||||
}
|
||||
status, support := lessonStatusSupport(t, d, reproduced.ID)
|
||||
if status != "retired" || support != 0 {
|
||||
t.Fatalf("reproduced retest lesson after FP: status=%s support=%d", status, support)
|
||||
}
|
||||
status, support = lessonStatusSupport(t, d, confirmed.ID)
|
||||
if status != "retired" || support != 0 {
|
||||
t.Fatalf("confirmed lesson after FP: status=%s support=%d", status, support)
|
||||
}
|
||||
neg := findLessonByAction(t, d, task.ID, "negative", "Do not re-report", label)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, neg.ID) })
|
||||
}
|
||||
|
||||
func TestExtractLessonsDoesNotUnretireWithEvidence(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
task, err := d.CreateTask(fmt.Sprintf("lesson keep retired %d", stamp), "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "task", d.DeleteTask(task.ID)) })
|
||||
|
||||
cue := fmt.Sprintf("xss-keep-retired-%d", stamp)
|
||||
label := fmt.Sprintf("reflected in search %d", stamp)
|
||||
var fid int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS',$3,'high','confirmed')
|
||||
RETURNING id`, task.ID, cue, label).Scan(&fid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pos := findLessonByAction(t, d, task.ID, "positive", "Confirmed", label)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, pos.ID) })
|
||||
|
||||
if _, err := d.Exec(`UPDATE rsi_lessons SET status='retired' WHERE id=$1`, pos.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
status, support := lessonStatusSupport(t, d, pos.ID)
|
||||
if status != "retired" {
|
||||
t.Fatalf("operator-retired lesson with evidence became %s", status)
|
||||
}
|
||||
if support != 1 {
|
||||
t.Fatalf("support=%d, want 1", support)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLessonsLeavesAdopted(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
task, err := d.CreateTask(fmt.Sprintf("lesson keep adopted %d", stamp), "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "task", d.DeleteTask(task.ID)) })
|
||||
|
||||
cue := fmt.Sprintf("xss-keep-adopted-%d", stamp)
|
||||
label := fmt.Sprintf("reflected in search %d", stamp)
|
||||
var fid int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS',$3,'high','confirmed')
|
||||
RETURNING id`, task.ID, cue, label).Scan(&fid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pos := findLessonByAction(t, d, task.ID, "positive", "Confirmed", label)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, pos.ID) })
|
||||
|
||||
if _, err := d.Exec(`UPDATE rsi_lessons SET status='adopted' WHERE id=$1`, pos.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
status, _ := lessonStatusSupport(t, d, pos.ID)
|
||||
if status != "adopted" {
|
||||
t.Fatalf("adopted lesson became %s", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLessonsReopenRetractsNothing(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
task, err := d.CreateTask(fmt.Sprintf("lesson reopen %d", stamp), "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "task", d.DeleteTask(task.ID)) })
|
||||
|
||||
cue := fmt.Sprintf("xss-reopen-%d", stamp)
|
||||
label := fmt.Sprintf("reflected in search %d", stamp)
|
||||
var fid int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS',$3,'high','confirmed')
|
||||
RETURNING id`, task.ID, cue, label).Scan(&fid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pos := findLessonByAction(t, d, task.ID, "positive", "Confirmed", label)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, pos.ID) })
|
||||
|
||||
if _, err := d.SetFindingStatus(fid, FindingPending); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
status, support := lessonStatusSupport(t, d, pos.ID)
|
||||
if status != "candidate" || support != 1 {
|
||||
t.Fatalf("reopened finding retracted the lesson: status=%s support=%d", status, support)
|
||||
}
|
||||
}
|
||||
|
||||
func insertLessonRetest(t *testing.T, d *DB, findingID int64, verdict string) int64 {
|
||||
t.Helper()
|
||||
var id int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO finding_retests(finding_id,status,verdict,snapshot)
|
||||
VALUES ($1,'completed',$2,'{}'::jsonb)
|
||||
RETURNING id`, findingID, verdict).Scan(&id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM finding_retests WHERE id=$1`, id) })
|
||||
return id
|
||||
}
|
||||
|
||||
func findLessonByAction(t *testing.T, d *DB, taskID int64, kind, actionPrefix, label string) *Lesson {
|
||||
t.Helper()
|
||||
lessons, err := d.TaskLessons(t.Context(), taskID, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range lessons {
|
||||
if lessons[i].Kind == kind && strings.Contains(lessons[i].Action, actionPrefix) && strings.Contains(lessons[i].Action, label) {
|
||||
return &lessons[i]
|
||||
}
|
||||
}
|
||||
t.Fatalf("no %s lesson with %q and %q: %+v", kind, actionPrefix, label, lessons)
|
||||
return nil
|
||||
}
|
||||
|
||||
func positiveRetestEvidence(t *testing.T, d *DB, retestID int64) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := d.QueryRow(`
|
||||
SELECT COUNT(*) FROM rsi_lesson_evidence e
|
||||
JOIN rsi_lessons l ON l.id=e.lesson_id
|
||||
WHERE e.source_kind='retest' AND e.source_id=$1 AND l.kind='positive'`, retestID).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func lessonStatusSupport(t *testing.T, d *DB, lessonID int64) (status string, support int) {
|
||||
t.Helper()
|
||||
if err := d.QueryRow(`SELECT status,
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id)
|
||||
FROM rsi_lessons WHERE id=$1`, lessonID).Scan(&status, &support); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return status, support
|
||||
}
|
||||
|
||||
+2
-2
@@ -137,9 +137,9 @@ func assertEmbeddedApplied(t *testing.T, d *DB) []appliedMigration {
|
||||
version int
|
||||
name string
|
||||
}{{1, llmRecordsMigration}, {2, llmUsageMigration}, {3, "0003_task_scope_icp_index.sql"}, {4, "0004_llm_usage_ts_index.sql"},
|
||||
{5, "0005_traffic_rewrite_rules.sql"}}
|
||||
{5, "0005_traffic_rewrite_rules.sql"}, {6, "0006_activity_outbox.sql"}}
|
||||
if len(applied) != len(want) {
|
||||
t.Fatalf("schema_migrations = %+v, want versions 1 to 5", applied)
|
||||
t.Fatalf("schema_migrations = %+v, want versions 1 to 6", applied)
|
||||
}
|
||||
for i, w := range want {
|
||||
content, err := fs.ReadFile(migrationsFS(), w.name)
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
-- Activities that could not be inserted (the exploration row was missing, or
|
||||
-- the write failed after retries) are kept here and replayed into activity
|
||||
-- once the parent row accepts them. No foreign key: the row has to survive
|
||||
-- the missing exploration that caused the original failure.
|
||||
CREATE TABLE IF NOT EXISTS activity_outbox (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
exploration_id BIGINT NOT NULL,
|
||||
payload JSONB NOT NULL,
|
||||
last_error TEXT,
|
||||
attempts INT NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS activity_outbox_exploration_idx
|
||||
ON activity_outbox (exploration_id, id);
|
||||
+15
-6
@@ -32,9 +32,15 @@ Extraction runs automatically when a task reaches a terminal status
|
||||
POST /api/tasks/{id}/lessons/extract
|
||||
```
|
||||
|
||||
It is idempotent. `rsi_lesson_evidence` is keyed by `(lesson, source_kind,
|
||||
source_id)`, so a second run for the same task adds no evidence and cannot
|
||||
inflate support.
|
||||
It is idempotent for the same disposition. `rsi_lesson_evidence` is keyed by
|
||||
`(lesson, source_kind, source_id)`, so a second run for the same finding adds
|
||||
no evidence and cannot inflate support. A later extract after a reversed
|
||||
disposition retracts that finding's evidence from the opposite-kind lesson,
|
||||
including every retest of the finding, and retires the lesson when its
|
||||
support hits zero. Confirming again restores the evidence; a retired lesson
|
||||
is un-retired only when it has no remaining evidence. Extraction does not
|
||||
run on status PATCH; call the endpoint after triage. Reopening a finding
|
||||
(`pending` / `in_progress`) produces nothing and retracts nothing.
|
||||
|
||||
Rules:
|
||||
|
||||
@@ -84,10 +90,13 @@ What this layer deliberately does not do:
|
||||
|
||||
- no LLM summarization in this phase: rules only, so every lesson traces to a
|
||||
finding or retest id;
|
||||
- no adoption: `status` is `candidate`; `adopted` and `retired` are reserved for
|
||||
a later, owner-approved phase;
|
||||
- no prompt mutation, no benchmark probing, no lesson editing through the UI;
|
||||
- no decay or deletion; the evidence table is the audit trail.
|
||||
- `status` starts as `candidate`. Extraction auto-retires a lesson when its last
|
||||
evidence is retracted. It un-retires only a retired lesson with no remaining
|
||||
evidence. Adopted stays adopted; a retired lesson that still has evidence
|
||||
stays retired;
|
||||
- no time-based decay; reversed dispositions retract that finding's evidence
|
||||
from the opposite-kind lesson on the next extract.
|
||||
|
||||
## Measuring it
|
||||
|
||||
|
||||
@@ -3,9 +3,10 @@
|
||||
A rewrite rule matches a request or a response that goes through the capture proxy and describes
|
||||
how to change it: the method, the path and query, headers, the body, the status code.
|
||||
|
||||
**Status: rules are stored, validated and compiled. The proxy does not apply them yet.** Creating
|
||||
or enabling a rule changes nothing in what the proxy forwards or records. Applying the rules, the
|
||||
per-exchange trace, the archive format and the web UI come in a later change.
|
||||
**Status: the capture proxy applies enabled rules to HTTP traffic (request and response phases),
|
||||
and records what it did.** Not yet in this change: archive format, a per-replay rule toggle, the web UI,
|
||||
and the proof flag. See [Application in the capture proxy](#application-in-the-capture-proxy) for
|
||||
what is covered and what is not.
|
||||
|
||||
Rules are a convenience for the operator, not a security control: a rule that fails leaves the
|
||||
message unchanged (fail open).
|
||||
@@ -145,3 +146,66 @@ cannot restore them, and ARTEX refuses to start under a key that cannot open the
|
||||
|
||||
The server compiles the enabled rules into an immutable snapshot at startup and after every rule
|
||||
change, and hands it to the proxy without a restart.
|
||||
|
||||
## Application in the capture proxy
|
||||
|
||||
The proxy applies the snapshot to each exchange in two phases. The request phase runs before the
|
||||
request is sent to the target; the response phase runs before the response is returned to the
|
||||
client. Each rule sees the output of the previous one. Only rules whose phase matches the step
|
||||
are run, and a rule that fails leaves the message as it was (fail open).
|
||||
|
||||
### What is recorded
|
||||
|
||||
For a rewritten exchange, the record keeps the request as the client sent it and the response as
|
||||
the target sent it. The rewritten values are not stored as the exchange's original data.
|
||||
|
||||
- Each action that ran (applied, skipped or failed) is written in the SQLite table
|
||||
`exchange_rewrites`, in the same transaction as the exchange itself, with the rule id, a copy of
|
||||
the rule as it was applied, the action, the outcome and the reason.
|
||||
- The before value is kept as its SHA-256. The after value is kept as the value itself, inline or
|
||||
in the blob store when it is large, with its SHA-256. A sensitive action stores neither value nor
|
||||
plain hash: its after value is not stored, and the hashes it leaves are a keyed HMAC when a key is
|
||||
available (empty otherwise). The same HMAC applies to later actions on a field a sensitive action
|
||||
wrote.
|
||||
- `traffic_get` shows a `=== REWRITTEN ===` line naming the rule ids when the exchange was changed.
|
||||
The line says that the shown values are the originals.
|
||||
- `traffic_search` searches only the original values. It never finds a value that a rule wrote.
|
||||
|
||||
Reasons a rule is not applied are recorded in the trace, for example `not rewritten: binary`,
|
||||
`not rewritten: stream`, or `not rewritten: snapshot capacity`.
|
||||
|
||||
### Streams: SSE and WebSocket
|
||||
|
||||
- **SSE** (`text/event-stream`): status and headers are rewritten at the start of the stream
|
||||
(SSEStart), since they are sent once. Later event frames pass unchanged.
|
||||
- **WebSocket**: not rewritten. The capture library calls its start hook after the upgrade is
|
||||
already done, and it short-circuits the upgrade before the request and response hooks run, so
|
||||
the handshake status and headers cannot be changed. Frames pass unchanged. The exchange is
|
||||
traced `not rewritten: stream`.
|
||||
|
||||
### Large responses
|
||||
|
||||
go-mitmproxy does not call the response hook for a body of 5 MiB or more (its `StreamLargeBodies`
|
||||
default). Such a response is forwarded to the client unchanged and is not recorded in full, and no
|
||||
response rule runs on it.
|
||||
|
||||
### Snapshot limits
|
||||
|
||||
Each exchange with a rewrite trace (at least one rule matched and produced a line: applied, skipped or failed) keeps an original copy in memory until it finishes. Limits:
|
||||
|
||||
| Limit | Value |
|
||||
|---|---|
|
||||
| Entries holding an original request | 1024 |
|
||||
| Bytes held by those entries | 64 MiB |
|
||||
| Tombstones (slots whose data was released) | 4096 |
|
||||
| Time after which the slot of a live flow may be turned into a tombstone (see below) | 15 minutes |
|
||||
|
||||
When there is no room, the request is not rewritten. The exchange is traced `not rewritten: snapshot capacity` when a tombstone slot is free; when the tombstone table is full, the request is refused with no trace line at all (the refusal is counted and logged, at most once a minute).
|
||||
The slot of a finished flow is swept when its connection closes (at most once per second) and at
|
||||
least once a minute while slots exist. The 15-minute TTL never deletes a live flow's slot: past it,
|
||||
the slot may be converted into a tombstone if a new reservation needs the room.
|
||||
|
||||
### Replay
|
||||
|
||||
Replay goes through the recording proxy like any other traffic, so the enabled rules apply to a
|
||||
replay and the resulting exchange is traced like a captured one. There is no per-replay toggle yet.
|
||||
|
||||
+91
-4
@@ -118,6 +118,7 @@ type Engine struct {
|
||||
// runs under one task-scoped context. Successful deletion cancels that context,
|
||||
// waits for all goroutines, then releases every task-level Engine reference.
|
||||
runtimeMu sync.Mutex
|
||||
draining bool
|
||||
runtimes map[string]*taskRuntime
|
||||
|
||||
// per-task execution context: each planner.Plan / worker.Execute runs under it,
|
||||
@@ -162,7 +163,7 @@ type Engine struct {
|
||||
|
||||
type taskRuntime struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
cancel context.CancelCauseFunc
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
@@ -231,9 +232,12 @@ func (e *Engine) IsDeleting(taskID string) bool {
|
||||
func (e *Engine) registerTaskRoutines(parent context.Context, taskID string, count int) *taskRuntime {
|
||||
e.runtimeMu.Lock()
|
||||
defer e.runtimeMu.Unlock()
|
||||
if e.draining {
|
||||
return nil
|
||||
}
|
||||
rt := e.runtimes[taskID]
|
||||
if rt == nil {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
ctx, cancel := context.WithCancelCause(parent)
|
||||
rt = &taskRuntime{ctx: ctx, cancel: cancel}
|
||||
e.runtimes[taskID] = rt
|
||||
}
|
||||
@@ -248,6 +252,61 @@ func runTaskRoutine(rt *taskRuntime, fn func(context.Context)) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait blocks until the task goroutines registered at the moment of the call,
|
||||
// and every admitted task operation, have finished, or until ctx ends. It
|
||||
// refuses new registrations and new beginTaskOperation calls first. Goroutines
|
||||
// that were already running are cancelled with AbortShutdown. That cancel does
|
||||
// not write a terminal task status, and an in-flight intent claim stays running
|
||||
// so the next start can reset it to open. Detached chat runs are not in a task
|
||||
// WaitGroup; they are counted in inflight and Wait stays until they return.
|
||||
func (e *Engine) Wait(ctx context.Context) error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
e.runtimeMu.Lock()
|
||||
e.draining = true
|
||||
groups := make([]*sync.WaitGroup, 0, len(e.runtimes))
|
||||
for _, rt := range e.runtimes {
|
||||
if rt == nil {
|
||||
continue
|
||||
}
|
||||
if rt.cancel != nil {
|
||||
rt.cancel(agent.AbortShutdown)
|
||||
}
|
||||
groups = append(groups, &rt.wg)
|
||||
}
|
||||
e.runtimeMu.Unlock()
|
||||
for _, wg := range groups {
|
||||
if err := waitGroup(ctx, wg); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for e.inflightTotal() > 0 {
|
||||
timer := time.NewTimer(20 * time.Millisecond)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func waitGroup(ctx context.Context, wg *sync.WaitGroup) error {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// StopTask permanently stops every long-lived goroutine and removes all Engine
|
||||
// state for a successfully deleted task. The delete barrier remains installed
|
||||
// until cleanup finishes, so no new task operation can race the teardown.
|
||||
@@ -260,7 +319,7 @@ func (e *Engine) StopTask(taskID string) {
|
||||
e.runtimeMu.Lock()
|
||||
rt := e.runtimes[taskID]
|
||||
if rt != nil {
|
||||
rt.cancel()
|
||||
rt.cancel(agent.AbortTaskDeleted)
|
||||
}
|
||||
e.runtimeMu.Unlock()
|
||||
if rt != nil {
|
||||
@@ -664,6 +723,23 @@ func (e *Engine) emitActivity(t *Task, r db.Activity) db.Activity {
|
||||
}
|
||||
log.Printf("[activity] task %s dropping activity record (%d dropped for this task so far) worker=%s kind=%s tool=%s tuid=%s reason=%s summary=%q: %v%s",
|
||||
t.ID, n, r.Worker, r.Kind, r.Tool, r.ToolUseID, dropReason(err), preview(r.Summary, 80), err, diag)
|
||||
if t.Store != nil {
|
||||
if qerr := t.Store.EnqueueActivity(r, err.Error()); qerr != nil {
|
||||
log.Printf("[activity] task %s outbox: %v", t.ID, qerr)
|
||||
}
|
||||
}
|
||||
// The row has no database id. Publish an error copy so a live session
|
||||
// does not stay on "Running" while the original return value is unchanged.
|
||||
notice := r
|
||||
notice.IsError = true
|
||||
if notice.Summary == "" {
|
||||
notice.Summary = "activity was not stored"
|
||||
} else {
|
||||
notice.Summary += " (not stored)"
|
||||
}
|
||||
if e.bc != nil {
|
||||
e.bc.Publish(t.ID, notice)
|
||||
}
|
||||
e.touch(t.ID)
|
||||
return r
|
||||
}
|
||||
@@ -762,6 +838,10 @@ func (e *Engine) Run(ctx context.Context, t *Task) {
|
||||
}
|
||||
rt := e.registerTaskRoutines(ctx, t.ID, 2+workers)
|
||||
e.deleteMu.RUnlock()
|
||||
if rt == nil {
|
||||
e.started.Delete(t.ID)
|
||||
return
|
||||
}
|
||||
e.touch(t.ID)
|
||||
runTaskRoutine(rt, func(loopCtx context.Context) { e.plannerLoop(loopCtx, t) })
|
||||
runTaskRoutine(rt, func(loopCtx context.Context) { e.claimSweepLoop(loopCtx, t) })
|
||||
@@ -1266,6 +1346,13 @@ func (e *Engine) runIntent(ctx context.Context, t *Task, name string, worker *ag
|
||||
e.touch(t.ID)
|
||||
return true
|
||||
}
|
||||
// Process shutdown cancels the task context. Leave the claim running: AbortShutdown
|
||||
// says restart resets leftover running intents to open. Settling done or blocked
|
||||
// here would make that reset miss the intent, so the task would resume without it.
|
||||
if ectx.Err() != nil && errors.Is(context.Cause(ectx), agent.AbortShutdown) {
|
||||
log.Printf("[worker %s] task %s intent #%d shutdown: leaving the claim running", name, t.ID, intent.ID)
|
||||
return true
|
||||
}
|
||||
// killed by the planner: mark stopped (don't write back results, don't auto-reclaim).
|
||||
if killed {
|
||||
if err := t.Store.SettleIntentClaim(intent.ID, token, "stopped"); err != nil {
|
||||
@@ -1324,7 +1411,7 @@ func (e *Engine) runIntent(ctx context.Context, t *Task, name string, worker *ag
|
||||
// be started; the intent is left untouched in that case.
|
||||
func (e *Engine) runDetachedIntent(ctx context.Context, t *Task, intentID int64, requestID, message, agentMessage string) error {
|
||||
if !e.beginTaskOperation(t.ID) {
|
||||
return fmt.Errorf("task is being deleted")
|
||||
return fmt.Errorf("task is not accepting work")
|
||||
}
|
||||
release := true
|
||||
defer func() {
|
||||
|
||||
@@ -9,6 +9,60 @@ import (
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
)
|
||||
|
||||
func TestWaitBlocksUntilDetachedInflightEnds(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
if !e.beginTaskOperation("t1") {
|
||||
t.Fatal("detached work was not admitted")
|
||||
}
|
||||
release := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
go func() {
|
||||
<-release
|
||||
e.decInflight("t1")
|
||||
close(finished)
|
||||
}()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.Wait(context.Background()) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("Wait returned while the detached operation was still running: %v", err)
|
||||
case <-time.After(80 * time.Millisecond):
|
||||
}
|
||||
if e.beginTaskOperation("t1") {
|
||||
t.Fatal("Wait admitted new work while draining")
|
||||
}
|
||||
close(release)
|
||||
<-finished
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Wait did not return after the detached operation ended")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitCancelsRuntimeWithShutdown(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
rt := e.registerTaskRoutines(context.Background(), "t1", 1)
|
||||
if rt == nil {
|
||||
t.Fatal("runtime was not registered")
|
||||
}
|
||||
started := make(chan struct{})
|
||||
runTaskRoutine(rt, func(ctx context.Context) {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
})
|
||||
<-started
|
||||
if err := e.Wait(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !errors.Is(context.Cause(rt.ctx), agent.AbortShutdown) {
|
||||
t.Fatalf("cause=%v, want shutdown", context.Cause(rt.ctx))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPauseCancelsWithNamedCause(t *testing.T) {
|
||||
e := NewEngine(nil)
|
||||
execCtx := e.execContextFor(context.Background(), "t1")
|
||||
|
||||
@@ -46,14 +46,20 @@ func (e *Engine) inflightCounter(taskID string) *int64 {
|
||||
}
|
||||
|
||||
// beginTaskOperation atomically registers a task-owned operation unless deletion
|
||||
// has already installed its barrier. The delete handler can therefore wait for
|
||||
// inflight==0 without a check-then-start race recreating files after cleanup.
|
||||
// has already installed its barrier, or the engine is draining for shutdown.
|
||||
// The delete handler and Wait can therefore wait for inflight==0 without a
|
||||
// check-then-start race admitting work that outlives the drain.
|
||||
func (e *Engine) beginTaskOperation(taskID string) bool {
|
||||
e.deleteMu.RLock()
|
||||
defer e.deleteMu.RUnlock()
|
||||
if e.IsDeleting(taskID) {
|
||||
return false
|
||||
}
|
||||
e.runtimeMu.Lock()
|
||||
defer e.runtimeMu.Unlock()
|
||||
if e.draining {
|
||||
return false
|
||||
}
|
||||
atomic.AddInt64(e.inflightCounter(taskID), 1)
|
||||
return true
|
||||
}
|
||||
@@ -63,6 +69,17 @@ func (e *Engine) inflightCount(taskID string) int64 {
|
||||
return atomic.LoadInt64(e.inflightCounter(taskID))
|
||||
}
|
||||
|
||||
// inflightTotal is the sum of every task's admitted operations, including
|
||||
// detached chat runs that are not members of a task WaitGroup.
|
||||
func (e *Engine) inflightTotal() int64 {
|
||||
var n int64
|
||||
e.inflight.Range(func(_, v any) bool {
|
||||
n += atomic.LoadInt64(v.(*int64))
|
||||
return true
|
||||
})
|
||||
return n
|
||||
}
|
||||
|
||||
// ---------- deadline ----------
|
||||
|
||||
// taskDeadline returns the task's absolute deadline (unix). Prefers the in-process
|
||||
@@ -161,6 +178,10 @@ func (e *Engine) startDeadlineCoordinator(ctx context.Context, t *Task) {
|
||||
}
|
||||
rt := e.registerTaskRoutines(ctx, t.ID, 1)
|
||||
e.deleteMu.RUnlock()
|
||||
if rt == nil {
|
||||
e.coordStarted.Delete(t.ID)
|
||||
return
|
||||
}
|
||||
runTaskRoutine(rt, func(loopCtx context.Context) { e.deadlineCoordinator(loopCtx, t) })
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,16 @@ func (s *Server) seedFindingWorkflowTools() {
|
||||
_ = s.m.pg.SetSetting(hostSearchDescriptionFlag, "true")
|
||||
}
|
||||
}
|
||||
const getRewriteDescriptionFlag = "traffic_get_rewrite_description_v1"
|
||||
if value, _, _ := s.m.pg.GetSetting(getRewriteDescriptionFlag); value != "true" {
|
||||
// Same rule: only the original built-in text is replaced.
|
||||
legacy := "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling. A WebSocket (status 101) or SSE (text/event-stream) record shows only its handshake here; read its messages with traffic_messages."
|
||||
if _, err := s.m.pg.Exec(`UPDATE tools SET description=$1,updated_at=now() WHERE key='traffic_get' AND system AND description=$2`, traffic.TrafficGetDescription, legacy); err != nil {
|
||||
log.Printf("[evidence] upgrade traffic_get description: %v", err)
|
||||
} else {
|
||||
_ = s.m.pg.SetSetting(getRewriteDescriptionFlag, "true")
|
||||
}
|
||||
}
|
||||
const flag = "finding_workflow_tools_v2_reporter"
|
||||
if value, _, _ := s.m.pg.GetSetting(flag); value == "true" {
|
||||
return
|
||||
|
||||
+32
-1
@@ -296,9 +296,16 @@ func (s *Server) restoreTaskRuntimes() {
|
||||
lifecycle := t.lifecycleSnapshot()
|
||||
// clear stale 'running' intents from a prior crash/restart (no live worker
|
||||
// owns them) so they re-claim instead of spinning forever in the UI.
|
||||
if n, _ := t.Store.ResetRunningIntents(); n > 0 {
|
||||
if n, err := t.Store.ResetRunningIntents(); err != nil {
|
||||
log.Printf("[engine] task %s reset leftover running intents: %v", t.ID, err)
|
||||
} else if n > 0 {
|
||||
log.Printf("[engine] task %s reset %d leftover running intents to open", t.ID, n)
|
||||
}
|
||||
if n, err := t.Store.FlushActivityOutbox(); err != nil {
|
||||
log.Printf("[activity] task %s outbox: %v", t.ID, err)
|
||||
} else if n > 0 {
|
||||
log.Printf("[activity] task %s stored %d activities recovered from the outbox", t.ID, n)
|
||||
}
|
||||
if lifecycle.Paused {
|
||||
s.engine.Pause(t.ID, agent.AbortPausedOnReload)
|
||||
}
|
||||
@@ -320,6 +327,30 @@ func (s *Server) restoreTaskRuntimes() {
|
||||
}
|
||||
}
|
||||
|
||||
// Wait blocks until the task goroutines snapshotted now, and the archive
|
||||
// worker, have finished, or until ctx ends.
|
||||
func (s *Server) Wait(ctx context.Context) error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
if s.engine != nil {
|
||||
if err := s.engine.Wait(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
s.archiveWG.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// agentMaxTurns returns the configured max_turns for an agent key (0 = unlimited,
|
||||
// also the fallback when no DB or no row).
|
||||
func (s *Server) agentMaxTurns(key string) int {
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/traffic"
|
||||
)
|
||||
|
||||
// The traffic_get description that says traffic_search never finds a value a
|
||||
// rewrite rule wrote replaces the original built-in text once, and never a
|
||||
// description the user edited.
|
||||
func TestTrafficGetDescriptionUpgrade(t *testing.T) {
|
||||
s, _, _ := trafficEvidenceServer(t)
|
||||
pg := s.m.pg
|
||||
old, err := pg.GetTool("traffic_get")
|
||||
if err != nil || old == nil {
|
||||
t.Fatal("missing tool traffic_get", err)
|
||||
}
|
||||
bindings, _ := json.Marshal(old.Agents)
|
||||
t.Cleanup(func() { pg.UpdateTool("traffic_get", old.Description, old.Schema, bindings, old.Enabled) })
|
||||
const flag = "traffic_get_rewrite_description_v1"
|
||||
legacy := "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling. A WebSocket (status 101) or SSE (text/event-stream) record shows only its handshake here; read its messages with traffic_messages."
|
||||
|
||||
for _, c := range []struct{ stored, want string }{
|
||||
{legacy, traffic.TrafficGetDescription},
|
||||
{"USER DESCRIPTION", "USER DESCRIPTION"},
|
||||
} {
|
||||
if err := pg.UpdateTool("traffic_get", c.stored, old.Schema, bindings, old.Enabled); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pg.SetSetting(flag, "false"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.seedFindingWorkflowTools()
|
||||
row, _ := pg.GetTool("traffic_get")
|
||||
if row.Description != c.want {
|
||||
t.Fatalf("stored %q: description %q, want %q", c.stored, row.Description, c.want)
|
||||
}
|
||||
if v, _, _ := pg.GetSetting(flag); v != "true" {
|
||||
t.Fatal("flag not set")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -76,6 +76,11 @@ func (t *Traffic) insertExchange(id string, ts int64, host, method, urlTemplate,
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // no-op once committed
|
||||
|
||||
// An id can repeat (seconds plus a rolling counter); the trace of the
|
||||
// exchange it replaces must not stay attached to this one.
|
||||
if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id=?`, id); err != nil {
|
||||
return fmt.Errorf("clearing rewrite trace: %w", err)
|
||||
}
|
||||
res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,'')`,
|
||||
id, ts, host, method, urlTemplate, rawURL, status, contentType, reqLen, respLen)
|
||||
|
||||
+5
-1
@@ -40,11 +40,15 @@ func (s *sink) Requestheaders(f *mproxy.Flow) {
|
||||
}
|
||||
}
|
||||
|
||||
// ClientDisconnected forgets a closed connection's mark.
|
||||
// ClientDisconnected forgets a closed connection's mark, and asks for a sweep
|
||||
// of the request snapshots its flows may have left (rewrite_flow.go).
|
||||
func (s *sink) ClientDisconnected(c *mproxy.ClientConn) {
|
||||
if c != nil {
|
||||
s.paced.Delete(c.Id)
|
||||
}
|
||||
if s.t != nil {
|
||||
s.t.snaps.requestSweep()
|
||||
}
|
||||
}
|
||||
|
||||
// pace waits for the request's host under the current noise profile, bounded
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
|
||||
)
|
||||
@@ -65,4 +66,23 @@ func TestMaybePassthroughFlagsHostOnce(t *testing.T) {
|
||||
if _, tunnel := tr.pass.Load(hostOnly("target.test:443")); !tunnel {
|
||||
t.Fatal("flagged host must be recognized for the CONNECT form with port")
|
||||
}
|
||||
if !tr.tunneling("target.test") {
|
||||
t.Fatal("a fresh passthrough mark must tunnel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPassthroughMarkExpires(t *testing.T) {
|
||||
tr := &Traffic{}
|
||||
host := "expired.example"
|
||||
tr.pass.Store(host, time.Now().Add(-time.Second))
|
||||
if tr.tunneling(host) {
|
||||
t.Fatal("expired passthrough mark still tunnels")
|
||||
}
|
||||
if _, ok := tr.pass.Load(host); ok {
|
||||
t.Fatal("expired mark was kept")
|
||||
}
|
||||
tr.pass.Store(host, time.Now().Add(time.Minute))
|
||||
if !tr.tunneling(host) {
|
||||
t.Fatal("fresh passthrough mark should tunnel")
|
||||
}
|
||||
}
|
||||
|
||||
+1
-2
@@ -86,8 +86,7 @@ func (t *Traffic) replayRecorded(scheme, host string) bool {
|
||||
if scheme != "https" {
|
||||
return true
|
||||
}
|
||||
_, tunneled := t.pass.Load(hostOnly(host))
|
||||
return !tunneled
|
||||
return !t.tunneling(hostOnly(host))
|
||||
}
|
||||
|
||||
func replayInvalid(format string, args ...any) error {
|
||||
|
||||
+10
-2
@@ -199,7 +199,10 @@ func TestReplayPassthroughHostIsNotIntercepted(t *testing.T) {
|
||||
tr, _, _ := startProxy(t)
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(echoHandler))
|
||||
t.Cleanup(srv.Close)
|
||||
tr.pass.Store("localhost", struct{}{})
|
||||
tr.pass.Store("localhost", time.Now().Add(passthroughTTL))
|
||||
if !tr.tunneling("localhost") {
|
||||
t.Fatal("passthrough mark was not recognized before the replay")
|
||||
}
|
||||
target := localhostURL(srv.URL) + "/tunneled"
|
||||
_, err := tr.Replay(t.Context(), ReplayRequest{Method: "GET", URL: target})
|
||||
if err == nil || !strings.Contains(err.Error(), "certificate") {
|
||||
@@ -249,7 +252,11 @@ func TestReplayRejectsInvalidRequests(t *testing.T) {
|
||||
// Only an https host in passthrough is reported as not recorded.
|
||||
func TestReplayRecordedHelper(t *testing.T) {
|
||||
tr := &Traffic{}
|
||||
tr.pass.Store("localhost", struct{}{})
|
||||
tr.pass.Store("localhost", time.Now().Add(passthroughTTL))
|
||||
if !tr.tunneling("localhost") {
|
||||
t.Fatal("fresh passthrough mark was not tunneling")
|
||||
}
|
||||
tr.pass.Store("expired.example", time.Now().Add(-time.Second))
|
||||
for _, c := range []struct {
|
||||
scheme, host string
|
||||
want bool
|
||||
@@ -257,6 +264,7 @@ func TestReplayRecordedHelper(t *testing.T) {
|
||||
{"https", "localhost:443", false},
|
||||
{"https", "example.com:443", true},
|
||||
{"http", "localhost:80", true},
|
||||
{"https", "expired.example:443", true},
|
||||
} {
|
||||
if got := tr.replayRecorded(c.scheme, c.host); got != c.want {
|
||||
t.Errorf("replayRecorded(%s, %s) = %v, want %v", c.scheme, c.host, got, c.want)
|
||||
|
||||
@@ -80,6 +80,14 @@ func (s *Set) LoadErrors() map[int64]string {
|
||||
return s.loadErrors
|
||||
}
|
||||
|
||||
// NoteError records a run-time error of a rule seen outside the engine, such
|
||||
// as the transport failure of a request the rule rewrote.
|
||||
func (s *Set) NoteError(id int64, class string) {
|
||||
if s != nil {
|
||||
s.errors.record(id, class)
|
||||
}
|
||||
}
|
||||
|
||||
// LastError is the most recent run-time error of a rule. It lives in memory
|
||||
// only; the per-exchange trace keeps the history.
|
||||
type LastError struct {
|
||||
|
||||
@@ -30,6 +30,9 @@ const (
|
||||
ErrClassInvalidURL = "invalid_url" // a rewritten path and query is not a valid request URI
|
||||
ErrClassExpansion = "expansion_limit" // a replace result exceeds MaxExpansion
|
||||
ErrClassPanic = "panic"
|
||||
// ErrClassTransport is recorded by the proxy, not the engine: the request a
|
||||
// rule rewrote could not be sent or answered.
|
||||
ErrClassTransport = "transport_error"
|
||||
)
|
||||
|
||||
// TraceLine is one row of an exchange's rewrite trace, in order. An exchange is
|
||||
|
||||
@@ -0,0 +1,663 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/traffic/rewrite"
|
||||
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
|
||||
)
|
||||
|
||||
// Applying the rewrite rules to the proxied flows.
|
||||
//
|
||||
// The request phase runs in the Request hook. A request a rule rewrote leaves
|
||||
// the proxy changed, but its exchange records what the client sent: the
|
||||
// original is kept in a per-flow request snapshot until the Response hook
|
||||
// records the exchange. The response phase runs at the start of the Response
|
||||
// hook (status and headers only at SSEStart), and the exchange records the
|
||||
// response as the target sent it. The trace of both phases is written in the
|
||||
// exchange's own transaction (exchange_rewrites).
|
||||
//
|
||||
// A request is rewritten only once its snapshot slot is reserved, and a slot
|
||||
// stays until the flow's exchange is recorded or the flow is finished, so the
|
||||
// Response hook never takes a rewritten request for the client's.
|
||||
|
||||
// Limits of the request snapshots.
|
||||
const (
|
||||
maxSnapEntries = 1024 // entries holding an original request
|
||||
maxSnapBytes = 64 << 20 // what those entries hold
|
||||
maxSnapTombs = 4096 // tombstones: slots whose data was released
|
||||
// snapTTL deletes nothing: past it, the entry of a live flow may be turned
|
||||
// into a tombstone when a new reservation finds no room.
|
||||
snapTTL = 15 * time.Minute
|
||||
sweepDebounce = time.Second // at most one sweep per second
|
||||
sweepPeriod = time.Minute // a sweep at least this often while slots exist
|
||||
// libraryStreamBody is go-mitmproxy's StreamLargeBodies default: a body this
|
||||
// large reaches neither the Request nor the Response hook.
|
||||
libraryStreamBody = 5 << 20
|
||||
|
||||
reasonSnapshotCapacity = "not rewritten: snapshot capacity"
|
||||
reasonOriginalUnavailable = "original request unavailable"
|
||||
)
|
||||
|
||||
// testRuleHook, when a test sets it, runs inside each rule that checks a body
|
||||
// ("rule") and in the sink right after the engine ("request", "response"), so
|
||||
// a test can make a rule, or the sink itself, panic.
|
||||
var testRuleHook atomic.Pointer[func(stage string)]
|
||||
|
||||
func runTestHook(stage string) {
|
||||
if h := testRuleHook.Load(); h != nil {
|
||||
(*h)(stage)
|
||||
}
|
||||
}
|
||||
|
||||
// rulesIsBinary is the binary predicate handed to the engine: the recorder's
|
||||
// own, so "not rewritten: binary" follows what the recorder treats as binary.
|
||||
func rulesIsBinary(contentType string, body []byte) bool {
|
||||
runTestHook("rule")
|
||||
return isBinaryBody(contentType, body)
|
||||
}
|
||||
|
||||
type flowKey [16]byte
|
||||
|
||||
func keyOf(f *mproxy.Flow) flowKey { return flowKey(f.Id) }
|
||||
|
||||
// reqSnap is one slot of the table: an entry, which holds the original of a
|
||||
// request, or a tombstone, which holds only why it has none.
|
||||
type reqSnap struct {
|
||||
// done is the flow's Done() channel: closed, no hook will come for the
|
||||
// slot any more. The flow itself is not referenced.
|
||||
done <-chan struct{}
|
||||
rules *rewrite.Set // the snapshot of rules the flow runs with, both phases
|
||||
host string // host name: the only part of the request a log shows
|
||||
tomb bool
|
||||
|
||||
// Entry.
|
||||
created time.Time
|
||||
cost int64
|
||||
rewritten bool // the request phase changed the request
|
||||
method string
|
||||
url *url.URL
|
||||
header http.Header
|
||||
rawHost string
|
||||
body []byte // the client's body, kept when a body action applied
|
||||
keepBody bool
|
||||
lines []rewrite.TraceLine // request-phase trace
|
||||
|
||||
// Tombstone.
|
||||
reason string // reasonSnapshotCapacity or reasonOriginalUnavailable
|
||||
ruleIDs []int64 // rules that had rewritten the request
|
||||
}
|
||||
|
||||
// requestRewritten reports a flow whose request left the proxy changed.
|
||||
func (s *reqSnap) requestRewritten() bool {
|
||||
if s.tomb {
|
||||
return s.reason == reasonOriginalUnavailable
|
||||
}
|
||||
return s.rewritten
|
||||
}
|
||||
|
||||
func (s *reqSnap) appliedRuleIDs() []int64 {
|
||||
if s.tomb {
|
||||
return s.ruleIDs
|
||||
}
|
||||
_, ids := appliedRules(s.lines)
|
||||
return ids
|
||||
}
|
||||
|
||||
// snapTable holds the slots, keyed by flow id. Its mutex is a leaf lock.
|
||||
type snapTable struct {
|
||||
mu sync.Mutex
|
||||
slots map[flowKey]*reqSnap
|
||||
entries int
|
||||
bytes int64
|
||||
tombs int
|
||||
lastSweep time.Time
|
||||
lastFullLog time.Time
|
||||
sweepWanted atomic.Bool
|
||||
|
||||
now func() time.Time
|
||||
maxEntries int
|
||||
maxBytes int64
|
||||
maxTombs int
|
||||
ttl time.Duration
|
||||
|
||||
sweeps, capacity, tombDrops, conversions, largeResponses atomic.Int64
|
||||
}
|
||||
|
||||
func newSnapTable() *snapTable {
|
||||
return &snapTable{slots: map[flowKey]*reqSnap{}, now: time.Now,
|
||||
maxEntries: maxSnapEntries, maxBytes: maxSnapBytes, maxTombs: maxSnapTombs, ttl: snapTTL}
|
||||
}
|
||||
|
||||
func doneClosed(ch <-chan struct{}) bool {
|
||||
if ch == nil {
|
||||
return false // a nil channel is never ready: the flow counts as live
|
||||
}
|
||||
select {
|
||||
case <-ch:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// reserve stores the entry of a rewritten request. On failure the request must
|
||||
// leave unchanged: a capacity tombstone then stands for the flow, when there is
|
||||
// room for one.
|
||||
func (t *snapTable) reserve(k flowKey, e *reqSnap) bool {
|
||||
if t == nil {
|
||||
return false
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
t.removeLocked(k)
|
||||
if !t.fitsLocked(e.cost) {
|
||||
// A failed reservation asks for an immediate sweep, under the same
|
||||
// debounce; a sweep refused by the debounce fails the reservation.
|
||||
now := t.now()
|
||||
if now.Sub(t.lastSweep) < sweepDebounce {
|
||||
t.refuseLocked(k, e)
|
||||
return false
|
||||
}
|
||||
t.sweepLocked(now)
|
||||
if !t.fitsLocked(e.cost) && !t.convertLocked(e.cost) {
|
||||
t.refuseLocked(k, e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
e.created = t.now()
|
||||
t.slots[k] = e
|
||||
t.entries++
|
||||
t.bytes += e.cost
|
||||
return true
|
||||
}
|
||||
|
||||
func (t *snapTable) fitsLocked(cost int64) bool {
|
||||
return t.entries < t.maxEntries && t.bytes+cost <= t.maxBytes
|
||||
}
|
||||
|
||||
// convertLocked turns the oldest entry of a live flow older than the TTL into
|
||||
// a tombstone, in place, so the flow is never without a slot. It does so only
|
||||
// when releasing that entry lets a reservation of cost succeed, and when a
|
||||
// tombstone place is free (after evicting tombstones of finished flows only).
|
||||
// Otherwise nothing changes and it reports false.
|
||||
func (t *snapTable) convertLocked(cost int64) bool {
|
||||
now := t.now()
|
||||
var victim *reqSnap
|
||||
for _, s := range t.slots {
|
||||
if s.tomb || doneClosed(s.done) || now.Sub(s.created) < t.ttl {
|
||||
continue
|
||||
}
|
||||
if t.entries-1 >= t.maxEntries || t.bytes-s.cost+cost > t.maxBytes {
|
||||
continue // released, it would still leave no room
|
||||
}
|
||||
if victim == nil || s.created.Before(victim.created) {
|
||||
victim = s
|
||||
}
|
||||
}
|
||||
if victim == nil {
|
||||
return false
|
||||
}
|
||||
if t.tombs >= t.maxTombs {
|
||||
t.evictFinishedTombsLocked()
|
||||
}
|
||||
if t.tombs >= t.maxTombs {
|
||||
return false
|
||||
}
|
||||
ids := victim.appliedRuleIDs()
|
||||
t.entries--
|
||||
t.bytes -= victim.cost
|
||||
t.tombs++
|
||||
*victim = reqSnap{done: victim.done, rules: victim.rules, host: victim.host, tomb: true, reason: reasonOriginalUnavailable, ruleIDs: ids}
|
||||
t.conversions.Add(1)
|
||||
log.Printf("[traffic] rewrite snapshot of a request to %s (rules %v) released under pressure: its exchange will be recorded as original request unavailable", victim.host, ids)
|
||||
return true
|
||||
}
|
||||
|
||||
// refuseLocked is the capacity path: the request is not rewritten, and a
|
||||
// tombstone records why, when a place is free.
|
||||
func (t *snapTable) refuseLocked(k flowKey, e *reqSnap) {
|
||||
t.capacity.Add(1)
|
||||
if t.tombs >= t.maxTombs {
|
||||
t.evictFinishedTombsLocked()
|
||||
}
|
||||
full := t.tombs >= t.maxTombs
|
||||
if full {
|
||||
t.tombDrops.Add(1)
|
||||
} else {
|
||||
t.slots[k] = &reqSnap{done: e.done, rules: e.rules, host: e.host, tomb: true, reason: reasonSnapshotCapacity}
|
||||
t.tombs++
|
||||
}
|
||||
if now := t.now(); now.Sub(t.lastFullLog) >= time.Minute {
|
||||
t.lastFullLog = now
|
||||
log.Printf("[traffic] rewrite snapshot table full: requests are sent without rewriting (%d refused, %d without a trace so far)", t.capacity.Load(), t.tombDrops.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func (t *snapTable) evictFinishedTombsLocked() {
|
||||
for k, s := range t.slots {
|
||||
if s.tomb && doneClosed(s.done) {
|
||||
t.removeLocked(k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sweepLocked deletes every slot whose flow is finished. The slot of a live
|
||||
// flow is never touched, whatever its age.
|
||||
func (t *snapTable) sweepLocked(now time.Time) {
|
||||
for k, s := range t.slots {
|
||||
if doneClosed(s.done) {
|
||||
t.removeLocked(k)
|
||||
}
|
||||
}
|
||||
t.lastSweep = now
|
||||
t.sweepWanted.Store(false)
|
||||
t.sweeps.Add(1)
|
||||
}
|
||||
|
||||
func (t *snapTable) removeLocked(k flowKey) {
|
||||
s := t.slots[k]
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
if s.tomb {
|
||||
t.tombs--
|
||||
} else {
|
||||
t.entries--
|
||||
t.bytes -= s.cost
|
||||
}
|
||||
delete(t.slots, k)
|
||||
}
|
||||
|
||||
// take removes and returns the slot of a flow, nil when it has none.
|
||||
func (t *snapTable) take(k flowKey) *reqSnap {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
s := t.slots[k]
|
||||
t.removeLocked(k)
|
||||
return s
|
||||
}
|
||||
|
||||
// requestSweep asks the sweeper for a pass; it runs at most once a second.
|
||||
func (t *snapTable) requestSweep() {
|
||||
if t != nil {
|
||||
t.sweepWanted.Store(true)
|
||||
}
|
||||
}
|
||||
|
||||
// tick is one beat of the sweeper: a sweep when one was asked for and the last
|
||||
// is a second old, or when the last is a minute old. Nothing on an empty table.
|
||||
func (t *snapTable) tick() {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if len(t.slots) == 0 {
|
||||
t.sweepWanted.Store(false)
|
||||
return
|
||||
}
|
||||
now := t.now()
|
||||
since := now.Sub(t.lastSweep)
|
||||
if (t.sweepWanted.Load() && since >= sweepDebounce) || since >= sweepPeriod {
|
||||
t.sweepLocked(now)
|
||||
}
|
||||
}
|
||||
|
||||
// counts reports the entries and tombstones in the table.
|
||||
func (t *snapTable) counts() (entries, tombs int) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return t.entries, t.tombs
|
||||
}
|
||||
|
||||
// sweepSnapshots is the single sweeper goroutine, started by Open and stopped
|
||||
// by Close.
|
||||
func (t *Traffic) sweepSnapshots() {
|
||||
tick := time.NewTicker(time.Second)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-t.closed:
|
||||
return
|
||||
case <-tick.C:
|
||||
t.snaps.tick()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RewriteStats are the proxy-side counters of the rewrite rules since start.
|
||||
type RewriteStats struct {
|
||||
// LargeResponses counts flows with a snapshot whose response reached the
|
||||
// library's 5 MiB stream threshold: passed through, not recorded.
|
||||
LargeResponses int64 `json:"large_responses"`
|
||||
// CapacityRefusals counts requests sent without rewriting for lack of room.
|
||||
CapacityRefusals int64 `json:"capacity_refusals"`
|
||||
// TombstoneDrops counts those of them that got no capacity tombstone either.
|
||||
TombstoneDrops int64 `json:"tombstone_drops"`
|
||||
// Conversions counts entries released under pressure.
|
||||
Conversions int64 `json:"conversions"`
|
||||
Sweeps int64 `json:"sweeps"`
|
||||
}
|
||||
|
||||
func (t *Traffic) RewriteStats() RewriteStats {
|
||||
s := t.snaps
|
||||
if s == nil {
|
||||
return RewriteStats{}
|
||||
}
|
||||
return RewriteStats{LargeResponses: s.largeResponses.Load(), CapacityRefusals: s.capacity.Load(),
|
||||
TombstoneDrops: s.tombDrops.Load(), Conversions: s.conversions.Load(), Sweeps: s.sweeps.Load()}
|
||||
}
|
||||
|
||||
// newEntry builds the snapshot of a request before the rules' result is put on
|
||||
// the flow: URL and headers cloned, the body referenced, and only when a body
|
||||
// action applied (otherwise the flow's body is still the client's).
|
||||
func newEntry(f *mproxy.Flow, set *rewrite.Set, res rewrite.Result) *reqSnap {
|
||||
r := f.Request
|
||||
u := *r.URL
|
||||
e := &reqSnap{done: f.Done(), rules: set, host: r.URL.Hostname(), rewritten: res.Changed,
|
||||
method: r.Method, url: &u, header: r.Header.Clone(), lines: res.Lines}
|
||||
if raw := r.Raw(); raw != nil {
|
||||
e.rawHost = raw.Host
|
||||
}
|
||||
for _, l := range res.Lines {
|
||||
if l.Outcome == rewrite.OutcomeApplied && l.Field == "body" && len(r.Body) <= rewrite.MaxBodySize {
|
||||
e.body, e.keepBody = r.Body, true
|
||||
break
|
||||
}
|
||||
}
|
||||
e.cost = int64(len(e.method) + len(u.String()) + len(e.rawHost) + len(e.body))
|
||||
for k, vs := range e.header {
|
||||
for _, v := range vs {
|
||||
e.cost += int64(len(k) + len(v))
|
||||
}
|
||||
}
|
||||
for _, l := range e.lines {
|
||||
e.cost += int64(len(l.After) + len(l.RuleSnapshot))
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// originalRequest is the request an exchange records, with the request-phase
|
||||
// trace: the flow's own request when no rule rewrote it, the entry's copy when
|
||||
// one did, and only what Raw() still knows when the entry was released.
|
||||
func originalRequest(f *mproxy.Flow, snap *reqSnap) (requestView, []rewrite.TraceLine) {
|
||||
switch {
|
||||
case snap == nil:
|
||||
return flowRequestView(f.Request), nil
|
||||
case !snap.tomb:
|
||||
v := requestView{method: snap.method, url: snap.url, proto: f.Request.Proto, header: snap.header, rawHost: snap.rawHost, body: f.Request.Body}
|
||||
if snap.keepBody {
|
||||
v.body = snap.body
|
||||
}
|
||||
return v, snap.lines
|
||||
case snap.reason == reasonSnapshotCapacity:
|
||||
// The request was not rewritten: the flow's request is the client's.
|
||||
return flowRequestView(f.Request), []rewrite.TraceLine{{Phase: rewrite.PhaseRequest, Outcome: rewrite.OutcomeSkipped, Reason: reasonSnapshotCapacity}}
|
||||
default:
|
||||
// Rewritten, original released: never record the rewritten request as
|
||||
// the client's. Raw() is the client's request line and Host.
|
||||
v := requestView{unavailable: true}
|
||||
if raw := f.Request.Raw(); raw != nil {
|
||||
v.method, v.url, v.proto, v.rawHost = raw.Method, raw.URL, raw.Proto, raw.Host
|
||||
}
|
||||
lines := []rewrite.TraceLine{{Phase: rewrite.PhaseRequest, Outcome: rewrite.OutcomeError, Reason: reasonOriginalUnavailable}}
|
||||
for _, id := range snap.ruleIDs {
|
||||
lines = append(lines, rewrite.TraceLine{Phase: rewrite.PhaseRequest, RuleID: id, Outcome: rewrite.OutcomeApplied})
|
||||
}
|
||||
return v, lines
|
||||
}
|
||||
}
|
||||
|
||||
func engineRequest(r *mproxy.Request) rewrite.Request {
|
||||
return rewrite.Request{Method: r.Method, URL: r.URL, Header: r.Header, Body: r.Body}
|
||||
}
|
||||
|
||||
// Request runs the request phase. The result goes on the flow only once the
|
||||
// request's snapshot slot is reserved; without one the request leaves as the
|
||||
// client sent it. Rules fail open: a panic leaves the request unchanged.
|
||||
func (s *sink) Request(f *mproxy.Flow) {
|
||||
if s.t == nil || f.Request == nil || f.Request.URL == nil {
|
||||
return
|
||||
}
|
||||
set := s.t.rewriteRules.Load()
|
||||
if set.Len(rewrite.PhaseRequest) == 0 {
|
||||
return
|
||||
}
|
||||
host := f.Request.URL.Hostname()
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
log.Printf("[traffic] rewrite rules stopped on a request to %s (class %s): it is sent as the client sent it", host, rewrite.ErrClassPanic)
|
||||
}
|
||||
}()
|
||||
res := set.Apply(rewrite.PhaseRequest, rewrite.Input{Request: engineRequest(f.Request), IsBinary: rulesIsBinary})
|
||||
runTestHook("request")
|
||||
if len(res.Lines) == 0 {
|
||||
return
|
||||
}
|
||||
if !s.t.snaps.reserve(keyOf(f), newEntry(f, set, res)) {
|
||||
return
|
||||
}
|
||||
if res.Changed {
|
||||
r := f.Request
|
||||
r.Method, r.URL, r.Header, r.Body = res.Request.Method, res.Request.URL, res.Request.Header, res.Request.Body
|
||||
}
|
||||
}
|
||||
|
||||
// Response records the exchange and runs the response phase: the exchange
|
||||
// keeps the client's request and the target's response, the client gets the
|
||||
// rewritten response. Without a slot and without response rules this is the
|
||||
// plain record.
|
||||
func (s *sink) Response(f *mproxy.Flow) {
|
||||
if f.Request == nil || f.Response == nil {
|
||||
return
|
||||
}
|
||||
snap := s.t.snaps.take(keyOf(f))
|
||||
set := s.t.rewriteRules.Load()
|
||||
if snap != nil {
|
||||
set = snap.rules
|
||||
}
|
||||
if snap == nil && set.Len(rewrite.PhaseResponse) == 0 {
|
||||
s.t.record(f)
|
||||
return
|
||||
}
|
||||
rq, lines := originalRequest(f, snap)
|
||||
rs := flowResponseView(f.Response)
|
||||
res := s.applyResponse(f, set, rewrite.Input{
|
||||
Request: engineRequest(f.Request), // as sent to the target
|
||||
Response: rewrite.Response{Status: rs.status, Header: rs.header, Body: rs.body},
|
||||
IsBinary: rulesIsBinary,
|
||||
})
|
||||
s.t.recordViews(rq, rs, append(lines, res.Lines...))
|
||||
if res.Changed {
|
||||
f.Response.StatusCode, f.Response.Header, f.Response.Body = res.Response.Status, res.Response.Header, res.Response.Body
|
||||
}
|
||||
}
|
||||
|
||||
// applyResponse runs the response phase; a panic yields no change and no line.
|
||||
func (s *sink) applyResponse(f *mproxy.Flow, set *rewrite.Set, in rewrite.Input) (res rewrite.Result) {
|
||||
if set.Len(rewrite.PhaseResponse) == 0 {
|
||||
return rewrite.Result{}
|
||||
}
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
res = rewrite.Result{}
|
||||
log.Printf("[traffic] rewrite rules stopped on a response from %s (class %s): it is sent as the target sent it", f.Request.URL.Hostname(), rewrite.ErrClassPanic)
|
||||
}
|
||||
}()
|
||||
r := set.Apply(rewrite.PhaseResponse, in)
|
||||
runTestHook("response")
|
||||
return r
|
||||
}
|
||||
|
||||
// rewriteStream prepares the exchange of a stream at SSEStart or
|
||||
// WebSocketStart: the client's request, the target's status and headers (SSE),
|
||||
// and the trace. For SSE, the response phase runs in stream mode: status and
|
||||
// header actions apply (res, put on the flow by applyStreamResponse), rules
|
||||
// that read or write the response body are traced "not rewritten: stream".
|
||||
// WebSocketStart runs once go-mitmproxy has already answered the client's
|
||||
// upgrade, with no response on the flow: nothing of the handshake can change
|
||||
// there, so every candidate rule is traced "not rewritten: stream".
|
||||
func (s *sink) rewriteStream(f *mproxy.Flow, kind string) (rq requestView, rs responseView, lines []rewrite.TraceLine, res rewrite.Result) {
|
||||
snap := s.t.snaps.take(keyOf(f))
|
||||
set := s.t.rewriteRules.Load()
|
||||
if snap != nil {
|
||||
set = snap.rules
|
||||
}
|
||||
rq, lines = originalRequest(f, snap)
|
||||
if kind == "sse" {
|
||||
rs = flowResponseView(f.Response)
|
||||
}
|
||||
cur := engineRequest(f.Request)
|
||||
if kind != "websocket" {
|
||||
res = s.applyResponse(f, set, rewrite.Input{Request: cur, Response: rewrite.Response{Status: rs.status, Header: rs.header}, Stream: true, IsBinary: rulesIsBinary})
|
||||
return rq, rs, append(lines, res.Lines...), res
|
||||
}
|
||||
if set.Len(rewrite.PhaseRequest)+set.Len(rewrite.PhaseResponse) == 0 {
|
||||
return rq, rs, lines, rewrite.Result{}
|
||||
}
|
||||
// The engine never changes a request with Upgrade; a ws:// stream arrives
|
||||
// on its CONNECT flow, which has none.
|
||||
if cur.Header.Get("Upgrade") == "" {
|
||||
cur.Header = cur.Header.Clone()
|
||||
if cur.Header == nil {
|
||||
cur.Header = http.Header{}
|
||||
}
|
||||
cur.Header.Set("Upgrade", "websocket")
|
||||
}
|
||||
if snap == nil {
|
||||
lines = append(lines, s.applyRequestTrace(f, set, cur)...)
|
||||
}
|
||||
r := s.applyResponse(f, set, rewrite.Input{Request: cur, Response: rewrite.Response{Status: http.StatusSwitchingProtocols}, Stream: true, IsBinary: rulesIsBinary})
|
||||
return rq, rs, append(lines, r.Lines...), rewrite.Result{}
|
||||
}
|
||||
|
||||
// applyRequestTrace runs the request phase for its trace only.
|
||||
func (s *sink) applyRequestTrace(f *mproxy.Flow, set *rewrite.Set, req rewrite.Request) (lines []rewrite.TraceLine) {
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
lines = nil
|
||||
log.Printf("[traffic] rewrite rules stopped on a request to %s (class %s)", f.Request.URL.Hostname(), rewrite.ErrClassPanic)
|
||||
}
|
||||
}()
|
||||
return set.Apply(rewrite.PhaseRequest, rewrite.Input{Request: req, IsBinary: rulesIsBinary}).Lines
|
||||
}
|
||||
|
||||
// applyStreamResponse puts the rewritten status and headers of an SSE
|
||||
// response on the flow, before go-mitmproxy replies to the client.
|
||||
func (s *sink) applyStreamResponse(f *mproxy.Flow, res rewrite.Result) {
|
||||
if res.Changed && f.Response != nil {
|
||||
f.Response.StatusCode, f.Response.Header = res.Response.Status, res.Response.Header
|
||||
}
|
||||
}
|
||||
|
||||
// RequestError fires when a request through an established MITM tunnel fails. If
|
||||
// the failure looks proxy/protocol-caused (h2 quirks, HEAD-with-body, protocol
|
||||
// errors) and not a plain target-unreachable error, the host is tunneled for
|
||||
// passthroughTTL: this request has already failed, later requests in the window
|
||||
// connect straight through, and the first one after the window tries
|
||||
// interception again. A request a rule rewrote never flags its host: the rule,
|
||||
// not the interception, may be at fault. Its log names the host, the error class
|
||||
// and the rules, never the URL or the error text.
|
||||
func (s *sink) RequestError(f *mproxy.Flow, err error) {
|
||||
if f != nil {
|
||||
if snap := s.t.snaps.take(keyOf(f)); snap != nil && snap.requestRewritten() {
|
||||
ids := snap.appliedRuleIDs()
|
||||
for _, id := range ids {
|
||||
snap.rules.NoteError(id, rewrite.ErrClassTransport)
|
||||
}
|
||||
log.Printf("[traffic] a request to %s rewritten by rules %v failed (class %s); the host stays intercepted", snap.host, ids, rewrite.ErrClassTransport)
|
||||
return
|
||||
}
|
||||
}
|
||||
s.t.maybePassthrough(f, err)
|
||||
}
|
||||
|
||||
// StreamResponseModifier passes every body through as received, nil included,
|
||||
// without wrapping it. A response that reached the library's stream threshold
|
||||
// is never recorded: the slot of its flow, if any, is released here.
|
||||
func (s *sink) StreamResponseModifier(f *mproxy.Flow, in io.Reader) (out io.Reader) {
|
||||
out = in
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
out = in
|
||||
}
|
||||
}()
|
||||
if s == nil || s.t == nil || f == nil || !f.Stream || f.SSE != nil {
|
||||
return in
|
||||
}
|
||||
snap := s.t.snaps.take(keyOf(f))
|
||||
if snap == nil {
|
||||
return in
|
||||
}
|
||||
s.t.snaps.largeResponses.Add(1)
|
||||
log.Printf("[traffic] the response to a request to %s with rewrite rules %v reached %d MiB: passed through, not recorded", snap.host, snap.appliedRuleIDs(), libraryStreamBody>>20)
|
||||
return in
|
||||
}
|
||||
|
||||
// spillTrace stores the values of the trace lines as record stores bodies.
|
||||
// Called under wmu, before the transaction.
|
||||
func (t *Traffic) spillTrace(lines []rewrite.TraceLine) []storedBody {
|
||||
out := make([]storedBody, len(lines))
|
||||
for i, l := range lines {
|
||||
out[i] = t.spill(l.After, "")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// writeTrace inserts an exchange's trace rows and the blob references of their
|
||||
// spilled values, inside the exchange's transaction.
|
||||
func writeTrace(tx *sql.Tx, id string, lines []rewrite.TraceLine, trace []storedBody) error {
|
||||
for _, b := range trace {
|
||||
if b.hash == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, b.hash, id); err != nil {
|
||||
return &recordError{"registering blob reference", err}
|
||||
}
|
||||
}
|
||||
for i, l := range lines {
|
||||
var ruleID, after any
|
||||
if l.RuleID != 0 {
|
||||
ruleID = l.RuleID
|
||||
}
|
||||
if trace[i].inline != nil {
|
||||
after = trace[i].inline
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO exchange_rewrites(exchange_id,seq,phase,rule_id,rule_snapshot,action,field,outcome,reason,before_hash,after_hash,after_inline,after_blob)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
id, i, string(l.Phase), ruleID, nullIfEmpty(l.RuleSnapshot), nullIfEmpty(l.Action), nullIfEmpty(l.Field), l.Outcome, l.Reason,
|
||||
nullIfEmpty(l.BeforeHash), nullIfEmpty(l.AfterHash), after, nullIfEmpty(trace[i].hash)); err != nil {
|
||||
return &recordError{"writing rewrite trace", err}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RewriteInfo reports whether rules rewrote an exchange (at least one applied
|
||||
// trace line) and which rules did, by id.
|
||||
func (t *Traffic) RewriteInfo(id string) (rewritten bool, ruleIDs []int64, err error) {
|
||||
if err := t.db.QueryRow(`SELECT EXISTS(SELECT 1 FROM exchange_rewrites r WHERE r.exchange_id=? AND r.outcome='applied')`, id).Scan(&rewritten); err != nil || !rewritten {
|
||||
return false, nil, err
|
||||
}
|
||||
rows, err := t.db.Query(`SELECT DISTINCT rule_id FROM exchange_rewrites WHERE exchange_id=? AND outcome='applied' AND rule_id IS NOT NULL ORDER BY rule_id`, id)
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var rid int64
|
||||
if err := rows.Scan(&rid); err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
ruleIDs = append(ruleIDs, rid)
|
||||
}
|
||||
return true, ruleIDs, rows.Err()
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,102 +0,0 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/traffic/rewrite"
|
||||
)
|
||||
|
||||
// The proxy stores the compiled rewrite rules and does not apply them. With a
|
||||
// snapshot whose rules match every exchange, what the target receives, what the
|
||||
// client receives and what is recorded are the same as with no rule at all.
|
||||
func TestRewriteRulesStoredNotApplied(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var seen []string
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
mu.Lock()
|
||||
seen = append(seen, r.Method+" "+r.RequestURI+" X-Client="+r.Header.Get("X-Client")+" "+string(body))
|
||||
mu.Unlock()
|
||||
w.Header().Set("X-Target", "t")
|
||||
w.WriteHeader(http.StatusTeapot)
|
||||
io.WriteString(w, "target body")
|
||||
}))
|
||||
t.Cleanup(target.Close)
|
||||
|
||||
set := rewrite.Compile([]rewrite.Rule{
|
||||
{ID: 1, Name: "request", Enabled: true, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetMethod, Value: "PUT"},
|
||||
{Type: rewrite.ActionSetPathQuery, Value: "/rewritten"},
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Client", Value: "rewritten"},
|
||||
{Type: rewrite.ActionSetBody, Value: "rewritten"}}},
|
||||
{ID: 2, Name: "response", Enabled: true, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetStatus, Value: "200"},
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Rewritten", Value: "1"},
|
||||
{Type: rewrite.ActionSetBody, Value: "rewritten"}}},
|
||||
}, rewrite.Options{})
|
||||
if set.Len(rewrite.PhaseRequest) != 1 || set.Len(rewrite.PhaseResponse) != 1 || len(set.LoadErrors()) != 0 {
|
||||
t.Fatalf("the snapshot must hold both rules: %v", set.LoadErrors())
|
||||
}
|
||||
|
||||
// canon makes a recorded message comparable: the recorder writes headers in
|
||||
// map order, and the target dates its response.
|
||||
canon := func(raw string) string {
|
||||
head, body, _ := strings.Cut(raw, "\n\n")
|
||||
lines := strings.Split(head, "\n")
|
||||
sort.Strings(lines[1:])
|
||||
lines = slices.DeleteFunc(lines, func(l string) bool { return strings.HasPrefix(l, "Date: ") })
|
||||
return strings.Join(lines, "\n") + "\n\n" + body
|
||||
}
|
||||
// exchange sends one request through a fresh proxy holding set and returns
|
||||
// what the client received and what the proxy recorded.
|
||||
exchange := func(set *rewrite.Set) (client, recorded string) {
|
||||
tr, _, proxyURL := startProxy(t)
|
||||
if tr.RewriteRules() != nil {
|
||||
t.Fatal("Open published rewrite rules")
|
||||
}
|
||||
tr.SetRewriteRules(set)
|
||||
if tr.RewriteRules() != set {
|
||||
t.Fatal("RewriteRules must return the snapshot last published")
|
||||
}
|
||||
req, err := http.NewRequest("POST", target.URL+"/path?q=1", strings.NewReader("client body"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Set("X-Client", "c")
|
||||
resp, err := proxyClient(t, tr, proxyURL).Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
client = resp.Status + " X-Target=" + resp.Header.Get("X-Target") + " X-Rewritten=" + resp.Header.Get("X-Rewritten") + " " + string(body)
|
||||
reqText, respText, err := tr.Get(onlyExchangeID(t, tr))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return client, canon(reqText) + "\n--\n" + canon(respText)
|
||||
}
|
||||
|
||||
baseClient, baseRecorded := exchange(nil)
|
||||
client, recorded := exchange(set)
|
||||
if want := "418 I'm a teapot X-Target=t X-Rewritten= target body"; baseClient != want || client != want {
|
||||
t.Fatalf("client received %q without rules and %q with rules, want %q", baseClient, client, want)
|
||||
}
|
||||
if recorded != baseRecorded {
|
||||
t.Fatalf("the recorded exchange differs with rules stored:\n without: %s\n with: %s", baseRecorded, recorded)
|
||||
}
|
||||
if !strings.Contains(recorded, "client body") || !strings.Contains(recorded, "target body") || strings.Contains(recorded, "rewritten") {
|
||||
t.Fatalf("recorded exchange: %s", recorded)
|
||||
}
|
||||
want := "POST /path?q=1 X-Client=c client body"
|
||||
if len(seen) != 2 || seen[0] != want || seen[1] != want {
|
||||
t.Fatalf("the target received %q, want %q twice", seen, want)
|
||||
}
|
||||
}
|
||||
+60
-27
@@ -16,6 +16,7 @@ import (
|
||||
"unsafe"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/traffic/rewrite"
|
||||
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
|
||||
)
|
||||
|
||||
@@ -144,17 +145,35 @@ func (s *sink) startStream(f *mproxy.Flow, kind string) {
|
||||
if f.Request == nil || f.Request.URL == nil || (kind == "sse" && f.Response == nil) {
|
||||
return
|
||||
}
|
||||
id, err := s.t.recordStreamStart(f, kind)
|
||||
rq, rs, lines, res := s.rewriteStream(f, kind)
|
||||
id, err := s.t.recordStreamStart(rq, rs, kind, lines)
|
||||
s.applyStreamResponse(f, res)
|
||||
if err != nil {
|
||||
if rewritten, ruleIDs := appliedRules(lines); rewritten {
|
||||
var re *recordError
|
||||
errors.As(err, &re)
|
||||
log.Printf("[traffic] failed to record a rewritten %s stream of %s (%s; rules %v)", kind, f.Request.URL.Hostname(), re.step, ruleIDs)
|
||||
return
|
||||
}
|
||||
log.Printf("[traffic] failed to record %s stream %s: %v", kind, f.Request.URL, err)
|
||||
return
|
||||
}
|
||||
st.id = id
|
||||
if kind == "sse" {
|
||||
st.reqBytes = int64(len(f.Request.Body))
|
||||
st.reqBytes = int64(len(rq.body))
|
||||
}
|
||||
}
|
||||
|
||||
// recordError names the step of a recording that failed. Its text is the step
|
||||
// and the cause, as recordStreamStart always reported.
|
||||
type recordError struct {
|
||||
step string
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *recordError) Error() string { return e.step + ": " + e.err.Error() }
|
||||
func (e *recordError) Unwrap() error { return e.err }
|
||||
|
||||
// libMutex returns the unexported mu sync.Mutex that go-mitmproxy holds while it
|
||||
// appends to WebSocketData.Messages / SSEData.Events. The library exposes no
|
||||
// accessor, and the two WebSocket goroutines append concurrently, so reading the
|
||||
@@ -252,9 +271,9 @@ func sseEventsSince(sse *mproxy.SSEData, st *streamState) []streamRow {
|
||||
// streamDisplayURL is the URL shown for a WebSocket exchange: http→ws and
|
||||
// https→wss. A plain ws:// connection reaches the WebSocket hooks through its
|
||||
// CONNECT tunnel, where only the target authority is known, not the path.
|
||||
func streamDisplayURL(f *mproxy.Flow, kind string) (method, display, tmpl string) {
|
||||
u := *f.Request.URL
|
||||
method = f.Request.Method
|
||||
func streamDisplayURL(reqMethod string, reqURL *url.URL, kind string) (method, display, tmpl string) {
|
||||
u := *reqURL
|
||||
method = reqMethod
|
||||
if kind == "websocket" && strings.EqualFold(method, "CONNECT") {
|
||||
host := u.Host
|
||||
if host == "" {
|
||||
@@ -270,63 +289,77 @@ func streamDisplayURL(f *mproxy.Flow, kind string) (method, display, tmpl string
|
||||
u.Scheme = "wss"
|
||||
}
|
||||
}
|
||||
return method, u.String(), db.TemplatePath(f.Request.URL.EscapedPath())
|
||||
return method, u.String(), db.TemplatePath(reqURL.EscapedPath())
|
||||
}
|
||||
|
||||
// recordStreamStart writes the stream's exchange: metadata, request head and
|
||||
// body, the response head (synthetic 101 for WebSocket), and the header text in
|
||||
// the full-text index. Messages are never indexed.
|
||||
func (t *Traffic) recordStreamStart(f *mproxy.Flow, kind string) (string, error) {
|
||||
// body, the response head (synthetic 101 for WebSocket), the rewrite trace,
|
||||
// and the header text in the full-text index. Messages are never indexed. rq
|
||||
// is the request the client sent and rs (SSE only) the status and headers the
|
||||
// target sent, whatever the rules changed.
|
||||
func (t *Traffic) recordStreamStart(rq requestView, rs responseView, kind string, lines []rewrite.TraceLine) (string, error) {
|
||||
now := time.Now()
|
||||
id := t.newExchangeID(now)
|
||||
method, display, tmpl := streamDisplayURL(f, kind)
|
||||
host := f.Request.URL.Hostname()
|
||||
target := f.Request.URL.RequestURI()
|
||||
if strings.EqualFold(f.Request.Method, "CONNECT") {
|
||||
target = f.Request.URL.Host // authority-form, as on the wire
|
||||
method, display, tmpl, host, target := rq.method, "", "", "", ""
|
||||
if rq.url != nil {
|
||||
method, display, tmpl = streamDisplayURL(rq.method, rq.url, kind)
|
||||
host = rq.url.Hostname()
|
||||
target = rq.url.RequestURI()
|
||||
if strings.EqualFold(rq.method, "CONNECT") {
|
||||
target = rq.url.Host // authority-form, as on the wire
|
||||
}
|
||||
}
|
||||
reqHead := fmt.Sprintf("%s %s %s\n%s", f.Request.Method, target, f.Request.Proto, requestHeaderLines(f.Request))
|
||||
reqHead := requestHead(rq, target)
|
||||
status, ct, respHead := 101, "websocket", "HTTP/1.1 101 Switching Protocols\n"
|
||||
if kind == "sse" {
|
||||
status = f.Response.StatusCode
|
||||
ct = f.Response.Header.Get("Content-Type")
|
||||
respHead = fmt.Sprintf("HTTP %d\n%s", status, headerLines(f.Response.Header))
|
||||
status = rs.status
|
||||
ct = rs.header.Get("Content-Type")
|
||||
respHead = fmt.Sprintf("HTTP %d\n%s", status, headerLines(rs.header))
|
||||
}
|
||||
|
||||
t.wmu.Lock()
|
||||
defer t.wmu.Unlock()
|
||||
reqB := t.spill(f.Request.Body, f.Request.Header.Get("Content-Type"))
|
||||
reqB := t.spill(rq.body, rq.header.Get("Content-Type"))
|
||||
trace := t.spillTrace(lines)
|
||||
tx, err := t.db.Begin()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("beginning transaction: %w", err)
|
||||
return "", &recordError{"beginning transaction", err}
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // no-op once committed
|
||||
// An id can repeat (seconds plus a rolling counter); the trace of the
|
||||
// exchange it replaces must not stay attached to this one.
|
||||
if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id=?`, id); err != nil {
|
||||
return "", &recordError{"clearing rewrite trace", err}
|
||||
}
|
||||
res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,'')`, id, now.Unix(), host, method, tmpl, display, status, ct, len(f.Request.Body), 0)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,'')`, id, now.Unix(), host, method, tmpl, display, status, ct, len(rq.body), 0)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("writing index: %w", err)
|
||||
return "", &recordError{"writing index", err}
|
||||
}
|
||||
rowid, err := res.LastInsertId()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fetching rowid: %w", err)
|
||||
return "", &recordError{"fetching rowid", err}
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT OR REPLACE INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob)
|
||||
VALUES(?,?,?,?,?,NULL,NULL)`, id, reqHead, reqB.inline, nullIfEmpty(reqB.hash), respHead); err != nil {
|
||||
return "", fmt.Errorf("writing body: %w", err)
|
||||
return "", &recordError{"writing body", err}
|
||||
}
|
||||
if reqB.hash != "" {
|
||||
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, reqB.hash, id); err != nil {
|
||||
return "", fmt.Errorf("registering blob reference: %w", err)
|
||||
return "", &recordError{"registering blob reference", err}
|
||||
}
|
||||
}
|
||||
if err := writeTrace(tx, id, lines, trace); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if t.fts {
|
||||
idx := strings.Join([]string{display, reqHead, reqB.index, respHead}, "\n")
|
||||
if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowid, idx); err != nil {
|
||||
return "", fmt.Errorf("writing full-text index: %w", err)
|
||||
return "", &recordError{"writing full-text index", err}
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return "", fmt.Errorf("committing: %w", err)
|
||||
return "", &recordError{"committing", err}
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
+221
-51
@@ -22,6 +22,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -92,6 +93,26 @@ CREATE TABLE IF NOT EXISTS stream_messages (
|
||||
PRIMARY KEY (exchange_id, seq)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_stream_messages_ex ON stream_messages(exchange_id, seq);
|
||||
|
||||
-- Rewrite-rule trace of an exchange, in order. The exchange row keeps what the
|
||||
-- client sent and what the target answered; these rows say what the rules
|
||||
-- changed on the way. An exchange is rewritten when it has an 'applied' row.
|
||||
CREATE TABLE IF NOT EXISTS exchange_rewrites (
|
||||
exchange_id TEXT NOT NULL,
|
||||
seq INTEGER NOT NULL,
|
||||
phase TEXT NOT NULL, -- 'request' | 'response'
|
||||
rule_id INTEGER, -- NULL on a global line
|
||||
rule_snapshot TEXT, -- JSON, no sensitive action value
|
||||
action TEXT,
|
||||
field TEXT, -- 'method' | 'url' | 'header:<Name>' | 'body' | 'status'
|
||||
outcome TEXT NOT NULL, -- 'applied' | 'skipped' | 'error'
|
||||
reason TEXT NOT NULL DEFAULT '',
|
||||
before_hash TEXT,
|
||||
after_hash TEXT,
|
||||
after_inline BLOB,
|
||||
after_blob TEXT,
|
||||
PRIMARY KEY (exchange_id, seq)
|
||||
);
|
||||
`
|
||||
|
||||
// ftsSchema is applied separately from indexSchema: a driver build without FTS5
|
||||
@@ -160,6 +181,10 @@ const (
|
||||
// and the catalog migration cannot drift apart.
|
||||
const TrafficSearchDescription = "Query the target traffic the recording proxy has captured (host is required; accepts a bare host, host:port, or full URL, optionally filtered further by URL substring or body keyword). When a port is specified, only that service's traffic is returned, avoiding cross-talk between different ports on the same IP. body_contains runs a full-text search over the captured request/response headers and bodies, supporting any substring including Chinese (at least 3 characters). Returns only an ultra-light index (id/method/url/status/resp_len) with no response content; once results are non-empty, use traffic_get to verify the request/response of each one, then hand the IDs that actually support the current vulnerability to bind_finding_traffic. Returns 3 by default, at most 10 per page; use page to paginate when there are more results."
|
||||
|
||||
// TrafficGetDescription is the traffic_get description, kept here for the same
|
||||
// reason as TrafficSearchDescription.
|
||||
const TrafficGetDescription = "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling. A WebSocket (status 101) or SSE (text/event-stream) record shows only its handshake here; read its messages with traffic_messages. When proxy rewrite rules changed the exchange, the result starts with a REWRITTEN line naming the rule ids, and shows the request as the client sent it and the response as the target sent it. traffic_search searches only these original values: it never finds a value a rewrite rule wrote."
|
||||
|
||||
// Traffic runs the recording proxy and owns the file tree + index.
|
||||
type Traffic struct {
|
||||
dir string
|
||||
@@ -190,16 +215,23 @@ type Traffic struct {
|
||||
closeOnce sync.Once
|
||||
// pass is the set of hosts whose MITM interception failed for a proxy/protocol
|
||||
// reason; connections to them are tunneled transparently (fail-open) so the
|
||||
// request still reaches the target — unrecorded — instead of being killed.
|
||||
pass sync.Map // hostname(string) -> struct{}
|
||||
// request still reaches the target, unrecorded, instead of being killed.
|
||||
// The value is the time the mark expires. After that, interception is tried again.
|
||||
pass sync.Map // hostname(string) -> time.Time
|
||||
// upstream is the global egress proxy every captured request is forwarded
|
||||
// through (nil = dial targets directly). Both the intercepted and the
|
||||
// transparent-passthrough paths honor it (go-mitmproxy's getUpstreamConn),
|
||||
// so no host escapes it. Hot-swappable at runtime via SetUpstreamProxy.
|
||||
upstream atomic.Pointer[url.URL]
|
||||
// rewriteRules is the compiled snapshot of the rewrite rules, replaced as a
|
||||
// whole by SetRewriteRules. The proxy does not apply it yet.
|
||||
// whole by SetRewriteRules and applied by the sink (rewrite_flow.go).
|
||||
rewriteRules atomic.Pointer[rewrite.Set]
|
||||
// snaps keeps the original of each request a rule rewrote until its
|
||||
// exchange is recorded (rewrite_flow.go). Nil on the zero value.
|
||||
snaps *snapTable
|
||||
// sweeping tracks the snapshot sweeper, which runs until Close. It is kept
|
||||
// apart from reaping, which only ever holds work that finishes on its own.
|
||||
sweeping sync.WaitGroup
|
||||
}
|
||||
|
||||
// Open initializes the traffic tree, blob store and SQLite index under dir.
|
||||
@@ -225,7 +257,7 @@ func Open(dir, addr string) (*Traffic, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t := &Traffic{dir: dir, addr: addr, db: db, closed: make(chan struct{})}
|
||||
t := &Traffic{dir: dir, addr: addr, db: db, closed: make(chan struct{}), snaps: newSnapTable()}
|
||||
if err := t.initIndex(); err != nil {
|
||||
db.Close()
|
||||
return nil, err
|
||||
@@ -253,11 +285,11 @@ func Open(dir, addr string) (*Traffic, error) {
|
||||
// can't intercept without breaking (see maybePassthrough). Those are tunneled
|
||||
// transparently so the request still reaches the target instead of being killed.
|
||||
p.SetShouldInterceptRule(func(req *http.Request) bool {
|
||||
_, tunnel := t.pass.Load(hostOnly(req.Host))
|
||||
return !tunnel
|
||||
return !t.tunneling(hostOnly(req.Host))
|
||||
})
|
||||
p.AddAddon(&sink{t: t})
|
||||
t.proxy = p
|
||||
t.sweeping.Go(t.sweepSnapshots) // stops on t.closed; Close waits for it
|
||||
return t, nil
|
||||
}
|
||||
|
||||
@@ -358,8 +390,8 @@ func (t *Traffic) SetUpstreamProxy(raw string) error {
|
||||
}
|
||||
|
||||
// SetRewriteRules publishes a compiled snapshot of the rewrite rules. The swap
|
||||
// is atomic; nil clears the rules. The snapshot is stored only: the proxy does
|
||||
// not apply rewrite rules yet, so capture is unchanged whatever it holds.
|
||||
// is atomic; nil clears the rules. A flow already rewritten keeps the snapshot
|
||||
// its request saw for its response as well.
|
||||
func (t *Traffic) SetRewriteRules(s *rewrite.Set) { t.rewriteRules.Store(s) }
|
||||
|
||||
// RewriteRules returns the snapshot last published, nil when there is none.
|
||||
@@ -406,6 +438,7 @@ func (t *Traffic) Close() error {
|
||||
t.closeOnce.Do(func() { close(t.closed) })
|
||||
}
|
||||
t.reaping.Wait()
|
||||
t.sweeping.Wait()
|
||||
return t.db.Close()
|
||||
}
|
||||
|
||||
@@ -430,19 +463,6 @@ type sink struct {
|
||||
streams sync.Map // *mproxy.Flow of a live WebSocket/SSE stream -> *streamState (stream.go)
|
||||
}
|
||||
|
||||
func (s *sink) Response(f *mproxy.Flow) {
|
||||
if f.Request == nil || f.Response == nil {
|
||||
return
|
||||
}
|
||||
s.t.record(f)
|
||||
}
|
||||
|
||||
// RequestError fires when a request through an established MITM tunnel fails. If
|
||||
// the failure looks proxy/protocol-caused (h2 quirks, HEAD-with-body, protocol
|
||||
// errors) — not a plain target-unreachable error — we flag the host for
|
||||
// transparent passthrough so future requests to it succeed instead of dying.
|
||||
func (s *sink) RequestError(f *mproxy.Flow, err error) { s.t.maybePassthrough(f, err) }
|
||||
|
||||
// maybePassthrough marks a host to be tunneled transparently on the next
|
||||
// connection, but only for errors the proxy itself caused — a target that is
|
||||
// simply down/filtered would fail without us too, and must stay MITM'd+recorded.
|
||||
@@ -454,9 +474,34 @@ func (t *Traffic) maybePassthrough(f *mproxy.Flow, err error) {
|
||||
if host == "" {
|
||||
return
|
||||
}
|
||||
if _, loaded := t.pass.LoadOrStore(host, struct{}{}); !loaded {
|
||||
log.Printf("[traffic] MITM with %s failed, falling back to passthrough (this host now connects straight to the target and is no longer recorded, but requests proceed normally): %v", host, err)
|
||||
if t.tunneling(host) {
|
||||
return
|
||||
}
|
||||
until := time.Now().Add(passthroughTTL)
|
||||
if _, loaded := t.pass.LoadOrStore(host, until); !loaded {
|
||||
log.Printf("[traffic] MITM with %s failed, falling back to passthrough for %s (this request failed; the host connects straight to the target and is not recorded until then; the first request after the window fails again if interception still cannot handle it): %v", host, passthroughTTL, err)
|
||||
}
|
||||
}
|
||||
|
||||
// passthroughTTL is how long a host stays tunneled after a proxy-caused MITM
|
||||
// failure. A host whose interception always fails pays one lost request per
|
||||
// window: one per host per minute, where a mark that never expired cost one
|
||||
// per process. The window is what lets a stale bypass end.
|
||||
const passthroughTTL = time.Minute
|
||||
|
||||
// tunneling reports whether host is inside an unexpired passthrough mark.
|
||||
// An expired mark is removed so the next connection tries interception again.
|
||||
func (t *Traffic) tunneling(host string) bool {
|
||||
v, ok := t.pass.Load(host)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
until, ok := v.(time.Time)
|
||||
if !ok || !time.Now().Before(until) {
|
||||
t.pass.CompareAndDelete(host, v)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// proxyCausedErr reports whether err indicates the interception layer (not the
|
||||
@@ -475,52 +520,121 @@ func proxyCausedErr(err error) bool {
|
||||
// full-text index. Nothing is written to a per-request directory — only bodies
|
||||
// above maxInlineBody spill to the content-addressed blob store.
|
||||
func (t *Traffic) record(f *mproxy.Flow) {
|
||||
t.recordViews(flowRequestView(f.Request), flowResponseView(f.Response), nil)
|
||||
}
|
||||
|
||||
// requestView is the request an exchange records: what the client sent, which
|
||||
// is not the flow's request once a rule rewrote it.
|
||||
type requestView struct {
|
||||
method string
|
||||
url *url.URL // nil when the original request is unavailable and Raw() is nil
|
||||
proto string
|
||||
header http.Header
|
||||
// rawHost is the client's Host header (Raw().Host), "" when unknown.
|
||||
rawHost string
|
||||
body []byte
|
||||
// unavailable marks a request rewritten by a rule whose original was not
|
||||
// kept: only its method, URL and Host are known.
|
||||
unavailable bool
|
||||
}
|
||||
|
||||
// responseView is the response an exchange records: what the target answered.
|
||||
type responseView struct {
|
||||
status int
|
||||
header http.Header
|
||||
body []byte
|
||||
}
|
||||
|
||||
func flowRequestView(r *mproxy.Request) requestView {
|
||||
v := requestView{method: r.Method, url: r.URL, proto: r.Proto, header: r.Header, body: r.Body}
|
||||
if raw := r.Raw(); raw != nil {
|
||||
v.rawHost = raw.Host
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func flowResponseView(r *mproxy.Response) responseView {
|
||||
return responseView{status: r.StatusCode, header: r.Header, body: r.Body}
|
||||
}
|
||||
|
||||
// unavailableMarker stands for the headers and body of a request whose
|
||||
// original was not kept (see rewrite_flow.go).
|
||||
const unavailableMarker = "[original request unavailable]\n"
|
||||
|
||||
// recordViews writes one exchange from explicit views, with its rewrite trace,
|
||||
// in one transaction: the exchange row, its bodies, blob references, the trace
|
||||
// and the full-text index of the views only. A failure rolls everything back,
|
||||
// so there is never a rewritten exchange without its trace. Without trace
|
||||
// lines this is exactly what record always wrote.
|
||||
func (t *Traffic) recordViews(rq requestView, rs responseView, lines []rewrite.TraceLine) {
|
||||
// The write lock covers blob + index writes, so DeleteHost (and its blob GC)
|
||||
// can run under the same lock without racing a concurrent record.
|
||||
t.wmu.Lock()
|
||||
defer t.wmu.Unlock()
|
||||
host := f.Request.URL.Hostname()
|
||||
method := f.Request.Method
|
||||
tmpl := db.TemplatePath(f.Request.URL.EscapedPath())
|
||||
var host, tmpl, url, uri string
|
||||
if rq.url != nil {
|
||||
host = rq.url.Hostname()
|
||||
tmpl = db.TemplatePath(rq.url.EscapedPath())
|
||||
url = rq.url.String()
|
||||
uri = rq.url.RequestURI()
|
||||
}
|
||||
method := rq.method
|
||||
now := time.Now()
|
||||
id := t.newExchangeID(now)
|
||||
|
||||
ct := f.Response.Header.Get("Content-Type")
|
||||
url := f.Request.URL.String()
|
||||
reqHead := fmt.Sprintf("%s %s %s\n%s", method, f.Request.URL.RequestURI(), f.Request.Proto, requestHeaderLines(f.Request))
|
||||
respHead := fmt.Sprintf("HTTP %d\n%s", f.Response.StatusCode, headerLines(f.Response.Header))
|
||||
ct := rs.header.Get("Content-Type")
|
||||
reqHead := requestHead(rq, uri)
|
||||
respHead := fmt.Sprintf("HTTP %d\n%s", rs.status, headerLines(rs.header))
|
||||
// Bodies are spilled before the transaction opens: blob writes are filesystem
|
||||
// work and must not sit inside the SQLite write lock.
|
||||
reqB := t.spill(f.Request.Body, f.Request.Header.Get("Content-Type"))
|
||||
respB := t.spill(f.Response.Body, ct)
|
||||
reqB := t.spill(rq.body, rq.header.Get("Content-Type"))
|
||||
respB := t.spill(rs.body, ct)
|
||||
trace := t.spillTrace(lines)
|
||||
|
||||
// A rewritten exchange is logged by host, step and rule ids only: its URL
|
||||
// and the error text may carry a value a rule wrote.
|
||||
rewritten, ruleIDs := appliedRules(lines)
|
||||
fail := func(step string, err error) {
|
||||
if rewritten {
|
||||
log.Printf("[traffic] failed to record a rewritten exchange of %s (%s; rules %v)", host, step, ruleIDs)
|
||||
return
|
||||
}
|
||||
log.Printf("[traffic] failed to record %s (%s): %v", url, step, err)
|
||||
}
|
||||
|
||||
tx, err := t.db.Begin()
|
||||
if err != nil {
|
||||
log.Printf("[traffic] failed to record %s (beginning transaction): %v", url, err)
|
||||
fail("beginning transaction", err)
|
||||
return
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // no-op once committed
|
||||
|
||||
// An id can repeat (seconds plus a rolling counter); the trace of the
|
||||
// exchange it replaces must not stay attached to this one.
|
||||
if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id=?`, id); err != nil {
|
||||
fail("clearing rewrite trace", err)
|
||||
return
|
||||
}
|
||||
// path stays empty for db-resident exchanges; a non-empty path marks a legacy
|
||||
// row whose bodies still live in the old on-disk tree (see Get).
|
||||
res, err := tx.Exec(`INSERT OR REPLACE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,'')`,
|
||||
id, now.Unix(), host, method, tmpl, url, f.Response.StatusCode, ct,
|
||||
len(f.Request.Body), len(f.Response.Body))
|
||||
id, now.Unix(), host, method, tmpl, url, rs.status, ct,
|
||||
len(rq.body), len(rs.body))
|
||||
if err != nil {
|
||||
log.Printf("[traffic] failed to record %s (writing index): %v", url, err)
|
||||
fail("writing index", err)
|
||||
return
|
||||
}
|
||||
rowid, err := res.LastInsertId()
|
||||
if err != nil {
|
||||
log.Printf("[traffic] failed to record %s (fetching rowid): %v", url, err)
|
||||
fail("fetching rowid", err)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(`INSERT OR REPLACE INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob)
|
||||
VALUES(?,?,?,?,?,?,?)`,
|
||||
id, reqHead, reqB.inline, nullIfEmpty(reqB.hash), respHead, respB.inline, nullIfEmpty(respB.hash)); err != nil {
|
||||
log.Printf("[traffic] failed to record %s (writing body): %v", url, err)
|
||||
fail("writing body", err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -529,26 +643,51 @@ VALUES(?,?,?,?,?,?,?)`,
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, h, id); err != nil {
|
||||
log.Printf("[traffic] failed to record %s (registering blob reference): %v", url, err)
|
||||
fail("registering blob reference", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := writeTrace(tx, id, lines, trace); err != nil {
|
||||
var re *recordError
|
||||
errors.As(err, &re)
|
||||
fail(re.step, re.err)
|
||||
return
|
||||
}
|
||||
|
||||
if t.fts {
|
||||
// The index is fed from memory, so a body that spilled to _blobs is still
|
||||
// fully searchable even though only its preview is stored inline.
|
||||
// fully searchable even though only its preview is stored inline. Only the
|
||||
// recorded views are indexed, never a value a rule wrote.
|
||||
idx := strings.Join([]string{url, reqHead, reqB.index, respHead, respB.index}, "\n")
|
||||
if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowid, idx); err != nil {
|
||||
log.Printf("[traffic] failed to record %s (writing full-text index): %v", url, err)
|
||||
fail("writing full-text index", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
log.Printf("[traffic] failed to record %s (committing): %v", url, err)
|
||||
fail("committing", err)
|
||||
}
|
||||
}
|
||||
|
||||
// appliedRules reports whether a trace rewrote its exchange, with the ids of
|
||||
// the rules that did, in order of first appearance.
|
||||
func appliedRules(lines []rewrite.TraceLine) (bool, []int64) {
|
||||
rewritten := false
|
||||
var ids []int64
|
||||
for _, l := range lines {
|
||||
if l.Outcome != rewrite.OutcomeApplied {
|
||||
continue
|
||||
}
|
||||
rewritten = true
|
||||
if l.RuleID != 0 && !slices.Contains(ids, l.RuleID) {
|
||||
ids = append(ids, l.RuleID)
|
||||
}
|
||||
}
|
||||
return rewritten, ids
|
||||
}
|
||||
|
||||
// newExchangeID names a recorded exchange: unix seconds plus a rolling sequence.
|
||||
func (t *Traffic) newExchangeID(now time.Time) string {
|
||||
n := t.seq.Add(1)
|
||||
@@ -699,15 +838,35 @@ func headerLines(h map[string][]string) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// requestHeaderLines restores the HTTP Host header, which net/http stores on
|
||||
// Request.Host rather than in Header. Keeping it in request.http makes the raw
|
||||
// capture complete and directly replayable.
|
||||
// requestHeaderLines is requestHeaderLinesOf for a flow's request.
|
||||
func requestHeaderLines(req *mproxy.Request) string {
|
||||
headers := req.Header.Clone()
|
||||
v := flowRequestView(req)
|
||||
return requestHeaderLinesOf(v.header, req.URL.Host, v.rawHost)
|
||||
}
|
||||
|
||||
// requestHead is the head an exchange records for a request view: request
|
||||
// line and headers, or only the Host when the original request is unavailable.
|
||||
func requestHead(rq requestView, target string) string {
|
||||
urlHost := ""
|
||||
if rq.url != nil {
|
||||
urlHost = rq.url.Host
|
||||
}
|
||||
if rq.unavailable {
|
||||
return fmt.Sprintf("%s %s %s\n%s%s", rq.method, target, rq.proto, requestHeaderLinesOf(nil, urlHost, rq.rawHost), unavailableMarker)
|
||||
}
|
||||
return fmt.Sprintf("%s %s %s\n%s", rq.method, target, rq.proto, requestHeaderLinesOf(rq.header, urlHost, rq.rawHost))
|
||||
}
|
||||
|
||||
// requestHeaderLinesOf restores the HTTP Host header, which net/http stores on
|
||||
// Request.Host rather than in Header. Keeping it in request.http makes the raw
|
||||
// capture complete and directly replayable. rawHost is the client's Host
|
||||
// header (Raw().Host); urlHost stands in when it is empty.
|
||||
func requestHeaderLinesOf(header http.Header, urlHost, rawHost string) string {
|
||||
headers := header.Clone()
|
||||
headers.Del("Host")
|
||||
host := req.URL.Host
|
||||
if raw := req.Raw(); raw != nil && strings.TrimSpace(raw.Host) != "" {
|
||||
host = raw.Host
|
||||
host := urlHost
|
||||
if strings.TrimSpace(rawHost) != "" {
|
||||
host = rawHost
|
||||
}
|
||||
var b strings.Builder
|
||||
if host = strings.TrimSpace(host); host != "" {
|
||||
@@ -1244,6 +1403,9 @@ func (t *Traffic) deleteWhere(tx *sql.Tx, where string, args ...any) (int64, err
|
||||
if _, err := tx.Exec(`DELETE FROM stream_messages WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil {
|
||||
return 0, fmt.Errorf("delete stream messages: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM exchange_rewrites WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil {
|
||||
return 0, fmt.Errorf("delete rewrite trace: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM blob_refs WHERE exchange_id IN (SELECT id FROM exchanges WHERE `+where+`)`, args...); err != nil {
|
||||
return 0, fmt.Errorf("delete blob references: %w", err)
|
||||
}
|
||||
@@ -2111,7 +2273,7 @@ func (t *Traffic) Tools() []actool.CoreTool {
|
||||
|
||||
get := actool.Build(actool.Spec{
|
||||
Name: "traffic_get",
|
||||
Description: "Fetch the raw request/response of one captured traffic record by id (truncated when too large). Use together with traffic_search to avoid re-curling. A WebSocket (status 101) or SSE (text/event-stream) record shows only its handshake here; read its messages with traffic_messages.",
|
||||
Description: TrafficGetDescription,
|
||||
Schema: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{"id": map[string]any{"type": "string", "description": "the id returned by traffic_search"}},
|
||||
@@ -2126,7 +2288,15 @@ func (t *Traffic) Tools() []actool.CoreTool {
|
||||
if err != nil {
|
||||
return actool.Errorf(err.Error()), nil
|
||||
}
|
||||
return actool.Text("=== REQUEST ===\n" + clip(req, 2500) + "\n\n=== RESPONSE ===\n" + clip(resp, 4000)), nil
|
||||
rewritten, ruleIDs, err := t.RewriteInfo(a.ID)
|
||||
if err != nil {
|
||||
return actool.Errorf(err.Error()), nil
|
||||
}
|
||||
head := ""
|
||||
if rewritten {
|
||||
head = fmt.Sprintf("=== REWRITTEN === rewrite rules %v changed this exchange in the proxy. Shown below: the request as the client sent it and the response as the target sent it, not the rewritten values.\n\n", ruleIDs)
|
||||
}
|
||||
return actool.Text(head + "=== REQUEST ===\n" + clip(req, 2500) + "\n\n=== RESPONSE ===\n" + clip(resp, 4000)), nil
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user