Merge remote-tracking branch 'upstream/main' into codex/upload-500mib

This commit is contained in:
Chubboy Ren
2026-11-17 01:55:45 +00:00
75 changed files with 8451 additions and 103 deletions
+10 -3
View File
@@ -82,9 +82,16 @@ func applyTaskArchiveBlocker(dto *TaskDTO, blockers map[int64]int64) {
if err != nil {
return
}
if dependentID := blockers[taskID]; dependentID > 0 {
dto.ArchiveBlockedBy = strconv.FormatInt(dependentID, 10)
applyTaskArchiveBlockerID(dto, blockers[taskID])
}
// applyTaskArchiveBlockerID records dependentID as the task's archive blocker;
// 0 (nothing blocks it) leaves the DTO unchanged.
func applyTaskArchiveBlockerID(dto *TaskDTO, dependentID int64) {
if dto == nil || dependentID <= 0 {
return
}
dto.ArchiveBlockedBy = strconv.FormatInt(dependentID, 10)
}
// TokenTotalDTO is a whole-task (all agents) token aggregate.
@@ -132,7 +139,7 @@ func taskDTO(t *Task, status string) TaskDTO {
LLMProfileID: llmState.ProfileID,
LLMProfileIDs: profileIDs,
ActiveLLMProfileID: llmState.ActiveID,
LLMRotate: t.LLMRotate,
LLMRotate: llmState.Rotate,
LLMFailoverState: llmState.FailoverState,
LLMFailoverReason: llmState.FailoverReason,
SourceTaskIDs: sourceIDs,
+2 -4
View File
@@ -1109,10 +1109,8 @@ func (e *Engine) runIntent(ctx context.Context, t *Task, name string, worker *ag
// Per-intent LLM rotation (#85): each claimed worker intent runs on the
// next chain head. The pin travels through the run context into the
// provider resolver; non-worker roles and failover keep the active head.
if t.LLMRotate {
if head, ok := t.nextRotatedProfile(); ok {
ctx = withLLMProfilePin(ctx, head)
}
if head, ok := t.nextRotatedProfile(); ok {
ctx = withLLMProfilePin(ctx, head)
}
hasChatMessage := message != ""
e.stampFirstRun(t) // first real execution → stamp first_run_at + compute the deadline (only for tasks with a timeout)
+1 -1
View File
@@ -88,7 +88,7 @@ func (s *Server) seedFindingWorkflowTools() {
// Upgrade only the original default binding. Customized lists and enabled
// flags survive; the one-time flag also preserves future user unbinding.
readers := `["worker","reporter"]`
for _, key := range []string{"traffic_search", "traffic_get", "traffic_blob"} {
for _, key := range []string{"traffic_search", "traffic_get", "traffic_blob", "traffic_messages"} {
if _, err := s.m.pg.Exec(`UPDATE tools SET agents=$2::jsonb WHERE key=$1 AND system AND (agents='["worker"]'::jsonb OR (agents @> '["worker","planner","mainagent","auto","pentest"]'::jsonb AND jsonb_array_length(agents)=5))`, key, readers); err != nil {
return
}
+20 -5
View File
@@ -24,6 +24,7 @@ import (
"github.com/Autumn-27/artex/guard"
"github.com/Autumn-27/artex/intercept"
"github.com/Autumn-27/artex/traffic"
"github.com/Autumn-27/artex/traffic/rewrite"
actool "github.com/Autumn-27/norma/tool"
)
@@ -161,6 +162,7 @@ func (t *Task) updateLifecycle(update func(*taskLifecycleState)) {
}
type taskLLMState struct {
Rotate bool
ProfileID *int64
ProfileIDs []int64
ActiveID *int64
@@ -181,6 +183,7 @@ func (t *Task) llmStateSnapshot() taskLLMState {
t.llmMu.RLock()
defer t.llmMu.RUnlock()
return taskLLMState{
Rotate: t.LLMRotate,
ProfileID: cloneInt64Ptr(t.LLMProfileID),
ProfileIDs: append(make([]int64, 0, len(t.LLMProfileIDs)), t.LLMProfileIDs...),
ActiveID: cloneInt64Ptr(t.ActiveLLMProfileID),
@@ -195,10 +198,12 @@ func (t *Task) llmStateSnapshot() taskLLMState {
// chain. The active head stays authoritative for every non-worker role and for
// failover ordering; rotation only distributes worker lanes across heads.
func (t *Task) nextRotatedProfile() (int64, bool) {
t.llmMu.RLock()
defer t.llmMu.RUnlock()
t.rotateMu.Lock()
defer t.rotateMu.Unlock()
ids := t.LLMProfileIDs
if len(ids) == 0 {
if !t.LLMRotate || len(ids) == 0 {
return 0, false
}
pick := ids[t.rotateNext%len(ids)]
@@ -206,12 +211,15 @@ func (t *Task) nextRotatedProfile() (int64, bool) {
return pick, true
}
func (t *Task) setLLMState(profileID, activeID *int64, profileIDs []int64, revision int64, state, reason string) bool {
func (t *Task) setLLMState(profileID, activeID *int64, profileIDs []int64, revision int64, state, reason string, rotate ...bool) bool {
t.llmMu.Lock()
defer t.llmMu.Unlock()
if revision < t.LLMChainRevision {
return false
}
if len(rotate) > 0 {
t.LLMRotate = rotate[0]
}
t.LLMProfileID = cloneInt64Ptr(profileID)
t.ActiveLLMProfileID = cloneInt64Ptr(activeID)
t.LLMProfileIDs = append(t.LLMProfileIDs[:0], profileIDs...)
@@ -253,6 +261,12 @@ type Manager struct {
enrich *enrich.Engine // engine-side asset auto-completion (DNS/HTTP)
interceptor *intercept.Interceptor // user-configured tool-call interception rules
// Capture-proxy rewrite rules: rewriteMu serializes reloads, rewriteSet is
// the snapshot last compiled, rewriteErrors outlives the reloads.
rewriteMu sync.Mutex
rewriteSet atomic.Pointer[rewrite.Set]
rewriteErrors rewrite.ErrorLog
companyMu sync.Mutex // serializes task/company-scope commits with live handle registration
// taskStateMu preserves commit order between PostgreSQL lifecycle writes and
// their in-memory mirrors. lifecycleMu makes snapshots race-free, but without
@@ -442,6 +456,7 @@ func NewManager(dir, proxyAddr string) (*Manager, error) {
}
if tr != nil {
m.traffic = tr
m.reloadRewriteRules() // in place before the proxy accepts a connection
go func() {
log.Printf("[traffic] recording proxy on %s (set HTTP_PROXY=%s + trust _ca CA)", proxyAddr, tr.ProxyAddr())
if err := tr.Start(); err != nil {
@@ -1132,12 +1147,12 @@ func (m *Manager) DeleteCompanyWithAssets(id int64, deleteAssets bool) (int64, e
// ReplaceTaskLLMProfiles resets a task's ordered provider chain and mirrors the
// committed state onto the live task handle. Terminal tasks are editable too —
// their main-agent conversation keeps running on the chain after the task finishes.
func (m *Manager) ReplaceTaskLLMProfiles(id string, profileIDs []int64, activeProfileID int64) (int64, error) {
func (m *Manager) ReplaceTaskLLMProfiles(id string, profileIDs []int64, activeProfileID int64, rotate ...*bool) (int64, error) {
n, err := strconv.ParseInt(id, 10, 64)
if err != nil {
return 0, err
}
if err := m.pg.ReplaceTaskLLMProfiles(n, profileIDs, activeProfileID); err != nil {
if err := m.pg.ReplaceTaskLLMProfiles(n, profileIDs, activeProfileID, rotate...); err != nil {
return 0, err
}
pt, err := m.pg.GetTask(n)
@@ -1146,7 +1161,7 @@ func (m *Manager) ReplaceTaskLLMProfiles(id string, profileIDs []int64, activePr
}
m.mu.Lock()
if task := m.tasks[id]; task != nil {
task.setLLMState(pt.LLMProfileID, pt.ActiveLLMProfileID, pt.LLMProfileIDs, pt.LLMChainRevision, pt.LLMFailoverState, pt.LLMFailoverReason)
task.setLLMState(pt.LLMProfileID, pt.ActiveLLMProfileID, pt.LLMProfileIDs, pt.LLMChainRevision, pt.LLMFailoverState, pt.LLMFailoverReason, pt.LLMRotate)
}
m.mu.Unlock()
// Terminal tasks do not reopen budget-blocked intents: no worker is running for
+137 -6
View File
@@ -2,6 +2,7 @@ package server
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
@@ -823,8 +824,10 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("DELETE /api/traffic/hosts", s.deleteTrafficHosts)
mux.HandleFunc("DELETE /api/traffic/all", s.deleteAllTraffic)
mux.HandleFunc("GET /api/traffic/exchange", s.getTrafficExchange)
mux.HandleFunc("GET /api/traffic/stream", s.getTrafficStream)
mux.HandleFunc("GET /api/traffic/blob", s.getTrafficBlob)
mux.HandleFunc("POST /api/traffic/ingest", s.ingestTraffic)
mux.HandleFunc("POST /api/traffic/replay", s.replayTraffic)
mux.HandleFunc("GET /api/commands", s.pgListCommands)
mux.HandleFunc("GET /api/commands/stats", s.pgToolStats) // aggregate call counts per tool
mux.HandleFunc("GET /api/llm/records", s.pgListLLMRecords)
@@ -950,6 +953,13 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("DELETE /api/intercept/rules/{id}", s.interceptDeleteRule)
mux.HandleFunc("POST /api/intercept/rules/{id}/toggle", s.interceptToggleRule)
// capture-proxy rewrite rule management (stored and compiled; not applied by the proxy yet)
mux.HandleFunc("GET /api/traffic/rewrite-rules", s.rewriteListRules)
mux.HandleFunc("POST /api/traffic/rewrite-rules", s.rewriteCreateRule)
mux.HandleFunc("PUT /api/traffic/rewrite-rules/{id}", s.rewriteUpdateRule)
mux.HandleFunc("DELETE /api/traffic/rewrite-rules/{id}", s.rewriteDeleteRule)
mux.HandleFunc("POST /api/traffic/rewrite-rules/{id}/toggle", s.rewriteToggleRule)
// asset interception rule management (global blocklist: domains/IPs/URLs/CIDRs)
mux.HandleFunc("GET /api/asset-intercept/rules", s.assetInterceptListRules)
mux.HandleFunc("POST /api/asset-intercept/rules", s.assetInterceptCreateRule)
@@ -1614,7 +1624,7 @@ type createTaskReq struct {
SourceTaskIDs []string `json:"source_task_ids,omitempty"` // direct, read-only inherited source tasks
CompanyIDs []int64 `json:"company_ids,omitempty"` // links the company scope and snapshots its current assets; does not copy assets or force intent generation
TimeoutSeconds int `json:"timeout_seconds"`
LLMRotate *bool `json:"llm_rotate"` // task-level timeout (seconds); 0/omitted = no limit
LLMRotate *bool `json:"llm_rotate"` // distribute worker intents across the ordered profiles
PlanHeartbeatSeconds int `json:"plan_heartbeat_seconds"` // planner heartbeat interval (seconds); 0/omitted = default 600 (10min); floor = default = 600, lower values are raised to 600
PlanQuietMode *bool `json:"plan_quiet_mode,omitempty"` // planner quiet-mode switch (#86); omitted = default on; false = edge rounds always plan
TargetGate *bool `json:"target_gate,omitempty"` // approval gate for asset targets, opt-in; omitted/null = off
@@ -1755,6 +1765,7 @@ func (s *Server) updateTaskLLMProfiles(w http.ResponseWriter, r *http.Request) {
var req struct {
LLMProfileIDs []int64 `json:"llm_profile_ids"`
ActiveLLMProfileID *int64 `json:"active_llm_profile_id"`
LLMRotate *bool `json:"llm_rotate"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeErr(w, 400, "bad json: "+err.Error())
@@ -1768,7 +1779,7 @@ func (s *Server) updateTaskLLMProfiles(w http.ResponseWriter, r *http.Request) {
if req.ActiveLLMProfileID != nil {
active = *req.ActiveLLMProfileID
}
reopened, err := s.m.ReplaceTaskLLMProfiles(t.ID, req.LLMProfileIDs, active)
reopened, err := s.m.ReplaceTaskLLMProfiles(t.ID, req.LLMProfileIDs, active, req.LLMRotate)
if err != nil {
writeErr(w, 400, err.Error())
return
@@ -1782,6 +1793,7 @@ func (s *Server) updateTaskLLMProfiles(w http.ResponseWriter, r *http.Request) {
llmState := t.llmStateSnapshot()
result := map[string]any{
"id": t.ID, "llm_profile_ids": llmState.ProfileIDs,
"llm_rotate": llmState.Rotate,
"active_llm_profile_id": llmState.ActiveID,
"llm_failover_state": llmState.FailoverState, "reopened_intents": reopened,
}
@@ -1932,8 +1944,10 @@ func (s *Server) getTask(w http.ResponseWriter, r *http.Request) {
log.Printf("[tasks] detail metrics query failed for %s — metrics zeroed: %v", t.ID, err)
}
s.applyTaskMetrics(&dto, t, metrics)
archiveBlockers, _ := s.m.PG().TaskArchiveBlockers()
applyTaskArchiveBlocker(&dto, archiveBlockers)
// Best effort: only this task's blocker, not the whole relation table.
taskID, _ := strconv.ParseInt(t.ID, 10, 64)
blocker, _ := s.m.PG().TaskArchiveBlocker(taskID)
applyTaskArchiveBlockerID(&dto, blocker)
writeJSON(w, 200, dto)
}
@@ -3449,7 +3463,103 @@ func (s *Server) getTrafficExchange(w http.ResponseWriter, r *http.Request) {
writeErr(w, 404, err.Error())
return
}
writeJSON(w, 200, map[string]any{"req": req, "resp": resp})
// A WebSocket or SSE exchange carries its messages separately
// (GET /api/traffic/stream); stream_kind is "" for an ordinary exchange.
count, kind, err := tr.StreamInfo(id)
if err != nil {
log.Printf("[traffic] stream info for %s: %v", id, err)
}
writeJSON(w, 200, map[string]any{"req": req, "resp": resp, "stream_count": count, "stream_kind": kind})
}
// getTrafficStream returns one page of a WebSocket/SSE exchange's messages in
// order (limit 1..200, default 50).
func (s *Server) getTrafficStream(w http.ResponseWriter, r *http.Request) {
tr := s.m.Traffic()
if tr == nil {
writeErr(w, 404, "traffic disabled")
return
}
q := r.URL.Query()
id := strings.TrimSpace(q.Get("exchange"))
if id == "" {
writeErr(w, 400, "missing exchange")
return
}
offset, _ := strconv.Atoi(q.Get("offset"))
limit, _ := strconv.Atoi(q.Get("limit"))
msgs, total, err := tr.StreamMessages(id, offset, limit)
if err != nil {
writeErr(w, 500, err.Error())
return
}
writeJSON(w, 200, map[string]any{"items": msgs, "total": total})
}
// maxReplayRequestBytes bounds the replay request body (JSON, with the payload
// possibly base64-encoded); traffic.Replay applies its own body limit after
// decoding.
const maxReplayRequestBytes = 16 << 20
// replayTraffic sends an edited request through the recording proxy and returns
// the response; the proxy records the exchange like any other.
//
// Operator-only: this is the equivalent of pointing a browser at the proxy, and
// no agent tool exposes it. If an agent tool is ever added, it must go through
// the target gate and the task's scope like every other outbound request.
func (s *Server) replayTraffic(w http.ResponseWriter, r *http.Request) {
tr := s.m.Traffic()
if tr == nil {
writeErr(w, 503, "traffic disabled")
return
}
r.Body = http.MaxBytesReader(w, r.Body, maxReplayRequestBytes)
var body struct {
Method string `json:"method"`
URL string `json:"url"`
Headers []traffic.ReplayHeader `json:"headers"`
Body string `json:"body"`
BodyBase64 bool `json:"body_base64"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
writeErr(w, 413, "replay request too large")
return
}
writeErr(w, 400, "bad json: "+err.Error())
return
}
payload := []byte(body.Body)
if body.BodyBase64 {
decoded, err := base64.StdEncoding.DecodeString(body.Body)
if err != nil {
writeErr(w, 400, "body is not valid base64: "+err.Error())
return
}
payload = decoded
}
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
defer cancel()
res, err := tr.Replay(ctx, traffic.ReplayRequest{Method: body.Method, URL: body.URL, Headers: body.Headers, Body: payload})
switch {
case errors.Is(err, traffic.ErrReplayTooLarge):
writeErr(w, 413, err.Error())
return
case errors.Is(err, traffic.ErrReplayInvalid):
writeErr(w, 400, err.Error())
return
case errors.Is(err, context.DeadlineExceeded):
writeErr(w, 504, "replay timed out")
return
case err != nil:
writeErr(w, 502, err.Error())
return
}
writeJSON(w, 200, map[string]any{
"status": res.Status, "headers": res.Headers, "raw": res.Raw, "duration_ms": res.DurationMS,
"truncated": res.Truncated, "recorded": res.Recorded,
})
}
// getTrafficBlob streams one oversized body by its sha256. Bodies past the inline
@@ -4118,14 +4228,35 @@ func (s *Server) getReport(w http.ResponseWriter, r *http.Request) {
counts[ty] = len(ns)
}
}
taskID, _ := strconv.ParseInt(t.ID, 10, 64)
var ledger *db.CoverageLedger
if t.CoverageEnabled {
taskID, _ := strconv.ParseInt(t.ID, 10, 64)
ledger, _ = s.m.Assets().CoverageLedger(taskID)
}
// The exploration node payload carries only name/summary/evidence/severity/
// vulnclass. Everything the report says about proof strength (poe level,
// verification state and note, counterevidence, confidence rationale, both
// evidence versions, the reporter-written body) lives on the standalone findings
// row, joined by node_id, and one report-only query fetches all of it. The
// findings list keeps its own lean meta query: folding report bodies into it
// would fetch multi-KB text for rows that never render it.
//
// A lookup failure is not an absence. The report is generated on demand and can
// be retried, so a failed read fails loudly instead of rendering every finding as
// "not assessed" — a database error must never become a claim about the finding.
quality := map[int64]db.FindingQuality{} // pg-less deployments have no standalone rows at all
if s.m.pg != nil {
var err error
if quality, err = s.m.pg.FindingQualityByNodeID(taskID); err != nil {
log.Printf("[report] task=%s finding quality: %v", t.ID, err)
writeErr(w, 500, "report unavailable: the finding quality lookup failed, retry")
return
}
}
md := report.Markdown(report.Input{
Title: t.Description, Goal: t.Goal, GeneratedAt: time.Now(),
AssetCounts: counts, Findings: findings, Coverage: ledger,
Quality: quality,
})
w.Header().Set("Content-Type", "text/markdown; charset=utf-8")
w.WriteHeader(200)
+2 -2
View File
@@ -206,7 +206,7 @@ func (r *taskLLMRuntime) currentFor(ctx context.Context) (taskLLMSelection, erro
}
sel.profileID = *pt.ActiveLLMProfileID
if pin, ok := llmProfilePinFrom(ctx); ok && pin != sel.profileID {
for _, id := range pt.LLMProfileIDs {
for _, id := range pt.ReadyLLMProfileIDs {
if id == pin {
sel.profileID = pin
break
@@ -794,7 +794,7 @@ func (s *Server) syncTaskLLMState(pt *db.Task) {
id := fmt.Sprintf("%d", pt.ID)
s.m.mu.Lock()
if task := s.m.tasks[id]; task != nil {
task.setLLMState(pt.LLMProfileID, pt.ActiveLLMProfileID, pt.LLMProfileIDs, pt.LLMChainRevision, pt.LLMFailoverState, pt.LLMFailoverReason)
task.setLLMState(pt.LLMProfileID, pt.ActiveLLMProfileID, pt.LLMProfileIDs, pt.LLMChainRevision, pt.LLMFailoverState, pt.LLMFailoverReason, pt.LLMRotate)
}
s.m.mu.Unlock()
}
+182 -1
View File
@@ -2,7 +2,15 @@ package server
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/norma/llm"
)
func TestLLMProfilePinRoundTrip(t *testing.T) {
@@ -17,8 +25,177 @@ func TestLLMProfilePinRoundTrip(t *testing.T) {
}
}
// Use two real HTTP provider adapters and a barrier: provider B must finish and
// another intent must start while provider A's first request is still blocked.
func TestRotationConcurrentProvidersAndPrecedence(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable: %v", err)
}
defer m.Close()
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
started := make(chan int, 3)
release := []chan struct{}{make(chan struct{}), make(chan struct{})}
ids := []int64{}
for i := 0; i < 2; i++ {
i := i
endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
started <- i
select {
case <-release[i]:
case <-r.Context().Done():
return
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"choices":[{"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}`)
}))
defer endpoint.Close()
id, err := m.pg.SaveProfile(&db.LLMProfile{Name: fmt.Sprintf("rotation-http-%d-%d", time.Now().UnixNano(), i), Format: "openai", BaseURL: endpoint.URL + "/v1", Model: "test", APIKey: "test-key"})
if err != nil {
t.Fatal(err)
}
ids = append(ids, id)
defer m.pg.DeleteProfile(id)
}
// Release blocked handlers even when an assertion fails.
defer func() {
for _, ch := range release {
select {
case <-ch:
default:
close(ch)
}
}
}()
on := true
task, err := m.CreateTaskWithOptions("rotation HTTP", "test", db.TaskCreateOptions{LLMProfileIDs: ids, LLMRotate: &on})
if err != nil {
t.Fatal(err)
}
defer m.DeleteTask(task.ID, DeleteTaskOptions{})
s := &Server{m: m, provByProfile: map[int64]*provEntry{}}
runtime := &taskLLMRuntime{s: s, taskID: task.ID, agentKey: "worker"}
done := make(chan error, 3)
dispatch := func(want int) {
t.Helper()
head, _ := task.nextRotatedProfile()
go func() {
_, _, _, err := runtime.Complete(withLLMProfilePin(ctx, head), llm.CompletionRequest{})
done <- err
}()
select {
case got := <-started:
if got != want {
t.Fatalf("provider=%d want=%d", got, want)
}
case <-ctx.Done():
t.Fatal("provider request did not start concurrently")
}
}
dispatch(0)
dispatch(1)
close(release[1])
select {
case err := <-done:
if err != nil {
t.Fatal(err)
}
case <-ctx.Done():
t.Fatal(ctx.Err())
}
dispatch(0)
close(release[0])
for i := 0; i < 2; i++ {
select {
case err := <-done:
if err != nil {
t.Fatal(err)
}
case <-ctx.Done():
t.Fatal(ctx.Err())
}
}
// An exhausted pin must fall back rather than retry the failed provider forever.
n, _ := parseTaskID(task.ID)
if _, err := m.pg.MarkTaskLLMProfileQuotaExhausted(n, ids[1], "insufficient_quota"); err != nil {
t.Fatal(err)
}
sel, err := runtime.currentFor(withLLMProfilePin(ctx, ids[1]))
if err != nil || sel.profileID != ids[0] {
t.Fatalf("exhausted pin selected: %+v, %v", sel, err)
}
if err := m.pg.SetAgentLLMProfile("worker", &ids[1]); err != nil {
t.Fatal(err)
}
defer m.pg.SetAgentLLMProfile("worker", nil)
sel, err = runtime.currentFor(withLLMProfilePin(ctx, ids[0]))
if err != nil || sel.profileID != 0 {
t.Fatalf("fixed binding did not outrank rotation: %+v, %v", sel, err)
}
off := false
if _, err := m.ReplaceTaskLLMProfiles(task.ID, []int64{ids[1], ids[0]}, 0, &off); err != nil {
t.Fatal(err)
}
if task.llmStateSnapshot().Rotate || taskDTO(task, "idle").LLMRotate {
t.Fatal("live task or API response retained disabled rotation")
}
if _, err := m.ReplaceTaskLLMProfiles(task.ID, ids, 0, &on); err != nil {
t.Fatal(err)
}
pt, err := m.pg.GetTask(n)
if err != nil {
t.Fatal(err)
}
reloaded := taskFromPG(pt, task.Store, m.interceptor)
if !reloaded.llmStateSnapshot().Rotate {
t.Fatal("task reload lost enabled rotation")
}
}
func TestRotationConcurrentDispatchAndUpdates(t *testing.T) {
task := &Task{LLMProfileIDs: []int64{1, 2}, LLMRotate: true}
const workers = 100
counts := make(chan int64, workers)
var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
id, ok := task.nextRotatedProfile()
if !ok {
t.Error("missing assignment")
return
}
counts <- id
}()
}
wg.Wait()
close(counts)
got := map[int64]int{}
for id := range counts {
got[id]++
}
if got[1] != workers/2 || got[2] != workers/2 {
t.Fatalf("unbalanced assignments: %v", got)
}
// Replace the chain while dispatching, exercising the shared list lock.
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < workers; i++ {
task.setLLMState(nil, nil, []int64{3, 4}, int64(i+1), "ready", "", i%2 == 0)
}
}()
for i := 0; i < workers; i++ {
task.nextRotatedProfile()
task.llmStateSnapshot()
}
wg.Wait()
}
func TestNextRotatedProfileRoundRobin(t *testing.T) {
tk := &Task{LLMProfileIDs: []int64{3, 1, 2}}
tk := &Task{LLMProfileIDs: []int64{3, 1, 2}, LLMRotate: true}
want := []int64{3, 1, 2, 3, 1}
for i, w := range want {
got, ok := tk.nextRotatedProfile()
@@ -29,4 +206,8 @@ func TestNextRotatedProfileRoundRobin(t *testing.T) {
if _, ok := (&Task{}).nextRotatedProfile(); ok {
t.Fatal("empty chain must not rotate")
}
tk.setLLMState(nil, nil, []int64{3, 1, 2}, 1, "ready", "", false)
if _, ok := tk.nextRotatedProfile(); ok {
t.Fatal("disabled rotation must not assign a pin")
}
}
+12
View File
@@ -184,6 +184,18 @@ func TestApplyTaskArchiveBlocker(t *testing.T) {
if !strings.Contains(string(raw), `"archive_blocked_by_task_id":"11"`) {
t.Fatalf("archive blocker missing from payload: %s", raw)
}
byID := taskDTO(&Task{ID: "7"}, "paused")
applyTaskArchiveBlockerID(&byID, 12)
if byID.ArchiveBlockedBy != "12" {
t.Fatalf("archive blocker by id=%q, want 12", byID.ArchiveBlockedBy)
}
none := taskDTO(&Task{ID: "7"}, "paused")
applyTaskArchiveBlockerID(&none, 0)
if none.ArchiveBlockedBy != "" {
t.Fatalf("no blocker set %q", none.ArchiveBlockedBy)
}
applyTaskArchiveBlockerID(nil, 12) // must not panic
}
func TestTaskLLMStateSnapshotIsConcurrentAndDetached(t *testing.T) {
+163
View File
@@ -0,0 +1,163 @@
package server
import (
"context"
"encoding/base64"
"encoding/json"
"io"
"net"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"time"
"github.com/Autumn-27/artex/traffic"
)
// replayServer wires a Server like ingestServer. With traffic off,
// s.m.Traffic() is nil. With trafficOn but not live, the store is open and the
// proxy is not listening. With live, NewManager opens the store and starts the
// recording proxy on a free loopback port, as in production; the proxy has no
// exported stop, so it keeps listening until the test binary exits.
func replayServer(t *testing.T, trafficOn, live bool) (*Server, func(body string) *httptest.ResponseRecorder) {
t.Helper()
skipWithoutDSN(t)
proxyAddr := ""
if live {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
proxyAddr = ln.Addr().String()
ln.Close()
}
m, err := NewManager(t.TempDir(), proxyAddr)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { m.Close() })
if live {
if m.traffic == nil {
t.Fatal("NewManager did not open the traffic store")
}
deadline := time.Now().Add(5 * time.Second)
for {
c, err := net.Dial("tcp", proxyAddr)
if err == nil {
c.Close()
break
}
if time.Now().After(deadline) {
t.Fatalf("proxy did not listen on %s: %v", proxyAddr, err)
}
time.Sleep(10 * time.Millisecond)
}
} else if trafficOn {
if m.traffic, err = traffic.Open(filepath.Join(m.dir, "traffic"), ":0"); err != nil {
t.Fatal(err)
}
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
s.archiveWG.Wait()
<-s.side.done
token, err := signJWT(s.jwtKey)
if err != nil {
t.Fatal(err)
}
return s, func(body string) *httptest.ResponseRecorder {
r := httptest.NewRequest("POST", "/api/traffic/replay", strings.NewReader(body))
r.Header.Set("Authorization", "Bearer "+token)
r.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
s.Handler().ServeHTTP(w, r)
return w
}
}
func TestTrafficReplayRouteTrafficOff(t *testing.T) {
_, replay := replayServer(t, false, false)
if w := replay(`{"method":"GET","url":"http://example.com/"}`); w.Code != 503 {
t.Fatalf("traffic off = %d %s, want 503", w.Code, w.Body)
}
}
func TestTrafficReplayRouteValidation(t *testing.T) {
_, replay := replayServer(t, true, false)
for name, c := range map[string]struct {
body string
want int
}{
"bad json": {`{"method":`, 400},
"CONNECT": {`{"method":"CONNECT","url":"http://example.com/"}`, 400},
"ftp url": {`{"method":"GET","url":"ftp://example.com/x"}`, 400},
"no host": {`{"method":"GET","url":"http:///x"}`, 400},
"bad base64": {`{"method":"POST","url":"http://example.com/","body":"%%%","body_base64":true}`, 400},
"header CRLF": {`{"method":"GET","url":"http://example.com/","headers":[{"name":"X-A","value":"a\r\nb"}]}`, 400},
"too large": {`{"method":"POST","url":"http://example.com/","body":"` + strings.Repeat("a", 17<<20) + `"}`, 413},
"proxy down": {`{"method":"GET","url":"http://127.0.0.1:1/"}`, 502},
"body too big": {`{"method":"POST","url":"http://example.com/","body":"` + base64.StdEncoding.EncodeToString(make([]byte, 9<<20)) + `","body_base64":true}`, 413},
} {
if w := replay(c.body); w.Code != c.want {
t.Errorf("%s: %d %s, want %d", name, w.Code, strings.TrimSpace(w.Body.String()), c.want)
}
}
}
func TestTrafficReplayRouteEndToEnd(t *testing.T) {
s, replay := replayServer(t, true, true)
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b, _ := io.ReadAll(r.Body)
w.Header().Set("X-Seen", r.Method+" "+r.Header.Get("X-Custom"))
_, _ = w.Write(append([]byte("got:"), b...))
}))
t.Cleanup(target.Close)
url := target.URL + "/replayed?x=1"
req, _ := json.Marshal(map[string]any{
"method": "PUT", "url": url,
"headers": []map[string]string{{"name": "X-Custom", "value": "edited"}},
"body": base64.StdEncoding.EncodeToString([]byte("payload")), "body_base64": true,
})
w := replay(string(req))
if w.Code != 200 {
t.Fatalf("replay = %d %s", w.Code, w.Body)
}
var res struct {
Status int `json:"status"`
Headers []traffic.ReplayHeader `json:"headers"`
Raw string `json:"raw"`
DurationMS int64 `json:"duration_ms"`
Truncated bool `json:"truncated"`
Recorded bool `json:"recorded"`
}
if err := json.Unmarshal(w.Body.Bytes(), &res); err != nil {
t.Fatal(err)
}
if res.Status != 200 || res.Truncated || !res.Recorded || !strings.Contains(res.Raw, "X-Seen: PUT edited") || !strings.HasSuffix(res.Raw, "got:payload") {
t.Fatalf("result %+v", res)
}
seen := false
for _, h := range res.Headers {
seen = seen || (h.Name == "X-Seen" && h.Value == "PUT edited")
}
if !seen {
t.Fatalf("headers %+v lack X-Seen", res.Headers)
}
deadline := time.Now().Add(5 * time.Second)
for {
var n int
if err := s.m.Traffic().DB().QueryRow(`SELECT COUNT(*) FROM exchanges WHERE method='PUT' AND url=?`, url).Scan(&n); err != nil {
t.Fatal(err)
}
if n == 1 {
break
}
if time.Now().After(deadline) {
t.Fatalf("replayed exchange not recorded (%d rows)", n)
}
time.Sleep(20 * time.Millisecond)
}
}
+337
View File
@@ -0,0 +1,337 @@
package server
import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"strings"
"time"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/traffic/rewrite"
)
// --- capture-proxy rewrite rules ---
//
// Global rules, admin only, never an agent tool. This layer stores, validates
// and compiles them and hands the compiled snapshot to the proxy; the proxy
// does not apply rewrite rules yet.
// rewriteActionDTO is an action on the wire. The value of a sensitive action is
// never returned: a read leaves Value out and sets ValueSet. On update, a
// sensitive action sent without a value keeps the stored one, provided the
// stored rule has exactly one sensitive action of the same type, canonical
// header name and regex, wherever it stands; none or several is refused.
type rewriteActionDTO struct {
Type string `json:"type"`
Name string `json:"name,omitempty"`
Regex string `json:"regex,omitempty"`
Value *string `json:"value,omitempty"`
Sensitive bool `json:"sensitive,omitempty"`
ValueSet bool `json:"value_set,omitempty"`
}
type rewriteRuleReq struct {
Name string `json:"name"`
Enabled bool `json:"enabled"`
Phase rewrite.Phase `json:"phase"`
Priority int `json:"priority"`
Matchers rewrite.Matchers `json:"matchers"`
Actions []rewriteActionDTO `json:"actions"`
Note string `json:"note"`
}
type rewriteRuleDTO struct {
ID int64 `json:"id"`
Name string `json:"name"`
Enabled bool `json:"enabled"`
Phase string `json:"phase"`
Priority int `json:"priority"`
Matchers rewrite.Matchers `json:"matchers"`
Actions []rewriteActionDTO `json:"actions"`
Note string `json:"note"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
// LoadError is why an enabled rule is left out of the running snapshot.
LoadError string `json:"load_error,omitempty"`
// LastError is the rule's most recent run-time error, kept in memory only.
LastError *rewrite.LastError `json:"last_error,omitempty"`
}
// errRewriteInvalid marks a rule the validation refuses (HTTP 400).
var errRewriteInvalid = errors.New("invalid rewrite rule")
const rewriteUndecodable = "the stored rule cannot be decoded"
// decodeRewriteRule turns a stored row into the engine's rule.
func decodeRewriteRule(row db.TrafficRewriteRule) (rewrite.Rule, error) {
r := rewrite.Rule{ID: row.ID, Name: row.Name, Enabled: row.Enabled, Phase: rewrite.Phase(row.Phase), Priority: row.Priority, Note: row.Note}
if err := json.Unmarshal(row.Matchers, &r.Matchers); err != nil {
return r, err
}
return r, json.Unmarshal(row.Actions, &r.Actions)
}
func encodeRewriteRule(r rewrite.Rule) db.TrafficRewriteRule {
matchers, _ := json.Marshal(r.Matchers)
actions, _ := json.Marshal(r.Actions)
return db.TrafficRewriteRule{Name: r.Name, Enabled: r.Enabled, Phase: string(r.Phase), Priority: r.Priority, Matchers: matchers, Actions: actions, Note: r.Note}
}
// rule builds the rule a request describes. stored is the rule being updated,
// nil on create: it supplies the sensitive values the request left out.
//
// A stored sensitive value is not written to a new destination: when one is
// kept, the host matcher must not change (compared case-insensitively). The
// url_regex matcher is not compared, an accepted limit: a kept value can follow
// a rule whose url_regex changes on the same host.
func (req rewriteRuleReq) rule(stored *rewrite.Rule) (rewrite.Rule, error) {
r := rewrite.Rule{Name: req.Name, Enabled: req.Enabled, Phase: req.Phase, Priority: req.Priority, Matchers: req.Matchers, Note: req.Note}
kept := false
for i, a := range req.Actions {
out := rewrite.Action{Type: a.Type, Name: a.Name, Regex: a.Regex, Sensitive: a.Sensitive}
switch {
case a.Value != nil:
out.Value = *a.Value
case a.Sensitive && a.Type != rewrite.ActionDeleteHeader:
if stored == nil {
return r, fmt.Errorf("action %d (%s): a sensitive action needs a value", i+1, a.Type)
}
var match []rewrite.Action
for _, old := range stored.Actions {
if old.Sensitive && old.Type == a.Type && http.CanonicalHeaderKey(old.Name) == http.CanonicalHeaderKey(a.Name) && old.Regex == a.Regex {
match = append(match, old)
}
}
if len(match) != 1 {
return r, fmt.Errorf("action %d (%s): %d stored sensitive actions match its type, header name and regex, exactly one is needed to keep a value: send the value", i+1, a.Type, len(match))
}
out.Value = match[0].Value
kept = true
}
r.Actions = append(r.Actions, out)
}
if kept && !strings.EqualFold(stored.Matchers.Host, req.Matchers.Host) {
return r, errors.New("destination changed (host matcher) while a stored sensitive value was kept: send the value again or clear it; stored secrets are not written to a new destination")
}
return r, nil
}
// rewriteRuleDTO renders a stored row without the value of any sensitive action.
func (m *Manager) rewriteRuleDTO(row db.TrafficRewriteRule) rewriteRuleDTO {
out := rewriteRuleDTO{ID: row.ID, Name: row.Name, Enabled: row.Enabled, Phase: row.Phase, Priority: row.Priority,
Actions: []rewriteActionDTO{}, Note: row.Note, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt}
r, err := decodeRewriteRule(row)
if err != nil {
out.LoadError = rewriteUndecodable
return out
}
out.Matchers = r.Matchers
for _, a := range r.Actions {
d := rewriteActionDTO{Type: a.Type, Name: a.Name, Regex: a.Regex, Sensitive: a.Sensitive}
if a.Sensitive {
d.ValueSet = a.Type != rewrite.ActionDeleteHeader
} else {
v := a.Value
d.Value = &v
}
out.Actions = append(out.Actions, d)
}
out.LoadError = m.rewriteSet.Load().LoadErrors()[row.ID]
if e, ok := m.rewriteErrors.Get(row.ID); ok {
out.LastError = &e
}
return out
}
// reloadRewriteRules compiles the stored rules and publishes the snapshot to
// the proxy. It runs at startup once the traffic store is open, and after every
// rule change. On a read failure the previous snapshot stays.
func (m *Manager) reloadRewriteRules() {
m.rewriteMu.Lock()
defer m.rewriteMu.Unlock()
rows, err := m.pg.ListTrafficRewriteRules()
if err != nil {
log.Printf("[traffic] rewrite rules not reloaded: %v", err)
return
}
rules := make([]rewrite.Rule, 0, len(rows))
for _, row := range rows {
r, err := decodeRewriteRule(row)
if err != nil {
log.Printf("[traffic] rewrite rule %d not loaded: %s", row.ID, rewriteUndecodable)
continue
}
rules = append(rules, r)
}
set := rewrite.Compile(rules, rewrite.Options{TraceKey: db.TrafficTraceKey(), Errors: &m.rewriteErrors})
for id, reason := range set.LoadErrors() {
log.Printf("[traffic] rewrite rule %d not loaded: %s", id, reason)
}
m.rewriteSet.Store(set)
if m.traffic != nil {
m.traffic.SetRewriteRules(set)
}
}
// rewriteWriteErr answers a failed rule write.
func rewriteWriteErr(w http.ResponseWriter, err error) {
switch {
case errors.Is(err, sql.ErrNoRows):
writeErr(w, 404, "rewrite rule not found")
case errors.Is(err, errRewriteInvalid), errors.Is(err, db.ErrTrafficRewriteRuleLimit), errors.Is(err, db.ErrTrafficRewriteEnabledLimit):
writeErr(w, 400, err.Error())
default:
writeErr(w, 500, err.Error())
}
}
func rewriteInvalid(err error) error { return fmt.Errorf("%w: %v", errRewriteInvalid, err) }
func (s *Server) rewriteListRules(w http.ResponseWriter, r *http.Request) {
pg := s.pg(w)
if pg == nil {
return
}
rows, err := pg.ListTrafficRewriteRules()
if err != nil {
writeErr(w, 500, err.Error())
return
}
rules := make([]rewriteRuleDTO, 0, len(rows))
for _, row := range rows {
rules = append(rules, s.m.rewriteRuleDTO(row))
}
writeJSON(w, 200, map[string]any{"rules": rules})
}
// rewriteCreateRule creates a rule disabled, whatever the request says: a rule
// is enabled by the toggle or by an update.
func (s *Server) rewriteCreateRule(w http.ResponseWriter, r *http.Request) {
pg := s.pg(w)
if pg == nil {
return
}
var req rewriteRuleReq
if err := decode(r, &req); err != nil {
writeErr(w, 400, err.Error())
return
}
rule, err := req.rule(nil)
if err == nil {
rule.Enabled = false
err = rewrite.Validate(rule)
}
if err != nil {
rewriteWriteErr(w, rewriteInvalid(err))
return
}
row, err := pg.CreateTrafficRewriteRule(encodeRewriteRule(rule))
if err != nil {
rewriteWriteErr(w, err)
return
}
s.m.reloadRewriteRules()
writeJSON(w, 200, s.m.rewriteRuleDTO(row))
}
func (s *Server) rewriteUpdateRule(w http.ResponseWriter, r *http.Request) {
pg := s.pg(w)
if pg == nil {
return
}
id, ok := pathInt(r, "id")
if !ok {
writeErr(w, 400, "bad rule id")
return
}
var req rewriteRuleReq
if err := decode(r, &req); err != nil {
writeErr(w, 400, err.Error())
return
}
row, err := pg.UpdateTrafficRewriteRule(id, func(stored db.TrafficRewriteRule) (db.TrafficRewriteRule, error) {
var keep *rewrite.Rule
if old, err := decodeRewriteRule(stored); err == nil {
keep = &old
}
rule, err := req.rule(keep)
if err == nil {
err = rewrite.Validate(rule)
}
if err != nil {
return stored, rewriteInvalid(err)
}
return encodeRewriteRule(rule), nil
})
if err != nil {
rewriteWriteErr(w, err)
return
}
s.m.rewriteErrors.Forget(id)
s.m.reloadRewriteRules()
writeJSON(w, 200, s.m.rewriteRuleDTO(row))
}
func (s *Server) rewriteDeleteRule(w http.ResponseWriter, r *http.Request) {
pg := s.pg(w)
if pg == nil {
return
}
id, ok := pathInt(r, "id")
if !ok {
writeErr(w, 400, "bad rule id")
return
}
if err := pg.DeleteTrafficRewriteRule(id); err != nil {
rewriteWriteErr(w, err)
return
}
s.m.rewriteErrors.Forget(id)
s.m.reloadRewriteRules()
writeJSON(w, 200, map[string]any{"deleted": id})
}
func (s *Server) rewriteToggleRule(w http.ResponseWriter, r *http.Request) {
pg := s.pg(w)
if pg == nil {
return
}
id, ok := pathInt(r, "id")
if !ok {
writeErr(w, 400, "bad rule id")
return
}
var req struct {
Enabled bool `json:"enabled"`
}
if err := decode(r, &req); err != nil {
writeErr(w, 400, err.Error())
return
}
if req.Enabled {
// A rule saved under older validation rules is checked again before it runs.
stored, err := pg.GetTrafficRewriteRule(id)
if err != nil {
rewriteWriteErr(w, err)
return
}
rule, err := decodeRewriteRule(stored)
if err == nil {
err = rewrite.Validate(rule)
}
if err != nil {
rewriteWriteErr(w, rewriteInvalid(err))
return
}
}
if err := pg.ToggleTrafficRewriteRule(id, req.Enabled); err != nil {
rewriteWriteErr(w, err)
return
}
s.m.reloadRewriteRules()
writeJSON(w, 200, map[string]any{"ok": true, "enabled": req.Enabled})
}
+492
View File
@@ -0,0 +1,492 @@
package server
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"net"
"net/http"
"net/http/httptest"
"net/url"
"path/filepath"
"strings"
"testing"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/traffic"
"github.com/Autumn-27/artex/traffic/rewrite"
)
const rewriteAPI = "/api/traffic/rewrite-rules"
// cleanRewriteRules removes the rules of these tests, now and at the end, and
// skips when the table holds other rules: the limit test counts rows.
func cleanRewriteRules(t *testing.T, pg *db.DB) {
t.Helper()
clean := func() { pg.Exec(`DELETE FROM traffic_rewrite_rules WHERE name LIKE 't-rwapi-%'`) }
clean()
t.Cleanup(clean)
var others int
if err := pg.QueryRow(`SELECT count(*) FROM traffic_rewrite_rules`).Scan(&others); err != nil {
t.Fatal(err)
}
if others > 0 {
t.Skipf("traffic_rewrite_rules holds %d rule(s) that are not test fixtures — skipping", others)
}
}
// rewriteServer is a server with an open traffic store whose proxy does not
// listen. request sends an authenticated call when auth is true.
func rewriteServer(t *testing.T) (*Server, func(auth bool, method, path, body string) *httptest.ResponseRecorder) {
t.Helper()
skipWithoutDSN(t)
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { m.Close() })
cleanRewriteRules(t, m.pg)
if m.traffic, err = traffic.Open(filepath.Join(m.dir, "traffic"), ":0"); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
s.archiveWG.Wait()
<-s.side.done
token, err := signJWT(s.jwtKey)
if err != nil {
t.Fatal(err)
}
return s, func(auth bool, method, path, body string) *httptest.ResponseRecorder {
t.Helper()
r := httptest.NewRequest(method, path, strings.NewReader(body))
if auth {
r.Header.Set("Authorization", "Bearer "+token)
}
r.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
s.Handler().ServeHTTP(w, r)
return w
}
}
type rewriteRuleJSON struct {
ID int64
Name string
Enabled bool
Phase string
Priority int
Matchers rewrite.Matchers
Actions []struct {
Type, Name, Regex string
Value *string
Sensitive bool
ValueSet bool `json:"value_set"`
}
Note string
LoadError string `json:"load_error"`
}
func decodeRewriteRuleJSON(t *testing.T, resp *httptest.ResponseRecorder) rewriteRuleJSON {
t.Helper()
var r rewriteRuleJSON
if resp.Code != 200 {
t.Fatalf("HTTP %d %s", resp.Code, resp.Body.String())
}
if err := json.Unmarshal(resp.Body.Bytes(), &r); err != nil {
t.Fatal(err)
}
return r
}
func listRewriteRules(t *testing.T, request func(bool, string, string, string) *httptest.ResponseRecorder) (rules []rewriteRuleJSON, raw string) {
t.Helper()
resp := request(true, "GET", rewriteAPI, "")
var out struct{ Rules []rewriteRuleJSON }
if resp.Code != 200 || json.Unmarshal(resp.Body.Bytes(), &out) != nil || out.Rules == nil {
t.Fatalf("list: HTTP %d %s", resp.Code, resp.Body.String())
}
return out.Rules, resp.Body.String()
}
// sentToTarget applies the request phase of the proxy's snapshot to a request:
// what the stored rules would do, read from the snapshot itself.
func sentToTarget(s *Server) rewrite.Request {
u, _ := url.Parse("http://api.example/v1")
return s.m.traffic.RewriteRules().Apply(rewrite.PhaseRequest, rewrite.Input{Request: rewrite.Request{Method: "GET", URL: u, Header: http.Header{}}}).Request
}
func TestRewriteRulesRequireAuth(t *testing.T) {
_, request := rewriteServer(t)
for _, c := range [][2]string{{"GET", rewriteAPI}, {"POST", rewriteAPI}, {"PUT", rewriteAPI + "/1"}, {"DELETE", rewriteAPI + "/1"}, {"POST", rewriteAPI + "/1/toggle"}} {
if resp := request(false, c[0], c[1], `{}`); resp.Code != 401 {
t.Errorf("%s %s without a token: HTTP %d, want 401", c[0], c[1], resp.Code)
}
}
if rules, _ := listRewriteRules(t, request); len(rules) != 0 {
t.Fatalf("an unauthenticated call created a rule: %+v", rules)
}
}
func TestRewriteRulesCRUD(t *testing.T) {
s, request := rewriteServer(t)
pg := s.m.pg
const secret, secret2 = "Bearer rwapi-plain-0001", "Bearer rwapi-plain-0002"
body := func(name, auth string) string {
return `{"name":"` + name + `","enabled":true,"phase":"request","priority":3,"note":"n",
"matchers":{"host":"*.example","methods":["GET"]},
"actions":[{"type":"set_header","name":"Authorization",` + auth + `"sensitive":true},{"type":"set_header","name":"X-Plain","value":"visible"}]}`
}
if s.m.traffic.RewriteRules() != nil {
t.Fatal("a snapshot exists before any rule change on this fixture")
}
created := request(true, "POST", rewriteAPI, body("t-rwapi-crud", `"value":"`+secret+`",`))
r := decodeRewriteRuleJSON(t, created)
if r.ID == 0 || r.Name != "t-rwapi-crud" || r.Phase != "request" || r.Priority != 3 || r.Note != "n" || r.Matchers.Host != "*.example" {
t.Fatalf("created rule = %+v", r)
}
if r.Enabled {
t.Fatal("a rule must be created disabled, whatever the request says")
}
if a := r.Actions; len(a) != 2 || a[0].Value != nil || !a[0].ValueSet || !a[0].Sensitive || a[1].Value == nil || *a[1].Value != "visible" || a[1].ValueSet {
t.Fatalf("actions = %+v", a)
}
_, listed := listRewriteRules(t, request)
if strings.Contains(created.Body.String()+listed, "rwapi-plain") {
t.Fatal("the sensitive value was returned")
}
path := fmt.Sprintf("%s/%d", rewriteAPI, r.ID)
// Both columns are sealed at rest.
var matchers, actions string
if err := pg.QueryRow(`SELECT matchers::text, actions::text FROM traffic_rewrite_rules WHERE id=$1`, r.ID).Scan(&matchers, &actions); err != nil {
t.Fatal(err)
}
for _, col := range []string{matchers, actions} {
if !strings.Contains(col, `"__artex_enc"`) || strings.Contains(col, "rwapi-plain") || strings.Contains(col, "example") {
t.Fatalf("column not sealed at rest: %s", col)
}
}
// Every change republishes the snapshot. A disabled rule is not in it.
set := s.m.traffic.RewriteRules()
if set == nil || set.Len(rewrite.PhaseRequest) != 0 {
t.Fatalf("after create: the snapshot must exist and hold no rule")
}
if resp := request(true, "POST", path+"/toggle", `{"enabled":true}`); resp.Code != 200 {
t.Fatalf("toggle: HTTP %d %s", resp.Code, resp.Body.String())
}
if s.m.traffic.RewriteRules() == set || s.m.traffic.RewriteRules().Len(rewrite.PhaseRequest) != 1 {
t.Fatal("after toggle: the snapshot must be replaced and hold the rule")
}
if got := sentToTarget(s).Header.Get("Authorization"); got != secret {
t.Fatalf("the snapshot holds Authorization=%q, want the stored value", got)
}
// Update without the sensitive value: the stored one is kept.
updated := request(true, "PUT", path, body("t-rwapi-crud2", ""))
if r2 := decodeRewriteRuleJSON(t, updated); r2.Name != "t-rwapi-crud2" || !r2.Enabled || !r2.Actions[0].ValueSet || r2.Actions[0].Value != nil {
t.Fatalf("updated rule = %+v", r2)
}
if strings.Contains(updated.Body.String(), "rwapi-plain") {
t.Fatal("the sensitive value was returned by the update")
}
if got := sentToTarget(s).Header.Get("Authorization"); got != secret {
t.Fatalf("after an update without value: Authorization=%q, want the stored value kept", got)
}
// Update with a value: replaced. An explicit empty value is a value.
decodeRewriteRuleJSON(t, request(true, "PUT", path, body("t-rwapi-crud2", `"value":"`+secret2+`",`)))
if got := sentToTarget(s).Header.Get("Authorization"); got != secret2 {
t.Fatalf("after an update with a value: Authorization=%q", got)
}
decodeRewriteRuleJSON(t, request(true, "PUT", path, body("t-rwapi-crud2", `"value":"",`)))
if h := sentToTarget(s).Header; h.Get("Authorization") != "" || len(h.Values("Authorization")) != 1 {
t.Fatalf("after an update with an empty value: %v", h)
}
decodeRewriteRuleJSON(t, request(true, "PUT", path, body("t-rwapi-crud2", `"value":"`+secret2+`",`)))
// A kept value never moves to another action, and never comes back in clear.
for name, bad := range map[string]string{
"other type": `[{"type":"set_body","sensitive":true}]`,
"other header": `[{"type":"set_header","name":"X-Other","sensitive":true}]`,
} {
resp := request(true, "PUT", path, `{"name":"t-rwapi-crud2","phase":"request","actions":`+bad+`}`)
if resp.Code != 400 || strings.Contains(resp.Body.String(), "rwapi-plain") {
t.Fatalf("%s: HTTP %d %s, want 400", name, resp.Code, resp.Body.String())
}
}
unmasked := request(true, "PUT", path, `{"name":"t-rwapi-crud2","enabled":true,"phase":"request","actions":[{"type":"set_header","name":"Authorization"}]}`)
if r3 := decodeRewriteRuleJSON(t, unmasked); r3.Actions[0].Value == nil || *r3.Actions[0].Value != "" || strings.Contains(unmasked.Body.String(), "rwapi-plain") {
t.Fatalf("dropping the sensitive flag must not reveal the stored value: %s", unmasked.Body.String())
}
if stored, err := pg.GetTrafficRewriteRule(r.ID); err != nil || strings.Contains(string(stored.Actions), "rwapi-plain") {
t.Fatalf("the stored value must be gone once the action is saved in clear: %s (%v)", stored.Actions, err)
}
// Unknown rule: 404. Bad id: 400.
for _, c := range [][3]string{{"PUT", rewriteAPI + "/999999999", body("t-rwapi-x", `"value":"v",`)},
{"POST", rewriteAPI + "/999999999/toggle", `{"enabled":true}`}, {"POST", rewriteAPI + "/999999999/toggle", `{"enabled":false}`},
{"DELETE", rewriteAPI + "/999999999", ""}} {
if resp := request(true, c[0], c[1], c[2]); resp.Code != 404 {
t.Errorf("%s %s: HTTP %d, want 404", c[0], c[1], resp.Code)
}
}
if resp := request(true, "DELETE", rewriteAPI+"/abc", ""); resp.Code != 400 {
t.Errorf("bad id: HTTP %d, want 400", resp.Code)
}
if resp := request(true, "DELETE", path, ""); resp.Code != 200 {
t.Fatalf("delete: HTTP %d %s", resp.Code, resp.Body.String())
}
if rules, _ := listRewriteRules(t, request); len(rules) != 0 || s.m.traffic.RewriteRules().Len(rewrite.PhaseRequest) != 0 {
t.Fatalf("after delete: %d rule(s) listed, %d in the snapshot", len(rules), s.m.traffic.RewriteRules().Len(rewrite.PhaseRequest))
}
}
// An update that keeps a stored sensitive value cannot change the host matcher:
// the value is not written to a new destination.
func TestRewriteRulesKeptValueHostChange400(t *testing.T) {
s, request := rewriteServer(t)
const secret = "Bearer rwapi-plain-host"
body := func(host, value string) string {
return `{"name":"t-rwapi-host","enabled":true,"phase":"request","matchers":{"host":"` + host + `"},
"actions":[{"type":"set_header","name":"Authorization",` + value + `"sensitive":true}]}`
}
r := decodeRewriteRuleJSON(t, request(true, "POST", rewriteAPI, body("api.example", `"value":"`+secret+`",`)))
path := fmt.Sprintf("%s/%d", rewriteAPI, r.ID)
if resp := request(true, "POST", path+"/toggle", `{"enabled":true}`); resp.Code != 200 {
t.Fatalf("toggle: HTTP %d %s", resp.Code, resp.Body.String())
}
for _, host := range []string{"evil.example", "*.example", ""} {
resp := request(true, "PUT", path, body(host, ""))
if resp.Code != 400 || !strings.Contains(resp.Body.String(), "destination changed (host matcher)") || strings.Contains(resp.Body.String(), "rwapi-plain") {
t.Fatalf("host %q with the stored value kept: HTTP %d %s, want 400", host, resp.Code, resp.Body.String())
}
}
if stored, _ := listRewriteRules(t, request); len(stored) != 1 || stored[0].Matchers.Host != "api.example" {
t.Fatalf("a refused update changed the rule: %+v", stored)
}
if got := sentToTarget(s).Header.Get("Authorization"); got != secret {
t.Fatalf("after the refused updates: Authorization=%q, want the stored value", got)
}
// Same host in another case: kept. A new host with the value sent again: accepted.
decodeRewriteRuleJSON(t, request(true, "PUT", path, body("API.Example", "")))
if got := sentToTarget(s).Header.Get("Authorization"); got != secret {
t.Fatalf("host in another case: Authorization=%q, want the stored value kept", got)
}
if r2 := decodeRewriteRuleJSON(t, request(true, "PUT", path, body("evil.example", `"value":"other",`))); r2.Matchers.Host != "evil.example" {
t.Fatalf("new host with the value sent again: %+v", r2)
}
}
// A kept value is matched on the action's type, header name and regex, not on
// its position: reordering two sensitive actions keeps each value on its own
// action.
func TestRewriteRulesKeptValuesFollowReorder(t *testing.T) {
s, request := rewriteServer(t)
const auth, key = "Bearer rwapi-plain-auth", "rwapi-plain-key"
r := decodeRewriteRuleJSON(t, request(true, "POST", rewriteAPI, `{"name":"t-rwapi-order","phase":"request","matchers":{"host":"api.example"},
"actions":[{"type":"set_header","name":"Authorization","value":"`+auth+`","sensitive":true},{"type":"set_header","name":"X-Api-Key","value":"`+key+`","sensitive":true}]}`))
path := fmt.Sprintf("%s/%d", rewriteAPI, r.ID)
reordered := request(true, "PUT", path, `{"name":"t-rwapi-order","enabled":true,"phase":"request","matchers":{"host":"api.example"},
"actions":[{"type":"set_header","name":"x-api-key","sensitive":true},{"type":"set_header","name":"Authorization","sensitive":true}]}`)
r2 := decodeRewriteRuleJSON(t, reordered)
if len(r2.Actions) != 2 || r2.Actions[0].Name != "x-api-key" || r2.Actions[1].Name != "Authorization" || !r2.Actions[0].ValueSet || !r2.Actions[1].ValueSet {
t.Fatalf("reordered rule = %+v", r2)
}
if strings.Contains(reordered.Body.String(), "rwapi-plain") {
t.Fatal("the sensitive values were returned by the update")
}
if h := sentToTarget(s).Header; h.Get("Authorization") != auth || h.Get("X-Api-Key") != key {
t.Fatalf("after the reorder: Authorization=%q X-Api-Key=%q, want each value on its own action", h.Get("Authorization"), h.Get("X-Api-Key"))
}
}
// Two stored sensitive actions of the same type, header name and regex: an
// action sent without its value cannot be told apart, the update is refused.
func TestRewriteRulesKeptValueAmbiguous400(t *testing.T) {
s, request := rewriteServer(t)
action := func(value string) string {
return `{"type":"replace_header","name":"Authorization","regex":"^Bearer (.*)$",` + value + `"sensitive":true}`
}
r := decodeRewriteRuleJSON(t, request(true, "POST", rewriteAPI, `{"name":"t-rwapi-ambiguous","phase":"request","matchers":{"host":"api.example"},
"actions":[`+action(`"value":"rwapi-plain-a",`)+`,`+action(`"value":"rwapi-plain-b",`)+`]}`))
path := fmt.Sprintf("%s/%d", rewriteAPI, r.ID)
for name, actions := range map[string]string{
"both omitted": action("") + `,` + action(""),
"one omitted": action(`"value":"rwapi-new",`) + `,` + action(""),
"single, omitted": action(""),
} {
resp := request(true, "PUT", path, `{"name":"t-rwapi-ambiguous","phase":"request","matchers":{"host":"api.example"},"actions":[`+actions+`]}`)
if resp.Code != 400 || !strings.Contains(resp.Body.String(), "send the value") || strings.Contains(resp.Body.String(), "rwapi-plain") {
t.Fatalf("%s: HTTP %d %s, want 400", name, resp.Code, resp.Body.String())
}
}
stored, err := s.m.pg.GetTrafficRewriteRule(r.ID)
if err != nil {
t.Fatal(err)
}
if a := string(stored.Actions); !strings.Contains(a, "rwapi-plain-a") || !strings.Contains(a, "rwapi-plain-b") || strings.Contains(a, "rwapi-new") {
t.Fatalf("a refused update changed the stored actions: %s", a)
}
}
// The validation refuses a rule with HTTP 400 on create and on update, and
// stores nothing.
func TestRewriteRulesValidation400(t *testing.T) {
s, request := rewriteServer(t)
valid := `{"name":"t-rwapi-valid","phase":"request","actions":[{"type":"set_header","name":"X-A","value":"v"}]}`
r := decodeRewriteRuleJSON(t, request(true, "POST", rewriteAPI, valid))
path := fmt.Sprintf("%s/%d", rewriteAPI, r.ID)
rule := func(phase, matchers, actions string) string {
return `{"name":"t-rwapi-bad","phase":"` + phase + `","matchers":` + matchers + `,"actions":` + actions + `}`
}
cases := map[string]string{
"malformed JSON": `{"name":`,
"empty name": `{"phase":"request","actions":[{"type":"set_header","name":"X-A","value":"v"}]}`,
"bad phase": rule("both", `{}`, `[{"type":"set_header","name":"X-A","value":"v"}]`),
"no action": rule("request", `{}`, `[]`),
"unknown action": rule("request", `{}`, `[{"type":"set_host","value":"evil.example"}]`),
"Host header": rule("request", `{}`, `[{"type":"set_header","name":"Host","value":"evil.example"}]`),
"Content-Length": rule("response", `{}`, `[{"type":"delete_header","name":"content-length"}]`),
"Proxy- header": rule("request", `{}`, `[{"type":"set_header","name":"Proxy-Authorization","value":"v"}]`),
"hop-by-hop header": rule("request", `{}`, `[{"type":"set_header","name":"Connection","value":"close"}]`),
"header line break": rule("request", `{}`, `[{"type":"set_header","name":"X-A","value":"a\r\nX-B: b"}]`),
"header name": rule("request", `{}`, `[{"type":"set_header","name":"X A","value":"v"}]`),
"absolute URL": rule("request", `{}`, `[{"type":"set_path_query","value":"http://evil.example/x"}]`),
"method CONNECT": rule("request", `{}`, `[{"type":"set_method","value":"CONNECT"}]`),
"status 204": rule("response", `{}`, `[{"type":"set_status","value":"204"}]`),
"status 600": rule("response", `{}`, `[{"type":"set_status","value":"600"}]`),
"status in request": rule("request", `{}`, `[{"type":"set_status","value":"200"}]`),
"host glob": rule("request", `{"host":"a.*.b"}`, `[{"type":"set_header","name":"X-A","value":"v"}]`),
"bad regex": rule("request", `{"url_regex":"("}`, `[{"type":"set_header","name":"X-A","value":"v"}]`),
"regex over 1 KiB": rule("request", `{"url_regex":"`+strings.Repeat("a", rewrite.MaxPatternSize+1)+`"}`, `[{"type":"set_header","name":"X-A","value":"v"}]`),
"value over 1 MiB": rule("request", `{}`, `[{"type":"set_body","value":"`+strings.Repeat("a", rewrite.MaxValueSize+1)+`"}]`),
"sensitive, no value": rule("request", `{}`, `[{"type":"set_body","sensitive":true}]`),
"response matcher": rule("request", `{"status":["200"]}`, `[{"type":"set_header","name":"X-A","value":"v"}]`),
"replace without expr": rule("request", `{}`, `[{"type":"replace_body","value":"v"}]`),
}
for name, body := range cases {
for _, c := range [][2]string{{"POST", rewriteAPI}, {"PUT", path}} {
if resp := request(true, c[0], c[1], body); resp.Code != 400 {
t.Errorf("%s: %s answered HTTP %d, want 400", name, c[0], resp.Code)
}
}
}
rules, _ := listRewriteRules(t, request)
if len(rules) != 1 || rules[0].Name != "t-rwapi-valid" || rules[0].Actions[0].Name != "X-A" {
t.Fatalf("a refused request changed the table: %+v", rules)
}
if resp := request(true, "POST", path+"/toggle", `{"enabled":`); resp.Code != 400 {
t.Errorf("toggle with malformed JSON: HTTP %d, want 400", resp.Code)
}
// A stored rule the validation no longer accepts: reported by load_error,
// kept out of the snapshot, and refused by the toggle.
old, err := s.m.pg.CreateTrafficRewriteRule(db.TrafficRewriteRule{Name: "t-rwapi-old", Enabled: true, Phase: "request",
Matchers: json.RawMessage(`{}`), Actions: json.RawMessage(`[{"type":"set_header","name":"Host","value":"evil.example"}]`)})
if err != nil {
t.Fatal(err)
}
s.m.reloadRewriteRules()
if set := s.m.traffic.RewriteRules(); set.Len(rewrite.PhaseRequest) != 0 || !strings.Contains(set.LoadErrors()[old.ID], "Host") {
t.Fatalf("snapshot: %d rule(s), load errors %v", set.Len(rewrite.PhaseRequest), set.LoadErrors())
}
rules, _ = listRewriteRules(t, request)
for _, r := range rules {
if (r.ID == old.ID) != (r.LoadError != "") {
t.Errorf("rule %s: load_error=%q", r.Name, r.LoadError)
}
}
oldPath := fmt.Sprintf("%s/%d/toggle", rewriteAPI, old.ID)
if resp := request(true, "POST", oldPath, `{"enabled":true}`); resp.Code != 400 {
t.Errorf("enabling an invalid stored rule: HTTP %d, want 400", resp.Code)
}
if resp := request(true, "POST", oldPath, `{"enabled":false}`); resp.Code != 200 {
t.Errorf("disabling an invalid stored rule: HTTP %d, want 200", resp.Code)
}
}
// 100 rules at most, 50 of them enabled.
func TestRewriteRulesLimits(t *testing.T) {
s, request := rewriteServer(t)
body := func(i int) string {
return fmt.Sprintf(`{"name":"t-rwapi-limit-%03d","phase":"response","actions":[{"type":"set_header","name":"X-A","value":"v"}]}`, i)
}
var ids []int64
for i := 0; i < db.MaxTrafficRewriteRules; i++ {
ids = append(ids, decodeRewriteRuleJSON(t, request(true, "POST", rewriteAPI, body(i))).ID)
}
if resp := request(true, "POST", rewriteAPI, body(100)); resp.Code != 400 || !strings.Contains(resp.Body.String(), "100") {
t.Fatalf("101st rule: HTTP %d %s, want 400", resp.Code, resp.Body.String())
}
toggle := func(id int64) *httptest.ResponseRecorder {
return request(true, "POST", fmt.Sprintf("%s/%d/toggle", rewriteAPI, id), `{"enabled":true}`)
}
for _, id := range ids[:db.MaxEnabledTrafficRewriteRules] {
if resp := toggle(id); resp.Code != 200 {
t.Fatalf("toggle: HTTP %d %s", resp.Code, resp.Body.String())
}
}
over := ids[db.MaxEnabledTrafficRewriteRules]
if resp := toggle(over); resp.Code != 400 || !strings.Contains(resp.Body.String(), "50") {
t.Fatalf("51st enabled rule by toggle: HTTP %d %s, want 400", resp.Code, resp.Body.String())
}
update := `{"name":"t-rwapi-limit-upd","enabled":true,"phase":"response","actions":[{"type":"set_header","name":"X-A","value":"v"}]}`
if resp := request(true, "PUT", fmt.Sprintf("%s/%d", rewriteAPI, over), update); resp.Code != 400 {
t.Fatalf("51st enabled rule by update: HTTP %d %s, want 400", resp.Code, resp.Body.String())
}
if n := s.m.traffic.RewriteRules().Len(rewrite.PhaseResponse); n != db.MaxEnabledTrafficRewriteRules {
t.Fatalf("the snapshot holds %d rules, want %d", n, db.MaxEnabledTrafficRewriteRules)
}
rules, _ := listRewriteRules(t, request)
if len(rules) != db.MaxTrafficRewriteRules {
t.Fatalf("%d rules listed, want %d", len(rules), db.MaxTrafficRewriteRules)
}
}
// NewManager loads the rules once the traffic store is open, and refuses to
// start under a key other than the one that sealed them.
func TestRewriteRulesLoadedAtStartup(t *testing.T) {
_, request := rewriteServer(t)
r := decodeRewriteRuleJSON(t, request(true, "POST", rewriteAPI,
`{"name":"t-rwapi-startup","phase":"request","actions":[{"type":"set_header","name":"X-Startup","value":"1"}]}`))
if resp := request(true, "POST", fmt.Sprintf("%s/%d/toggle", rewriteAPI, r.ID), `{"enabled":true}`); resp.Code != 200 {
t.Fatalf("toggle: HTTP %d", resp.Code)
}
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
proxyAddr := ln.Addr().String()
ln.Close()
m, err := NewManager(t.TempDir(), proxyAddr)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { m.Close() })
if m.traffic == nil {
t.Fatal("NewManager did not open the traffic store")
}
set := m.traffic.RewriteRules()
if set == nil || set.Len(rewrite.PhaseRequest) != 1 || set.Len(rewrite.PhaseResponse) != 0 || len(set.LoadErrors()) != 0 {
t.Fatalf("snapshot at startup = %+v", set)
}
// Not the key that sealed the rule.
t.Setenv("ARTEX_SECRET_KEY", base64.StdEncoding.EncodeToString([]byte(strings.Repeat("f", 32))))
if m2, err := NewManager(t.TempDir(), ""); err == nil {
m2.Close()
t.Fatal("NewManager started under a key that cannot open the stored rules")
} else if !strings.Contains(err.Error(), "secrets at rest") || !strings.Contains(err.Error(), "secret key check") {
t.Fatalf("startup error = %v", err)
}
}
+89
View File
@@ -0,0 +1,89 @@
package server
import (
"encoding/json"
"testing"
)
func TestTrafficStreamRoute(t *testing.T) {
s, request := ingestServer(t)
db := s.m.Traffic().DB()
if _, err := db.Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
VALUES('1-0001',1,'ws.example.com','GET','/ws','wss://ws.example.com/ws',101,'websocket',0,0,'')`); err != nil {
t.Fatal(err)
}
if _, err := db.Exec(`INSERT INTO exchange_bodies(id,req_head,resp_head) VALUES('1-0001','GET /ws HTTP/1.1\n','HTTP/1.1 101 Switching Protocols\n')`); err != nil {
t.Fatal(err)
}
const total = 250
tx, err := db.Begin()
if err != nil {
t.Fatal(err)
}
for i := 1; i <= total; i++ {
kind, payload := 1, []byte("hello")
if i%2 == 0 {
kind, payload = 2, []byte{0, 1, 2}
}
if _, err := tx.Exec(`INSERT INTO stream_messages(exchange_id,seq,ts,direction,kind,size,payload) VALUES('1-0001',?,1,'c2s',?,?,?)`, i, kind, len(payload), payload); err != nil {
t.Fatal(err)
}
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
type page struct {
Items []struct {
Seq int64 `json:"seq"`
Type string `json:"type"`
Payload string `json:"payload"`
Encoding string `json:"encoding"`
} `json:"items"`
Total int `json:"total"`
}
get := func(path string, wantStatus int) page {
t.Helper()
w := request("GET", path, "")
if w.Code != wantStatus {
t.Fatalf("GET %s = %d %s, want %d", path, w.Code, w.Body, wantStatus)
}
var p page
if wantStatus == 200 {
if err := json.Unmarshal(w.Body.Bytes(), &p); err != nil {
t.Fatalf("GET %s: %v in %s", path, err, w.Body)
}
}
return p
}
get("/api/traffic/stream", 400)
if p := get("/api/traffic/stream?exchange=nope", 200); p.Items == nil || len(p.Items) != 0 || p.Total != 0 {
t.Fatalf("unknown exchange = %+v, want items [] and total 0", p)
}
p := get("/api/traffic/stream?exchange=1-0001", 200)
if p.Total != total || len(p.Items) != 50 || p.Items[0].Seq != 1 {
t.Fatalf("default page: total %d, %d items, first seq %d", p.Total, len(p.Items), p.Items[0].Seq)
}
if first, second := p.Items[0], p.Items[1]; first.Type != "text" || first.Payload != "hello" || second.Type != "binary" || second.Encoding != "base64" || second.Payload != "AAEC" {
t.Fatalf("payloads %+v %+v", first, second)
}
if p := get("/api/traffic/stream?exchange=1-0001&limit=500", 200); len(p.Items) != 200 {
t.Fatalf("limit=500 returned %d items, want the 200 cap", len(p.Items))
}
if p := get("/api/traffic/stream?exchange=1-0001&offset=-5&limit=1", 200); len(p.Items) != 1 || p.Items[0].Seq != 1 {
t.Fatalf("negative offset: %+v, want seq 1", p.Items)
}
if p := get("/api/traffic/stream?exchange=1-0001&offset=248&limit=10", 200); len(p.Items) != 2 || p.Items[0].Seq != 249 {
t.Fatalf("last page: %+v, want seqs 249 and 250", p.Items)
}
w := request("GET", "/api/traffic/exchange?id=1-0001", "")
var ex struct {
StreamCount int `json:"stream_count"`
StreamKind string `json:"stream_kind"`
}
if err := json.Unmarshal(w.Body.Bytes(), &ex); err != nil || w.Code != 200 || ex.StreamCount != total || ex.StreamKind != "websocket" {
t.Fatalf("exchange = %d %s", w.Code, w.Body)
}
}