From cf369dec4fb66f1602b2a6a275c961f7dca7d12a Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Mon, 19 Oct 2026 00:28:27 +0200 Subject: [PATCH] feat(server): per-task outcome card for RSI baselines Adds GET /api/tasks/{id}/outcome: finding disposition and severity, retest verdicts, result-row token totals, tool calls, skill misses, and wall time in one row. This is the measurement layer an RSI loop needs before claiming a change helped; it alters no agent behavior. Metric selection follows harness2 (Google Research, Apache-2.0, arXiv 2610.14224) and the task-gain / retained-gain / improver-gain taxonomy in the awesome-rsi list. --- db/task_outcome.go | 119 ++++++++++++++++++++++++++++++++++++ server/server.go | 1 + server/task_outcome.go | 31 ++++++++++ server/task_outcome_test.go | 114 ++++++++++++++++++++++++++++++++++ 4 files changed, 265 insertions(+) create mode 100644 db/task_outcome.go create mode 100644 server/task_outcome.go create mode 100644 server/task_outcome_test.go diff --git a/db/task_outcome.go b/db/task_outcome.go new file mode 100644 index 0000000..0757202 --- /dev/null +++ b/db/task_outcome.go @@ -0,0 +1,119 @@ +package db + +import "context" + +// The per-task outcome card is the baseline layer for RSI comparisons. It +// consolidates what a task produced and what it cost so a later harness or +// lesson change can be judged against a number instead of an impression. +// Metric selection follows harness2 (Google Research, Apache-2.0, +// arXiv 2610.14224) and the task-gain / retained-gain / improver-gain taxonomy +// used by the awesome-rsi list (lobehub). + +// TaskOutcomeFindings is the finding disposition and severity split. +type TaskOutcomeFindings struct { + Total int `json:"total"` + Confirmed int `json:"confirmed"` // confirmed, resolved, or fixed + Open int `json:"open"` // pending or in_progress + FalsePositive int `json:"false_positive"` + Duplicate int `json:"duplicate"` + Critical int `json:"critical"` // confirmed only + High int `json:"high"` + Medium int `json:"medium"` + Low int `json:"low"` +} + +// TaskOutcomeRetests is the retest verdict split for this task's findings. +type TaskOutcomeRetests struct { + Reproduced int `json:"reproduced"` + Fixed int `json:"fixed"` + Inconclusive int `json:"inconclusive"` +} + +// TaskOutcomeTokens is the task's result-row token total. +type TaskOutcomeTokens struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + CacheReadTokens int `json:"cache_read_tokens"` + CacheWriteTokens int `json:"cache_write_tokens"` +} + +// TaskOutcome is one task's production and cost baseline. +type TaskOutcome struct { + TaskID int64 `json:"task_id"` + Status string `json:"status"` + CategoryID int64 `json:"category_id"` + CompanyID int64 `json:"company_id"` + Findings TaskOutcomeFindings `json:"findings"` + Retests TaskOutcomeRetests `json:"retests"` + Tokens TaskOutcomeTokens `json:"tokens"` + ToolCalls int `json:"tool_calls"` + SkillMisses int `json:"skill_misses"` + WallSeconds int64 `json:"wall_seconds"` +} + +// TaskOutcome returns the outcome card for one live task. sql.ErrNoRows means +// the task does not exist or is deleted. +func (d *DB) TaskOutcome(ctx context.Context, taskID int64) (TaskOutcome, error) { + var o TaskOutcome + err := d.QueryRowContext(ctx, ` + SELECT t.id, t.status, + COALESCE(t.category_id,0), COALESCE(t.company_id,0), + COALESCE(f.total,0), COALESCE(f.confirmed,0), COALESCE(f.open,0), + COALESCE(f.false_positive,0), COALESCE(f.duplicate,0), + COALESCE(f.critical,0), COALESCE(f.high,0), + COALESCE(f.medium,0), COALESCE(f.low,0), + COALESCE(r.reproduced,0), COALESCE(r.fixed,0), COALESCE(r.inconclusive,0), + COALESCE(a.input_tokens,0), COALESCE(a.output_tokens,0), + COALESCE(a.cache_read_tokens,0), COALESCE(a.cache_write_tokens,0), + COALESCE(tu.calls,0), COALESCE(su.misses,0), + GREATEST(0, EXTRACT(EPOCH FROM + (COALESCE(t.completed_at, now()) - COALESCE(t.first_run_at, t.created_at)))::bigint) + FROM tasks t + LEFT JOIN LATERAL ( + SELECT COUNT(*) AS total, + COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed')) AS confirmed, + COUNT(*) FILTER (WHERE status IN ('pending','in_progress')) AS open, + COUNT(*) FILTER (WHERE status='false_positive') AS false_positive, + COUNT(*) FILTER (WHERE status='duplicate') AS duplicate, + COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='critical') AS critical, + COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='high') AS high, + COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='medium') AS medium, + COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='low') AS low + FROM findings + WHERE task_id=t.id + ) f ON true + LEFT JOIN LATERAL ( + SELECT COUNT(*) FILTER (WHERE fr.verdict='reproduced') AS reproduced, + COUNT(*) FILTER (WHERE fr.verdict='fixed') AS fixed, + COUNT(*) FILTER (WHERE fr.verdict='inconclusive') AS inconclusive + FROM finding_retests fr + JOIN findings f2 ON f2.id=fr.finding_id + WHERE f2.task_id=t.id + ) r ON true + LEFT JOIN LATERAL ( + SELECT SUM(input_tokens) AS input_tokens, + SUM(output_tokens) AS output_tokens, + SUM(cache_read_tokens) AS cache_read_tokens, + SUM(cache_write_tokens) AS cache_write_tokens + FROM activity + WHERE exploration_id=t.exploration_id AND kind='result' + ) a ON true + LEFT JOIN LATERAL ( + SELECT COUNT(*) AS calls FROM tool_usage WHERE task_id=t.id + ) tu ON true + LEFT JOIN LATERAL ( + SELECT COUNT(*) AS misses FROM skill_usage WHERE task_id=t.id AND NOT found + ) su ON true + WHERE t.id=$1 AND t.deleted_at IS NULL`, taskID).Scan( + &o.TaskID, &o.Status, + &o.CategoryID, &o.CompanyID, + &o.Findings.Total, &o.Findings.Confirmed, &o.Findings.Open, + &o.Findings.FalsePositive, &o.Findings.Duplicate, + &o.Findings.Critical, &o.Findings.High, &o.Findings.Medium, &o.Findings.Low, + &o.Retests.Reproduced, &o.Retests.Fixed, &o.Retests.Inconclusive, + &o.Tokens.InputTokens, &o.Tokens.OutputTokens, + &o.Tokens.CacheReadTokens, &o.Tokens.CacheWriteTokens, + &o.ToolCalls, &o.SkillMisses, &o.WallSeconds, + ) + return o, err +} diff --git a/server/server.go b/server/server.go index 85f837e..e3fd936 100644 --- a/server/server.go +++ b/server/server.go @@ -683,6 +683,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("PATCH /api/task-templates/{id}", s.pgUpdateTaskTemplate) mux.HandleFunc("DELETE /api/task-templates/{id}", s.pgDeleteTaskTemplate) mux.HandleFunc("GET /api/tasks/{id}", s.getTask) + mux.HandleFunc("GET /api/tasks/{id}/outcome", s.taskOutcome) mux.HandleFunc("PATCH /api/tasks/{id}", s.updateTaskMetadata) mux.HandleFunc("PATCH /api/tasks/{id}/category", s.updateTaskCategory) // task-level asset interception/allow rules diff --git a/server/task_outcome.go b/server/task_outcome.go new file mode 100644 index 0000000..8e12015 --- /dev/null +++ b/server/task_outcome.go @@ -0,0 +1,31 @@ +package server + +import ( + "database/sql" + "errors" + "log" + "net/http" + "strconv" +) + +// taskOutcome returns one task's production and cost baseline. It is the +// measurement layer for later RSI comparisons (harness2, Google Research, +// arXiv 2610.14224): nothing here changes agent behavior. +func (s *Server) taskOutcome(w http.ResponseWriter, r *http.Request) { + id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) + if err != nil || id <= 0 { + writeErr(w, 400, "invalid task id") + return + } + outcome, err := s.m.PG().TaskOutcome(r.Context(), id) + if errors.Is(err, sql.ErrNoRows) { + writeErr(w, 404, "task not found") + return + } + if err != nil { + log.Printf("[tasks] outcome query failed for %d: %v", id, err) + writeErr(w, 500, "outcome query failed") + return + } + writeJSON(w, 200, outcome) +} diff --git a/server/task_outcome_test.go b/server/task_outcome_test.go new file mode 100644 index 0000000..4ffaa51 --- /dev/null +++ b/server/task_outcome_test.go @@ -0,0 +1,114 @@ +package server + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "github.com/Autumn-27/artex/db" +) + +func TestTaskOutcomeCard(t *testing.T) { + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Skipf("postgres unavailable (%v) — skipping", err) + } + defer m.Close() + task, err := m.CreateTask("outcome card", "goal", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + taskID, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { _ = m.pg.DeleteTask(taskID) }() + + if _, err := m.pg.Exec(` +INSERT INTO findings(task_id,vulnclass,name,severity,status) VALUES + ($1,'xss','A','high','confirmed'), + ($1,'sqli','B','critical','fixed'), + ($1,'csrf','C','medium','pending'), + ($1,'xss','D','low','false_positive'), + ($1,'idor','E','high','duplicate')`, taskID); err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec(` +INSERT INTO finding_retests(finding_id,status,verdict,snapshot) +SELECT id,'completed','reproduced','{}'::jsonb FROM findings WHERE task_id=$1 AND name='A'`, taskID); err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec(` +INSERT INTO finding_retests(finding_id,status,verdict,snapshot) +SELECT id,'completed','fixed','{}'::jsonb FROM findings WHERE task_id=$1 AND name='B'`, taskID); err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec(` +INSERT INTO activity(exploration_id,kind,input_tokens,output_tokens,cache_read_tokens,cache_write_tokens) +VALUES ($1,'result',11,7,3,2), ($1,'tool_result',999,999,999,999)`, task.ExpID); err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec(` +INSERT INTO tool_usage(tool_key,task_id) VALUES ('traffic_search',$1),('report_finding',$1)`, taskID); err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec(` +INSERT INTO skill_usage(skill,task_id,found) VALUES ('api-recon',$1,true),('graphql-audit',$1,false)`, taskID); err != nil { + t.Fatal(err) + } + + s := New(context.Background(), m, t.TempDir(), t.TempDir(), t.TempDir()) + token, err := signJWT(s.jwtKey) + if err != nil { + t.Fatal(err) + } + get := func(path string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, path, nil) + req.Header.Set("Authorization", "Bearer "+token) + rec := httptest.NewRecorder() + s.Handler().ServeHTTP(rec, req) + return rec + } + + rec := get("/api/tasks/" + task.ID + "/outcome") + if rec.Code != http.StatusOK { + t.Fatalf("outcome status=%d body=%s", rec.Code, rec.Body.String()) + } + var got db.TaskOutcome + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.TaskID != taskID || got.Status != "created" { + t.Fatalf("task identity: %+v", got) + } + wantFindings := db.TaskOutcomeFindings{ + Total: 5, Confirmed: 2, Open: 1, FalsePositive: 1, Duplicate: 1, + Critical: 1, High: 1, Medium: 0, Low: 0, + } + if got.Findings != wantFindings { + t.Fatalf("findings=%+v want %+v", got.Findings, wantFindings) + } + if got.Retests.Reproduced != 1 || got.Retests.Fixed != 1 || got.Retests.Inconclusive != 0 { + t.Fatalf("retests=%+v", got.Retests) + } + if got.Tokens.InputTokens != 11 || got.Tokens.OutputTokens != 7 || + got.Tokens.CacheReadTokens != 3 || got.Tokens.CacheWriteTokens != 2 { + t.Fatalf("tokens=%+v (non-result rows must be excluded)", got.Tokens) + } + if got.ToolCalls != 2 { + t.Fatalf("tool_calls=%d want 2", got.ToolCalls) + } + if got.SkillMisses != 1 { + t.Fatalf("skill_misses=%d want 1", got.SkillMisses) + } + if got.WallSeconds < 0 { + t.Fatalf("wall_seconds=%d", got.WallSeconds) + } + + if rec := get("/api/tasks/999999999/outcome"); rec.Code != http.StatusNotFound { + t.Fatalf("unknown task status=%d body=%s", rec.Code, rec.Body.String()) + } + if rec := get("/api/tasks/not-a-number/outcome"); rec.Code != http.StatusBadRequest { + t.Fatalf("bad id status=%d body=%s", rec.Code, rec.Body.String()) + } +}