feat(server): per-task outcome card for RSI baselines

Adds GET /api/tasks/{id}/outcome: finding disposition and severity, retest verdicts, result-row token totals, tool calls, skill misses, and wall time in one row. This is the measurement layer an RSI loop needs before claiming a change helped; it alters no agent behavior. Metric selection follows harness2 (Google Research, Apache-2.0, arXiv 2610.14224) and the task-gain / retained-gain / improver-gain taxonomy in the awesome-rsi list.
2026-11-13 11:40:53 +08:00 · 2026-10-19 00:28:27 +02:00
parent bba7989903
commit cf369dec4f
4 changed files with 265 additions and 0 deletions
@@ -0,0 +1,119 @@
+package db
+
+import "context"
+
+// The per-task outcome card is the baseline layer for RSI comparisons. It
+// consolidates what a task produced and what it cost so a later harness or
+// lesson change can be judged against a number instead of an impression.
+// Metric selection follows harness2 (Google Research, Apache-2.0,
+// arXiv 2610.14224) and the task-gain / retained-gain / improver-gain taxonomy
+// used by the awesome-rsi list (lobehub).
+
+// TaskOutcomeFindings is the finding disposition and severity split.
+type TaskOutcomeFindings struct {
+	Total         int `json:"total"`
+	Confirmed     int `json:"confirmed"` // confirmed, resolved, or fixed
+	Open          int `json:"open"`      // pending or in_progress
+	FalsePositive int `json:"false_positive"`
+	Duplicate     int `json:"duplicate"`
+	Critical      int `json:"critical"` // confirmed only
+	High          int `json:"high"`
+	Medium        int `json:"medium"`
+	Low           int `json:"low"`
+}
+
+// TaskOutcomeRetests is the retest verdict split for this task's findings.
+type TaskOutcomeRetests struct {
+	Reproduced   int `json:"reproduced"`
+	Fixed        int `json:"fixed"`
+	Inconclusive int `json:"inconclusive"`
+}
+
+// TaskOutcomeTokens is the task's result-row token total.
+type TaskOutcomeTokens struct {
+	InputTokens      int `json:"input_tokens"`
+	OutputTokens     int `json:"output_tokens"`
+	CacheReadTokens  int `json:"cache_read_tokens"`
+	CacheWriteTokens int `json:"cache_write_tokens"`
+}
+
+// TaskOutcome is one task's production and cost baseline.
+type TaskOutcome struct {
+	TaskID      int64               `json:"task_id"`
+	Status      string              `json:"status"`
+	CategoryID  int64               `json:"category_id"`
+	CompanyID   int64               `json:"company_id"`
+	Findings    TaskOutcomeFindings `json:"findings"`
+	Retests     TaskOutcomeRetests  `json:"retests"`
+	Tokens      TaskOutcomeTokens   `json:"tokens"`
+	ToolCalls   int                 `json:"tool_calls"`
+	SkillMisses int                 `json:"skill_misses"`
+	WallSeconds int64               `json:"wall_seconds"`
+}
+
+// TaskOutcome returns the outcome card for one live task. sql.ErrNoRows means
+// the task does not exist or is deleted.
+func (d *DB) TaskOutcome(ctx context.Context, taskID int64) (TaskOutcome, error) {
+	var o TaskOutcome
+	err := d.QueryRowContext(ctx, `
+		SELECT t.id, t.status,
+		       COALESCE(t.category_id,0), COALESCE(t.company_id,0),
+		       COALESCE(f.total,0), COALESCE(f.confirmed,0), COALESCE(f.open,0),
+		       COALESCE(f.false_positive,0), COALESCE(f.duplicate,0),
+		       COALESCE(f.critical,0), COALESCE(f.high,0),
+		       COALESCE(f.medium,0), COALESCE(f.low,0),
+		       COALESCE(r.reproduced,0), COALESCE(r.fixed,0), COALESCE(r.inconclusive,0),
+		       COALESCE(a.input_tokens,0), COALESCE(a.output_tokens,0),
+		       COALESCE(a.cache_read_tokens,0), COALESCE(a.cache_write_tokens,0),
+		       COALESCE(tu.calls,0), COALESCE(su.misses,0),
+		       GREATEST(0, EXTRACT(EPOCH FROM
+		           (COALESCE(t.completed_at, now()) - COALESCE(t.first_run_at, t.created_at)))::bigint)
+		FROM tasks t
+		LEFT JOIN LATERAL (
+			SELECT COUNT(*) AS total,
+			       COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed')) AS confirmed,
+			       COUNT(*) FILTER (WHERE status IN ('pending','in_progress'))       AS open,
+			       COUNT(*) FILTER (WHERE status='false_positive')                   AS false_positive,
+			       COUNT(*) FILTER (WHERE status='duplicate')                        AS duplicate,
+			       COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='critical') AS critical,
+			       COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='high')     AS high,
+			       COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='medium')   AS medium,
+			       COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='low')      AS low
+			FROM findings
+			WHERE task_id=t.id
+		) f ON true
+		LEFT JOIN LATERAL (
+			SELECT COUNT(*) FILTER (WHERE fr.verdict='reproduced')   AS reproduced,
+			       COUNT(*) FILTER (WHERE fr.verdict='fixed')        AS fixed,
+			       COUNT(*) FILTER (WHERE fr.verdict='inconclusive') AS inconclusive
+			FROM finding_retests fr
+			JOIN findings f2 ON f2.id=fr.finding_id
+			WHERE f2.task_id=t.id
+		) r ON true
+		LEFT JOIN LATERAL (
+			SELECT SUM(input_tokens) AS input_tokens,
+			       SUM(output_tokens) AS output_tokens,
+			       SUM(cache_read_tokens) AS cache_read_tokens,
+			       SUM(cache_write_tokens) AS cache_write_tokens
+			FROM activity
+			WHERE exploration_id=t.exploration_id AND kind='result'
+		) a ON true
+		LEFT JOIN LATERAL (
+			SELECT COUNT(*) AS calls FROM tool_usage WHERE task_id=t.id
+		) tu ON true
+		LEFT JOIN LATERAL (
+			SELECT COUNT(*) AS misses FROM skill_usage WHERE task_id=t.id AND NOT found
+		) su ON true
+		WHERE t.id=$1 AND t.deleted_at IS NULL`, taskID).Scan(
+		&o.TaskID, &o.Status,
+		&o.CategoryID, &o.CompanyID,
+		&o.Findings.Total, &o.Findings.Confirmed, &o.Findings.Open,
+		&o.Findings.FalsePositive, &o.Findings.Duplicate,
+		&o.Findings.Critical, &o.Findings.High, &o.Findings.Medium, &o.Findings.Low,
+		&o.Retests.Reproduced, &o.Retests.Fixed, &o.Retests.Inconclusive,
+		&o.Tokens.InputTokens, &o.Tokens.OutputTokens,
+		&o.Tokens.CacheReadTokens, &o.Tokens.CacheWriteTokens,
+		&o.ToolCalls, &o.SkillMisses, &o.WallSeconds,
+	)
+	return o, err
+}
@@ -683,6 +683,7 @@ func (s *Server) Handler() http.Handler {
 	mux.HandleFunc("PATCH /api/task-templates/{id}", s.pgUpdateTaskTemplate)
 	mux.HandleFunc("DELETE /api/task-templates/{id}", s.pgDeleteTaskTemplate)
 	mux.HandleFunc("GET /api/tasks/{id}", s.getTask)
+	mux.HandleFunc("GET /api/tasks/{id}/outcome", s.taskOutcome)
 	mux.HandleFunc("PATCH /api/tasks/{id}", s.updateTaskMetadata)
 	mux.HandleFunc("PATCH /api/tasks/{id}/category", s.updateTaskCategory)
 	// task-level asset interception/allow rules
@@ -0,0 +1,31 @@
+package server
+
+import (
+	"database/sql"
+	"errors"
+	"log"
+	"net/http"
+	"strconv"
+)
+
+// taskOutcome returns one task's production and cost baseline. It is the
+// measurement layer for later RSI comparisons (harness2, Google Research,
+// arXiv 2610.14224): nothing here changes agent behavior.
+func (s *Server) taskOutcome(w http.ResponseWriter, r *http.Request) {
+	id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
+	if err != nil || id <= 0 {
+		writeErr(w, 400, "invalid task id")
+		return
+	}
+	outcome, err := s.m.PG().TaskOutcome(r.Context(), id)
+	if errors.Is(err, sql.ErrNoRows) {
+		writeErr(w, 404, "task not found")
+		return
+	}
+	if err != nil {
+		log.Printf("[tasks] outcome query failed for %d: %v", id, err)
+		writeErr(w, 500, "outcome query failed")
+		return
+	}
+	writeJSON(w, 200, outcome)
+}
@@ -0,0 +1,114 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+
+	"github.com/Autumn-27/artex/db"
+)
+
+func TestTaskOutcomeCard(t *testing.T) {
+	m, err := NewManager(t.TempDir(), "")
+	if err != nil {
+		t.Skipf("postgres unavailable (%v) — skipping", err)
+	}
+	defer m.Close()
+	task, err := m.CreateTask("outcome card", "goal", nil, 0, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	taskID, _ := strconv.ParseInt(task.ID, 10, 64)
+	defer func() { _ = m.pg.DeleteTask(taskID) }()
+
+	if _, err := m.pg.Exec(`
+INSERT INTO findings(task_id,vulnclass,name,severity,status) VALUES
+ ($1,'xss','A','high','confirmed'),
+ ($1,'sqli','B','critical','fixed'),
+ ($1,'csrf','C','medium','pending'),
+ ($1,'xss','D','low','false_positive'),
+ ($1,'idor','E','high','duplicate')`, taskID); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := m.pg.Exec(`
+INSERT INTO finding_retests(finding_id,status,verdict,snapshot)
+SELECT id,'completed','reproduced','{}'::jsonb FROM findings WHERE task_id=$1 AND name='A'`, taskID); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := m.pg.Exec(`
+INSERT INTO finding_retests(finding_id,status,verdict,snapshot)
+SELECT id,'completed','fixed','{}'::jsonb FROM findings WHERE task_id=$1 AND name='B'`, taskID); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := m.pg.Exec(`
+INSERT INTO activity(exploration_id,kind,input_tokens,output_tokens,cache_read_tokens,cache_write_tokens)
+VALUES ($1,'result',11,7,3,2), ($1,'tool_result',999,999,999,999)`, task.ExpID); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := m.pg.Exec(`
+INSERT INTO tool_usage(tool_key,task_id) VALUES ('traffic_search',$1),('report_finding',$1)`, taskID); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := m.pg.Exec(`
+INSERT INTO skill_usage(skill,task_id,found) VALUES ('api-recon',$1,true),('graphql-audit',$1,false)`, taskID); err != nil {
+		t.Fatal(err)
+	}
+
+	s := New(context.Background(), m, t.TempDir(), t.TempDir(), t.TempDir())
+	token, err := signJWT(s.jwtKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	get := func(path string) *httptest.ResponseRecorder {
+		req := httptest.NewRequest(http.MethodGet, path, nil)
+		req.Header.Set("Authorization", "Bearer "+token)
+		rec := httptest.NewRecorder()
+		s.Handler().ServeHTTP(rec, req)
+		return rec
+	}
+
+	rec := get("/api/tasks/" + task.ID + "/outcome")
+	if rec.Code != http.StatusOK {
+		t.Fatalf("outcome status=%d body=%s", rec.Code, rec.Body.String())
+	}
+	var got db.TaskOutcome
+	if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
+		t.Fatal(err)
+	}
+	if got.TaskID != taskID || got.Status != "created" {
+		t.Fatalf("task identity: %+v", got)
+	}
+	wantFindings := db.TaskOutcomeFindings{
+		Total: 5, Confirmed: 2, Open: 1, FalsePositive: 1, Duplicate: 1,
+		Critical: 1, High: 1, Medium: 0, Low: 0,
+	}
+	if got.Findings != wantFindings {
+		t.Fatalf("findings=%+v want %+v", got.Findings, wantFindings)
+	}
+	if got.Retests.Reproduced != 1 || got.Retests.Fixed != 1 || got.Retests.Inconclusive != 0 {
+		t.Fatalf("retests=%+v", got.Retests)
+	}
+	if got.Tokens.InputTokens != 11 || got.Tokens.OutputTokens != 7 ||
+		got.Tokens.CacheReadTokens != 3 || got.Tokens.CacheWriteTokens != 2 {
+		t.Fatalf("tokens=%+v (non-result rows must be excluded)", got.Tokens)
+	}
+	if got.ToolCalls != 2 {
+		t.Fatalf("tool_calls=%d want 2", got.ToolCalls)
+	}
+	if got.SkillMisses != 1 {
+		t.Fatalf("skill_misses=%d want 1", got.SkillMisses)
+	}
+	if got.WallSeconds < 0 {
+		t.Fatalf("wall_seconds=%d", got.WallSeconds)
+	}
+
+	if rec := get("/api/tasks/999999999/outcome"); rec.Code != http.StatusNotFound {
+		t.Fatalf("unknown task status=%d body=%s", rec.Code, rec.Body.String())
+	}
+	if rec := get("/api/tasks/not-a-number/outcome"); rec.Code != http.StatusBadRequest {
+		t.Fatalf("bad id status=%d body=%s", rec.Code, rec.Body.String())
+	}
+}