feat(server): capability-gap report from skill misses

GET /api/skill-gaps aggregates skill_usage rows with found=false by
skill, agent, and task category. Hits are returned for the same key so a
miss rate needs no second query; hit-only skills are excluded. The source
rows already existed: a miss is written when the model names a skill the
platform does not provide.

This is the evidence-backed capability backlog, not an auto-fixer: it
creates no skill and changes no prompt. Motivated by the harness and
skill-evolution sections of awesome-rsi (lobehub).

db/skill_gaps.go

@@ -0,0 +1,51 @@
+package db
+
+import "context"
+
+// SkillGap is one skill the agent asked for but the platform did not provide,
+// aggregated by skill, agent, and task category. A miss row is written when the
+// model names a skill that does not exist (skill_usage.found=false), so this
+// report is the evidence-backed capability backlog rather than a guess.
+type SkillGap struct {
+	Skill      string `json:"skill"`
+	AgentKey   string `json:"agent_key"`
+	CategoryID int64  `json:"category_id"`
+	Misses     int    `json:"misses"`
+	Hits       int    `json:"hits"`
+	LastSeen   int64  `json:"last_seen"`
+}
+
+// SkillGaps returns the aggregated miss report, highest miss count first.
+// Hits are the successful calls for the same key, so callers can compute a
+// miss rate without a second query.
+func (d *DB) SkillGaps(ctx context.Context, limit int) ([]SkillGap, error) {
+	if limit <= 0 {
+		limit = 50
+	}
+	rows, err := d.QueryContext(ctx, `
+		SELECT su.skill,
+		       COALESCE(su.agent_key,''),
+		       COALESCE(t.category_id,0),
+		       COUNT(*) FILTER (WHERE NOT su.found) AS misses,
+		       COUNT(*) FILTER (WHERE su.found)     AS hits,
+		       EXTRACT(EPOCH FROM MAX(su.ts))::bigint
+		FROM skill_usage su
+		LEFT JOIN tasks t ON t.id=su.task_id
+		GROUP BY 1,2,3
+		HAVING COUNT(*) FILTER (WHERE NOT su.found) > 0
+		ORDER BY misses DESC, 6 DESC, 1
+		LIMIT $1`, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := []SkillGap{}
+	for rows.Next() {
+		var g SkillGap
+		if err := rows.Scan(&g.Skill, &g.AgentKey, &g.CategoryID, &g.Misses, &g.Hits, &g.LastSeen); err != nil {
+			return nil, err
+		}
+		out = append(out, g)
+	}
+	return out, rows.Err()
+}

db/task_outcome.go

@@ -1,119 +0,0 @@
-package db
-
-import "context"
-
-// The per-task outcome card is the baseline layer for RSI comparisons. It
-// consolidates what a task produced and what it cost so a later harness or
-// lesson change can be judged against a number instead of an impression.
-// Metric selection follows harness2 (Google Research, Apache-2.0,
-// arXiv 2610.14224) and the task-gain / retained-gain / improver-gain taxonomy
-// used by the awesome-rsi list (lobehub).
-
-// TaskOutcomeFindings is the finding disposition and severity split.
-type TaskOutcomeFindings struct {
-	Total         int `json:"total"`
-	Confirmed     int `json:"confirmed"` // confirmed, resolved, or fixed
-	Open          int `json:"open"`      // pending or in_progress
-	FalsePositive int `json:"false_positive"`
-	Duplicate     int `json:"duplicate"`
-	Critical      int `json:"critical"` // confirmed only
-	High          int `json:"high"`
-	Medium        int `json:"medium"`
-	Low           int `json:"low"`
-}
-
-// TaskOutcomeRetests is the retest verdict split for this task's findings.
-type TaskOutcomeRetests struct {
-	Reproduced   int `json:"reproduced"`
-	Fixed        int `json:"fixed"`
-	Inconclusive int `json:"inconclusive"`
-}
-
-// TaskOutcomeTokens is the task's result-row token total.
-type TaskOutcomeTokens struct {
-	InputTokens      int `json:"input_tokens"`
-	OutputTokens     int `json:"output_tokens"`
-	CacheReadTokens  int `json:"cache_read_tokens"`
-	CacheWriteTokens int `json:"cache_write_tokens"`
-}
-
-// TaskOutcome is one task's production and cost baseline.
-type TaskOutcome struct {
-	TaskID      int64               `json:"task_id"`
-	Status      string              `json:"status"`
-	CategoryID  int64               `json:"category_id"`
-	CompanyID   int64               `json:"company_id"`
-	Findings    TaskOutcomeFindings `json:"findings"`
-	Retests     TaskOutcomeRetests  `json:"retests"`
-	Tokens      TaskOutcomeTokens   `json:"tokens"`
-	ToolCalls   int                 `json:"tool_calls"`
-	SkillMisses int                 `json:"skill_misses"`
-	WallSeconds int64               `json:"wall_seconds"`
-}
-
-// TaskOutcome returns the outcome card for one live task. sql.ErrNoRows means
-// the task does not exist or is deleted.
-func (d *DB) TaskOutcome(ctx context.Context, taskID int64) (TaskOutcome, error) {
-	var o TaskOutcome
-	err := d.QueryRowContext(ctx, `
-		SELECT t.id, t.status,
-		       COALESCE(t.category_id,0), COALESCE(t.company_id,0),
-		       COALESCE(f.total,0), COALESCE(f.confirmed,0), COALESCE(f.open,0),
-		       COALESCE(f.false_positive,0), COALESCE(f.duplicate,0),
-		       COALESCE(f.critical,0), COALESCE(f.high,0),
-		       COALESCE(f.medium,0), COALESCE(f.low,0),
-		       COALESCE(r.reproduced,0), COALESCE(r.fixed,0), COALESCE(r.inconclusive,0),
-		       COALESCE(a.input_tokens,0), COALESCE(a.output_tokens,0),
-		       COALESCE(a.cache_read_tokens,0), COALESCE(a.cache_write_tokens,0),
-		       COALESCE(tu.calls,0), COALESCE(su.misses,0),
-		       GREATEST(0, EXTRACT(EPOCH FROM
-		           (COALESCE(t.completed_at, now()) - COALESCE(t.first_run_at, t.created_at)))::bigint)
-		FROM tasks t
-		LEFT JOIN LATERAL (
-			SELECT COUNT(*) AS total,
-			       COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed')) AS confirmed,
-			       COUNT(*) FILTER (WHERE status IN ('pending','in_progress'))       AS open,
-			       COUNT(*) FILTER (WHERE status='false_positive')                   AS false_positive,
-			       COUNT(*) FILTER (WHERE status='duplicate')                        AS duplicate,
-			       COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='critical') AS critical,
-			       COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='high')     AS high,
-			       COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='medium')   AS medium,
-			       COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='low')      AS low
-			FROM findings
-			WHERE task_id=t.id
-		) f ON true
-		LEFT JOIN LATERAL (
-			SELECT COUNT(*) FILTER (WHERE fr.verdict='reproduced')   AS reproduced,
-			       COUNT(*) FILTER (WHERE fr.verdict='fixed')        AS fixed,
-			       COUNT(*) FILTER (WHERE fr.verdict='inconclusive') AS inconclusive
-			FROM finding_retests fr
-			JOIN findings f2 ON f2.id=fr.finding_id
-			WHERE f2.task_id=t.id
-		) r ON true
-		LEFT JOIN LATERAL (
-			SELECT SUM(input_tokens) AS input_tokens,
-			       SUM(output_tokens) AS output_tokens,
-			       SUM(cache_read_tokens) AS cache_read_tokens,
-			       SUM(cache_write_tokens) AS cache_write_tokens
-			FROM activity
-			WHERE exploration_id=t.exploration_id AND kind='result'
-		) a ON true
-		LEFT JOIN LATERAL (
-			SELECT COUNT(*) AS calls FROM tool_usage WHERE task_id=t.id
-		) tu ON true
-		LEFT JOIN LATERAL (
-			SELECT COUNT(*) AS misses FROM skill_usage WHERE task_id=t.id AND NOT found
-		) su ON true
-		WHERE t.id=$1 AND t.deleted_at IS NULL`, taskID).Scan(
-		&o.TaskID, &o.Status,
-		&o.CategoryID, &o.CompanyID,
-		&o.Findings.Total, &o.Findings.Confirmed, &o.Findings.Open,
-		&o.Findings.FalsePositive, &o.Findings.Duplicate,
-		&o.Findings.Critical, &o.Findings.High, &o.Findings.Medium, &o.Findings.Low,
-		&o.Retests.Reproduced, &o.Retests.Fixed, &o.Retests.Inconclusive,
-		&o.Tokens.InputTokens, &o.Tokens.OutputTokens,
-		&o.Tokens.CacheReadTokens, &o.Tokens.CacheWriteTokens,
-		&o.ToolCalls, &o.SkillMisses, &o.WallSeconds,
-	)
-	return o, err
-}

server/server.go

@@ -673,6 +673,7 @@ func (s *Server) Handler() http.Handler {
 
 	mux.HandleFunc("GET /api/tasks", s.listTasks)
 	mux.HandleFunc("POST /api/tasks", s.createTask)
+	mux.HandleFunc("GET /api/skill-gaps", s.skillGaps)
 	mux.HandleFunc("GET /api/task-categories", s.pgListTaskCategories)
 	mux.HandleFunc("POST /api/task-categories", s.pgCreateTaskCategory)
 	mux.HandleFunc("PATCH /api/task-categories/{id}", s.pgRenameTaskCategory)
@@ -683,7 +684,6 @@ func (s *Server) Handler() http.Handler {
 	mux.HandleFunc("PATCH /api/task-templates/{id}", s.pgUpdateTaskTemplate)
 	mux.HandleFunc("DELETE /api/task-templates/{id}", s.pgDeleteTaskTemplate)
 	mux.HandleFunc("GET /api/tasks/{id}", s.getTask)
-	mux.HandleFunc("GET /api/tasks/{id}/outcome", s.taskOutcome)
 	mux.HandleFunc("PATCH /api/tasks/{id}", s.updateTaskMetadata)
 	mux.HandleFunc("PATCH /api/tasks/{id}/category", s.updateTaskCategory)
 	// task-level asset interception/allow rules

server/skill_gaps.go

@@ -0,0 +1,30 @@
+package server
+
+import (
+	"log"
+	"net/http"
+	"strconv"
+)
+
+// skillGaps reports skills the agent asked for but the platform did not
+// provide, aggregated by skill, agent, and task category. The source is the
+// existing skill_usage miss rows, so the backlog is evidence-backed rather than
+// inferred. Read-only: it creates no skill and changes no prompt.
+func (s *Server) skillGaps(w http.ResponseWriter, r *http.Request) {
+	limit := 50
+	if raw := r.URL.Query().Get("limit"); raw != "" {
+		n, err := strconv.Atoi(raw)
+		if err != nil || n < 1 || n > 500 {
+			writeErr(w, 400, "limit must be an integer between 1 and 500")
+			return
+		}
+		limit = n
+	}
+	gaps, err := s.m.PG().SkillGaps(r.Context(), limit)
+	if err != nil {
+		log.Printf("[skills] gap query failed: %v", err)
+		writeErr(w, 500, "skill gap query failed")
+		return
+	}
+	writeJSON(w, 200, map[string]any{"gaps": gaps, "count": len(gaps)})
+}

server/skill_gaps_test.go

@@ -0,0 +1,107 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+)
+
+func TestSkillGapsReport(t *testing.T) {
+	m, err := NewManager(t.TempDir(), "")
+	if err != nil {
+		t.Skipf("postgres unavailable (%v) — skipping", err)
+	}
+	defer m.Close()
+	task, err := m.CreateTask("skill gaps", "goal", nil, 0, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	taskID, _ := strconv.ParseInt(task.ID, 10, 64)
+	defer func() { _ = m.pg.DeleteTask(taskID) }()
+	defer func() { _, _ = m.pg.Exec(`DELETE FROM skill_usage WHERE task_id=$1`, taskID) }()
+
+	if _, err := m.pg.Exec(`
+INSERT INTO skill_usage(skill,agent_key,task_id,found) VALUES
+ ('rsi-test-graphql','worker',$1,false),
+ ('rsi-test-graphql','worker',$1,false),
+ ('rsi-test-graphql','worker',$1,true),
+ ('rsi-test-cloud','worker',$1,false),
+ ('rsi-test-known','worker',$1,true)`, taskID); err != nil {
+		t.Fatal(err)
+	}
+
+	s := New(context.Background(), m, t.TempDir(), t.TempDir(), t.TempDir())
+	token, err := signJWT(s.jwtKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	get := func(path string) *httptest.ResponseRecorder {
+		req := httptest.NewRequest(http.MethodGet, path, nil)
+		req.Header.Set("Authorization", "Bearer "+token)
+		rec := httptest.NewRecorder()
+		s.Handler().ServeHTTP(rec, req)
+		return rec
+	}
+
+	rec := get("/api/skill-gaps")
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
+	}
+	var out struct {
+		Gaps []struct {
+			Skill    string `json:"skill"`
+			AgentKey string `json:"agent_key"`
+			Misses   int    `json:"misses"`
+			Hits     int    `json:"hits"`
+		} `json:"gaps"`
+		Count int `json:"count"`
+	}
+	if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
+		t.Fatal(err)
+	}
+	bySkill := map[string]struct {
+		misses, hits int
+		agent        string
+	}{}
+	for _, g := range out.Gaps {
+		bySkill[g.Skill] = struct {
+			misses, hits int
+			agent        string
+		}{g.Misses, g.Hits, g.AgentKey}
+	}
+	if got := bySkill["rsi-test-graphql"]; got.misses != 2 || got.hits != 1 || got.agent != "worker" {
+		t.Fatalf("graphql gap=%+v want misses=2 hits=1 agent=worker", got)
+	}
+	if got := bySkill["rsi-test-cloud"]; got.misses != 1 {
+		t.Fatalf("cloud gap=%+v want misses=1", got)
+	}
+	if _, ok := bySkill["rsi-test-known"]; ok {
+		t.Fatal("a hit-only skill must not appear in the miss report")
+	}
+	first := out.Gaps[0].Skill
+	if first != "rsi-test-graphql" {
+		t.Fatalf("ordering: first=%q want rsi-test-graphql (highest misses)", first)
+	}
+
+	rec = get("/api/skill-gaps?limit=1")
+	if rec.Code != http.StatusOK {
+		t.Fatalf("limit status=%d body=%s", rec.Code, rec.Body.String())
+	}
+	var limited struct {
+		Gaps []struct {
+			Skill string `json:"skill"`
+		} `json:"gaps"`
+	}
+	if err := json.Unmarshal(rec.Body.Bytes(), &limited); err != nil {
+		t.Fatal(err)
+	}
+	if len(limited.Gaps) != 1 || limited.Gaps[0].Skill != "rsi-test-graphql" {
+		t.Fatalf("limit=1 returned %+v", limited.Gaps)
+	}
+	if rec := get("/api/skill-gaps?limit=abc"); rec.Code != http.StatusBadRequest {
+		t.Fatalf("bad limit status=%d want 400", rec.Code)
+	}
+}

server/task_outcome.go

@@ -1,31 +0,0 @@
-package server
-
-import (
-	"database/sql"
-	"errors"
-	"log"
-	"net/http"
-	"strconv"
-)
-
-// taskOutcome returns one task's production and cost baseline. It is the
-// measurement layer for later RSI comparisons (harness2, Google Research,
-// arXiv 2610.14224): nothing here changes agent behavior.
-func (s *Server) taskOutcome(w http.ResponseWriter, r *http.Request) {
-	id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
-	if err != nil || id <= 0 {
-		writeErr(w, 400, "invalid task id")
-		return
-	}
-	outcome, err := s.m.PG().TaskOutcome(r.Context(), id)
-	if errors.Is(err, sql.ErrNoRows) {
-		writeErr(w, 404, "task not found")
-		return
-	}
-	if err != nil {
-		log.Printf("[tasks] outcome query failed for %d: %v", id, err)
-		writeErr(w, 500, "outcome query failed")
-		return
-	}
-	writeJSON(w, 200, outcome)
-}

server/task_outcome_test.go

@@ -1,114 +0,0 @@
-package server
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"strconv"
-	"testing"
-
-	"github.com/Autumn-27/artex/db"
-)
-
-func TestTaskOutcomeCard(t *testing.T) {
-	m, err := NewManager(t.TempDir(), "")
-	if err != nil {
-		t.Skipf("postgres unavailable (%v) — skipping", err)
-	}
-	defer m.Close()
-	task, err := m.CreateTask("outcome card", "goal", nil, 0, 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-	taskID, _ := strconv.ParseInt(task.ID, 10, 64)
-	defer func() { _ = m.pg.DeleteTask(taskID) }()
-
-	if _, err := m.pg.Exec(`
-INSERT INTO findings(task_id,vulnclass,name,severity,status) VALUES
- ($1,'xss','A','high','confirmed'),
- ($1,'sqli','B','critical','fixed'),
- ($1,'csrf','C','medium','pending'),
- ($1,'xss','D','low','false_positive'),
- ($1,'idor','E','high','duplicate')`, taskID); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := m.pg.Exec(`
-INSERT INTO finding_retests(finding_id,status,verdict,snapshot)
-SELECT id,'completed','reproduced','{}'::jsonb FROM findings WHERE task_id=$1 AND name='A'`, taskID); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := m.pg.Exec(`
-INSERT INTO finding_retests(finding_id,status,verdict,snapshot)
-SELECT id,'completed','fixed','{}'::jsonb FROM findings WHERE task_id=$1 AND name='B'`, taskID); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := m.pg.Exec(`
-INSERT INTO activity(exploration_id,kind,input_tokens,output_tokens,cache_read_tokens,cache_write_tokens)
-VALUES ($1,'result',11,7,3,2), ($1,'tool_result',999,999,999,999)`, task.ExpID); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := m.pg.Exec(`
-INSERT INTO tool_usage(tool_key,task_id) VALUES ('traffic_search',$1),('report_finding',$1)`, taskID); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := m.pg.Exec(`
-INSERT INTO skill_usage(skill,task_id,found) VALUES ('api-recon',$1,true),('graphql-audit',$1,false)`, taskID); err != nil {
-		t.Fatal(err)
-	}
-
-	s := New(context.Background(), m, t.TempDir(), t.TempDir(), t.TempDir())
-	token, err := signJWT(s.jwtKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	get := func(path string) *httptest.ResponseRecorder {
-		req := httptest.NewRequest(http.MethodGet, path, nil)
-		req.Header.Set("Authorization", "Bearer "+token)
-		rec := httptest.NewRecorder()
-		s.Handler().ServeHTTP(rec, req)
-		return rec
-	}
-
-	rec := get("/api/tasks/" + task.ID + "/outcome")
-	if rec.Code != http.StatusOK {
-		t.Fatalf("outcome status=%d body=%s", rec.Code, rec.Body.String())
-	}
-	var got db.TaskOutcome
-	if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
-		t.Fatal(err)
-	}
-	if got.TaskID != taskID || got.Status != "created" {
-		t.Fatalf("task identity: %+v", got)
-	}
-	wantFindings := db.TaskOutcomeFindings{
-		Total: 5, Confirmed: 2, Open: 1, FalsePositive: 1, Duplicate: 1,
-		Critical: 1, High: 1, Medium: 0, Low: 0,
-	}
-	if got.Findings != wantFindings {
-		t.Fatalf("findings=%+v want %+v", got.Findings, wantFindings)
-	}
-	if got.Retests.Reproduced != 1 || got.Retests.Fixed != 1 || got.Retests.Inconclusive != 0 {
-		t.Fatalf("retests=%+v", got.Retests)
-	}
-	if got.Tokens.InputTokens != 11 || got.Tokens.OutputTokens != 7 ||
-		got.Tokens.CacheReadTokens != 3 || got.Tokens.CacheWriteTokens != 2 {
-		t.Fatalf("tokens=%+v (non-result rows must be excluded)", got.Tokens)
-	}
-	if got.ToolCalls != 2 {
-		t.Fatalf("tool_calls=%d want 2", got.ToolCalls)
-	}
-	if got.SkillMisses != 1 {
-		t.Fatalf("skill_misses=%d want 1", got.SkillMisses)
-	}
-	if got.WallSeconds < 0 {
-		t.Fatalf("wall_seconds=%d", got.WallSeconds)
-	}
-
-	if rec := get("/api/tasks/999999999/outcome"); rec.Code != http.StatusNotFound {
-		t.Fatalf("unknown task status=%d body=%s", rec.Code, rec.Body.String())
-	}
-	if rec := get("/api/tasks/not-a-number/outcome"); rec.Code != http.StatusBadRequest {
-		t.Fatalf("bad id status=%d body=%s", rec.Code, rec.Body.String())
-	}
-}