mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-13 11:40:53 +08:00
Merge pull request #106 from BaaRai/rsi/skill-gaps
feat(server): capability-gap report from skill misses
This commit is contained in:
BaaRai
@@ -0,0 +1,51 @@
|
||||
package db
|
||||
|
||||
import "context"
|
||||
|
||||
// SkillGap is one skill the agent asked for but the platform did not provide,
|
||||
// aggregated by skill, agent, and task category. A miss row is written when the
|
||||
// model names a skill that does not exist (skill_usage.found=false), so this
|
||||
// report is the evidence-backed capability backlog rather than a guess.
|
||||
type SkillGap struct {
|
||||
Skill string `json:"skill"`
|
||||
AgentKey string `json:"agent_key"`
|
||||
CategoryID int64 `json:"category_id"`
|
||||
Misses int `json:"misses"`
|
||||
Hits int `json:"hits"`
|
||||
LastSeen int64 `json:"last_seen"`
|
||||
}
|
||||
|
||||
// SkillGaps returns the aggregated miss report, highest miss count first.
|
||||
// Hits are the successful calls for the same key, so callers can compute a
|
||||
// miss rate without a second query.
|
||||
func (d *DB) SkillGaps(ctx context.Context, limit int) ([]SkillGap, error) {
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
rows, err := d.QueryContext(ctx, `
|
||||
SELECT su.skill,
|
||||
COALESCE(su.agent_key,''),
|
||||
COALESCE(t.category_id,0),
|
||||
COUNT(*) FILTER (WHERE NOT su.found) AS misses,
|
||||
COUNT(*) FILTER (WHERE su.found) AS hits,
|
||||
EXTRACT(EPOCH FROM MAX(su.ts))::bigint
|
||||
FROM skill_usage su
|
||||
LEFT JOIN tasks t ON t.id=su.task_id
|
||||
GROUP BY 1,2,3
|
||||
HAVING COUNT(*) FILTER (WHERE NOT su.found) > 0
|
||||
ORDER BY misses DESC, 6 DESC, 1
|
||||
LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []SkillGap{}
|
||||
for rows.Next() {
|
||||
var g SkillGap
|
||||
if err := rows.Scan(&g.Skill, &g.AgentKey, &g.CategoryID, &g.Misses, &g.Hits, &g.LastSeen); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, g)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -673,6 +673,7 @@ func (s *Server) Handler() http.Handler {
|
||||
|
||||
mux.HandleFunc("GET /api/tasks", s.listTasks)
|
||||
mux.HandleFunc("POST /api/tasks", s.createTask)
|
||||
mux.HandleFunc("GET /api/skill-gaps", s.skillGaps)
|
||||
mux.HandleFunc("GET /api/task-categories", s.pgListTaskCategories)
|
||||
mux.HandleFunc("POST /api/task-categories", s.pgCreateTaskCategory)
|
||||
mux.HandleFunc("PATCH /api/task-categories/{id}", s.pgRenameTaskCategory)
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// skillGaps reports skills the agent asked for but the platform did not
|
||||
// provide, aggregated by skill, agent, and task category. The source is the
|
||||
// existing skill_usage miss rows, so the backlog is evidence-backed rather than
|
||||
// inferred. Read-only: it creates no skill and changes no prompt.
|
||||
func (s *Server) skillGaps(w http.ResponseWriter, r *http.Request) {
|
||||
limit := 50
|
||||
if raw := r.URL.Query().Get("limit"); raw != "" {
|
||||
n, err := strconv.Atoi(raw)
|
||||
if err != nil || n < 1 || n > 500 {
|
||||
writeErr(w, 400, "limit must be an integer between 1 and 500")
|
||||
return
|
||||
}
|
||||
limit = n
|
||||
}
|
||||
gaps, err := s.m.PG().SkillGaps(r.Context(), limit)
|
||||
if err != nil {
|
||||
log.Printf("[skills] gap query failed: %v", err)
|
||||
writeErr(w, 500, "skill gap query failed")
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"gaps": gaps, "count": len(gaps)})
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSkillGapsReport(t *testing.T) {
|
||||
m, err := NewManager(t.TempDir(), "")
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
defer m.Close()
|
||||
task, err := m.CreateTask("skill gaps", "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
|
||||
defer func() { _ = m.pg.DeleteTask(taskID) }()
|
||||
defer func() { _, _ = m.pg.Exec(`DELETE FROM skill_usage WHERE task_id=$1`, taskID) }()
|
||||
|
||||
if _, err := m.pg.Exec(`
|
||||
INSERT INTO skill_usage(skill,agent_key,task_id,found) VALUES
|
||||
('rsi-test-graphql','worker',$1,false),
|
||||
('rsi-test-graphql','worker',$1,false),
|
||||
('rsi-test-graphql','worker',$1,true),
|
||||
('rsi-test-cloud','worker',$1,false),
|
||||
('rsi-test-known','worker',$1,true)`, taskID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s := New(context.Background(), m, t.TempDir(), t.TempDir(), t.TempDir())
|
||||
token, err := signJWT(s.jwtKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
get := func(path string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
rec := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
rec := get("/api/skill-gaps")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var out struct {
|
||||
Gaps []struct {
|
||||
Skill string `json:"skill"`
|
||||
AgentKey string `json:"agent_key"`
|
||||
Misses int `json:"misses"`
|
||||
Hits int `json:"hits"`
|
||||
} `json:"gaps"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bySkill := map[string]struct {
|
||||
misses, hits int
|
||||
agent string
|
||||
}{}
|
||||
for _, g := range out.Gaps {
|
||||
bySkill[g.Skill] = struct {
|
||||
misses, hits int
|
||||
agent string
|
||||
}{g.Misses, g.Hits, g.AgentKey}
|
||||
}
|
||||
if got := bySkill["rsi-test-graphql"]; got.misses != 2 || got.hits != 1 || got.agent != "worker" {
|
||||
t.Fatalf("graphql gap=%+v want misses=2 hits=1 agent=worker", got)
|
||||
}
|
||||
if got := bySkill["rsi-test-cloud"]; got.misses != 1 {
|
||||
t.Fatalf("cloud gap=%+v want misses=1", got)
|
||||
}
|
||||
if _, ok := bySkill["rsi-test-known"]; ok {
|
||||
t.Fatal("a hit-only skill must not appear in the miss report")
|
||||
}
|
||||
first := out.Gaps[0].Skill
|
||||
if first != "rsi-test-graphql" {
|
||||
t.Fatalf("ordering: first=%q want rsi-test-graphql (highest misses)", first)
|
||||
}
|
||||
|
||||
rec = get("/api/skill-gaps?limit=1")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("limit status=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var limited struct {
|
||||
Gaps []struct {
|
||||
Skill string `json:"skill"`
|
||||
} `json:"gaps"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &limited); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(limited.Gaps) != 1 || limited.Gaps[0].Skill != "rsi-test-graphql" {
|
||||
t.Fatalf("limit=1 returned %+v", limited.Gaps)
|
||||
}
|
||||
if rec := get("/api/skill-gaps?limit=abc"); rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("bad limit status=%d want 400", rec.Code)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user