Compare commits

...
Author SHA1 Message Date
BaaRai cf369dec4f feat(server): per-task outcome card for RSI baselines
Adds GET /api/tasks/{id}/outcome: finding disposition and severity,
retest verdicts, result-row token totals, tool calls, skill misses, and
wall time in one row. This is the measurement layer an RSI loop needs
before claiming a change helped; it alters no agent behavior.

Metric selection follows harness2 (Google Research, Apache-2.0,
arXiv 2610.14224) and the task-gain / retained-gain / improver-gain
taxonomy in the awesome-rsi list.
2026-10-19 00:28:27 +02:00
BaaRai bba7989903 Merge pull request #98 from BaaRai/fix/drost-dynamic-port
fix(bench): discover the challenge port dynamically and tear down on deploy failure
2026-10-16 14:31:42 +02:00
BaaRai 89ece8eb8d fix(bench): discover the challenge port dynamically and tear down on deploy failure
The suite does not use one container port: most challenges publish 8080
but some (051, 091, 100) publish 5000, and the adapter had 8080
hardcoded, so deploy failed on those and the failure happened before
the teardown block, leaving the challenge running.

- the host port now comes from the app container port mapping (any
  container port, via docker inspect), never from an assumed 8080
- the teardown finally now wraps the deploy as well: a failure halfway
  through deploy no longer leaves containers behind (--keep and
  --deploy-only still opt out)
2026-10-16 14:29:00 +02:00
BaaRai 623e5e58a7 Merge pull request #92 from BaaRai/fix/task-detail-metrics
fix(server): fill task metrics on the detail endpoint
2026-10-16 13:38:10 +02:00
BaaRai 0777edfb8f Merge pull request #97 from BaaRai/fix/drost-finding-detail-resilience
fix(bench): survive finding-detail 404s and always tear the corpus down
2026-10-16 12:17:54 +02:00
BaaRai 67971ef7da fix(bench): survive finding-detail 404s and always tear the corpus down
The first live drost run died right after the task completed: the
findings detail route keys on finding_id (the findings-table row), not
on the exploration node id the list also carries, so every detail fetch
404ed; fetch_findings then re-raised the SystemExit from the API
wrapper, aborting scoring, the flag scan, the report and the teardown.

- details are fetched with finding_id (node id as fallback) and a
  missing detail no longer kills the run: the list fields remain and the
  finding stays gradeable
- the stored finding carries finding_id so the deep flag scan can
  re-fetch untruncated text
- the deep scan tolerates the same failures
- teardown runs in a finally around the whole task phase, so a crash in
  scoring or reporting can no longer leave the challenge running
  (--keep still opts out deliberately)
2026-10-16 12:15:02 +02:00
BaaRai 57cac93d49 Merge pull request #96 from BaaRai/fix/drostbench-goal-target
fix(bench): name the target URL in the DrostBench task goal
2026-10-16 12:04:46 +02:00
BaaRai b269d3e590 fix(server): fill task metrics on the detail endpoint
GET /api/tasks/{id} returned the metrics block zeroed (tokens, goal
progress, in_flight, findings, last_activity) because only listTasks
populated those DTO fields. Extract the fill into applyTaskMetrics and
call it from both handlers so the two endpoints report identical
numbers from the same TaskListMetricsAll query. The swallowed query
error in listTasks is now logged instead of silently zeroing the table.

Test: TestTaskDetailReturnsListMetrics seeds result-activity tokens and
goal nodes, asserts the detail endpoint reports them and matches the
list endpoint; fails on main, passes with the fix.

Fixes #91
2026-10-13 14:49:47 +02:00
7 changed files with 521 additions and 95 deletions
+119
View File
@@ -0,0 +1,119 @@
package db
import "context"
// The per-task outcome card is the baseline layer for RSI comparisons. It
// consolidates what a task produced and what it cost so a later harness or
// lesson change can be judged against a number instead of an impression.
// Metric selection follows harness2 (Google Research, Apache-2.0,
// arXiv 2610.14224) and the task-gain / retained-gain / improver-gain taxonomy
// used by the awesome-rsi list (lobehub).
// TaskOutcomeFindings is the finding disposition and severity split.
type TaskOutcomeFindings struct {
Total int `json:"total"`
Confirmed int `json:"confirmed"` // confirmed, resolved, or fixed
Open int `json:"open"` // pending or in_progress
FalsePositive int `json:"false_positive"`
Duplicate int `json:"duplicate"`
Critical int `json:"critical"` // confirmed only
High int `json:"high"`
Medium int `json:"medium"`
Low int `json:"low"`
}
// TaskOutcomeRetests is the retest verdict split for this task's findings.
type TaskOutcomeRetests struct {
Reproduced int `json:"reproduced"`
Fixed int `json:"fixed"`
Inconclusive int `json:"inconclusive"`
}
// TaskOutcomeTokens is the task's result-row token total.
type TaskOutcomeTokens struct {
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
CacheReadTokens int `json:"cache_read_tokens"`
CacheWriteTokens int `json:"cache_write_tokens"`
}
// TaskOutcome is one task's production and cost baseline.
type TaskOutcome struct {
TaskID int64 `json:"task_id"`
Status string `json:"status"`
CategoryID int64 `json:"category_id"`
CompanyID int64 `json:"company_id"`
Findings TaskOutcomeFindings `json:"findings"`
Retests TaskOutcomeRetests `json:"retests"`
Tokens TaskOutcomeTokens `json:"tokens"`
ToolCalls int `json:"tool_calls"`
SkillMisses int `json:"skill_misses"`
WallSeconds int64 `json:"wall_seconds"`
}
// TaskOutcome returns the outcome card for one live task. sql.ErrNoRows means
// the task does not exist or is deleted.
func (d *DB) TaskOutcome(ctx context.Context, taskID int64) (TaskOutcome, error) {
var o TaskOutcome
err := d.QueryRowContext(ctx, `
SELECT t.id, t.status,
COALESCE(t.category_id,0), COALESCE(t.company_id,0),
COALESCE(f.total,0), COALESCE(f.confirmed,0), COALESCE(f.open,0),
COALESCE(f.false_positive,0), COALESCE(f.duplicate,0),
COALESCE(f.critical,0), COALESCE(f.high,0),
COALESCE(f.medium,0), COALESCE(f.low,0),
COALESCE(r.reproduced,0), COALESCE(r.fixed,0), COALESCE(r.inconclusive,0),
COALESCE(a.input_tokens,0), COALESCE(a.output_tokens,0),
COALESCE(a.cache_read_tokens,0), COALESCE(a.cache_write_tokens,0),
COALESCE(tu.calls,0), COALESCE(su.misses,0),
GREATEST(0, EXTRACT(EPOCH FROM
(COALESCE(t.completed_at, now()) - COALESCE(t.first_run_at, t.created_at)))::bigint)
FROM tasks t
LEFT JOIN LATERAL (
SELECT COUNT(*) AS total,
COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed')) AS confirmed,
COUNT(*) FILTER (WHERE status IN ('pending','in_progress')) AS open,
COUNT(*) FILTER (WHERE status='false_positive') AS false_positive,
COUNT(*) FILTER (WHERE status='duplicate') AS duplicate,
COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='critical') AS critical,
COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='high') AS high,
COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='medium') AS medium,
COUNT(*) FILTER (WHERE status IN ('confirmed','resolved','fixed') AND severity='low') AS low
FROM findings
WHERE task_id=t.id
) f ON true
LEFT JOIN LATERAL (
SELECT COUNT(*) FILTER (WHERE fr.verdict='reproduced') AS reproduced,
COUNT(*) FILTER (WHERE fr.verdict='fixed') AS fixed,
COUNT(*) FILTER (WHERE fr.verdict='inconclusive') AS inconclusive
FROM finding_retests fr
JOIN findings f2 ON f2.id=fr.finding_id
WHERE f2.task_id=t.id
) r ON true
LEFT JOIN LATERAL (
SELECT SUM(input_tokens) AS input_tokens,
SUM(output_tokens) AS output_tokens,
SUM(cache_read_tokens) AS cache_read_tokens,
SUM(cache_write_tokens) AS cache_write_tokens
FROM activity
WHERE exploration_id=t.exploration_id AND kind='result'
) a ON true
LEFT JOIN LATERAL (
SELECT COUNT(*) AS calls FROM tool_usage WHERE task_id=t.id
) tu ON true
LEFT JOIN LATERAL (
SELECT COUNT(*) AS misses FROM skill_usage WHERE task_id=t.id AND NOT found
) su ON true
WHERE t.id=$1 AND t.deleted_at IS NULL`, taskID).Scan(
&o.TaskID, &o.Status,
&o.CategoryID, &o.CompanyID,
&o.Findings.Total, &o.Findings.Confirmed, &o.Findings.Open,
&o.Findings.FalsePositive, &o.Findings.Duplicate,
&o.Findings.Critical, &o.Findings.High, &o.Findings.Medium, &o.Findings.Low,
&o.Retests.Reproduced, &o.Retests.Fixed, &o.Retests.Inconclusive,
&o.Tokens.InputTokens, &o.Tokens.OutputTokens,
&o.Tokens.CacheReadTokens, &o.Tokens.CacheWriteTokens,
&o.ToolCalls, &o.SkillMisses, &o.WallSeconds,
)
return o, err
}
+22 -4
View File
@@ -36,7 +36,6 @@ FLAG_RE = re.compile(r"(?:FLAG\{[0-9a-f]{64}\}|flag\{[0-9a-f-]{36}\})")
CHALLENGE_RE = re.compile(r"DBEN-v1-\d{3}")
APP_SERVICE = "app"
APP_CONTAINER_PORT = "8080"
def random_flag() -> str:
@@ -80,13 +79,33 @@ def compose_project(challenge: str, run_id: str) -> str:
def host_port(output: str) -> int:
"""Parse the host port out of `docker compose port app 8080` output."""
"""Parse the host port out of `docker compose port` output."""
line = output.strip().splitlines()[-1] if output.strip() else ""
if ":" not in line:
raise SystemExit(f"could not parse a host port from docker compose output: {output!r}")
return int(line.rsplit(":", 1)[1])
def container_host_port(project: str, cdir: pathlib.Path) -> int:
"""Host port published by the app service, whatever container port the
challenge declares (most challenges use 8080, some use 5000: never assume)."""
res = _compose(project, cdir, ["ps", "-q", APP_SERVICE], timeout=60)
cid = res.stdout.strip().splitlines()[0] if res.stdout.strip() else ""
if not cid:
raise SystemExit(f"no running {APP_SERVICE} container for project {project}")
insp = subprocess.run(
["docker", "inspect", "--format",
"{{range $p, $conf := .NetworkSettings.Ports}}{{(index $conf 0).HostPort}}\n{{end}}",
cid],
capture_output=True, text=True, timeout=60)
for token in insp.stdout.split():
if token.isdigit():
return int(token)
raise SystemExit(
f"no published host port on container {cid} "
f"({insp.stderr.strip()[:200] or 'inspect returned no port mapping'})")
def derive_ground_truth(repo: str, challenge: str) -> dict:
"""Ground truth from the challenge's own benchmark.json.
@@ -168,8 +187,7 @@ def deploy(repo: str, challenge: str, flag: str, run_id: str) -> dict:
if up.returncode != 0:
print("[drost] up --wait failed; falling back to up -d and polling")
_compose(project, cdir, ["up", "-d"], timeout=900)
res = _compose(project, cdir, ["port", APP_SERVICE, APP_CONTAINER_PORT], timeout=60)
url = f"http://127.0.0.1:{host_port(res.stdout)}"
url = f"http://127.0.0.1:{container_host_port(project, cdir)}"
for _ in range(60):
try:
urllib.request.urlopen(url, timeout=5)
+88 -73
View File
@@ -189,18 +189,21 @@ def fetch_findings(api: Api, tid) -> list[dict]:
out = []
for f in items:
fid = f.get("id") or f.get("node_id")
# The detail route keys on finding_id (the findings-table row), not on
# the exploration node id the list also carries; asking with the node
# id 404s. Fall back to the node id so older payloads still resolve.
did = f.get("finding_id") or fid
detail = {}
try:
detail = api.get(f"/api/exploration/findings/{fid}")
except SystemExit:
raise
except Exception:
pass
detail = api.get(f"/api/exploration/findings/{did}")
except (SystemExit, Exception):
pass # a missing detail must not kill the run; list fields remain
report = detail.get("report") or f.get("report") or ""
summary = f.get("summary") or detail.get("summary") or ""
name = f.get("name") or f.get("title") or summary[:80]
out.append({
"id": fid,
"finding_id": str(f.get("finding_id") or ""),
"name": str(name)[:200],
"severity": str(f.get("severity") or detail.get("severity") or "?"),
"summary": str(summary)[:600],
@@ -217,13 +220,14 @@ def deep_evidence_texts(api: Api, tid, findings: list[dict]) -> list[str]:
common path."""
texts = []
for f in findings:
did = f.get("finding_id") or f.get("id")
try:
texts.append(json.dumps(api.get(f"/api/exploration/findings/{f['id']}")))
except Exception:
texts.append(json.dumps(api.get(f"/api/exploration/findings/{did}")))
except (SystemExit, Exception):
continue
try:
texts.append(json.dumps(api.get(f"/api/exploration/nodes?task={tid}&kind=fact&size=200")))
except Exception:
except (SystemExit, Exception):
pass
try:
r = urllib.request.Request(f"{api.base}/api/report?task={tid}")
@@ -468,72 +472,83 @@ def main() -> None:
gt = json.loads(gt_path.read_text())
gt_findings = gt["findings"]
dep = deploy_corpus(args)
corpus_url = dep["url"]
flag = dep.get("flag")
if args.deploy_only:
print(f"[drost] deploy-only: up at {corpus_url}, flag {flag}")
print(f"[drost] stop it later with: docker compose -p {dep['project']} down --remove-orphans"
f" (cwd {dep['challenge_dir']})")
return
task_url = corpus_url
if args.target_host != "127.0.0.1":
task_url = f"http://{args.target_host}:{corpus_url.rsplit(':', 1)[1]}"
print(f"[corpus] task target rewritten for ARTEX reachability: {task_url}")
api = Api(args.artex_url, token)
task_info = create_task(api, args, task_url, gt)
status = wait_task(api, task_info["id"], args.task_timeout + 1800)
findings = fetch_findings(api, task_info["id"])
invalid_ids = [f["id"] for f in findings if not (f["summary"].strip() or f["report"].strip())]
usable = [f for f in findings if f["id"] not in invalid_ids]
flag_captured = None
if flag:
# Objective oracle first pass over the already-fetched fields; the deep
# scan (full details, facts, report) runs only before declaring a miss.
flag_captured = drost.flag_in_texts(flag, drost.finding_texts(findings))
if not flag_captured:
flag_captured = drost.flag_in_texts(flag, deep_evidence_texts(api, task_info["id"], findings))
print(f"[flag] captured: {flag_captured}")
if judge_enabled:
if not usable:
print("[judge] no usable findings; scoring zeros")
verdicts = [{"gt_id": g["id"], "detected": False, "matched_finding_id": None,
"gates": {"component": False, "class": False, "keywords": False, "proof": False},
"reason": "no usable scanner findings"} for g in gt_findings]
else:
print("[judge] running falsification battery")
judge_battery(args)
print("[judge] grading")
verdicts = run_judge(args, gt_findings, usable)
result = tally(gt_findings, verdicts, findings, invalid_ids)
else:
print("[judge] flag-only run: no judge configured, analysis gates not measured")
verdicts = None
result = {
"mode": "flag-only",
"scanner_findings_total": len(findings),
"invalid_findings_excluded": invalid_ids,
"discovery_candidates": [{"id": f["id"], "name": f["name"]} for f in usable],
"note": "analysis gates not measured: no judge configured",
}
if flag:
result["flag_captured"] = flag_captured
result["flag_used"] = flag
out_dir = pathlib.Path("benchmark-results") / args.corpus / args.run_id
write_report(out_dir, args.corpus, args, task_info, status, findings, verdicts, result)
print(json.dumps(result, indent=2)[:1500])
# drost project/dir are deterministic, so they are known before deploy and
# the finally below can clean up even when the deploy itself fails halfway.
drost_project = drost_cdir = None
if args.corpus_type == "drost":
if not args.keep:
drost.teardown(dep["project"], dep["challenge_dir"])
elif not (args.keep or args.corpus_url):
subprocess.run(["docker", "rm", "-f", f"bench-{args.corpus}-{args.run_id}"], capture_output=True)
drost_project = drost.compose_project(args.drost_id, args.run_id)
drost_cdir = str(drost.challenge_dir(args.drost_repo, args.drost_id))
try:
dep = deploy_corpus(args)
corpus_url = dep["url"]
flag = dep.get("flag")
if args.deploy_only:
print(f"[drost] deploy-only: up at {corpus_url}, flag {flag}")
print(f"[drost] stop it later with: docker compose -p {dep['project']} down --remove-orphans"
f" (cwd {dep['challenge_dir']})")
return
task_url = corpus_url
if args.target_host != "127.0.0.1":
task_url = f"http://{args.target_host}:{corpus_url.rsplit(':', 1)[1]}"
print(f"[corpus] task target rewritten for ARTEX reachability: {task_url}")
api = Api(args.artex_url, token)
task_info = create_task(api, args, task_url, gt)
status = wait_task(api, task_info["id"], args.task_timeout + 1800)
findings = fetch_findings(api, task_info["id"])
invalid_ids = [f["id"] for f in findings if not (f["summary"].strip() or f["report"].strip())]
usable = [f for f in findings if f["id"] not in invalid_ids]
flag_captured = None
if flag:
# Objective oracle first pass over the already-fetched fields; the deep
# scan (full details, facts, report) runs only before declaring a miss.
flag_captured = drost.flag_in_texts(flag, drost.finding_texts(findings))
if not flag_captured:
flag_captured = drost.flag_in_texts(flag, deep_evidence_texts(api, task_info["id"], findings))
print(f"[flag] captured: {flag_captured}")
if judge_enabled:
if not usable:
print("[judge] no usable findings; scoring zeros")
verdicts = [{"gt_id": g["id"], "detected": False, "matched_finding_id": None,
"gates": {"component": False, "class": False, "keywords": False, "proof": False},
"reason": "no usable scanner findings"} for g in gt_findings]
else:
print("[judge] running falsification battery")
judge_battery(args)
print("[judge] grading")
verdicts = run_judge(args, gt_findings, usable)
result = tally(gt_findings, verdicts, findings, invalid_ids)
else:
print("[judge] flag-only run: no judge configured, analysis gates not measured")
verdicts = None
result = {
"mode": "flag-only",
"scanner_findings_total": len(findings),
"invalid_findings_excluded": invalid_ids,
"discovery_candidates": [{"id": f["id"], "name": f["name"]} for f in usable],
"note": "analysis gates not measured: no judge configured",
}
if flag:
result["flag_captured"] = flag_captured
result["flag_used"] = flag
out_dir = pathlib.Path("benchmark-results") / args.corpus / args.run_id
write_report(out_dir, args.corpus, args, task_info, status, findings, verdicts, result)
print(json.dumps(result, indent=2)[:1500])
finally:
# The corpus is torn down even when a step fails halfway: a crash in
# deploy, scoring or reporting must not leave the challenge running
# (--keep and --deploy-only opt out deliberately).
if args.corpus_type == "drost":
if not (args.keep or args.deploy_only):
drost.teardown(drost_project, drost_cdir)
elif not (args.keep or args.corpus_url):
subprocess.run(["docker", "rm", "-f", f"bench-{args.corpus}-{args.run_id}"], capture_output=True)
if __name__ == "__main__":
+34 -18
View File
@@ -683,6 +683,7 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("PATCH /api/task-templates/{id}", s.pgUpdateTaskTemplate)
mux.HandleFunc("DELETE /api/task-templates/{id}", s.pgDeleteTaskTemplate)
mux.HandleFunc("GET /api/tasks/{id}", s.getTask)
mux.HandleFunc("GET /api/tasks/{id}/outcome", s.taskOutcome)
mux.HandleFunc("PATCH /api/tasks/{id}", s.updateTaskMetadata)
mux.HandleFunc("PATCH /api/tasks/{id}/category", s.updateTaskCategory)
// task-level asset interception/allow rules
@@ -1044,34 +1045,44 @@ func (s *Server) listTasks(w http.ResponseWriter, r *http.Request) {
active = t.ID
}
list := s.m.List()
metrics, _ := s.m.PG().TaskListMetricsAll()
metrics, err := s.m.PG().TaskListMetricsAll()
if err != nil {
log.Printf("[tasks] list metrics query failed — metrics zeroed: %v", err)
}
archiveBlockers, _ := s.m.PG().TaskArchiveBlockers()
dtos := make([]TaskDTO, 0, len(list))
for _, t := range list {
dto := taskDTO(t, s.resolvedTaskStatus(t))
applyTaskArchiveBlocker(&dto, archiveBlockers)
metric := metrics[t.ExpID]
dto.Tokens = tokenTotalDTO(metric.Tokens)
// prefer the live in-memory heartbeat (fresher) and fall back to the
// persisted max activity time (survives restarts) for run-duration display.
dto.LastActivity = metric.LastActivity
if live := s.engine.LastActivity(t.ID); live > dto.LastActivity {
dto.LastActivity = live
}
dto.GoalsTotal = metric.Goals.Total
dto.GoalsMet = metric.Goals.Met
dto.InFlight = metric.RunningIntents
dto.Findings = FindingSeverityDTO{
Critical: metric.Findings.Critical,
High: metric.Findings.High,
Medium: metric.Findings.Medium,
Low: metric.Findings.Low,
}
s.applyTaskMetrics(&dto, t, metrics[t.ExpID])
dtos = append(dtos, dto)
}
writeJSON(w, 200, map[string]any{"tasks": dtos, "active": active})
}
// applyTaskMetrics fills the aggregate DTO fields (tokens, goal progress,
// running intents, findings, last activity) from the list-metrics query so the
// detail and list endpoints report identical numbers. A zero metric leaves the
// DTO zeroed, which is why callers log the query error instead of assuming it.
func (s *Server) applyTaskMetrics(dto *TaskDTO, t *Task, metric db.TaskListMetrics) {
dto.Tokens = tokenTotalDTO(metric.Tokens)
// prefer the live in-memory heartbeat (fresher) and fall back to the
// persisted max activity time (survives restarts) for run-duration display.
dto.LastActivity = metric.LastActivity
if live := s.engine.LastActivity(t.ID); live > dto.LastActivity {
dto.LastActivity = live
}
dto.GoalsTotal = metric.Goals.Total
dto.GoalsMet = metric.Goals.Met
dto.InFlight = metric.RunningIntents
dto.Findings = FindingSeverityDTO{
Critical: metric.Findings.Critical,
High: metric.Findings.High,
Medium: metric.Findings.Medium,
Low: metric.Findings.Low,
}
}
func (s *Server) resolvedTaskStatus(t *Task) string {
if t == nil {
return "created"
@@ -1798,6 +1809,11 @@ func (s *Server) getTask(w http.ResponseWriter, r *http.Request) {
return
}
dto := taskDTO(t, s.resolvedTaskStatus(t))
metrics, err := s.m.PG().TaskListMetricsAll()
if err != nil {
log.Printf("[tasks] detail metrics query failed for %s — metrics zeroed: %v", t.ID, err)
}
s.applyTaskMetrics(&dto, t, metrics[t.ExpID])
archiveBlockers, _ := s.m.PG().TaskArchiveBlockers()
applyTaskArchiveBlocker(&dto, archiveBlockers)
writeJSON(w, 200, dto)
+113
View File
@@ -0,0 +1,113 @@
package server
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strconv"
"testing"
)
func TestTaskDetailReturnsListMetrics(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable (%v) — skipping", err)
}
defer m.Close()
task, err := m.CreateTask("detail metrics", "goal", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() { _ = m.pg.DeleteTask(taskID) }()
// Seed the aggregates the metrics query reads: one result-activity with
// tokens, one non-result row that must be excluded, and two goals (1 met).
if _, err := m.pg.Exec(`
INSERT INTO activity(
exploration_id, kind, input_tokens, output_tokens,
cache_read_tokens, cache_write_tokens, created_at
) VALUES ($1,'result',11,7,3,2,now()), ($1,'tool_result',999,999,999,999,now())`,
task.ExpID); err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exec(`
INSERT INTO exploration_nodes(exploration_id, kind, payload, state)
VALUES ($1,'goal','{}','met'), ($1,'goal','{}','open')`, task.ExpID); err != nil {
t.Fatal(err)
}
s := New(context.Background(), m, t.TempDir(), t.TempDir(), t.TempDir())
token, err := signJWT(s.jwtKey)
if err != nil {
t.Fatal(err)
}
var probe struct {
Tokens struct {
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
CacheReadTokens int `json:"cache_read_tokens"`
CacheWriteTokens int `json:"cache_write_tokens"`
} `json:"tokens"`
GoalsMet int `json:"goals_met"`
GoalsTotal int `json:"goals_total"`
}
req := httptest.NewRequest(http.MethodGet, "/api/tasks/"+task.ID, nil)
req.Header.Set("Authorization", "Bearer "+token)
rec := httptest.NewRecorder()
s.Handler().ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("detail status=%d body=%s", rec.Code, rec.Body.String())
}
if err := json.Unmarshal(rec.Body.Bytes(), &probe); err != nil {
t.Fatal(err)
}
if probe.Tokens.InputTokens != 11 || probe.Tokens.OutputTokens != 7 ||
probe.Tokens.CacheReadTokens != 3 || probe.Tokens.CacheWriteTokens != 2 {
t.Fatalf("detail tokens not filled from metrics: %+v", probe.Tokens)
}
if probe.GoalsMet != 1 || probe.GoalsTotal != 2 {
t.Fatalf("detail goal progress not filled from metrics: %d/%d", probe.GoalsMet, probe.GoalsTotal)
}
// The list endpoint must report the same numbers for the same task.
listReq := httptest.NewRequest(http.MethodGet, "/api/tasks", nil)
listReq.Header.Set("Authorization", "Bearer "+token)
listRec := httptest.NewRecorder()
s.Handler().ServeHTTP(listRec, listReq)
if listRec.Code != http.StatusOK {
t.Fatalf("list status=%d body=%s", listRec.Code, listRec.Body.String())
}
var listed struct {
Tasks []struct {
ID string `json:"id"`
Tokens struct {
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
} `json:"tokens"`
GoalsMet int `json:"goals_met"`
GoalsTotal int `json:"goals_total"`
} `json:"tasks"`
}
if err := json.Unmarshal(listRec.Body.Bytes(), &listed); err != nil {
t.Fatal(err)
}
found := false
for _, lt := range listed.Tasks {
if lt.ID != task.ID {
continue
}
found = true
if lt.Tokens.InputTokens != probe.Tokens.InputTokens ||
lt.Tokens.OutputTokens != probe.Tokens.OutputTokens ||
lt.GoalsMet != probe.GoalsMet || lt.GoalsTotal != probe.GoalsTotal {
t.Fatalf("list and detail metrics disagree: list=%+v/%d detail=%+v/%d",
lt.Tokens, lt.GoalsMet, probe.Tokens, probe.GoalsTotal)
}
}
if !found {
t.Fatalf("task %s missing from list response", task.ID)
}
}
+31
View File
@@ -0,0 +1,31 @@
package server
import (
"database/sql"
"errors"
"log"
"net/http"
"strconv"
)
// taskOutcome returns one task's production and cost baseline. It is the
// measurement layer for later RSI comparisons (harness2, Google Research,
// arXiv 2610.14224): nothing here changes agent behavior.
func (s *Server) taskOutcome(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
if err != nil || id <= 0 {
writeErr(w, 400, "invalid task id")
return
}
outcome, err := s.m.PG().TaskOutcome(r.Context(), id)
if errors.Is(err, sql.ErrNoRows) {
writeErr(w, 404, "task not found")
return
}
if err != nil {
log.Printf("[tasks] outcome query failed for %d: %v", id, err)
writeErr(w, 500, "outcome query failed")
return
}
writeJSON(w, 200, outcome)
}
+114
View File
@@ -0,0 +1,114 @@
package server
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"github.com/Autumn-27/artex/db"
)
func TestTaskOutcomeCard(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable (%v) — skipping", err)
}
defer m.Close()
task, err := m.CreateTask("outcome card", "goal", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() { _ = m.pg.DeleteTask(taskID) }()
if _, err := m.pg.Exec(`
INSERT INTO findings(task_id,vulnclass,name,severity,status) VALUES
($1,'xss','A','high','confirmed'),
($1,'sqli','B','critical','fixed'),
($1,'csrf','C','medium','pending'),
($1,'xss','D','low','false_positive'),
($1,'idor','E','high','duplicate')`, taskID); err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exec(`
INSERT INTO finding_retests(finding_id,status,verdict,snapshot)
SELECT id,'completed','reproduced','{}'::jsonb FROM findings WHERE task_id=$1 AND name='A'`, taskID); err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exec(`
INSERT INTO finding_retests(finding_id,status,verdict,snapshot)
SELECT id,'completed','fixed','{}'::jsonb FROM findings WHERE task_id=$1 AND name='B'`, taskID); err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exec(`
INSERT INTO activity(exploration_id,kind,input_tokens,output_tokens,cache_read_tokens,cache_write_tokens)
VALUES ($1,'result',11,7,3,2), ($1,'tool_result',999,999,999,999)`, task.ExpID); err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exec(`
INSERT INTO tool_usage(tool_key,task_id) VALUES ('traffic_search',$1),('report_finding',$1)`, taskID); err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exec(`
INSERT INTO skill_usage(skill,task_id,found) VALUES ('api-recon',$1,true),('graphql-audit',$1,false)`, taskID); err != nil {
t.Fatal(err)
}
s := New(context.Background(), m, t.TempDir(), t.TempDir(), t.TempDir())
token, err := signJWT(s.jwtKey)
if err != nil {
t.Fatal(err)
}
get := func(path string) *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodGet, path, nil)
req.Header.Set("Authorization", "Bearer "+token)
rec := httptest.NewRecorder()
s.Handler().ServeHTTP(rec, req)
return rec
}
rec := get("/api/tasks/" + task.ID + "/outcome")
if rec.Code != http.StatusOK {
t.Fatalf("outcome status=%d body=%s", rec.Code, rec.Body.String())
}
var got db.TaskOutcome
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
t.Fatal(err)
}
if got.TaskID != taskID || got.Status != "created" {
t.Fatalf("task identity: %+v", got)
}
wantFindings := db.TaskOutcomeFindings{
Total: 5, Confirmed: 2, Open: 1, FalsePositive: 1, Duplicate: 1,
Critical: 1, High: 1, Medium: 0, Low: 0,
}
if got.Findings != wantFindings {
t.Fatalf("findings=%+v want %+v", got.Findings, wantFindings)
}
if got.Retests.Reproduced != 1 || got.Retests.Fixed != 1 || got.Retests.Inconclusive != 0 {
t.Fatalf("retests=%+v", got.Retests)
}
if got.Tokens.InputTokens != 11 || got.Tokens.OutputTokens != 7 ||
got.Tokens.CacheReadTokens != 3 || got.Tokens.CacheWriteTokens != 2 {
t.Fatalf("tokens=%+v (non-result rows must be excluded)", got.Tokens)
}
if got.ToolCalls != 2 {
t.Fatalf("tool_calls=%d want 2", got.ToolCalls)
}
if got.SkillMisses != 1 {
t.Fatalf("skill_misses=%d want 1", got.SkillMisses)
}
if got.WallSeconds < 0 {
t.Fatalf("wall_seconds=%d", got.WallSeconds)
}
if rec := get("/api/tasks/999999999/outcome"); rec.Code != http.StatusNotFound {
t.Fatalf("unknown task status=%d body=%s", rec.Code, rec.Body.String())
}
if rec := get("/api/tasks/not-a-number/outcome"); rec.Code != http.StatusBadRequest {
t.Fatalf("bad id status=%d body=%s", rec.Code, rec.Body.String())
}
}