mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-13 11:40:53 +08:00
fix(bench): survive finding-detail 404s and always tear the corpus down
The first live drost run died right after the task completed: the findings detail route keys on finding_id (the findings-table row), not on the exploration node id the list also carries, so every detail fetch 404ed; fetch_findings then re-raised the SystemExit from the API wrapper, aborting scoring, the flag scan, the report and the teardown. - details are fetched with finding_id (node id as fallback) and a missing detail no longer kills the run: the list fields remain and the finding stays gradeable - the stored finding carries finding_id so the deep flag scan can re-fetch untruncated text - the deep scan tolerates the same failures - teardown runs in a finally around the whole task phase, so a crash in scoring or reporting can no longer leave the challenge running (--keep still opts out deliberately)
This commit is contained in:
BaaRai
@@ -189,18 +189,21 @@ def fetch_findings(api: Api, tid) -> list[dict]:
|
||||
out = []
|
||||
for f in items:
|
||||
fid = f.get("id") or f.get("node_id")
|
||||
# The detail route keys on finding_id (the findings-table row), not on
|
||||
# the exploration node id the list also carries; asking with the node
|
||||
# id 404s. Fall back to the node id so older payloads still resolve.
|
||||
did = f.get("finding_id") or fid
|
||||
detail = {}
|
||||
try:
|
||||
detail = api.get(f"/api/exploration/findings/{fid}")
|
||||
except SystemExit:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
detail = api.get(f"/api/exploration/findings/{did}")
|
||||
except (SystemExit, Exception):
|
||||
pass # a missing detail must not kill the run; list fields remain
|
||||
report = detail.get("report") or f.get("report") or ""
|
||||
summary = f.get("summary") or detail.get("summary") or ""
|
||||
name = f.get("name") or f.get("title") or summary[:80]
|
||||
out.append({
|
||||
"id": fid,
|
||||
"finding_id": str(f.get("finding_id") or ""),
|
||||
"name": str(name)[:200],
|
||||
"severity": str(f.get("severity") or detail.get("severity") or "?"),
|
||||
"summary": str(summary)[:600],
|
||||
@@ -217,13 +220,14 @@ def deep_evidence_texts(api: Api, tid, findings: list[dict]) -> list[str]:
|
||||
common path."""
|
||||
texts = []
|
||||
for f in findings:
|
||||
did = f.get("finding_id") or f.get("id")
|
||||
try:
|
||||
texts.append(json.dumps(api.get(f"/api/exploration/findings/{f['id']}")))
|
||||
except Exception:
|
||||
texts.append(json.dumps(api.get(f"/api/exploration/findings/{did}")))
|
||||
except (SystemExit, Exception):
|
||||
continue
|
||||
try:
|
||||
texts.append(json.dumps(api.get(f"/api/exploration/nodes?task={tid}&kind=fact&size=200")))
|
||||
except Exception:
|
||||
except (SystemExit, Exception):
|
||||
pass
|
||||
try:
|
||||
r = urllib.request.Request(f"{api.base}/api/report?task={tid}")
|
||||
@@ -478,62 +482,66 @@ def main() -> None:
|
||||
return
|
||||
|
||||
task_url = corpus_url
|
||||
if args.target_host != "127.0.0.1":
|
||||
task_url = f"http://{args.target_host}:{corpus_url.rsplit(':', 1)[1]}"
|
||||
print(f"[corpus] task target rewritten for ARTEX reachability: {task_url}")
|
||||
try:
|
||||
if args.target_host != "127.0.0.1":
|
||||
task_url = f"http://{args.target_host}:{corpus_url.rsplit(':', 1)[1]}"
|
||||
print(f"[corpus] task target rewritten for ARTEX reachability: {task_url}")
|
||||
|
||||
api = Api(args.artex_url, token)
|
||||
task_info = create_task(api, args, task_url, gt)
|
||||
status = wait_task(api, task_info["id"], args.task_timeout + 1800)
|
||||
findings = fetch_findings(api, task_info["id"])
|
||||
api = Api(args.artex_url, token)
|
||||
task_info = create_task(api, args, task_url, gt)
|
||||
status = wait_task(api, task_info["id"], args.task_timeout + 1800)
|
||||
findings = fetch_findings(api, task_info["id"])
|
||||
|
||||
invalid_ids = [f["id"] for f in findings if not (f["summary"].strip() or f["report"].strip())]
|
||||
usable = [f for f in findings if f["id"] not in invalid_ids]
|
||||
invalid_ids = [f["id"] for f in findings if not (f["summary"].strip() or f["report"].strip())]
|
||||
usable = [f for f in findings if f["id"] not in invalid_ids]
|
||||
|
||||
flag_captured = None
|
||||
if flag:
|
||||
# Objective oracle first pass over the already-fetched fields; the deep
|
||||
# scan (full details, facts, report) runs only before declaring a miss.
|
||||
flag_captured = drost.flag_in_texts(flag, drost.finding_texts(findings))
|
||||
if not flag_captured:
|
||||
flag_captured = drost.flag_in_texts(flag, deep_evidence_texts(api, task_info["id"], findings))
|
||||
print(f"[flag] captured: {flag_captured}")
|
||||
flag_captured = None
|
||||
if flag:
|
||||
# Objective oracle first pass over the already-fetched fields; the deep
|
||||
# scan (full details, facts, report) runs only before declaring a miss.
|
||||
flag_captured = drost.flag_in_texts(flag, drost.finding_texts(findings))
|
||||
if not flag_captured:
|
||||
flag_captured = drost.flag_in_texts(flag, deep_evidence_texts(api, task_info["id"], findings))
|
||||
print(f"[flag] captured: {flag_captured}")
|
||||
|
||||
if judge_enabled:
|
||||
if not usable:
|
||||
print("[judge] no usable findings; scoring zeros")
|
||||
verdicts = [{"gt_id": g["id"], "detected": False, "matched_finding_id": None,
|
||||
"gates": {"component": False, "class": False, "keywords": False, "proof": False},
|
||||
"reason": "no usable scanner findings"} for g in gt_findings]
|
||||
if judge_enabled:
|
||||
if not usable:
|
||||
print("[judge] no usable findings; scoring zeros")
|
||||
verdicts = [{"gt_id": g["id"], "detected": False, "matched_finding_id": None,
|
||||
"gates": {"component": False, "class": False, "keywords": False, "proof": False},
|
||||
"reason": "no usable scanner findings"} for g in gt_findings]
|
||||
else:
|
||||
print("[judge] running falsification battery")
|
||||
judge_battery(args)
|
||||
print("[judge] grading")
|
||||
verdicts = run_judge(args, gt_findings, usable)
|
||||
result = tally(gt_findings, verdicts, findings, invalid_ids)
|
||||
else:
|
||||
print("[judge] running falsification battery")
|
||||
judge_battery(args)
|
||||
print("[judge] grading")
|
||||
verdicts = run_judge(args, gt_findings, usable)
|
||||
result = tally(gt_findings, verdicts, findings, invalid_ids)
|
||||
else:
|
||||
print("[judge] flag-only run: no judge configured, analysis gates not measured")
|
||||
verdicts = None
|
||||
result = {
|
||||
"mode": "flag-only",
|
||||
"scanner_findings_total": len(findings),
|
||||
"invalid_findings_excluded": invalid_ids,
|
||||
"discovery_candidates": [{"id": f["id"], "name": f["name"]} for f in usable],
|
||||
"note": "analysis gates not measured: no judge configured",
|
||||
}
|
||||
if flag:
|
||||
result["flag_captured"] = flag_captured
|
||||
result["flag_used"] = flag
|
||||
print("[judge] flag-only run: no judge configured, analysis gates not measured")
|
||||
verdicts = None
|
||||
result = {
|
||||
"mode": "flag-only",
|
||||
"scanner_findings_total": len(findings),
|
||||
"invalid_findings_excluded": invalid_ids,
|
||||
"discovery_candidates": [{"id": f["id"], "name": f["name"]} for f in usable],
|
||||
"note": "analysis gates not measured: no judge configured",
|
||||
}
|
||||
if flag:
|
||||
result["flag_captured"] = flag_captured
|
||||
result["flag_used"] = flag
|
||||
|
||||
out_dir = pathlib.Path("benchmark-results") / args.corpus / args.run_id
|
||||
write_report(out_dir, args.corpus, args, task_info, status, findings, verdicts, result)
|
||||
print(json.dumps(result, indent=2)[:1500])
|
||||
|
||||
if args.corpus_type == "drost":
|
||||
if not args.keep:
|
||||
drost.teardown(dep["project"], dep["challenge_dir"])
|
||||
elif not (args.keep or args.corpus_url):
|
||||
subprocess.run(["docker", "rm", "-f", f"bench-{args.corpus}-{args.run_id}"], capture_output=True)
|
||||
out_dir = pathlib.Path("benchmark-results") / args.corpus / args.run_id
|
||||
write_report(out_dir, args.corpus, args, task_info, status, findings, verdicts, result)
|
||||
print(json.dumps(result, indent=2)[:1500])
|
||||
finally:
|
||||
# The corpus is torn down even when scoring fails halfway: a crash in
|
||||
# the report phase must not leave the challenge running (--keep opts
|
||||
# out deliberately).
|
||||
if args.corpus_type == "drost":
|
||||
if not args.keep:
|
||||
drost.teardown(dep["project"], dep["challenge_dir"])
|
||||
elif not (args.keep or args.corpus_url):
|
||||
subprocess.run(["docker", "rm", "-f", f"bench-{args.corpus}-{args.run_id}"], capture_output=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user