diff --git a/scripts/submission_review/README.md b/scripts/submission_review/README.md new file mode 100755 index 0000000..4cab7b6 --- /dev/null +++ b/scripts/submission_review/README.md @@ -0,0 +1,70 @@ +# Pre-submission report review + +The submission-validation loop: review a vulnerability report against a +program's eligibility rules BEFORE submitting, fix the gaps, re-review, submit +only on SUBMIT-READY. Catches locally what the program's triage would catch +later, while fixing it is still free. + +Uses the same measurement discipline as the benchmark harness: the reviewer is +an independent LLM that must pass a falsification battery before it is allowed +to grade (an obviously eligible report must pass, an obviously out-of-scope one +must fail), it refuses to review a report written by its own model, and every +verdict cites the rule it applied plus a concrete fix. + +## The loop + +``` +finding + report + | + v +review.py --report report.md --rules rules/.json + | + SUBMIT-READY? --no--> fix the cited gaps --> re-run + | + yes + v + submit +``` + +Exit codes: `0` submit-ready, `2` gaps found, `3` reviewer not trusted +(battery failed or configuration error). + +## Checks + +| Check | What it enforces | +|---|---| +| eligibility | class is in the rules' accepted list and not in the rejected list (cites both) | +| poc_remote | reproduction is a remote attacker over HTTP, not server-side access | +| reproduction | a third party can follow it; requests complete; expected vs actual stated | +| impact_measured | quantified with denominator or a demonstrated outcome, not adjectives | +| consolidation | one finding per report; same-type findings consolidated | +| version_current | component + exact version + latest-at-time stated | +| ban_risk | no untested claims presented as verified | + +## Usage + +```bash +REVIEW_API=https://api.example.com REVIEW_API_KEY=... REVIEW_MODEL= \ +python3 review.py --report my-finding.md --rules rules/strict-vdp.json \ + --subject-model +``` + +`--subject-model` exists so a model never reviews its own writing (pass +`--allow-self-review` to override knowingly). Verdicts land next to the report +as `.review.json` with every check, the applied rule, the reason, and +the suggested fix. + +## Rule files + +`rules/strict-vdp.json` is the default: a strict disclosure policy's shape +(accepted classes with chaining conditions, rejected classes, format +requirements, risk notes). Copy it per program and paste the program's exact +published criteria — the reviewer judges only against what the file says, so +fidelity to the real policy is the file's job, not the reviewer's. + +## Limits + +The reviewer is an LLM and can misjudge; every verdict carries its reason and +the rule text, so audit before trusting a pass. A single pass is one opinion: +for high-stakes submissions, re-run after every edit (the loop is cheap; a +rejection is not). diff --git a/scripts/submission_review/review.py b/scripts/submission_review/review.py new file mode 100755 index 0000000..2a00010 --- /dev/null +++ b/scripts/submission_review/review.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Pre-submission report review: validate a vulnerability report against a +program's eligibility rules BEFORE it is submitted, and refine on the feedback +(local oracle + refine-on-response). + +Why: strict vulnerability programs reject (and some ban) reports that miss +their criteria — wrong vulnerability class, PoC not written from a remote +attacker's perspective, no measurable impact, several findings of one type +split across reports. Reviewing that locally first, with an independent +reviewer that must pass a falsification battery, catches exactly the gaps a +program's triage would catch, while fixing them is still free. + +Discipline (same method as the benchmark harness in this repo): +- the reviewer is an independent LLM and must pass a battery before grading + (an obviously eligible report passes, an obviously out-of-scope one fails, + a vague one fails); a grader that cannot fail the wrong ones does not run +- every verdict carries the rule it cites and a suggested fix +- exit code 0 = submit-ready, 2 = gaps found, 3 = reviewer not trusted + +Usage: + REVIEW_API=https://api.example.com REVIEW_API_KEY=... REVIEW_MODEL=... \ + python3 review.py --report finding.md --rules rules/strict-vdp.json +""" +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import sys +import urllib.error +import urllib.request + +REVIEW_VERSION = "review-v1" + +REVIEWER_SYSTEM = """You are a strict vulnerability-program triager. You receive one security report and the program's eligibility rules. Judge the report EXACTLY against the rules, not against your own sense of severity. + +For each check below, output one object in a JSON array: +{"check": "", "pass": true|false, "rule": "", "reason": "1-2 sentences citing the report", "fix": "one concrete sentence to fix it, empty when pass"} + +Checks to run (ids in this order): eligibility, poc_remote, reproduction, impact_measured, consolidation, version_current, ban_risk. + +- eligibility: the vulnerability class is in the rules' accepted list AND not in the rejected list. Quote both entries you compared against. +- poc_remote: the reproduction steps are written from a REMOTE attacker's perspective over HTTP (requests an outside attacker can send), not requiring server-side access, local files, or vendor tooling on the server. +- reproduction: a third party could follow the steps and see the result; every request shows method, URL, headers that matter, and body; expected vs actual behavior is stated. +- impact_measured: the impact is quantified with its denominator or a concrete attacker outcome (data actually returned / command actually executed), not an adjective. "An attacker can dump the table" fails; "3 of 20 payload variants returned other-tenant rows" passes. +- consolidation: the report covers ONE finding; same-type findings the reporter knows about are consolidated here, not promised in separate reports. +- version_current: the report states the affected component, its exact version, and that it is the latest at the time of writing. +- ban_risk: the report contains no untested claims presented as verified (every claim traces to an executed request or output in the report). + +Output ONLY the JSON array.""" + + +def llm(base: str, key: str, model: str, user: str, max_tokens: int = 3000) -> str: + payload = json.dumps({ + "model": model, + "messages": [{"role": "system", "content": REVIEWER_SYSTEM}, {"role": "user", "content": user}], + "temperature": 0, "max_tokens": max_tokens, + }).encode() + r = urllib.request.Request(base.rstrip("/") + "/chat/completions", data=payload, method="POST") + r.add_header("Content-Type", "application/json") + r.add_header("Authorization", "Bearer " + key) + try: + with urllib.request.urlopen(r, timeout=int(os.environ.get("REVIEW_TIMEOUT", "900"))) as resp: + body = json.loads(resp.read().decode()) + return body["choices"][0]["message"]["content"] or "" + except urllib.error.HTTPError as exc: + raise SystemExit(f"reviewer call failed: HTTP {exc.code}: {exc.read().decode(errors='replace')[:200]}") + + +def parse_verdicts(raw: str) -> list[dict]: + start, end = raw.find("["), raw.rfind("]") + if start == -1 or end <= start: + raise ValueError("no JSON array in reviewer output") + verdicts = json.loads(raw[start:end + 1]) + if not isinstance(verdicts, list) or not verdicts: + raise ValueError("empty verdicts") + return verdicts + + +def battery(base: str, key: str, model: str, rules: dict) -> None: + """The reviewer must fail the wrong reports before it is allowed to grade.""" + good = "# SQL injection in item search\n\nPOST /search with q=' OR 1=1-- returns all rows.\n\n## Measured impact\n2 of 4 payload variants returned rows from other tenants; 2 rejected fail-closed. Proof basis: executed against the current released version 2.3.1 over HTTP." + bad = "# Username enumeration on login\n\nDifferent responses for existing vs unknown users let an attacker enumerate accounts." + cases = [ + ("eligible-must-pass", good, True, "eligibility"), + ("out-of-scope-must-fail", bad, False, "eligibility"), + ] + for label, report, expect, check in cases: + user = f"## Program rules\n{json.dumps(rules, indent=1)}\n\n## Report\n{report}\n\nRun the checks; output the JSON array." + raw = llm(base, key, model, user, 2000) + try: + v = {x.get("check"): x for x in parse_verdicts(raw)} + got = v.get(check, {}).get("pass") + except ValueError: + raise SystemExit(f"reviewer battery '{label}': unparseable output; reviewer not trusted") + if got != expect: + raise SystemExit(f"reviewer battery FAILED on '{label}' (expected {check} pass={expect}, got {got}); a reviewer that cannot fail the wrong reports does not run") + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--report", required=True, help="path to the report markdown (or '-' for stdin)") + ap.add_argument("--rules", default=str(pathlib.Path(__file__).parent / "rules" / "strict-vdp.json")) + ap.add_argument("--subject-model", help="model that wrote the report, to refuse self-review") + ap.add_argument("--allow-self-review", action="store_true") + ap.add_argument("--out", help="write the verdict JSON here (default: alongside the report)") + args = ap.parse_args() + + base = os.environ.get("REVIEW_API", "") + key = os.environ.get("REVIEW_API_KEY", "") + model = os.environ.get("REVIEW_MODEL", "") + if not (base and key and model): + raise SystemExit("need REVIEW_API, REVIEW_API_KEY, REVIEW_MODEL") + + if args.subject_model and args.subject_model == model and not args.allow_self_review: + raise SystemExit(f"reviewer model equals the report's model ({model}); refusing. Pass --allow-self-review to override knowingly.") + + rules = json.loads(pathlib.Path(args.rules).read_text()) + report = sys.stdin.read() if args.report == "-" else pathlib.Path(args.report).read_text() + if not report.strip(): + raise SystemExit("empty report") + + print("[battery] verifying the reviewer fails the wrong reports") + battery(base, key, model, rules) + + user = (f"## Program rules\n{json.dumps(rules, indent=1)}\n\n## Report\n{report}\n\n" + "Run the checks in order; output ONLY the JSON array.") + print("[review] grading") + verdicts = parse_verdicts(llm(base, key, model, user)) + + failed = [v for v in verdicts if not v.get("pass")] + doc = { + "review_version": REVIEW_VERSION, + "rules_file": os.path.basename(args.rules), + "reviewer_model": model, + "submit_ready": not failed, + "failed_checks": failed, + "verdicts": verdicts, + } + out = pathlib.Path(args.out) if args.out else pathlib.Path(args.report).with_suffix(".review.json") + out.write_text(json.dumps(doc, indent=2)) + + print(f"\n{'SUBMIT-READY' if not failed else 'GAPS FOUND'} ({len(verdicts) - len(failed)}/{len(verdicts)} checks pass)") + for v in failed: + print(f" FAIL {v.get('check')}: {v.get('reason')}") + if v.get("fix"): + print(f" fix: {v['fix']}") + print(f"\nverdicts: {out}") + print("fix the gaps, re-run, submit only on SUBMIT-READY. A rate without its basis is not a verdict: read the cited rules before trusting a pass.") + sys.exit(0 if not failed else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/submission_review/rules/strict-vdp.json b/scripts/submission_review/rules/strict-vdp.json new file mode 100755 index 0000000..531c427 --- /dev/null +++ b/scripts/submission_review/rules/strict-vdp.json @@ -0,0 +1,39 @@ +{ + "name": "strict-vdp", + "version": "v1", + "description": "Default rule set mirroring a strict vulnerability-disclosure policy (rewrite it to your program's exact published text; the reviewer only judges against what is written here).", + "accepted_classes": [ + "SQL injection", + "arbitrary file upload / deletion / download with full path and extension control", + "remote code execution", + "PHP object injection", + "significant settings change (impactful application options)", + "privilege escalation to contributor level or above", + "broken access control on genuinely sensitive objects only (API keys, secrets, password hashes, backup or SQL files)", + "IDOR with significant impact (personal-data leakage alone is NOT significant)", + "stored XSS that executes on page open with zero interaction, unauthenticated or low-privilege reachable", + "CSRF only when it chains into one of the accepted write actions above", + "denial of service only when the whole site crashes" + ], + "rejected_classes": [ + "user or account enumeration and other low-impact information disclosure", + "personal-data-limited broken access control or IDOR", + "CSRF that does not chain into an accepted write action", + "role prerequisites at or below contributor level", + "blind SSRF", + "CSV injection", + "open redirects", + "self-XSS or XSS requiring unusual interaction" + ], + "format_requirements": { + "language": "English", + "version": "the latest released version of the component at the time of writing", + "poc_perspective": "remote attacker over HTTP; steps a third party can follow end to end; the full PoC lives in the report body, not only in attachments", + "consolidation": "multiple findings of the same type go into ONE report", + "impact": "quantified with its denominator or a concrete demonstrated attacker outcome" + }, + "risk_notes": [ + "an untested or assumption-based report can earn an immediate temporary ban", + "a high rejection rate can remove the reporter from the program" + ] +}