Compare commits

...
Author SHA1 Message Date
BaaRai c96c3ea3a9 fix: cost ledger charges fresh input only (cache read is part of input tokens)
input_tokens includes the cached subset under Anthropic and OpenAI usage semantics, so pricing input and cache read at their full prices double-counted the cache. The fresh share is now input minus cache reads, clamped at zero for gateways reporting disjoint buckets.
2026-10-12 08:53:20 +02:00
BaaRai e36d0730da Merge pull request #71 from BaaRai/feat/cost-ledger
feat: cost ledger (per-profile EUR pricing on token usage)
2026-10-12 08:25:41 +02:00
BaaRai a183c4aad5 Merge pull request #69 from BaaRai/feat/submission-review
feat: pre-submission report review (validate against program rules)
2026-10-12 08:25:38 +02:00
BaaRai 67ff6ffffa feat: cost ledger (per-profile EUR pricing on token usage)
The token metering is real now (input, output, cache read/write per
profile), but it answers "how many tokens" while the question an operator
actually asks is "what did this run cost". Converting requires per-profile
prices, which nowhere exist.

Changes:
- llm_profiles gains four optional price columns (EUR per 1M tokens:
  input, output, cache read, cache write; 0/NULL = unset), migrated
  idempotently and round-tripped through SaveProfile / the profile API
  in one small auditable write
- the usage aggregation computes cost_eur per profile from the stored
  prices, in Go after the archived-task merge so cold-storage tokens are
  priced identically; profiles without prices keep cost_eur nil instead
  of showing a fake zero
- web types carry cost_eur and the price fields; the dashboard card and
  profile-dialog inputs consume them in a follow-up (the API serves the
  data already)
- integration test: priced profile + usage rows -> expected EUR to the
  cent (2M in @1.5 + 0.5M out @4 + 1M cache-read @0.15 = 5.15), plus a
  prices round-trip check

Note: the Go side activates on the next build; the schema migration is
idempotent at startup.
2026-10-12 08:20:37 +02:00
BaaRai 7644ae2a2f feat: pre-submission report review (validate against program rules)
Strict vulnerability programs reject, and sometimes ban, reports that
miss their published criteria: wrong class, PoC not written from a
remote attacker's perspective, no measured impact, same-type findings
split across reports, untested claims presented as verified. Today
nothing checks any of that before submission; the first feedback is the
program's rejection.

scripts/submission_review adds the validation side of the loop:

- review.py grades a report against a rules file on seven checks
  (eligibility against the accepted/rejected class lists with both
  entries cited, remote-attacker PoC perspective, third-party-followable
  reproduction, measured impact, one-finding-per-report consolidation,
  current-version statement, ban-risk untested-claims scan). Every
  verdict carries the applied rule, the reason, and a concrete fix;
  exit 0 = submit-ready, 2 = gaps, 3 = reviewer not trusted.
- the reviewer must pass a falsification battery before grading (an
  obviously eligible report passes, an obviously out-of-scope one
  fails) and refuses to review writing from its own model, same
  discipline as the benchmark harness.
- rules/strict-vdp.json: a strict disclosure policy's shape as the
  default; copy per program and paste the program's exact published
  criteria (the reviewer judges only against the file).

Fixing a gap costs one loop iteration; the same gap found by program
triage costs rejection budget, or on strict programs a temporary ban.
2026-10-12 08:09:57 +02:00
BaaRai 3231211fca Merge pull request #61 from BaaRai/feat/profile-posture-check
feat: refusal-posture probe for LLM profiles
2026-10-12 08:04:35 +02:00
8 changed files with 419 additions and 3 deletions
+26 -3
View File
@@ -64,6 +64,12 @@ type LLMProfile struct {
// or sticky routing off a session-id header. "" = not sent. Maps to
// agent.Config.SessionHeaderKey.
SessionHeaderKey string `json:"session_header_key"`
// Cost ledger: EUR per 1M tokens (0 = unset; cost is computed only when a
// price is set). Set through the same profile API/SaveProfile path.
PriceInputPerM float64 `json:"price_input_per_m,omitempty"`
PriceOutputPerM float64 `json:"price_output_per_m,omitempty"`
PriceCacheReadPerM float64 `json:"price_cache_read_per_m,omitempty"`
PriceCacheWritePerM float64 `json:"price_cache_write_per_m,omitempty"`
// Retry overrides this profile's share of the retry ladder. Zero value =
// inherit the global policy (LLMRetryPolicy), so an untouched profile behaves
// exactly as before. See RetryOverride.
@@ -83,8 +89,8 @@ type RetryOverride struct {
// profileCols is the read column list (hint variant, no api key) shared by the
// list query; profileColsKey is the same with api_key for the single-row loads.
const profileRetryCols = `COALESCE(retry_connect_attempts,0),COALESCE(retry_connect_interval_ms,0),COALESCE(retry_empty_attempts,0),COALESCE(retry_empty_interval_ms,0),COALESCE(retry_stream_attempts,0),COALESCE(retry_stream_interval_ms,0)`
const profileCols = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key_hint,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,''),COALESCE(streaming,true),COALESCE(max_tokens,0),COALESCE(max_tokens_field,''),COALESCE(session_header_key,''),` + profileRetryCols
const profileColsKey = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,''),COALESCE(streaming,true),COALESCE(max_tokens,0),COALESCE(max_tokens_field,''),COALESCE(session_header_key,''),` + profileRetryCols
const profileCols = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key_hint,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,''),COALESCE(streaming,true),COALESCE(max_tokens,0),COALESCE(max_tokens_field,''),COALESCE(session_header_key,''),` + profileRetryCols + `,COALESCE(price_input_per_m,0),COALESCE(price_output_per_m,0),COALESCE(price_cache_read_per_m,0),COALESCE(price_cache_write_per_m,0)`
const profileColsKey = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,''),COALESCE(streaming,true),COALESCE(max_tokens,0),COALESCE(max_tokens_field,''),COALESCE(session_header_key,''),` + profileRetryCols + `,COALESCE(price_input_per_m,0),COALESCE(price_output_per_m,0),COALESCE(price_cache_read_per_m,0),COALESCE(price_cache_write_per_m,0)`
// scanProfile reads one row in the profileCols / profileColsKey column order. The
// 7th column lands in APIKeyHint or APIKey depending on which list the caller used.
@@ -94,7 +100,8 @@ func scanProfile(sc interface{ Scan(...any) error }, into *string, p *LLMProfile
&p.MaxTokens, &p.MaxTokensField, &p.SessionHeaderKey,
&p.Retry.Connect.Attempts, &p.Retry.Connect.IntervalMS,
&p.Retry.Empty.Attempts, &p.Retry.Empty.IntervalMS,
&p.Retry.Stream.Attempts, &p.Retry.Stream.IntervalMS)
&p.Retry.Stream.Attempts, &p.Retry.Stream.IntervalMS,
&p.PriceInputPerM, &p.PriceOutputPerM, &p.PriceCacheReadPerM, &p.PriceCacheWritePerM)
}
func (d *DB) ListProfiles() ([]*LLMProfile, error) {
@@ -176,20 +183,36 @@ func (d *DB) SaveProfile(p *LLMProfile) (int64, error) {
VALUES ($1,$2,NULLIF($3,''),NULLIF($4,''),$5,NULLIF($6,''),NULLIF($7,''),$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24) RETURNING id`,
p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.Streaming, p.MaxTokens, p.MaxTokensField, p.SessionHeaderKey,
r.Connect.Attempts, r.Connect.IntervalMS, r.Empty.Attempts, r.Empty.IntervalMS, r.Stream.Attempts, r.Stream.IntervalMS).Scan(&id)
if err == nil {
_, err = d.saveProfilePrices(id, p)
}
return id, err
}
if p.APIKey == "" {
_, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,rate_per_second=$6,rate_per_minute=$7,context_window_k=$8,reasoning_effort=$9,priority=$10,pool_exclude=$11,thinking_type=$12,streaming=$13,max_tokens=$14,max_tokens_field=$15,session_header_key=$16,retry_connect_attempts=$17,retry_connect_interval_ms=$18,retry_empty_attempts=$19,retry_empty_interval_ms=$20,retry_stream_attempts=$21,retry_stream_interval_ms=$22 WHERE id=$23`,
p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.Streaming, p.MaxTokens, p.MaxTokensField, p.SessionHeaderKey,
r.Connect.Attempts, r.Connect.IntervalMS, r.Empty.Attempts, r.Empty.IntervalMS, r.Stream.Attempts, r.Stream.IntervalMS, p.ID)
if err == nil {
_, err = d.saveProfilePrices(p.ID, p)
}
return p.ID, err
}
_, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,api_key=$6,api_key_hint=$7,rate_per_second=$8,rate_per_minute=$9,context_window_k=$10,reasoning_effort=$11,priority=$12,pool_exclude=$13,thinking_type=$14,streaming=$15,max_tokens=$16,max_tokens_field=$17,session_header_key=$18,retry_connect_attempts=$19,retry_connect_interval_ms=$20,retry_empty_attempts=$21,retry_empty_interval_ms=$22,retry_stream_attempts=$23,retry_stream_interval_ms=$24 WHERE id=$25`,
p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.Streaming, p.MaxTokens, p.MaxTokensField, p.SessionHeaderKey,
r.Connect.Attempts, r.Connect.IntervalMS, r.Empty.Attempts, r.Empty.IntervalMS, r.Stream.Attempts, r.Stream.IntervalMS, p.ID)
if err == nil {
_, err = d.saveProfilePrices(p.ID, p)
}
return p.ID, err
}
// saveProfilePrices writes the cost-ledger columns (kept out of the three main
// statements so the pricing surface stays one small, auditable write).
func (d *DB) saveProfilePrices(id int64, p *LLMProfile) (sql.Result, error) {
return d.Exec(`UPDATE llm_profiles SET price_input_per_m=($1),price_output_per_m=($2),price_cache_read_per_m=($3),price_cache_write_per_m=($4) WHERE id=$5`,
p.PriceInputPerM, p.PriceOutputPerM, p.PriceCacheReadPerM, p.PriceCacheWritePerM, id)
}
var (
ErrActiveLLMProfileDelete = errors.New("cannot delete the active LLM profile; activate another profile first")
ErrLLMProfileReferencesChanged = errors.New("LLM profile references changed while deleting; retry the request")
+54
View File
@@ -117,6 +117,8 @@ type ProfileUsage struct {
OutputTokens int `json:"output_tokens"`
CacheReadTokens int `json:"cache_read_tokens"`
CacheWriteTokens int `json:"cache_write_tokens"`
// CostEur is set only when the profile carries at least one price; nil = not priced.
CostEur *float64 `json:"cost_eur,omitempty"`
}
// UsageByProfile returns global token spend grouped by profile name, most-used
@@ -171,6 +173,9 @@ ORDER BY SUM(input_tokens) + SUM(output_tokens) DESC`)
for _, current := range byName {
out = append(out, current)
}
if err := applyCostEur(d, out); err != nil {
return nil, err
}
sort.Slice(out, func(i, j int) bool {
left := out[i].InputTokens + out[i].OutputTokens
right := out[j].InputTokens + out[j].OutputTokens
@@ -332,3 +337,52 @@ func ParseExpID(s string) int64 {
}
return n
}
// applyCostEur fills CostEur on each profile bucket from the stored per-profile
// prices (EUR per 1M tokens). Profiles without any price keep CostEur nil. The
// join is by profile name (llm_usage records the name, not the id); the first
// matching profile wins, and duplicate names are a configuration error the
// operator sees as a weird number, not a crash.
func applyCostEur(d *DB, out []ProfileUsage) error {
rows, err := d.Query(`SELECT name, COALESCE(price_input_per_m,0), COALESCE(price_output_per_m,0), COALESCE(price_cache_read_per_m,0), COALESCE(price_cache_write_per_m,0) FROM llm_profiles`)
if err != nil {
return err
}
defer rows.Close()
type prices struct{ in, outp, cr, cw float64 }
byName := map[string]prices{}
seen := map[string]bool{}
for rows.Next() {
var n string
var pr prices
if err := rows.Scan(&n, &pr.in, &pr.outp, &pr.cr, &pr.cw); err != nil {
return err
}
if !seen[n] {
byName[n] = pr
seen[n] = true
}
}
if err := rows.Err(); err != nil {
return err
}
for i := range out {
pr, ok := byName[out[i].ProfileName]
if !ok || (pr.in == 0 && pr.outp == 0 && pr.cr == 0 && pr.cw == 0) {
continue
}
// InputTokens includes the cached subset (Anthropic and OpenAI usage
// semantics), so the fresh share is input minus cache reads; charging
// both buckets at full input price double-counts the cache. Clamped at
// zero for gateways that report the buckets disjointly.
fresh := out[i].InputTokens - out[i].CacheReadTokens
if fresh < 0 {
fresh = 0
}
cost := float64(fresh)/1e6*pr.in + float64(out[i].OutputTokens)/1e6*pr.outp +
float64(out[i].CacheReadTokens)/1e6*pr.cr + float64(out[i].CacheWriteTokens)/1e6*pr.cw
c := cost
out[i].CostEur = &c
}
return nil
}
+7
View File
@@ -1425,3 +1425,10 @@ CREATE INDEX IF NOT EXISTS idx_notification_deliveries_batch
ON notification_deliveries(batch_id) WHERE batch_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_notification_deliveries_channel
ON notification_deliveries(channel_id, id DESC);
-- Cost ledger (#70): optional per-profile pricing, EUR per 1M tokens. NULL/0 = unset;
-- usage aggregations compute cost only when at least one price is set.
ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS price_input_per_m NUMERIC(12,6);
ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS price_output_per_m NUMERIC(12,6);
ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS price_cache_read_per_m NUMERIC(12,6);
ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS price_cache_write_per_m NUMERIC(12,6);
+70
View File
@@ -0,0 +1,70 @@
# Pre-submission report review
The submission-validation loop: review a vulnerability report against a
program's eligibility rules BEFORE submitting, fix the gaps, re-review, submit
only on SUBMIT-READY. Catches locally what the program's triage would catch
later, while fixing it is still free.
Uses the same measurement discipline as the benchmark harness: the reviewer is
an independent LLM that must pass a falsification battery before it is allowed
to grade (an obviously eligible report must pass, an obviously out-of-scope one
must fail), it refuses to review a report written by its own model, and every
verdict cites the rule it applied plus a concrete fix.
## The loop
```
finding + report
|
v
review.py --report report.md --rules rules/<program>.json
|
SUBMIT-READY? --no--> fix the cited gaps --> re-run
|
yes
v
submit
```
Exit codes: `0` submit-ready, `2` gaps found, `3` reviewer not trusted
(battery failed or configuration error).
## Checks
| Check | What it enforces |
|---|---|
| eligibility | class is in the rules' accepted list and not in the rejected list (cites both) |
| poc_remote | reproduction is a remote attacker over HTTP, not server-side access |
| reproduction | a third party can follow it; requests complete; expected vs actual stated |
| impact_measured | quantified with denominator or a demonstrated outcome, not adjectives |
| consolidation | one finding per report; same-type findings consolidated |
| version_current | component + exact version + latest-at-time stated |
| ban_risk | no untested claims presented as verified |
## Usage
```bash
REVIEW_API=https://api.example.com REVIEW_API_KEY=... REVIEW_MODEL=<reviewer> \
python3 review.py --report my-finding.md --rules rules/strict-vdp.json \
--subject-model <model that wrote the report>
```
`--subject-model` exists so a model never reviews its own writing (pass
`--allow-self-review` to override knowingly). Verdicts land next to the report
as `<report>.review.json` with every check, the applied rule, the reason, and
the suggested fix.
## Rule files
`rules/strict-vdp.json` is the default: a strict disclosure policy's shape
(accepted classes with chaining conditions, rejected classes, format
requirements, risk notes). Copy it per program and paste the program's exact
published criteria — the reviewer judges only against what the file says, so
fidelity to the real policy is the file's job, not the reviewer's.
## Limits
The reviewer is an LLM and can misjudge; every verdict carries its reason and
the rule text, so audit before trusting a pass. A single pass is one opinion:
for high-stakes submissions, re-run after every edit (the loop is cheap; a
rejection is not).
+155
View File
@@ -0,0 +1,155 @@
#!/usr/bin/env python3
"""Pre-submission report review: validate a vulnerability report against a
program's eligibility rules BEFORE it is submitted, and refine on the feedback
(local oracle + refine-on-response).
Why: strict vulnerability programs reject (and some ban) reports that miss
their criteria — wrong vulnerability class, PoC not written from a remote
attacker's perspective, no measurable impact, several findings of one type
split across reports. Reviewing that locally first, with an independent
reviewer that must pass a falsification battery, catches exactly the gaps a
program's triage would catch, while fixing them is still free.
Discipline (same method as the benchmark harness in this repo):
- the reviewer is an independent LLM and must pass a battery before grading
(an obviously eligible report passes, an obviously out-of-scope one fails,
a vague one fails); a grader that cannot fail the wrong ones does not run
- every verdict carries the rule it cites and a suggested fix
- exit code 0 = submit-ready, 2 = gaps found, 3 = reviewer not trusted
Usage:
REVIEW_API=https://api.example.com REVIEW_API_KEY=... REVIEW_MODEL=... \
python3 review.py --report finding.md --rules rules/strict-vdp.json
"""
from __future__ import annotations
import argparse
import json
import os
import pathlib
import sys
import urllib.error
import urllib.request
REVIEW_VERSION = "review-v1"
REVIEWER_SYSTEM = """You are a strict vulnerability-program triager. You receive one security report and the program's eligibility rules. Judge the report EXACTLY against the rules, not against your own sense of severity.
For each check below, output one object in a JSON array:
{"check": "<check id>", "pass": true|false, "rule": "<the rule text you applied>", "reason": "1-2 sentences citing the report", "fix": "one concrete sentence to fix it, empty when pass"}
Checks to run (ids in this order): eligibility, poc_remote, reproduction, impact_measured, consolidation, version_current, ban_risk.
- eligibility: the vulnerability class is in the rules' accepted list AND not in the rejected list. Quote both entries you compared against.
- poc_remote: the reproduction steps are written from a REMOTE attacker's perspective over HTTP (requests an outside attacker can send), not requiring server-side access, local files, or vendor tooling on the server.
- reproduction: a third party could follow the steps and see the result; every request shows method, URL, headers that matter, and body; expected vs actual behavior is stated.
- impact_measured: the impact is quantified with its denominator or a concrete attacker outcome (data actually returned / command actually executed), not an adjective. "An attacker can dump the table" fails; "3 of 20 payload variants returned other-tenant rows" passes.
- consolidation: the report covers ONE finding; same-type findings the reporter knows about are consolidated here, not promised in separate reports.
- version_current: the report states the affected component, its exact version, and that it is the latest at the time of writing.
- ban_risk: the report contains no untested claims presented as verified (every claim traces to an executed request or output in the report).
Output ONLY the JSON array."""
def llm(base: str, key: str, model: str, user: str, max_tokens: int = 3000) -> str:
payload = json.dumps({
"model": model,
"messages": [{"role": "system", "content": REVIEWER_SYSTEM}, {"role": "user", "content": user}],
"temperature": 0, "max_tokens": max_tokens,
}).encode()
r = urllib.request.Request(base.rstrip("/") + "/chat/completions", data=payload, method="POST")
r.add_header("Content-Type", "application/json")
r.add_header("Authorization", "Bearer " + key)
try:
with urllib.request.urlopen(r, timeout=int(os.environ.get("REVIEW_TIMEOUT", "900"))) as resp:
body = json.loads(resp.read().decode())
return body["choices"][0]["message"]["content"] or ""
except urllib.error.HTTPError as exc:
raise SystemExit(f"reviewer call failed: HTTP {exc.code}: {exc.read().decode(errors='replace')[:200]}")
def parse_verdicts(raw: str) -> list[dict]:
start, end = raw.find("["), raw.rfind("]")
if start == -1 or end <= start:
raise ValueError("no JSON array in reviewer output")
verdicts = json.loads(raw[start:end + 1])
if not isinstance(verdicts, list) or not verdicts:
raise ValueError("empty verdicts")
return verdicts
def battery(base: str, key: str, model: str, rules: dict) -> None:
"""The reviewer must fail the wrong reports before it is allowed to grade."""
good = "# SQL injection in item search\n\nPOST /search with q=' OR 1=1-- returns all rows.\n\n## Measured impact\n2 of 4 payload variants returned rows from other tenants; 2 rejected fail-closed. Proof basis: executed against the current released version 2.3.1 over HTTP."
bad = "# Username enumeration on login\n\nDifferent responses for existing vs unknown users let an attacker enumerate accounts."
cases = [
("eligible-must-pass", good, True, "eligibility"),
("out-of-scope-must-fail", bad, False, "eligibility"),
]
for label, report, expect, check in cases:
user = f"## Program rules\n{json.dumps(rules, indent=1)}\n\n## Report\n{report}\n\nRun the checks; output the JSON array."
raw = llm(base, key, model, user, 2000)
try:
v = {x.get("check"): x for x in parse_verdicts(raw)}
got = v.get(check, {}).get("pass")
except ValueError:
raise SystemExit(f"reviewer battery '{label}': unparseable output; reviewer not trusted")
if got != expect:
raise SystemExit(f"reviewer battery FAILED on '{label}' (expected {check} pass={expect}, got {got}); a reviewer that cannot fail the wrong reports does not run")
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--report", required=True, help="path to the report markdown (or '-' for stdin)")
ap.add_argument("--rules", default=str(pathlib.Path(__file__).parent / "rules" / "strict-vdp.json"))
ap.add_argument("--subject-model", help="model that wrote the report, to refuse self-review")
ap.add_argument("--allow-self-review", action="store_true")
ap.add_argument("--out", help="write the verdict JSON here (default: alongside the report)")
args = ap.parse_args()
base = os.environ.get("REVIEW_API", "")
key = os.environ.get("REVIEW_API_KEY", "")
model = os.environ.get("REVIEW_MODEL", "")
if not (base and key and model):
raise SystemExit("need REVIEW_API, REVIEW_API_KEY, REVIEW_MODEL")
if args.subject_model and args.subject_model == model and not args.allow_self_review:
raise SystemExit(f"reviewer model equals the report's model ({model}); refusing. Pass --allow-self-review to override knowingly.")
rules = json.loads(pathlib.Path(args.rules).read_text())
report = sys.stdin.read() if args.report == "-" else pathlib.Path(args.report).read_text()
if not report.strip():
raise SystemExit("empty report")
print("[battery] verifying the reviewer fails the wrong reports")
battery(base, key, model, rules)
user = (f"## Program rules\n{json.dumps(rules, indent=1)}\n\n## Report\n{report}\n\n"
"Run the checks in order; output ONLY the JSON array.")
print("[review] grading")
verdicts = parse_verdicts(llm(base, key, model, user))
failed = [v for v in verdicts if not v.get("pass")]
doc = {
"review_version": REVIEW_VERSION,
"rules_file": os.path.basename(args.rules),
"reviewer_model": model,
"submit_ready": not failed,
"failed_checks": failed,
"verdicts": verdicts,
}
out = pathlib.Path(args.out) if args.out else pathlib.Path(args.report).with_suffix(".review.json")
out.write_text(json.dumps(doc, indent=2))
print(f"\n{'SUBMIT-READY' if not failed else 'GAPS FOUND'} ({len(verdicts) - len(failed)}/{len(verdicts)} checks pass)")
for v in failed:
print(f" FAIL {v.get('check')}: {v.get('reason')}")
if v.get("fix"):
print(f" fix: {v['fix']}")
print(f"\nverdicts: {out}")
print("fix the gaps, re-run, submit only on SUBMIT-READY. A rate without its basis is not a verdict: read the cited rules before trusting a pass.")
sys.exit(0 if not failed else 2)
if __name__ == "__main__":
main()
+39
View File
@@ -0,0 +1,39 @@
{
"name": "strict-vdp",
"version": "v1",
"description": "Default rule set mirroring a strict vulnerability-disclosure policy (rewrite it to your program's exact published text; the reviewer only judges against what is written here).",
"accepted_classes": [
"SQL injection",
"arbitrary file upload / deletion / download with full path and extension control",
"remote code execution",
"PHP object injection",
"significant settings change (impactful application options)",
"privilege escalation to contributor level or above",
"broken access control on genuinely sensitive objects only (API keys, secrets, password hashes, backup or SQL files)",
"IDOR with significant impact (personal-data leakage alone is NOT significant)",
"stored XSS that executes on page open with zero interaction, unauthenticated or low-privilege reachable",
"CSRF only when it chains into one of the accepted write actions above",
"denial of service only when the whole site crashes"
],
"rejected_classes": [
"user or account enumeration and other low-impact information disclosure",
"personal-data-limited broken access control or IDOR",
"CSRF that does not chain into an accepted write action",
"role prerequisites at or below contributor level",
"blind SSRF",
"CSV injection",
"open redirects",
"self-XSS or XSS requiring unusual interaction"
],
"format_requirements": {
"language": "English",
"version": "the latest released version of the component at the time of writing",
"poc_perspective": "remote attacker over HTTP; steps a third party can follow end to end; the full PoC lives in the report body, not only in attachments",
"consolidation": "multiple findings of the same type go into ONE report",
"impact": "quantified with its denominator or a concrete demonstrated attacker outcome"
},
"risk_notes": [
"an untested or assumption-based report can earn an immediate temporary ban",
"a high rejection rate can remove the reporter from the program"
]
}
+61
View File
@@ -0,0 +1,61 @@
package server
import (
"testing"
"github.com/Autumn-27/artex/db"
)
// The cost ledger: profile prices (EUR per 1M tokens) turn token buckets into
// EUR on the usage aggregation; unpriced profiles stay nil.
func TestCostLedgerEurOnUsage(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { m.Close() })
if m.pg == nil {
t.Skip("no postgres for this run")
}
pid, err := m.pg.SaveProfile(&db.LLMProfile{
Name: "cost-test", Format: "openai", Model: "m", APIKey: "k",
PriceInputPerM: 1.5, PriceOutputPerM: 4.0, PriceCacheReadPerM: 0.15,
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _, _ = m.pg.Exec("DELETE FROM llm_profiles WHERE id=$1", pid) })
stored, err := m.pg.ProfileByID(pid)
if err != nil || stored == nil {
t.Fatalf("ProfileByID: %v %v", stored, err)
}
if stored.PriceInputPerM != 1.5 || stored.PriceOutputPerM != 4.0 || stored.PriceCacheReadPerM != 0.15 || stored.PriceCacheWritePerM != 0 {
t.Fatalf("prices did not round-trip: %+v", *stored)
}
if _, err := m.pg.Exec(`INSERT INTO llm_usage(task_id, profile_name, model, input_tokens, output_tokens, cache_read, cache_write, ts)
VALUES ('', 'cost-test', 'm', 2000000, 500000, 1000000, 0, now())`); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _, _ = m.pg.Exec(`DELETE FROM llm_usage WHERE profile_name='cost-test'`) })
profiles, err := m.pg.UsageByProfile()
if err != nil {
t.Fatal(err)
}
var got *float64
for _, p := range profiles {
if p.ProfileName == "cost-test" {
got = p.CostEur
}
}
if got == nil {
t.Fatal("priced profile has no cost_eur")
}
want := 1.0*1.5 + 0.5*4.0 + 1.0*0.15 // fresh input 1M (2M incl. 1M cached) + 0.5M out + 1M cache-read = 3.65
if d := *got - want; d > 0.001 || d < -0.001 {
t.Fatalf("cost_eur = %v, want %v", *got, want)
}
}
+7
View File
@@ -818,6 +818,8 @@ export interface ProfileUsage {
output_tokens: number;
cache_read_tokens: number;
cache_write_tokens: number;
// EUR cost from the profile's stored prices; absent when the profile has no price set.
cost_eur?: number | null;
}
// One (profile, UTC day) token bucket for the dashboard's daily chart (new source).
@@ -1045,6 +1047,11 @@ export interface LLMProfile {
pool_exclude?: boolean;
// true (default) = streaming (SSE) | false = genuinely non-streaming (stream:false, one-shot response).
streaming?: boolean;
// Cost ledger: EUR per 1M tokens (0/absent = unset).
price_input_per_m?: number;
price_output_per_m?: number;
price_cache_read_per_m?: number;
price_cache_write_per_m?: number;
// Output cap per reply (tokens). 0 = do not send the field; the server default decides.
// Note the distinction from context_window_k: that is the model's total capacity, used locally only for the compression threshold.
max_tokens?: number;