mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-15 04:30:55 +08:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c96c3ea3a9 | ||
|
|
e36d0730da | ||
|
|
a183c4aad5 | ||
|
|
67ff6ffffa | ||
|
|
7644ae2a2f | ||
|
|
3231211fca |
+26
-3
@@ -64,6 +64,12 @@ type LLMProfile struct {
|
||||
// or sticky routing off a session-id header. "" = not sent. Maps to
|
||||
// agent.Config.SessionHeaderKey.
|
||||
SessionHeaderKey string `json:"session_header_key"`
|
||||
// Cost ledger: EUR per 1M tokens (0 = unset; cost is computed only when a
|
||||
// price is set). Set through the same profile API/SaveProfile path.
|
||||
PriceInputPerM float64 `json:"price_input_per_m,omitempty"`
|
||||
PriceOutputPerM float64 `json:"price_output_per_m,omitempty"`
|
||||
PriceCacheReadPerM float64 `json:"price_cache_read_per_m,omitempty"`
|
||||
PriceCacheWritePerM float64 `json:"price_cache_write_per_m,omitempty"`
|
||||
// Retry overrides this profile's share of the retry ladder. Zero value =
|
||||
// inherit the global policy (LLMRetryPolicy), so an untouched profile behaves
|
||||
// exactly as before. See RetryOverride.
|
||||
@@ -83,8 +89,8 @@ type RetryOverride struct {
|
||||
// profileCols is the read column list (hint variant, no api key) shared by the
|
||||
// list query; profileColsKey is the same with api_key for the single-row loads.
|
||||
const profileRetryCols = `COALESCE(retry_connect_attempts,0),COALESCE(retry_connect_interval_ms,0),COALESCE(retry_empty_attempts,0),COALESCE(retry_empty_interval_ms,0),COALESCE(retry_stream_attempts,0),COALESCE(retry_stream_interval_ms,0)`
|
||||
const profileCols = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key_hint,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,''),COALESCE(streaming,true),COALESCE(max_tokens,0),COALESCE(max_tokens_field,''),COALESCE(session_header_key,''),` + profileRetryCols
|
||||
const profileColsKey = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,''),COALESCE(streaming,true),COALESCE(max_tokens,0),COALESCE(max_tokens_field,''),COALESCE(session_header_key,''),` + profileRetryCols
|
||||
const profileCols = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key_hint,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,''),COALESCE(streaming,true),COALESCE(max_tokens,0),COALESCE(max_tokens_field,''),COALESCE(session_header_key,''),` + profileRetryCols + `,COALESCE(price_input_per_m,0),COALESCE(price_output_per_m,0),COALESCE(price_cache_read_per_m,0),COALESCE(price_cache_write_per_m,0)`
|
||||
const profileColsKey = `id,name,format,COALESCE(base_url,''),COALESCE(proxy,''),model,COALESCE(api_key,''),rate_per_second,rate_per_minute,context_window_k,COALESCE(reasoning_effort,''),is_default,priority,pool_exclude,COALESCE(thinking_type,''),COALESCE(streaming,true),COALESCE(max_tokens,0),COALESCE(max_tokens_field,''),COALESCE(session_header_key,''),` + profileRetryCols + `,COALESCE(price_input_per_m,0),COALESCE(price_output_per_m,0),COALESCE(price_cache_read_per_m,0),COALESCE(price_cache_write_per_m,0)`
|
||||
|
||||
// scanProfile reads one row in the profileCols / profileColsKey column order. The
|
||||
// 7th column lands in APIKeyHint or APIKey depending on which list the caller used.
|
||||
@@ -94,7 +100,8 @@ func scanProfile(sc interface{ Scan(...any) error }, into *string, p *LLMProfile
|
||||
&p.MaxTokens, &p.MaxTokensField, &p.SessionHeaderKey,
|
||||
&p.Retry.Connect.Attempts, &p.Retry.Connect.IntervalMS,
|
||||
&p.Retry.Empty.Attempts, &p.Retry.Empty.IntervalMS,
|
||||
&p.Retry.Stream.Attempts, &p.Retry.Stream.IntervalMS)
|
||||
&p.Retry.Stream.Attempts, &p.Retry.Stream.IntervalMS,
|
||||
&p.PriceInputPerM, &p.PriceOutputPerM, &p.PriceCacheReadPerM, &p.PriceCacheWritePerM)
|
||||
}
|
||||
|
||||
func (d *DB) ListProfiles() ([]*LLMProfile, error) {
|
||||
@@ -176,20 +183,36 @@ func (d *DB) SaveProfile(p *LLMProfile) (int64, error) {
|
||||
VALUES ($1,$2,NULLIF($3,''),NULLIF($4,''),$5,NULLIF($6,''),NULLIF($7,''),$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24) RETURNING id`,
|
||||
p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.Streaming, p.MaxTokens, p.MaxTokensField, p.SessionHeaderKey,
|
||||
r.Connect.Attempts, r.Connect.IntervalMS, r.Empty.Attempts, r.Empty.IntervalMS, r.Stream.Attempts, r.Stream.IntervalMS).Scan(&id)
|
||||
if err == nil {
|
||||
_, err = d.saveProfilePrices(id, p)
|
||||
}
|
||||
return id, err
|
||||
}
|
||||
if p.APIKey == "" {
|
||||
_, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,rate_per_second=$6,rate_per_minute=$7,context_window_k=$8,reasoning_effort=$9,priority=$10,pool_exclude=$11,thinking_type=$12,streaming=$13,max_tokens=$14,max_tokens_field=$15,session_header_key=$16,retry_connect_attempts=$17,retry_connect_interval_ms=$18,retry_empty_attempts=$19,retry_empty_interval_ms=$20,retry_stream_attempts=$21,retry_stream_interval_ms=$22 WHERE id=$23`,
|
||||
p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.Streaming, p.MaxTokens, p.MaxTokensField, p.SessionHeaderKey,
|
||||
r.Connect.Attempts, r.Connect.IntervalMS, r.Empty.Attempts, r.Empty.IntervalMS, r.Stream.Attempts, r.Stream.IntervalMS, p.ID)
|
||||
if err == nil {
|
||||
_, err = d.saveProfilePrices(p.ID, p)
|
||||
}
|
||||
return p.ID, err
|
||||
}
|
||||
_, err := d.Exec(`UPDATE llm_profiles SET name=$1,format=$2,base_url=NULLIF($3,''),proxy=NULLIF($4,''),model=$5,api_key=$6,api_key_hint=$7,rate_per_second=$8,rate_per_minute=$9,context_window_k=$10,reasoning_effort=$11,priority=$12,pool_exclude=$13,thinking_type=$14,streaming=$15,max_tokens=$16,max_tokens_field=$17,session_header_key=$18,retry_connect_attempts=$19,retry_connect_interval_ms=$20,retry_empty_attempts=$21,retry_empty_interval_ms=$22,retry_stream_attempts=$23,retry_stream_interval_ms=$24 WHERE id=$25`,
|
||||
p.Name, p.Format, p.BaseURL, p.Proxy, p.Model, p.APIKey, hint, p.RatePerSecond, p.RatePerMinute, p.ContextWindowK, p.ReasoningEffort, p.Priority, p.PoolExclude, p.ThinkingType, p.Streaming, p.MaxTokens, p.MaxTokensField, p.SessionHeaderKey,
|
||||
r.Connect.Attempts, r.Connect.IntervalMS, r.Empty.Attempts, r.Empty.IntervalMS, r.Stream.Attempts, r.Stream.IntervalMS, p.ID)
|
||||
if err == nil {
|
||||
_, err = d.saveProfilePrices(p.ID, p)
|
||||
}
|
||||
return p.ID, err
|
||||
}
|
||||
|
||||
// saveProfilePrices writes the cost-ledger columns (kept out of the three main
|
||||
// statements so the pricing surface stays one small, auditable write).
|
||||
func (d *DB) saveProfilePrices(id int64, p *LLMProfile) (sql.Result, error) {
|
||||
return d.Exec(`UPDATE llm_profiles SET price_input_per_m=($1),price_output_per_m=($2),price_cache_read_per_m=($3),price_cache_write_per_m=($4) WHERE id=$5`,
|
||||
p.PriceInputPerM, p.PriceOutputPerM, p.PriceCacheReadPerM, p.PriceCacheWritePerM, id)
|
||||
}
|
||||
|
||||
var (
|
||||
ErrActiveLLMProfileDelete = errors.New("cannot delete the active LLM profile; activate another profile first")
|
||||
ErrLLMProfileReferencesChanged = errors.New("LLM profile references changed while deleting; retry the request")
|
||||
|
||||
@@ -117,6 +117,8 @@ type ProfileUsage struct {
|
||||
OutputTokens int `json:"output_tokens"`
|
||||
CacheReadTokens int `json:"cache_read_tokens"`
|
||||
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||
// CostEur is set only when the profile carries at least one price; nil = not priced.
|
||||
CostEur *float64 `json:"cost_eur,omitempty"`
|
||||
}
|
||||
|
||||
// UsageByProfile returns global token spend grouped by profile name, most-used
|
||||
@@ -171,6 +173,9 @@ ORDER BY SUM(input_tokens) + SUM(output_tokens) DESC`)
|
||||
for _, current := range byName {
|
||||
out = append(out, current)
|
||||
}
|
||||
if err := applyCostEur(d, out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
left := out[i].InputTokens + out[i].OutputTokens
|
||||
right := out[j].InputTokens + out[j].OutputTokens
|
||||
@@ -332,3 +337,52 @@ func ParseExpID(s string) int64 {
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// applyCostEur fills CostEur on each profile bucket from the stored per-profile
|
||||
// prices (EUR per 1M tokens). Profiles without any price keep CostEur nil. The
|
||||
// join is by profile name (llm_usage records the name, not the id); the first
|
||||
// matching profile wins, and duplicate names are a configuration error the
|
||||
// operator sees as a weird number, not a crash.
|
||||
func applyCostEur(d *DB, out []ProfileUsage) error {
|
||||
rows, err := d.Query(`SELECT name, COALESCE(price_input_per_m,0), COALESCE(price_output_per_m,0), COALESCE(price_cache_read_per_m,0), COALESCE(price_cache_write_per_m,0) FROM llm_profiles`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
type prices struct{ in, outp, cr, cw float64 }
|
||||
byName := map[string]prices{}
|
||||
seen := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var n string
|
||||
var pr prices
|
||||
if err := rows.Scan(&n, &pr.in, &pr.outp, &pr.cr, &pr.cw); err != nil {
|
||||
return err
|
||||
}
|
||||
if !seen[n] {
|
||||
byName[n] = pr
|
||||
seen[n] = true
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range out {
|
||||
pr, ok := byName[out[i].ProfileName]
|
||||
if !ok || (pr.in == 0 && pr.outp == 0 && pr.cr == 0 && pr.cw == 0) {
|
||||
continue
|
||||
}
|
||||
// InputTokens includes the cached subset (Anthropic and OpenAI usage
|
||||
// semantics), so the fresh share is input minus cache reads; charging
|
||||
// both buckets at full input price double-counts the cache. Clamped at
|
||||
// zero for gateways that report the buckets disjointly.
|
||||
fresh := out[i].InputTokens - out[i].CacheReadTokens
|
||||
if fresh < 0 {
|
||||
fresh = 0
|
||||
}
|
||||
cost := float64(fresh)/1e6*pr.in + float64(out[i].OutputTokens)/1e6*pr.outp +
|
||||
float64(out[i].CacheReadTokens)/1e6*pr.cr + float64(out[i].CacheWriteTokens)/1e6*pr.cw
|
||||
c := cost
|
||||
out[i].CostEur = &c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1425,3 +1425,10 @@ CREATE INDEX IF NOT EXISTS idx_notification_deliveries_batch
|
||||
ON notification_deliveries(batch_id) WHERE batch_id IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_notification_deliveries_channel
|
||||
ON notification_deliveries(channel_id, id DESC);
|
||||
|
||||
-- Cost ledger (#70): optional per-profile pricing, EUR per 1M tokens. NULL/0 = unset;
|
||||
-- usage aggregations compute cost only when at least one price is set.
|
||||
ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS price_input_per_m NUMERIC(12,6);
|
||||
ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS price_output_per_m NUMERIC(12,6);
|
||||
ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS price_cache_read_per_m NUMERIC(12,6);
|
||||
ALTER TABLE llm_profiles ADD COLUMN IF NOT EXISTS price_cache_write_per_m NUMERIC(12,6);
|
||||
|
||||
Executable
+70
@@ -0,0 +1,70 @@
|
||||
# Pre-submission report review
|
||||
|
||||
The submission-validation loop: review a vulnerability report against a
|
||||
program's eligibility rules BEFORE submitting, fix the gaps, re-review, submit
|
||||
only on SUBMIT-READY. Catches locally what the program's triage would catch
|
||||
later, while fixing it is still free.
|
||||
|
||||
Uses the same measurement discipline as the benchmark harness: the reviewer is
|
||||
an independent LLM that must pass a falsification battery before it is allowed
|
||||
to grade (an obviously eligible report must pass, an obviously out-of-scope one
|
||||
must fail), it refuses to review a report written by its own model, and every
|
||||
verdict cites the rule it applied plus a concrete fix.
|
||||
|
||||
## The loop
|
||||
|
||||
```
|
||||
finding + report
|
||||
|
|
||||
v
|
||||
review.py --report report.md --rules rules/<program>.json
|
||||
|
|
||||
SUBMIT-READY? --no--> fix the cited gaps --> re-run
|
||||
|
|
||||
yes
|
||||
v
|
||||
submit
|
||||
```
|
||||
|
||||
Exit codes: `0` submit-ready, `2` gaps found, `3` reviewer not trusted
|
||||
(battery failed or configuration error).
|
||||
|
||||
## Checks
|
||||
|
||||
| Check | What it enforces |
|
||||
|---|---|
|
||||
| eligibility | class is in the rules' accepted list and not in the rejected list (cites both) |
|
||||
| poc_remote | reproduction is a remote attacker over HTTP, not server-side access |
|
||||
| reproduction | a third party can follow it; requests complete; expected vs actual stated |
|
||||
| impact_measured | quantified with denominator or a demonstrated outcome, not adjectives |
|
||||
| consolidation | one finding per report; same-type findings consolidated |
|
||||
| version_current | component + exact version + latest-at-time stated |
|
||||
| ban_risk | no untested claims presented as verified |
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
REVIEW_API=https://api.example.com REVIEW_API_KEY=... REVIEW_MODEL=<reviewer> \
|
||||
python3 review.py --report my-finding.md --rules rules/strict-vdp.json \
|
||||
--subject-model <model that wrote the report>
|
||||
```
|
||||
|
||||
`--subject-model` exists so a model never reviews its own writing (pass
|
||||
`--allow-self-review` to override knowingly). Verdicts land next to the report
|
||||
as `<report>.review.json` with every check, the applied rule, the reason, and
|
||||
the suggested fix.
|
||||
|
||||
## Rule files
|
||||
|
||||
`rules/strict-vdp.json` is the default: a strict disclosure policy's shape
|
||||
(accepted classes with chaining conditions, rejected classes, format
|
||||
requirements, risk notes). Copy it per program and paste the program's exact
|
||||
published criteria — the reviewer judges only against what the file says, so
|
||||
fidelity to the real policy is the file's job, not the reviewer's.
|
||||
|
||||
## Limits
|
||||
|
||||
The reviewer is an LLM and can misjudge; every verdict carries its reason and
|
||||
the rule text, so audit before trusting a pass. A single pass is one opinion:
|
||||
for high-stakes submissions, re-run after every edit (the loop is cheap; a
|
||||
rejection is not).
|
||||
Executable
+155
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Pre-submission report review: validate a vulnerability report against a
|
||||
program's eligibility rules BEFORE it is submitted, and refine on the feedback
|
||||
(local oracle + refine-on-response).
|
||||
|
||||
Why: strict vulnerability programs reject (and some ban) reports that miss
|
||||
their criteria — wrong vulnerability class, PoC not written from a remote
|
||||
attacker's perspective, no measurable impact, several findings of one type
|
||||
split across reports. Reviewing that locally first, with an independent
|
||||
reviewer that must pass a falsification battery, catches exactly the gaps a
|
||||
program's triage would catch, while fixing them is still free.
|
||||
|
||||
Discipline (same method as the benchmark harness in this repo):
|
||||
- the reviewer is an independent LLM and must pass a battery before grading
|
||||
(an obviously eligible report passes, an obviously out-of-scope one fails,
|
||||
a vague one fails); a grader that cannot fail the wrong ones does not run
|
||||
- every verdict carries the rule it cites and a suggested fix
|
||||
- exit code 0 = submit-ready, 2 = gaps found, 3 = reviewer not trusted
|
||||
|
||||
Usage:
|
||||
REVIEW_API=https://api.example.com REVIEW_API_KEY=... REVIEW_MODEL=... \
|
||||
python3 review.py --report finding.md --rules rules/strict-vdp.json
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
REVIEW_VERSION = "review-v1"
|
||||
|
||||
REVIEWER_SYSTEM = """You are a strict vulnerability-program triager. You receive one security report and the program's eligibility rules. Judge the report EXACTLY against the rules, not against your own sense of severity.
|
||||
|
||||
For each check below, output one object in a JSON array:
|
||||
{"check": "<check id>", "pass": true|false, "rule": "<the rule text you applied>", "reason": "1-2 sentences citing the report", "fix": "one concrete sentence to fix it, empty when pass"}
|
||||
|
||||
Checks to run (ids in this order): eligibility, poc_remote, reproduction, impact_measured, consolidation, version_current, ban_risk.
|
||||
|
||||
- eligibility: the vulnerability class is in the rules' accepted list AND not in the rejected list. Quote both entries you compared against.
|
||||
- poc_remote: the reproduction steps are written from a REMOTE attacker's perspective over HTTP (requests an outside attacker can send), not requiring server-side access, local files, or vendor tooling on the server.
|
||||
- reproduction: a third party could follow the steps and see the result; every request shows method, URL, headers that matter, and body; expected vs actual behavior is stated.
|
||||
- impact_measured: the impact is quantified with its denominator or a concrete attacker outcome (data actually returned / command actually executed), not an adjective. "An attacker can dump the table" fails; "3 of 20 payload variants returned other-tenant rows" passes.
|
||||
- consolidation: the report covers ONE finding; same-type findings the reporter knows about are consolidated here, not promised in separate reports.
|
||||
- version_current: the report states the affected component, its exact version, and that it is the latest at the time of writing.
|
||||
- ban_risk: the report contains no untested claims presented as verified (every claim traces to an executed request or output in the report).
|
||||
|
||||
Output ONLY the JSON array."""
|
||||
|
||||
|
||||
def llm(base: str, key: str, model: str, user: str, max_tokens: int = 3000) -> str:
|
||||
payload = json.dumps({
|
||||
"model": model,
|
||||
"messages": [{"role": "system", "content": REVIEWER_SYSTEM}, {"role": "user", "content": user}],
|
||||
"temperature": 0, "max_tokens": max_tokens,
|
||||
}).encode()
|
||||
r = urllib.request.Request(base.rstrip("/") + "/chat/completions", data=payload, method="POST")
|
||||
r.add_header("Content-Type", "application/json")
|
||||
r.add_header("Authorization", "Bearer " + key)
|
||||
try:
|
||||
with urllib.request.urlopen(r, timeout=int(os.environ.get("REVIEW_TIMEOUT", "900"))) as resp:
|
||||
body = json.loads(resp.read().decode())
|
||||
return body["choices"][0]["message"]["content"] or ""
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise SystemExit(f"reviewer call failed: HTTP {exc.code}: {exc.read().decode(errors='replace')[:200]}")
|
||||
|
||||
|
||||
def parse_verdicts(raw: str) -> list[dict]:
|
||||
start, end = raw.find("["), raw.rfind("]")
|
||||
if start == -1 or end <= start:
|
||||
raise ValueError("no JSON array in reviewer output")
|
||||
verdicts = json.loads(raw[start:end + 1])
|
||||
if not isinstance(verdicts, list) or not verdicts:
|
||||
raise ValueError("empty verdicts")
|
||||
return verdicts
|
||||
|
||||
|
||||
def battery(base: str, key: str, model: str, rules: dict) -> None:
|
||||
"""The reviewer must fail the wrong reports before it is allowed to grade."""
|
||||
good = "# SQL injection in item search\n\nPOST /search with q=' OR 1=1-- returns all rows.\n\n## Measured impact\n2 of 4 payload variants returned rows from other tenants; 2 rejected fail-closed. Proof basis: executed against the current released version 2.3.1 over HTTP."
|
||||
bad = "# Username enumeration on login\n\nDifferent responses for existing vs unknown users let an attacker enumerate accounts."
|
||||
cases = [
|
||||
("eligible-must-pass", good, True, "eligibility"),
|
||||
("out-of-scope-must-fail", bad, False, "eligibility"),
|
||||
]
|
||||
for label, report, expect, check in cases:
|
||||
user = f"## Program rules\n{json.dumps(rules, indent=1)}\n\n## Report\n{report}\n\nRun the checks; output the JSON array."
|
||||
raw = llm(base, key, model, user, 2000)
|
||||
try:
|
||||
v = {x.get("check"): x for x in parse_verdicts(raw)}
|
||||
got = v.get(check, {}).get("pass")
|
||||
except ValueError:
|
||||
raise SystemExit(f"reviewer battery '{label}': unparseable output; reviewer not trusted")
|
||||
if got != expect:
|
||||
raise SystemExit(f"reviewer battery FAILED on '{label}' (expected {check} pass={expect}, got {got}); a reviewer that cannot fail the wrong reports does not run")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--report", required=True, help="path to the report markdown (or '-' for stdin)")
|
||||
ap.add_argument("--rules", default=str(pathlib.Path(__file__).parent / "rules" / "strict-vdp.json"))
|
||||
ap.add_argument("--subject-model", help="model that wrote the report, to refuse self-review")
|
||||
ap.add_argument("--allow-self-review", action="store_true")
|
||||
ap.add_argument("--out", help="write the verdict JSON here (default: alongside the report)")
|
||||
args = ap.parse_args()
|
||||
|
||||
base = os.environ.get("REVIEW_API", "")
|
||||
key = os.environ.get("REVIEW_API_KEY", "")
|
||||
model = os.environ.get("REVIEW_MODEL", "")
|
||||
if not (base and key and model):
|
||||
raise SystemExit("need REVIEW_API, REVIEW_API_KEY, REVIEW_MODEL")
|
||||
|
||||
if args.subject_model and args.subject_model == model and not args.allow_self_review:
|
||||
raise SystemExit(f"reviewer model equals the report's model ({model}); refusing. Pass --allow-self-review to override knowingly.")
|
||||
|
||||
rules = json.loads(pathlib.Path(args.rules).read_text())
|
||||
report = sys.stdin.read() if args.report == "-" else pathlib.Path(args.report).read_text()
|
||||
if not report.strip():
|
||||
raise SystemExit("empty report")
|
||||
|
||||
print("[battery] verifying the reviewer fails the wrong reports")
|
||||
battery(base, key, model, rules)
|
||||
|
||||
user = (f"## Program rules\n{json.dumps(rules, indent=1)}\n\n## Report\n{report}\n\n"
|
||||
"Run the checks in order; output ONLY the JSON array.")
|
||||
print("[review] grading")
|
||||
verdicts = parse_verdicts(llm(base, key, model, user))
|
||||
|
||||
failed = [v for v in verdicts if not v.get("pass")]
|
||||
doc = {
|
||||
"review_version": REVIEW_VERSION,
|
||||
"rules_file": os.path.basename(args.rules),
|
||||
"reviewer_model": model,
|
||||
"submit_ready": not failed,
|
||||
"failed_checks": failed,
|
||||
"verdicts": verdicts,
|
||||
}
|
||||
out = pathlib.Path(args.out) if args.out else pathlib.Path(args.report).with_suffix(".review.json")
|
||||
out.write_text(json.dumps(doc, indent=2))
|
||||
|
||||
print(f"\n{'SUBMIT-READY' if not failed else 'GAPS FOUND'} ({len(verdicts) - len(failed)}/{len(verdicts)} checks pass)")
|
||||
for v in failed:
|
||||
print(f" FAIL {v.get('check')}: {v.get('reason')}")
|
||||
if v.get("fix"):
|
||||
print(f" fix: {v['fix']}")
|
||||
print(f"\nverdicts: {out}")
|
||||
print("fix the gaps, re-run, submit only on SUBMIT-READY. A rate without its basis is not a verdict: read the cited rules before trusting a pass.")
|
||||
sys.exit(0 if not failed else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"name": "strict-vdp",
|
||||
"version": "v1",
|
||||
"description": "Default rule set mirroring a strict vulnerability-disclosure policy (rewrite it to your program's exact published text; the reviewer only judges against what is written here).",
|
||||
"accepted_classes": [
|
||||
"SQL injection",
|
||||
"arbitrary file upload / deletion / download with full path and extension control",
|
||||
"remote code execution",
|
||||
"PHP object injection",
|
||||
"significant settings change (impactful application options)",
|
||||
"privilege escalation to contributor level or above",
|
||||
"broken access control on genuinely sensitive objects only (API keys, secrets, password hashes, backup or SQL files)",
|
||||
"IDOR with significant impact (personal-data leakage alone is NOT significant)",
|
||||
"stored XSS that executes on page open with zero interaction, unauthenticated or low-privilege reachable",
|
||||
"CSRF only when it chains into one of the accepted write actions above",
|
||||
"denial of service only when the whole site crashes"
|
||||
],
|
||||
"rejected_classes": [
|
||||
"user or account enumeration and other low-impact information disclosure",
|
||||
"personal-data-limited broken access control or IDOR",
|
||||
"CSRF that does not chain into an accepted write action",
|
||||
"role prerequisites at or below contributor level",
|
||||
"blind SSRF",
|
||||
"CSV injection",
|
||||
"open redirects",
|
||||
"self-XSS or XSS requiring unusual interaction"
|
||||
],
|
||||
"format_requirements": {
|
||||
"language": "English",
|
||||
"version": "the latest released version of the component at the time of writing",
|
||||
"poc_perspective": "remote attacker over HTTP; steps a third party can follow end to end; the full PoC lives in the report body, not only in attachments",
|
||||
"consolidation": "multiple findings of the same type go into ONE report",
|
||||
"impact": "quantified with its denominator or a concrete demonstrated attacker outcome"
|
||||
},
|
||||
"risk_notes": [
|
||||
"an untested or assumption-based report can earn an immediate temporary ban",
|
||||
"a high rejection rate can remove the reporter from the program"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// The cost ledger: profile prices (EUR per 1M tokens) turn token buckets into
|
||||
// EUR on the usage aggregation; unpriced profiles stay nil.
|
||||
func TestCostLedgerEurOnUsage(t *testing.T) {
|
||||
m, err := NewManager(t.TempDir(), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { m.Close() })
|
||||
if m.pg == nil {
|
||||
t.Skip("no postgres for this run")
|
||||
}
|
||||
|
||||
pid, err := m.pg.SaveProfile(&db.LLMProfile{
|
||||
Name: "cost-test", Format: "openai", Model: "m", APIKey: "k",
|
||||
PriceInputPerM: 1.5, PriceOutputPerM: 4.0, PriceCacheReadPerM: 0.15,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = m.pg.Exec("DELETE FROM llm_profiles WHERE id=$1", pid) })
|
||||
|
||||
stored, err := m.pg.ProfileByID(pid)
|
||||
if err != nil || stored == nil {
|
||||
t.Fatalf("ProfileByID: %v %v", stored, err)
|
||||
}
|
||||
if stored.PriceInputPerM != 1.5 || stored.PriceOutputPerM != 4.0 || stored.PriceCacheReadPerM != 0.15 || stored.PriceCacheWritePerM != 0 {
|
||||
t.Fatalf("prices did not round-trip: %+v", *stored)
|
||||
}
|
||||
|
||||
if _, err := m.pg.Exec(`INSERT INTO llm_usage(task_id, profile_name, model, input_tokens, output_tokens, cache_read, cache_write, ts)
|
||||
VALUES ('', 'cost-test', 'm', 2000000, 500000, 1000000, 0, now())`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = m.pg.Exec(`DELETE FROM llm_usage WHERE profile_name='cost-test'`) })
|
||||
|
||||
profiles, err := m.pg.UsageByProfile()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got *float64
|
||||
for _, p := range profiles {
|
||||
if p.ProfileName == "cost-test" {
|
||||
got = p.CostEur
|
||||
}
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatal("priced profile has no cost_eur")
|
||||
}
|
||||
want := 1.0*1.5 + 0.5*4.0 + 1.0*0.15 // fresh input 1M (2M incl. 1M cached) + 0.5M out + 1M cache-read = 3.65
|
||||
if d := *got - want; d > 0.001 || d < -0.001 {
|
||||
t.Fatalf("cost_eur = %v, want %v", *got, want)
|
||||
}
|
||||
}
|
||||
@@ -818,6 +818,8 @@ export interface ProfileUsage {
|
||||
output_tokens: number;
|
||||
cache_read_tokens: number;
|
||||
cache_write_tokens: number;
|
||||
// EUR cost from the profile's stored prices; absent when the profile has no price set.
|
||||
cost_eur?: number | null;
|
||||
}
|
||||
|
||||
// One (profile, UTC day) token bucket for the dashboard's daily chart (new source).
|
||||
@@ -1045,6 +1047,11 @@ export interface LLMProfile {
|
||||
pool_exclude?: boolean;
|
||||
// true (default) = streaming (SSE) | false = genuinely non-streaming (stream:false, one-shot response).
|
||||
streaming?: boolean;
|
||||
// Cost ledger: EUR per 1M tokens (0/absent = unset).
|
||||
price_input_per_m?: number;
|
||||
price_output_per_m?: number;
|
||||
price_cache_read_per_m?: number;
|
||||
price_cache_write_per_m?: number;
|
||||
// Output cap per reply (tokens). 0 = do not send the field; the server default decides.
|
||||
// Note the distinction from context_window_k: that is the model's total capacity, used locally only for the compression threshold.
|
||||
max_tokens?: number;
|
||||
|
||||
Reference in New Issue
Block a user