mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-21 15:40:54 +08:00
Compare commits
57
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7933df96f3 | ||
|
|
db9e121f68 | ||
|
|
680227b1ea | ||
|
|
76e936d8f3 | ||
|
|
d522a9cc47 | ||
|
|
f7779ff1de | ||
|
|
95d6d98307 | ||
|
|
f45d75bda5 | ||
|
|
3c2c6991ca | ||
|
|
bd2ee05caf | ||
|
|
9b84442a7f | ||
|
|
68e214c438 | ||
|
|
2d5906295c | ||
|
|
bd59d7d9d5 | ||
|
|
095d3a57d4 | ||
|
|
82691f1cc2 | ||
|
|
792e965f26 | ||
|
|
afa5d0e1a1 | ||
|
|
8b49362db4 | ||
|
|
551c14e644 | ||
|
|
19af5ba329 | ||
|
|
79119ae9bd | ||
|
|
b3b9334dc8 | ||
|
|
33e8d79426 | ||
|
|
42e8d3554d | ||
|
|
caa38f6ca8 | ||
|
|
0d85fc9b06 | ||
|
|
40a43580fe | ||
|
|
5def7bf57e | ||
|
|
268693066a | ||
|
|
0ffb0a68c4 | ||
|
|
892d1330f6 | ||
|
|
11adad32ce | ||
|
|
49c5e8fa8a | ||
|
|
f6f9575b19 | ||
|
|
280e99eace | ||
|
|
e2ca0118e7 | ||
|
|
1e9079267b | ||
|
|
aa8ba54efb | ||
|
|
5e1c55db4d | ||
|
|
5679618ea7 | ||
|
|
9ab895f2b7 | ||
|
|
f758b4d964 | ||
|
|
c5cb1afad7 | ||
|
|
3feeb37a3a | ||
|
|
bbe95d085b | ||
|
|
9be6b76420 | ||
|
|
ada0092168 | ||
|
|
2d6d5c4e2b | ||
|
|
12f0ec0432 | ||
|
|
2ff2fc59ac | ||
|
|
2db355389a | ||
|
|
6e906b4920 | ||
|
|
594d1918fd | ||
|
|
84c77021c4 | ||
|
|
5e979f102a | ||
|
|
cc465be782 |
@@ -21,6 +21,8 @@ ARTEX_LLM_PROVIDER=
|
||||
ARTEX_LLM_MODEL=
|
||||
ARTEX_LLM_BASE_URL=
|
||||
ARTEX_LLM_PROXY=
|
||||
# Allow the UI model list to reach a local Ollama, e.g. host.docker.internal:11434
|
||||
ARTEX_LLM_PRIVATE_ENDPOINTS=
|
||||
|
||||
# Untrusted-output boundary. Frames every tool output between nonce-tagged markers so
|
||||
# the model treats it as data, not instructions. On by default (unset or empty = on);
|
||||
|
||||
@@ -39,8 +39,6 @@ jobs:
|
||||
# - TestGraphOverviewExpandsAssociatedCompanyScope,
|
||||
# TestCaptureApprovalLifecycle: fail on unmodified upstream code with a
|
||||
# fresh database.
|
||||
# - TestSideUsageRecordedOnceOnConsumerCancellation: concurrency/cancellation
|
||||
# test that passes locally but is timing-sensitive on 2-core CI runners.
|
||||
# They now skip themselves in code (t.Skip with a pointer to the issue).
|
||||
- name: go test
|
||||
# -p 1: package test suites share the single postgres service; running
|
||||
|
||||
@@ -23,6 +23,10 @@ All notable changes to this project are documented in this file. The format is b
|
||||
- **`ask_operator` target warning**: a request whose question names a host or URL outside the goal targets is tagged as containing target-derived text (#197).
|
||||
- **Opt-in update banner**: `ARTEX_UPDATE_CHECK` makes the backend check the latest GitHub release at most once per 24 hours and the UI show a banner; off by default (#202).
|
||||
- **Release provenance**: release archives carry GitHub build provenance attestations and a CycloneDX SBOM (`gh attestation verify <zip> --repo BaaRai/ARTEX`); release notes are the matching `CHANGELOG.md` section (#219, #201).
|
||||
- **Opt-in constrained Go tampering hooks — backend, API & trace (P4.2 PR1)**: opt-in behind `ARTEX_TAMPER_HOOKS_ENABLED=1` (off by default; flag off loads zero hooks and keeps the no-hook baseline unchanged). Adds a small, deterministic, zero-dependency Go-subset evaluator (`traffic/hookvm`, parsed via `go/parser` with sorted map iteration, hard 100k step / 25 ms per-hook / 100 ms per-phase / 4 MiB memory bounds, and AST rejection of FS/network/exec/env/time/rand/goroutine/channel/defer/goto/global-var/pointer/switch constructs). Operator-only hooks (never an agent tool) are stored sealed at rest in `traffic_tamper_hooks` and `traffic_tamper_hook_versions` (migration `0011_traffic_tamper_hooks.sql`), managed via `/api/traffic/hooks*` (CRUD, toggle, version history, rollback, dry-run), prefiltered with `traffic/rewrite` matchers, executed after declarative rewrite rules per phase (fail-open only in 0.7; binary/compressed/>1 MiB bodies skipped; tainted sensitive fields stay HMAC-masked), and traced in SQLite `exchange_rewrites` (`source_kind`, `hook_id`, `hook_version`). See `docs/traffic-tamper-hooks.md`.
|
||||
- **Opt-in SOCKS5 UDP ASSOCIATE & raw UDP recording (P4.1c)**: when traffic capture is enabled, setting `ARTEX_SOCKS5_UDP_ENABLED=1` enables RFC 1928 `UDP ASSOCIATE` (`0x03`, IPv4 first) on the SOCKS5 listener (off by default: with flags off no listener starts; with SOCKS5 TCP on and UDP off, `UDP ASSOCIATE` returns `0x07` and no UDP socket is opened). Enforces per-datagram destination policy before forwarding using the existing target gate (`intercept.Interceptor.ResolveUDP4` / `intercept.PinnedUDP4Resolver`) and egress governor (`egress.Do`), dropping denied datagrams without passthrough; bounds associations (`UDPMaxDatagrams`, `UDPMaxBytes`, `UDPIdleTimeout`) and tears them down with the TCP control connection; drops `FRAG != 0` with no fragmentation reassembly; and records each forwarded datagram individually in SQLite `raw_udp_datagrams` with capped previews and atomic `_blobs` refs without writing HTTP `exchanges` rows or PG migrations. Transparent UDP (`IP_RECVORIGDSTADDR`) is deferred to C2. See `docs/traffic-socks5-udp.md`.
|
||||
- **Opt-in Linux transparent TCP inbound & raw TCP recording (P4.1b)**: on Linux when traffic capture is enabled, setting `ARTEX_TRANSPARENT_TCP_ENABLED=1` (loopback `127.0.0.1:1081` by default) or `ARTEX_TRANSPARENT_TCP_ADDR=<ipv4:port>` starts an IPv4 transparent TCP listener fed by operator-managed `iptables`/`nftables` `REDIRECT` rules (off by default; ARTEX never touches the host firewall; root/`CAP_NET_ADMIN` is required only for the operator's redirect rules; IPv6 is not covered). Recovers original IPv4 destinations via `SO_ORIGINAL_DST` with self-redirect loop protection, reuses the P4.1a target gate (`intercept.Interceptor.DialSOCKS5`), egress governor (`egress.Do`), half-close relay bounds, and SQLite `raw_tcp_streams` store (atomic `_blobs` spill, no HTTP `exchanges` row, no PostgreSQL migration). See `docs/traffic-transparent.md`.
|
||||
- **Opt-in SOCKS5 inbound proxy & raw TCP recording (P4.1a)**: when traffic capture is enabled, setting `ARTEX_SOCKS5_ENABLED=1` (loopback `127.0.0.1:1080` by default) or `ARTEX_SOCKS5_ADDR=<addr>` starts an RFC 1928 SOCKS5 `CONNECT` listener alongside the HTTP recording proxy (off by default; HTTP proxy and agent proxy env vars are unchanged; listener does not start if traffic recording is disabled). Accepts loopback no-auth and RFC 1929 username/password without verifying credentials (carrying `artex-pace` in the username; binding to a non-loopback address therefore exposes an unauthenticated proxy, though the target gate and governor still apply), rejects `BIND`/`UDP ASSOCIATE` with `0x07`, enforces the existing target gate (`intercept.Interceptor.DialSOCKS5` / `intercept.PinnedDialer`) and egress governor (`egress.Do`) before dialing, relays half-close-aware TCP with concurrency/byte/idle/handshake bounds, and records one row per connection in SQLite `raw_tcp_streams` with capped directional previews and atomic (temp+rename) `_blobs` refs without writing HTTP `exchanges` rows. See `docs/traffic-socks5.md`.
|
||||
- **Opt-in worker cache-prefix split (P3.5 Phase C)**: setting `ARTEX_CACHE_PREFIX=1` (off by default) splits the `worker` system prompt at `DynamicBoundary = 1` into a sibling-shareable static prefix (`body + untrustedContentTail + workerTrafficBlock`) and a per-intent dynamic suffix (`workerArtifactSpec(runDir)` through `<available-deferred-tools>`) without reordering sections or altering the visible joined prompt text. Planner, mainagent, chat, and goals are unchanged; custom worker prompts referencing `{{.Now}}` remain volatile in the static segment. See `docs/context-telemetry.md`.
|
||||
- **Opt-in LLM context size telemetry (P3.5 Phase A)**: setting `ARTEX_CONTEXT_TELEMETRY=1` (off by default) records per-call request section byte sizes (`system_static`/`system_dynamic` split at `DynamicBoundary`, serialized `tools`, `messages` by role, `tool_result` subset, and `last_message_bytes`), tool/message counts, and token/cache usage in the new `llm_context` table (migration `0009_llm_context.sql`), with zero prompt/behavior change and zero content stored, independent of `llm_record`. `GET /api/llm/context/summary` aggregates input percentiles (`p50`/`p90`/`p99`), pro-rata section shares over real `input_tokens`, `cache_read/input` ratios by `(worker, model)`, and top tasks. See `docs/context-telemetry.md`.
|
||||
- **HAR 1.2 traffic import and export**: `POST /api/traffic/import/har` imports raw HAR 1.2 JSON captures (up to 32 MiB / 2,000 entries / 32 MiB decoded bodies) with deterministic `har-<sha256>` IDs (and `_artexExchangeId` preservation for ARTEX exports so distinct same-second exchanges round-trip without collapsing), base64 decoding, `postData.params` and `cookies` array fallbacks, URL userinfo stripping, blob spilling above 256 KiB, and full-text indexing, without applying capture rewrite rules (`exchange_rewrites` stays empty); per-entry errors are reported alongside `imported`/`skipped`/`failed` counts. `GET /api/traffic/export/har` exports filtered traffic (reusing the `GET /api/traffic` filters) as a compliant HAR 1.2 document (with explicit unknown placeholders for unrecorded timings and header sizes) in stable `(ts, id)` order, encoding UTF-8 bodies as text and binary bodies as base64, capped at 1,000 entries / 20 MiB raw headers+bodies (31 MiB JSON backstop, guaranteed under the 32 MiB import limit) with explicit truncation reporting via `X-ARTEX-HAR-*` headers and `_truncated`/`_truncatedReason`/`_totalMatched` fields. The Traffic page adds **Import HAR** and **Export HAR** controls. See `docs/traffic-har.md`.
|
||||
@@ -53,6 +57,7 @@ All notable changes to this project are documented in this file. The format is b
|
||||
|
||||
### Fixed
|
||||
|
||||
- **LLM usage status on early stream exit**: when a consumer stops reading a stream before `message_stop`, the recorder now stores the `llm_usage` row as `error` at that point, whether or not the context was cancelled, instead of inferring the status from `ctx.Err()` in a deferred call. `TestSideUsageRecordedOnceOnConsumerCancellation` runs again (follow-up to #9).
|
||||
- **Workspace and skill symlink confinement**: workspace file operations, the skill file APIs and the skill loader resolve symlinks in every path component and refuse a target outside their root, so a link planted in the workspace or a skill directory cannot expose another file.
|
||||
- **Target gate behind proxies**: a target approval is name-only whenever a proxy is in the path, including the recording proxy, the global egress proxy and a custom tool's own proxy, so the gate never claims an IP binding the proxy does not enforce.
|
||||
- **Remote MCP response bound**: Streamable HTTP JSON-RPC bodies and SSE data payloads are refused above 8 MiB, so a remote MCP server cannot force an unbounded decode.
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
# Contributing
|
||||
|
||||
## First contribution
|
||||
|
||||
- **Share feedback.** Check [open issues](https://github.com/BaaRai/ARTEX/issues) for the same problem or proposal and add any new details there. For a new idea, explain what you were trying to do and what would make it easier.
|
||||
- **Report a problem.** Include a short reproduction and the relevant environment. For a UI issue, a sanitized screenshot and the browser version can help. Remove credentials, private URLs, and sensitive task data before posting.
|
||||
- **Report a security issue.** Do not open a public issue for a vulnerability; follow [SECURITY.md](SECURITY.md) to report it privately.
|
||||
- **Send a small change.** A documentation correction or a focused UI improvement is a good place to start. Check [open pull requests](https://github.com/BaaRai/ARTEX/pulls) for overlapping work and refer to the relevant issue in your PR so readers can find the discussion.
|
||||
|
||||
## Pull requests
|
||||
|
||||
- **At most 7 open pull requests per contributor.** The repository enforces this with GitHub's pull request creation limit, drafts included: when you reach it, close or merge one before opening another. Review is manual, one change at a time, so a larger batch makes the queue unmanageable and slows everyone down, including you. If you have more changes queued, open one issue that lists them and wait for the current ones to be reviewed.
|
||||
|
||||
@@ -545,6 +545,25 @@ a trusted peer can claim any client address.
|
||||
|
||||
## Operations
|
||||
|
||||
### Troubleshooting an empty tool call
|
||||
|
||||
In the task's session transcript, expand the affected tool call and compare **Command** and
|
||||
**Output**. The `UNTRUSTED_OUTPUT` markers identify the boundary around tool output. They are
|
||||
expected and should stay enabled while troubleshooting.
|
||||
|
||||
- **Command shows `{}`.** The recorded tool input is an empty JSON object. For a tool that
|
||||
requires a command, this means its command argument was missing. Check which model and version
|
||||
produced the call.
|
||||
- **Command shows `(empty)` or `(failed to load)`.** The transcript has no input detail to display
|
||||
or could not fetch it. If the session record contains a command, include that discrepancy in a
|
||||
report.
|
||||
- **Output shows `[]`.** This is the recorded result body. Check the row's success or failure
|
||||
indicator and its input separately before drawing conclusions about what ran.
|
||||
|
||||
For a recurring problem, share the ARTEX version or commit, the model and version, and a sanitized
|
||||
excerpt of the affected input and result. Remove credentials, target data, and other sensitive
|
||||
content before sharing.
|
||||
|
||||
### Backups
|
||||
|
||||
`scripts/backup.sh` dumps the Postgres database (and optionally the data directory) with
|
||||
|
||||
@@ -101,6 +101,14 @@ func guardPanic(t actool.CoreTool) actool.CoreTool { return &guardedTool{CoreToo
|
||||
|
||||
type guardedTool struct{ actool.CoreTool }
|
||||
|
||||
// AcceptsRawInput forwards the wrapped tool's opt-out of the harness schema
|
||||
// check, as overriddenTool does: this wrapper is the one the harness sees, and
|
||||
// the embedded interface alone would hide the opt-out from its type assertion.
|
||||
func (g *guardedTool) AcceptsRawInput() bool {
|
||||
r, ok := g.CoreTool.(interface{ AcceptsRawInput() bool })
|
||||
return ok && r.AcceptsRawInput()
|
||||
}
|
||||
|
||||
func (g *guardedTool) Call(ctx context.Context, in json.RawMessage, tc *actool.ToolContext) (res actool.Result, err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
|
||||
@@ -33,7 +33,7 @@ func TestTaskScopeLabelIsHonest(t *testing.T) {
|
||||
// denominator and list_untested_assets.
|
||||
func TestAutoScopeOutsideDeclaredRootExcluded(t *testing.T) {
|
||||
d := testDB(t)
|
||||
defer d.Close()
|
||||
t.Cleanup(func() { _ = d.Close() })
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
declared := fmt.Sprintf("app.f8-%d.invalid", stamp)
|
||||
|
||||
@@ -194,7 +194,7 @@ func companiesName(t *testing.T, companies *db.CompanyStore, companyID int64) st
|
||||
|
||||
func TestBlackboardToolsReadDirectSources(t *testing.T) {
|
||||
d := testDB(t)
|
||||
defer d.Close()
|
||||
t.Cleanup(func() { _ = d.Close() })
|
||||
|
||||
grand, err := d.CreateTask("grand", "grand goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
@@ -378,7 +378,7 @@ func TestBlackboardToolsReadDirectSources(t *testing.T) {
|
||||
// model which ids it deferred, after de-duplicating and dropping invalid ids.
|
||||
func TestGetWorkerTraceStepIDsDegradeGracefully(t *testing.T) {
|
||||
d := testDB(t)
|
||||
defer d.Close()
|
||||
t.Cleanup(func() { _ = d.Close() })
|
||||
|
||||
task, err := d.CreateTask("trace-cap", "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
// write happens even when the stored flag is off, which is the outage behaviour.
|
||||
func TestCoverageReadFailureKeepsToolsAndWritesAutoScope(t *testing.T) {
|
||||
d := testDB(t)
|
||||
defer d.Close()
|
||||
t.Cleanup(func() { _ = d.Close() })
|
||||
off := false
|
||||
task, err := d.CreateTaskWithOptions("coverage outage", "goal", db.TaskCreateOptions{CoverageEnabled: &off})
|
||||
if err != nil {
|
||||
|
||||
@@ -115,10 +115,9 @@ func TestRecordFactAssetAliases(t *testing.T) {
|
||||
func TestRecordFactAssetAliasSchema(t *testing.T) {
|
||||
schema := NewToolSet(nil, "").recordFact().InputSchema()
|
||||
props := schema["properties"].(map[string]any)
|
||||
for _, name := range []string{"asset_ids", "tested_asset_ids"} {
|
||||
if props[name] == nil {
|
||||
t.Fatalf("single schema missing %s", name)
|
||||
}
|
||||
// Facts-only contract: the alias lives on the items.
|
||||
if len(props) != 1 || props["facts"] == nil {
|
||||
t.Fatalf("top level must hold facts only: %v", props)
|
||||
}
|
||||
item := props["facts"].(map[string]any)["items"].(map[string]any)
|
||||
itemProps, _ := item["properties"].(map[string]any)
|
||||
|
||||
+88
-58
@@ -4,10 +4,12 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Match strings.TrimSpace, including Unicode whitespace, in the published schema.
|
||||
const factWhitespace = "\t\n\v\f\r \u0085\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000"
|
||||
const factWhitespace = "\t\n\v\f\r \u0085 -
"
|
||||
|
||||
func factAssetIDSchema() map[string]any {
|
||||
return map[string]any{
|
||||
@@ -16,10 +18,13 @@ func factAssetIDSchema() map[string]any {
|
||||
}
|
||||
}
|
||||
|
||||
// factItemSchema is one fact: the shape of every facts[] item. It is also the
|
||||
// shape of the legacy single-fact call (fields at the top level, no facts),
|
||||
// which the handler still accepts but the published schema no longer shows.
|
||||
func factItemSchema() map[string]any {
|
||||
schema := obj(map[string]any{
|
||||
"summary": str("Required on every fact: a non-empty one-sentence summary of this conclusion. Batch items do not inherit summary from the top level."),
|
||||
"intent_id": idp("The intent that produced this fact; only this field inherits from the top level in a batch."),
|
||||
"summary": str("Required: a non-empty one-sentence summary of this conclusion."),
|
||||
"intent_id": idp("The intent that produced this fact. Omit it to link the fact to the intent this worker is running; a batch may name a different intent per item."),
|
||||
"detail": str("Details of this fact, including all related observations."),
|
||||
"evidence": str("One line of key evidence: the command and the most proving output lines. Put longer output in detail."),
|
||||
"confidence": str("observed (seen directly in output) | inferred (inferred from phenomena). Label negative conclusions honestly."),
|
||||
@@ -34,68 +39,67 @@ func factItemSchema() map[string]any {
|
||||
return schema
|
||||
}
|
||||
|
||||
// recordFactFactsDescription documents the only top-level property of
|
||||
// record_fact. The wording tells a model that tends to fill every declared
|
||||
// property that there is nothing else to fill.
|
||||
const recordFactFactsDescription = "The facts to record, usually exactly one. Every item is a complete fact with its own summary. This is the only top-level property: never send summary, detail, evidence, confidence, source, intent_id or asset ids beside facts. Returned ids match item order; retry only failed indices."
|
||||
|
||||
// recordFactContractNote is appended to the record_fact description: the one
|
||||
// call shape the model is given.
|
||||
const recordFactContractNote = "\n★Call shape: exactly {\"facts\":[{...}]} with one item per distinct conclusion (usually one item). Each item carries its own summary, detail, evidence, confidence, source, asset_ids and, when it differs from the intent you are running, intent_id. Nothing else at the top level; a top-level fact field beside facts is refused and nothing is written."
|
||||
|
||||
// RecordFactContractNote exposes the text to the server's one-time refresh.
|
||||
func RecordFactContractNote() string { return recordFactContractNote }
|
||||
|
||||
// recordFactSchema is the published input contract: facts[] and nothing else.
|
||||
// Providers reject root-level anyOf, and the earlier contract (fact fields at
|
||||
// the top level beside facts[]) made models fill both; the handler alone keeps
|
||||
// the legacy shapes alive.
|
||||
func recordFactSchema() map[string]any {
|
||||
props := factItemSchema()["properties"].(map[string]any)
|
||||
props["facts"] = map[string]any{
|
||||
"type": "array", "minItems": 1, "items": factItemSchema(),
|
||||
"description": "A non-empty batch of complete facts. Every item needs its own summary. Only intent_id inherits from the top level. Returned ids match item order; retry only failed indices.",
|
||||
}
|
||||
schema := obj(props)
|
||||
// Providers reject root-level anyOf. The handler enforces single/batch mode.
|
||||
schema := obj(map[string]any{
|
||||
"facts": map[string]any{
|
||||
"type": "array", "minItems": 1, "items": factItemSchema(),
|
||||
"description": recordFactFactsDescription,
|
||||
},
|
||||
}, "facts")
|
||||
schema["additionalProperties"] = false
|
||||
return schema
|
||||
}
|
||||
|
||||
// RepairRecordFactSchema restores the single/batch input contract in stored or
|
||||
// cached schemas, preserving annotations only on fields the handler accepts.
|
||||
// RepairRecordFactSchema brings a stored or cached record_fact schema to the
|
||||
// facts-only contract. The structure always comes from the code; of the stored
|
||||
// schema only the annotations (description, default) of facts and of its known
|
||||
// item fields survive. Top-level fact fields of the earlier contract are
|
||||
// dropped: the handler still accepts them, but the model must not see them.
|
||||
func RepairRecordFactSchema(stored map[string]any) map[string]any {
|
||||
raw, _ := json.Marshal(stored)
|
||||
var out map[string]any
|
||||
_ = json.Unmarshal(raw, &out)
|
||||
if out == nil {
|
||||
out = map[string]any{}
|
||||
out := recordFactSchema()
|
||||
facts := out["properties"].(map[string]any)["facts"].(map[string]any)
|
||||
storedProps, _ := stored["properties"].(map[string]any)
|
||||
storedFacts, _ := storedProps["facts"].(map[string]any)
|
||||
if storedFacts == nil {
|
||||
return out
|
||||
}
|
||||
mergeFactContract(out, recordFactSchema())
|
||||
// A batch requires summaries on its items, never a redundant top-level one.
|
||||
delete(out, "required")
|
||||
delete(out, "anyOf")
|
||||
canonical := recordFactSchema()["properties"].(map[string]any)
|
||||
props := out["properties"].(map[string]any)
|
||||
for name := range props {
|
||||
if _, known := canonical[name]; !known {
|
||||
delete(props, name)
|
||||
if d, ok := storedFacts["description"].(string); ok && strings.TrimSpace(d) != "" {
|
||||
facts["description"] = d
|
||||
}
|
||||
storedItems, _ := storedFacts["items"].(map[string]any)
|
||||
storedItemProps, _ := storedItems["properties"].(map[string]any)
|
||||
itemProps := facts["items"].(map[string]any)["properties"].(map[string]any)
|
||||
for name, p := range itemProps {
|
||||
sp, _ := storedItemProps[name].(map[string]any)
|
||||
if sp == nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
items := props["facts"].(map[string]any)["items"].(map[string]any)
|
||||
itemProps := items["properties"].(map[string]any)
|
||||
for name := range itemProps {
|
||||
if _, known := canonical[name]; !known || name == "facts" {
|
||||
delete(itemProps, name)
|
||||
prop := p.(map[string]any)
|
||||
for _, key := range []string{"description", "default"} {
|
||||
if v, ok := sp[key]; ok && v != nil {
|
||||
prop[key] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mergeFactContract(dst, src map[string]any) {
|
||||
for key, value := range src {
|
||||
if key == "description" || key == "default" {
|
||||
if _, ok := dst[key]; ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if child, ok := value.(map[string]any); ok {
|
||||
target, _ := dst[key].(map[string]any)
|
||||
if target == nil {
|
||||
target = map[string]any{}
|
||||
dst[key] = target
|
||||
}
|
||||
mergeFactContract(target, child)
|
||||
} else {
|
||||
dst[key] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type recordFactInput struct {
|
||||
Facts []factItem `json:"facts"`
|
||||
factItem
|
||||
@@ -103,6 +107,9 @@ type recordFactInput struct {
|
||||
|
||||
// Validate exact field names before decoding: encoding/json otherwise treats
|
||||
// Summary, SUMMARY, etc. as aliases even with DisallowUnknownFields enabled.
|
||||
// Three input shapes reach the handler: the published one ({facts:[...]}), the
|
||||
// legacy single fact (fields at the top level, no facts) and the legacy batch
|
||||
// with a top-level intent_id as the default of its items.
|
||||
func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) {
|
||||
var out recordFactInput
|
||||
fields, err := exactFactFields(in, true)
|
||||
@@ -111,11 +118,6 @@ func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) {
|
||||
}
|
||||
_, batch := fields["facts"]
|
||||
if batch {
|
||||
for name := range fields {
|
||||
if name != "intent_id" && name != "facts" {
|
||||
return out, true, fmt.Errorf("field %q is not allowed at the top level in batch mode; only intent_id and facts are accepted", name)
|
||||
}
|
||||
}
|
||||
var items []json.RawMessage
|
||||
if err := json.Unmarshal(fields["facts"], &items); err != nil {
|
||||
return out, true, fmt.Errorf("facts must be an array: %w", err)
|
||||
@@ -123,10 +125,16 @@ func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) {
|
||||
if len(items) == 0 {
|
||||
return out, true, fmt.Errorf("facts must be a non-empty array of complete facts, each with its own summary")
|
||||
}
|
||||
itemFields := make([]map[string]json.RawMessage, len(items))
|
||||
for i, item := range items {
|
||||
if _, err := exactFactFields(item, false); err != nil {
|
||||
f, err := exactFactFields(item, false)
|
||||
if err != nil {
|
||||
return out, true, fmt.Errorf("facts[%d]: %w", i, err)
|
||||
}
|
||||
itemFields[i] = f
|
||||
}
|
||||
if err := checkBatchTopLevel(fields, itemFields); err != nil {
|
||||
return out, true, err
|
||||
}
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(in))
|
||||
@@ -137,6 +145,28 @@ func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) {
|
||||
return out, batch, nil
|
||||
}
|
||||
|
||||
// checkBatchTopLevel enforces the contract at the top level of a facts[] call,
|
||||
// where only facts and (legacy) intent_id carry meaning. Any other top-level
|
||||
// field is refused by name before anything is written, including empty
|
||||
// placeholders and copies of a fact's fields: a value there is at best
|
||||
// redundant and at worst a second, conflicting claim. The message names the
|
||||
// one accepted shape, so a model that filled every declared property of an
|
||||
// older schema learns where the fields belong.
|
||||
func checkBatchTopLevel(fields map[string]json.RawMessage, items []map[string]json.RawMessage) error {
|
||||
var extra []string
|
||||
for name := range fields {
|
||||
if name == "intent_id" || name == "facts" {
|
||||
continue
|
||||
}
|
||||
extra = append(extra, fmt.Sprintf("%q", name))
|
||||
}
|
||||
if len(extra) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Strings(extra)
|
||||
return fmt.Errorf("top-level field(s) %s are not allowed beside facts (%d item(s)): record_fact takes only {\"facts\":[...]}; put every field of a fact inside its facts item and send nothing else at the top level", strings.Join(extra, ", "), len(items))
|
||||
}
|
||||
|
||||
func exactFactFields(in json.RawMessage, topLevel bool) (map[string]json.RawMessage, error) {
|
||||
var fields map[string]json.RawMessage
|
||||
if err := json.Unmarshal(in, &fields); err != nil {
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package agent
|
||||
|
||||
// Texts of the earlier record_fact contract (fact fields at the top level
|
||||
// beside facts[]), kept verbatim so the server's one-time refresh
|
||||
// (server/fact_input_schema.go) can tell a stored default from an operator
|
||||
// edit. Nothing here is published to a model.
|
||||
|
||||
// recordFactLegacyDescription is the record_fact description seeded up to the
|
||||
// facts-only contract.
|
||||
const recordFactLegacyDescription = "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n" +
|
||||
"⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n" +
|
||||
"★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n" +
|
||||
"⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n" +
|
||||
" · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n" +
|
||||
" · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n" +
|
||||
" · source = where the content came from: target_response (content the target itself returned: page bodies, headers, error messages, banners, even when a tool fetched it) | tool_output (a tool's own result about the target: port states, scanner verdicts, status codes, timings) | third_party_intel (OSINT, CVE databases, public reports) | operator_input (given by the human operator) | inference (your own reasoning; the default when omitted). Any other value is refused.\n" +
|
||||
" · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen."
|
||||
|
||||
// recordFactReleasedDescription is the record_fact description seeded by the
|
||||
// releases v0.4.0 to v0.6.0 (agent/tools.go at those tags): the text above
|
||||
// without the source bullet.
|
||||
const recordFactReleasedDescription = "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n" +
|
||||
"⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n" +
|
||||
"★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n" +
|
||||
"⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n" +
|
||||
" · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n" +
|
||||
" · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n" +
|
||||
" · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen."
|
||||
|
||||
// recordFactLegacyItemDescriptions are the facts[] item descriptions of the
|
||||
// earlier contract whose wording referred to the top level; the refresh
|
||||
// replaces exactly these texts with the current ones.
|
||||
var recordFactLegacyItemDescriptions = map[string]string{
|
||||
"summary": "Required on every fact: a non-empty one-sentence summary of this conclusion. Batch items do not inherit summary from the top level.",
|
||||
"intent_id": "The intent that produced this fact; only this field inherits from the top level in a batch.",
|
||||
}
|
||||
|
||||
// recordFactLegacyFactsDescription is the facts property description of the
|
||||
// earlier contract.
|
||||
const recordFactLegacyFactsDescription = "A non-empty batch of complete facts. Every item needs its own summary. Only intent_id inherits from the top level. Returned ids match item order; retry only failed indices."
|
||||
|
||||
// recordFactReleasedFactsDescription is the facts property description seeded
|
||||
// by the releases v0.4.0 to v0.6.0. RepairRecordFactSchema keeps a stored facts
|
||||
// description, so the refresh has to know this text to replace it.
|
||||
const recordFactReleasedFactsDescription = "[use when there are several distinct conclusions] array of facts; element fields same as the top-level fields below (summary/detail/evidence/confidence/intent_id/asset_ids); omitting intent_id falls back to the top-level intent_id. The returned ids match this array in length and order."
|
||||
|
||||
// RecordFactLegacyDescriptions returns the description texts of the earlier
|
||||
// contract that the refresh may replace: the tool descriptions, the facts
|
||||
// property descriptions and the item descriptions.
|
||||
func RecordFactLegacyDescriptions() (tool, facts []string, items map[string]string) {
|
||||
return []string{recordFactLegacyDescription, recordFactReleasedDescription},
|
||||
[]string{recordFactLegacyFactsDescription, recordFactReleasedFactsDescription},
|
||||
recordFactLegacyItemDescriptions
|
||||
}
|
||||
+244
-16
@@ -3,9 +3,11 @@ package agent
|
||||
import (
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
actool "github.com/Autumn-27/norma/tool"
|
||||
)
|
||||
|
||||
@@ -47,6 +49,10 @@ func TestRecordFactSummaryContract(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A stored schema of the earlier contract (fact fields at the top level beside
|
||||
// facts[], operator annotations, a stray property) is republished as the
|
||||
// facts-only contract; annotations of facts and of its items survive, the top
|
||||
// level loses every fact field, and the result is idempotent.
|
||||
func TestRecordFactStoredContractRepair(t *testing.T) {
|
||||
old := map[string]any{
|
||||
"type": "object", "required": []any{"summary"},
|
||||
@@ -55,32 +61,40 @@ func TestRecordFactStoredContractRepair(t *testing.T) {
|
||||
"summary": map[string]any{"type": "string", "description": "operator summary"},
|
||||
"confidence": map[string]any{"type": "string", "default": "inferred"},
|
||||
"custom_field": map[string]any{"type": "string"},
|
||||
"facts": map[string]any{"type": "array", "items": map[string]any{"type": "object"}},
|
||||
"facts": map[string]any{"type": "array", "description": "operator facts", "items": map[string]any{"type": "object", "properties": map[string]any{
|
||||
"summary": map[string]any{"type": "string", "description": "operator item summary"},
|
||||
"confidence": map[string]any{"type": "string", "default": "observed"},
|
||||
"custom": map[string]any{"type": "string"},
|
||||
}}},
|
||||
},
|
||||
}
|
||||
base := NewToolSet(nil, "").recordFact()
|
||||
repaired := DecorateTool(base, "operator tool", old)
|
||||
schema := repaired.InputSchema()
|
||||
if _, required := schema["required"]; required {
|
||||
t.Fatal("batch still requires a top-level summary")
|
||||
if req, _ := schema["required"].([]string); len(req) != 1 || req[0] != "facts" {
|
||||
t.Fatalf("facts must be the only required property: %v", schema["required"])
|
||||
}
|
||||
if _, exists := schema["anyOf"]; exists {
|
||||
t.Fatal("root anyOf reaches the provider")
|
||||
}
|
||||
props := schema["properties"].(map[string]any)
|
||||
item := props["facts"].(map[string]any)["items"].(map[string]any)
|
||||
if len(props) != 1 || props["facts"] == nil || schema["additionalProperties"] != false {
|
||||
t.Fatalf("top level must hold facts only: %v", props)
|
||||
}
|
||||
facts := props["facts"].(map[string]any)
|
||||
item := facts["items"].(map[string]any)
|
||||
required := item["required"].([]string)
|
||||
if len(required) != 1 || required[0] != "summary" || props["facts"].(map[string]any)["minItems"] != 1 {
|
||||
if len(required) != 1 || required[0] != "summary" || facts["minItems"] != 1 {
|
||||
t.Fatal("batch must require a nonempty list with a summary per item")
|
||||
}
|
||||
if props["summary"].(map[string]any)["minLength"] != 1 ||
|
||||
item["properties"].(map[string]any)["summary"].(map[string]any)["minLength"] != 1 {
|
||||
itemProps := item["properties"].(map[string]any)
|
||||
if itemProps["summary"].(map[string]any)["minLength"] != 1 {
|
||||
t.Fatal("empty summaries permitted by schema")
|
||||
}
|
||||
if repaired.Description() != "operator tool" ||
|
||||
props["summary"].(map[string]any)["description"] != "operator summary" ||
|
||||
props["confidence"].(map[string]any)["default"] != "inferred" || props["custom_field"] != nil {
|
||||
t.Fatal("operator annotations lost")
|
||||
if repaired.Description() != "operator tool" || facts["description"] != "operator facts" ||
|
||||
itemProps["summary"].(map[string]any)["description"] != "operator item summary" ||
|
||||
itemProps["confidence"].(map[string]any)["default"] != "observed" || itemProps["custom"] != nil {
|
||||
t.Fatalf("item annotations lost or stray property kept: %v", itemProps)
|
||||
}
|
||||
if _, ok := old["properties"].(map[string]any)["tested_asset_ids"]; ok {
|
||||
t.Fatal("shared old schema mutated")
|
||||
@@ -94,9 +108,18 @@ func TestRecordFactStoredContractRepair(t *testing.T) {
|
||||
if err := json.Unmarshal(first, &roundTrip); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, input := range []string{`{"summary":"single"}`, `{"facts":[{"summary":"batch"}]}`} {
|
||||
if err := actool.ValidateInput(roundTrip, json.RawMessage(input)); err != nil {
|
||||
t.Fatalf("SDK rejects valid %s after catalog round trip: %v", input, err)
|
||||
if err := actool.ValidateInput(roundTrip, json.RawMessage(`{"facts":[{"summary":"batch"}]}`)); err != nil {
|
||||
t.Fatalf("SDK rejects the published shape after catalog round trip: %v", err)
|
||||
}
|
||||
// The legacy single shape no longer passes the published schema; the
|
||||
// handler validates its own input instead, also behind the catalog wrapper.
|
||||
if err := actool.ValidateInput(roundTrip, json.RawMessage(`{"summary":"single"}`)); err == nil {
|
||||
t.Fatal("published schema must require facts")
|
||||
}
|
||||
for _, tool := range []actool.CoreTool{base, repaired} {
|
||||
r, ok := tool.(interface{ AcceptsRawInput() bool })
|
||||
if !ok || !r.AcceptsRawInput() {
|
||||
t.Fatalf("%T must opt out of the harness schema check", tool)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -140,7 +163,7 @@ func TestRecordFactStrictFields(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRecordFactSchemaMatchesStrings(t *testing.T) {
|
||||
props := recordFactSchema()["properties"].(map[string]any)
|
||||
props := factItemSchema()["properties"].(map[string]any)
|
||||
summary := regexp.MustCompile(props["summary"].(map[string]any)["pattern"].(string))
|
||||
for _, value := range []string{"", " ", "\t\r\n", "\u00a0\u0085", "\u2003\u3000", "conclusion", " observed "} {
|
||||
if got, want := summary.MatchString(value), strings.TrimSpace(value) != ""; got != want {
|
||||
@@ -155,7 +178,7 @@ func TestRecordFactSchemaMatchesStrings(t *testing.T) {
|
||||
}
|
||||
}
|
||||
for _, input := range []string{
|
||||
`{"summary":"single","asset_ids":["1",2]}`,
|
||||
`{"facts":[{"summary":"batch","asset_ids":["1",2]}]}`,
|
||||
`{"facts":[{"summary":"batch","tested_asset_ids":["1",2]}]}`,
|
||||
} {
|
||||
if err := actool.ValidateInput(recordFactSchema(), json.RawMessage(input)); err != nil {
|
||||
@@ -163,3 +186,208 @@ func TestRecordFactSchemaMatchesStrings(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A model that fills every declared property of the earlier schema sends the
|
||||
// single fact inside facts[] and a copy (or a placeholder) of every fact field
|
||||
// beside it. Such a call is refused by name before any write, whether the
|
||||
// copies are identical, empty or dummies, and the message names the one shape
|
||||
// that is accepted. The published schema no longer declares those fields.
|
||||
func TestRecordFactRefusesTopLevelCopiesBesideFacts(t *testing.T) {
|
||||
store := guardTaskStore(t)
|
||||
intentID := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "fingerprint the service"}, "running")
|
||||
tools := NewToolSet(store, "worker")
|
||||
before, err := store.ListByKind(db.KindFact, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := strconv.FormatInt(intentID, 10)
|
||||
fact := `{"summary":"TLS 1.0 is offered on port 8443","detail":"the handshake succeeded with TLSv1.0","evidence":"openssl s_client -tls1 → Protocol: TLSv1","confidence":"observed","source":"tool_output","intent_id":` + id + `,"asset_ids":[]}`
|
||||
for _, tc := range []struct{ call, want string }{
|
||||
// an exact copy of every field beside facts[]
|
||||
{`{"summary":"TLS 1.0 is offered on port 8443","detail":"the handshake succeeded with TLSv1.0","evidence":"openssl s_client -tls1 → Protocol: TLSv1","confidence":"observed","source":"tool_output","intent_id":` + id + `,"asset_ids":[],"facts":[` + fact + `]}`, `"asset_ids", "confidence", "detail", "evidence", "source", "summary"`},
|
||||
// empty placeholders only
|
||||
{`{"summary":"","detail":"","evidence":"","confidence":"","source":"","asset_ids":[],"tested_asset_ids":[],"intent_id":` + id + `,"facts":[` + fact + `]}`, `"asset_ids", "confidence", "detail", "evidence", "source", "summary", "tested_asset_ids"`},
|
||||
// dummy values
|
||||
{`{"summary":"x","facts":[` + fact + `]}`, `"summary"`},
|
||||
{`{"summary":null,"facts":[` + fact + `]}`, `"summary"`},
|
||||
} {
|
||||
w := tools.Writes().Facts
|
||||
r := callGuardTool(t, tools.recordFact(), tc.call)
|
||||
if !r.IsError || !strings.Contains(r.Flatten(), tc.want) || !strings.Contains(r.Flatten(), `record_fact takes only {"facts":[...]}`) || tools.Writes().Facts != w {
|
||||
t.Errorf("%s: want refusal naming %s, got %s (writes %d→%d)", tc.call, tc.want, r.Flatten(), w, tools.Writes().Facts)
|
||||
}
|
||||
}
|
||||
if n, err := store.ListByKind(db.KindFact, 100); err != nil || len(n) != len(before) {
|
||||
t.Fatalf("refused calls wrote facts: %d → %d %v", len(before), len(n), err)
|
||||
}
|
||||
// The same fact in the published shape is written once and linked.
|
||||
res := callGuardTool(t, tools.recordFact(), `{"facts":[`+fact+`]}`)
|
||||
if res.IsError || tools.Writes().Facts != 1 {
|
||||
t.Fatalf("published shape refused: %s", res.Flatten())
|
||||
}
|
||||
var out struct {
|
||||
IDs []int64 `json:"ids"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(res.Flatten()), &out); err != nil || len(out.IDs) != 1 || out.IDs[0] <= 0 {
|
||||
t.Fatalf("batch result must carry exactly one id: %s", res.Flatten())
|
||||
}
|
||||
if yielded, err := store.FactsYielded(intentID); err != nil || len(yielded) != 1 || yielded[0] != out.IDs[0] {
|
||||
t.Fatalf("fact not linked to the intent: %v %v", yielded, err)
|
||||
}
|
||||
node, err := store.GetNode(out.IDs[0])
|
||||
if err != nil || node == nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var payload map[string]any
|
||||
_ = json.Unmarshal(node.Payload, &payload)
|
||||
if payload["summary"] != "TLS 1.0 is offered on port 8443" || payload["source"] != "tool_output" || payload["confidence"] != "observed" {
|
||||
t.Fatalf("fact payload: %s", node.Payload)
|
||||
}
|
||||
}
|
||||
|
||||
// Conflicting copies, placeholders, unknown fields, bad asset ids, missing
|
||||
// summaries and empty batches are refused, and nothing is written by a
|
||||
// refused call.
|
||||
func TestRecordFactBatchTopLevelConflicts(t *testing.T) {
|
||||
store := guardTaskStore(t)
|
||||
tools := NewToolSet(store, "worker")
|
||||
before, err := store.ListByKind(db.KindFact, 100) // the task's origin fact
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, tc := range []struct{ call, want string }{
|
||||
// dummy summaries a model may send after a refusal
|
||||
{`{"summary":"x","facts":[{"summary":"real conclusion"}]}`, `"summary"`},
|
||||
{`{"summary":"placeholder","detail":"test","evidence":"test","facts":[{"summary":"real","detail":"real detail","evidence":"real evidence"}]}`, `"detail", "evidence", "summary"`},
|
||||
{`{"summary":"contradictory","facts":[{"summary":"different"}]}`, `"summary"`},
|
||||
{`{"detail":"invented","facts":[{"summary":"legitimate"}]}`, `"detail"`},
|
||||
{`{"confidence":"observed","facts":[{"summary":"fact says inferred","confidence":"inferred"}]}`, `"confidence"`},
|
||||
{`{"source":"inference","facts":[{"summary":"fact says tool_output","source":"tool_output"}]}`, `"source"`},
|
||||
{`{"asset_ids":[1],"facts":[{"summary":"fact has other assets","asset_ids":[2]}]}`, `"asset_ids"`},
|
||||
{`{"summary":"first","facts":[{"summary":"first"},{"summary":"second"}]}`, `(2 item(s))`},
|
||||
{`{"summary":"x","typo":1,"facts":[{"summary":"unknown field"}]}`, `"typo"`},
|
||||
{`{"facts":[{"summary":"unknown nested","typo":1}]}`, `"typo"`},
|
||||
{`{"facts":[{"summary":"bad asset id","asset_ids":["abc"]}]}`, `asset`},
|
||||
{`{"summary":"must not inherit","facts":[{"detail":"missing summary"}]}`, `"summary"`},
|
||||
{`{"summary":"x","facts":[]}`, `non-empty`},
|
||||
{`{"facts":[]}`, `non-empty`},
|
||||
{`{"summary":"","facts":[]}`, `non-empty`},
|
||||
} {
|
||||
w := tools.Writes().Facts
|
||||
r := callGuardTool(t, tools.recordFact(), tc.call)
|
||||
if !r.IsError || !strings.Contains(r.Flatten(), tc.want) || tools.Writes().Facts != w {
|
||||
t.Errorf("%s: want refusal mentioning %s, got %s (writes %d→%d)", tc.call, tc.want, r.Flatten(), w, tools.Writes().Facts)
|
||||
}
|
||||
}
|
||||
if n, err := store.ListByKind(db.KindFact, 100); err != nil || len(n) != len(before) {
|
||||
t.Fatalf("refused calls wrote facts: %d → %d %v", len(before), len(n), err)
|
||||
}
|
||||
}
|
||||
|
||||
// The published schema holds facts[] and nothing else, and the description
|
||||
// names that one shape; the earlier "two shapes" wording is gone.
|
||||
func TestRecordFactSchemaIsFactsOnly(t *testing.T) {
|
||||
schema := recordFactSchema()
|
||||
props := schema["properties"].(map[string]any)
|
||||
if len(props) != 1 || props["facts"] == nil {
|
||||
t.Fatalf("top-level properties: %v", props)
|
||||
}
|
||||
if req, _ := schema["required"].([]string); len(req) != 1 || req[0] != "facts" || schema["additionalProperties"] != false {
|
||||
t.Fatalf("facts must be required and the only property: %v %v", schema["required"], schema["additionalProperties"])
|
||||
}
|
||||
facts := props["facts"].(map[string]any)
|
||||
if d, _ := facts["description"].(string); !strings.Contains(d, "never send summary") {
|
||||
t.Errorf("facts description must forbid top-level copies: %q", d)
|
||||
}
|
||||
item := facts["items"].(map[string]any)["properties"].(map[string]any)
|
||||
for _, name := range []string{"summary", "intent_id", "detail", "evidence", "confidence", "source", "asset_ids", "tested_asset_ids"} {
|
||||
if item[name] == nil {
|
||||
t.Errorf("facts[] item lacks %s", name)
|
||||
}
|
||||
}
|
||||
for name, d := range map[string]string{
|
||||
"summary": item["summary"].(map[string]any)["description"].(string),
|
||||
"intent_id": item["intent_id"].(map[string]any)["description"].(string),
|
||||
} {
|
||||
if strings.Contains(d, "top level") || strings.Contains(d, "top-level") {
|
||||
t.Errorf("facts[].%s still refers to a top level: %q", name, d)
|
||||
}
|
||||
}
|
||||
desc := NewToolSet(nil, "").recordFact().Description()
|
||||
if !strings.HasSuffix(desc, recordFactContractNote) || strings.Contains(desc, "top-level intent_id") {
|
||||
t.Fatalf("record_fact description must describe the facts-only contract: %q", desc)
|
||||
}
|
||||
// The seed the catalog stores is the same contract.
|
||||
for _, seed := range BuiltinToolSeeds() {
|
||||
if seed.Key != "record_fact" {
|
||||
continue
|
||||
}
|
||||
if seed.Desc != desc {
|
||||
t.Fatal("seed description differs from the tool")
|
||||
}
|
||||
if p := seed.Schema["properties"].(map[string]any); len(p) != 1 || p["facts"] == nil {
|
||||
t.Fatalf("seed schema is not facts-only: %v", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A facts[] item without intent_id is linked to the intent the worker runs
|
||||
// (SetOwnerNode); a legacy top-level intent_id still wins; a planner ToolSet
|
||||
// (owner = the task root, not an intent) writes an unlinked fact.
|
||||
func TestRecordFactDefaultsToRunningIntent(t *testing.T) {
|
||||
store := guardTaskStore(t)
|
||||
running := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "running intent"}, "running")
|
||||
other := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "other intent"}, "open")
|
||||
tools := NewToolSet(store, "worker")
|
||||
tools.SetOwnerNode(running)
|
||||
ids := func(res actool.Result) []int64 {
|
||||
t.Helper()
|
||||
var out struct {
|
||||
IDs []int64 `json:"ids"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(res.Flatten()), &out); err != nil || len(out.IDs) != 1 || out.IDs[0] <= 0 {
|
||||
t.Fatalf("batch result: %s", res.Flatten())
|
||||
}
|
||||
return out.IDs
|
||||
}
|
||||
res := callGuardTool(t, tools.recordFact(), `{"facts":[{"summary":"linked to the running intent","confidence":"observed","source":"tool_output"}]}`)
|
||||
if res.IsError {
|
||||
t.Fatal(res.Flatten())
|
||||
}
|
||||
first := ids(res)[0]
|
||||
if yielded, err := store.FactsYielded(running); err != nil || len(yielded) != 1 || yielded[0] != first {
|
||||
t.Fatalf("fact %d not linked to the running intent %d: %v %v", first, running, yielded, err)
|
||||
}
|
||||
res = callGuardTool(t, tools.recordFact(), `{"intent_id":`+strconv.FormatInt(other, 10)+`,"facts":[{"summary":"legacy top-level intent"}]}`)
|
||||
if res.IsError {
|
||||
t.Fatal(res.Flatten())
|
||||
}
|
||||
second := ids(res)[0]
|
||||
if yielded, err := store.FactsYielded(other); err != nil || len(yielded) != 1 || yielded[0] != second {
|
||||
t.Fatalf("legacy top-level intent_id must win: %v %v", yielded, err)
|
||||
}
|
||||
res = callGuardTool(t, tools.recordFact(), `{"facts":[{"summary":"item intent","intent_id":`+strconv.FormatInt(other, 10)+`}]}`)
|
||||
if res.IsError {
|
||||
t.Fatal(res.Flatten())
|
||||
}
|
||||
third := ids(res)[0]
|
||||
if yielded, _ := store.FactsYielded(other); len(yielded) != 2 || yielded[1] != third && yielded[0] != third {
|
||||
t.Fatalf("item intent_id must win: %v", yielded)
|
||||
}
|
||||
if yielded, _ := store.FactsYielded(running); len(yielded) != 1 {
|
||||
t.Fatalf("running intent gained facts it did not produce: %v", yielded)
|
||||
}
|
||||
planner := NewToolSet(store, "planner")
|
||||
origin, err := store.ListByKind(db.KindFact, 1)
|
||||
if err != nil || len(origin) == 0 {
|
||||
t.Fatalf("origin fact: %v", err)
|
||||
}
|
||||
planner.SetOwnerNode(origin[0].ID)
|
||||
res = callGuardTool(t, planner.recordFact(), `{"facts":[{"summary":"planner fact without intent"}]}`)
|
||||
if res.IsError || planner.Writes().Facts != 1 {
|
||||
t.Fatalf("planner write refused: %s", res.Flatten())
|
||||
}
|
||||
if yielded, _ := store.FactsYielded(running); len(yielded) != 1 {
|
||||
t.Fatalf("planner fact must not be linked to a worker intent: %v", yielded)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,7 +44,10 @@ func TestReportFindingStoresReasoningFields(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.DeleteTask(task.ID)
|
||||
defer func() {
|
||||
_, _ = d.DeleteFindingsByTask(task.ID)
|
||||
_ = d.DeleteTask(task.ID)
|
||||
}()
|
||||
ts := NewToolSet(nil, "")
|
||||
ts.ts, ts.taskID = d.Exploration(task.ExplorationID), task.ID
|
||||
for _, tc := range []struct{ extra, counter, rationale string }{
|
||||
|
||||
@@ -25,7 +25,10 @@ func TestReportFindingOptionalTrafficWithoutCapture(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.DeleteTask(task.ID)
|
||||
defer func() {
|
||||
_, _ = d.DeleteFindingsByTask(task.ID)
|
||||
_ = d.DeleteTask(task.ID)
|
||||
}()
|
||||
ts := NewToolSet(nil, "")
|
||||
ts.ts, ts.taskID = d.Exploration(task.ExplorationID), task.ID
|
||||
ts.SetFindingRecorder(evidence.New(d, nil, t.TempDir()))
|
||||
@@ -60,7 +63,10 @@ func TestReportFindingAtomicContract(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.DeleteTask(task.ID)
|
||||
defer func() {
|
||||
_, _ = d.DeleteFindingsByTask(task.ID)
|
||||
_ = d.DeleteTask(task.ID)
|
||||
}()
|
||||
ts := NewToolSet(nil, "")
|
||||
ts.ts = d.Exploration(task.ExplorationID)
|
||||
ts.taskID = task.ID
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// Synthetic replay of the record_fact call shapes a model produces, replayed
|
||||
// against the handler in the package test database. The shapes mirror what
|
||||
// was observed in production before the facts-only contract (a model filling
|
||||
// every declared top-level property beside facts[]) without using recorded
|
||||
// data. Checked per call: the verdict, that an accepted call writes exactly one
|
||||
// node per facts item linked to the intent and anchored to the assets, and
|
||||
// that a refused call writes nothing and names the top-level fields.
|
||||
func TestRecordFactShapesReplay(t *testing.T) {
|
||||
d := testDB(t)
|
||||
t.Cleanup(func() { _ = d.Close() })
|
||||
task, err := d.CreateTaskWithOptions("record_fact shapes", "replay synthetic record_fact call shapes", db.TaskCreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = d.DeleteTask(task.ID) })
|
||||
store := d.Exploration(task.ExplorationID)
|
||||
intentID := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "enumerate the service"}, "running")
|
||||
var assets []int64
|
||||
for i := 1; i <= 2; i++ {
|
||||
id, err := d.Assets().UpsertIP(db.UpsertIPReq{IP: fmt.Sprintf("203.0.113.%d", i), TaskID: task.ID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assets = append(assets, id)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = d.Assets().DeleteByIDs(assets) })
|
||||
tools := NewToolSet(store, "worker")
|
||||
tools.SetOwnerNode(intentID)
|
||||
tool := tools.recordFact()
|
||||
|
||||
iid := strconv.FormatInt(intentID, 10)
|
||||
a1, a2 := strconv.FormatInt(assets[0], 10), strconv.FormatInt(assets[1], 10)
|
||||
item := func(n int) string {
|
||||
return fmt.Sprintf(`{"summary":"conclusion %d","detail":"detail %d","evidence":"cmd %d → line","confidence":"observed","source":"tool_output","asset_ids":[%s]}`, n, n, n, a1)
|
||||
}
|
||||
type shape struct {
|
||||
name string
|
||||
call string
|
||||
accept bool
|
||||
items int
|
||||
fields string // named in the refusal
|
||||
}
|
||||
shapes := []shape{
|
||||
{"published: one item", `{"facts":[` + item(1) + `]}`, true, 1, ""},
|
||||
{"published: two items", `{"facts":[` + item(1) + `,` + item(2) + `]}`, true, 2, ""},
|
||||
{"published: item with its own intent_id and alias", `{"facts":[{"summary":"alias","tested_asset_ids":["` + a2 + `"],"intent_id":` + iid + `}]}`, true, 1, ""},
|
||||
{"legacy batch: top-level intent_id", `{"intent_id":` + iid + `,"facts":[` + item(3) + `]}`, true, 1, ""},
|
||||
{"legacy single", `{"summary":"single fact","confidence":"inferred","asset_ids":[` + a2 + `]}`, true, 1, ""},
|
||||
{"copies of every field beside facts", `{"summary":"conclusion 1","detail":"detail 1","evidence":"cmd 1 → line","confidence":"observed","source":"tool_output","asset_ids":[` + a1 + `],"facts":[` + item(1) + `]}`, false, 1, `"asset_ids", "confidence", "detail", "evidence", "source", "summary"`},
|
||||
{"empty placeholders beside facts", `{"summary":"","detail":"","asset_ids":[],"tested_asset_ids":[],"facts":[` + item(1) + `]}`, false, 1, `"asset_ids", "detail", "summary", "tested_asset_ids"`},
|
||||
{"dummy summary beside facts", `{"summary":"x","facts":[` + item(1) + `]}`, false, 1, `"summary"`},
|
||||
{"abbreviated copy beside facts", `{"summary":"short","detail":"short detail","facts":[` + item(1) + `]}`, false, 1, `"detail", "summary"`},
|
||||
{"conflicting source beside facts", `{"source":"inference","facts":[` + item(1) + `]}`, false, 1, `"source"`},
|
||||
{"top-level summary beside two items", `{"summary":"conclusion 1","facts":[` + item(1) + `,` + item(2) + `]}`, false, 2, `"summary"`},
|
||||
}
|
||||
for _, s := range shapes {
|
||||
before := tools.Writes().Facts
|
||||
linked, _ := store.FactsYielded(intentID)
|
||||
res := callGuardTool(t, tool, s.call)
|
||||
written := tools.Writes().Facts - before
|
||||
if !res.IsError != s.accept {
|
||||
t.Errorf("%s: accepted=%v want %v: %s", s.name, !res.IsError, s.accept, res.Flatten())
|
||||
continue
|
||||
}
|
||||
if !s.accept {
|
||||
if written != 0 || !strings.Contains(res.Flatten(), "top-level field(s) "+s.fields) || !strings.Contains(res.Flatten(), "no facts were written") {
|
||||
t.Errorf("%s: refusal must write nothing and name %s: %s (writes %d)", s.name, s.fields, res.Flatten(), written)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if written != s.items {
|
||||
t.Errorf("%s: wrote %d facts for %d items", s.name, written, s.items)
|
||||
}
|
||||
after, _ := store.FactsYielded(intentID)
|
||||
if len(after) != len(linked)+s.items {
|
||||
t.Errorf("%s: intent links %d → %d for %d items", s.name, len(linked), len(after), s.items)
|
||||
}
|
||||
ids := after[len(linked):]
|
||||
if strings.HasPrefix(s.call, `{"facts"`) || strings.HasPrefix(s.call, `{"intent_id"`) {
|
||||
var out struct {
|
||||
IDs []int64 `json:"ids"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(res.Flatten()), &out); err != nil || len(out.IDs) != s.items {
|
||||
t.Errorf("%s: ids: %s", s.name, res.Flatten())
|
||||
}
|
||||
}
|
||||
for _, id := range ids {
|
||||
node, err := store.GetNode(id)
|
||||
if err != nil || node == nil || node.Kind != db.KindFact {
|
||||
t.Fatalf("%s: fact %d: %v", s.name, id, err)
|
||||
}
|
||||
var payload map[string]any
|
||||
_ = json.Unmarshal(node.Payload, &payload)
|
||||
if sum, _ := payload["summary"].(string); sum == "" || sum == "x" {
|
||||
t.Errorf("%s: stored summary %q", s.name, sum)
|
||||
}
|
||||
if strings.Contains(s.call, "asset_ids") {
|
||||
anchored := false
|
||||
for _, asset := range assets {
|
||||
refs, err := store.AssetRefs(asset)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range refs {
|
||||
if r.ID == id {
|
||||
anchored = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !anchored {
|
||||
t.Errorf("%s: fact %d not anchored to an asset", s.name, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,7 @@ func TestIngestScanOutputPG(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const assetsWhere = `ip IN ('198.51.100.77','2001:db8::77') OR domain LIKE '%scan-ingest.example.com' OR url LIKE '%198.51.100.77%' OR url LIKE '%2001:db8::77%'`
|
||||
const assetsWhere = `ip IN ('198.51.100.77','2001:db8::77') OR domain IN ('198.51.100.77','2001:db8::77','example.com') OR domain LIKE '%scan-ingest.example.com' OR url LIKE '%198.51.100.77%' OR url LIKE '%2001:db8::77%'`
|
||||
t.Cleanup(func() {
|
||||
_ = d.DeleteTask(task.ID)
|
||||
_, _ = d.Exec(`DELETE FROM assets WHERE ` + assetsWhere)
|
||||
|
||||
@@ -180,6 +180,14 @@ type overriddenTool struct {
|
||||
func (o *overriddenTool) Description() string { return o.desc }
|
||||
func (o *overriddenTool) InputSchema() map[string]any { return o.schema }
|
||||
|
||||
// AcceptsRawInput forwards the wrapped tool's opt-out of the harness schema
|
||||
// check (record_fact validates its own input); the embedded interface alone
|
||||
// would hide it from the harness's type assertion.
|
||||
func (o *overriddenTool) AcceptsRawInput() bool {
|
||||
r, ok := o.CoreTool.(interface{ AcceptsRawInput() bool })
|
||||
return ok && r.AcceptsRawInput()
|
||||
}
|
||||
|
||||
func (o *overriddenTool) Call(ctx context.Context, in json.RawMessage, tc *actool.ToolContext) (actool.Result, error) {
|
||||
return o.CoreTool.Call(ctx, injectDefaults(in, o.schema), tc)
|
||||
}
|
||||
|
||||
+44
-5
@@ -394,8 +394,21 @@ func readTool(name, desc string, schema map[string]any, run func(context.Context
|
||||
}
|
||||
|
||||
func writeTool(name, desc string, schema map[string]any, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool {
|
||||
return buildWriteTool(name, desc, schema, false, run)
|
||||
}
|
||||
|
||||
// writeRawTool is writeTool for a handler that validates its own input: the
|
||||
// harness skips its schema check (Spec.RawInput), so the published schema can
|
||||
// be stricter (required properties) than the legacy shapes the handler still
|
||||
// accepts, and refusals carry the handler's own message instead of the
|
||||
// SDK's generic one.
|
||||
func writeRawTool(name, desc string, schema map[string]any, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool {
|
||||
return buildWriteTool(name, desc, schema, true, run)
|
||||
}
|
||||
|
||||
func buildWriteTool(name, desc string, schema map[string]any, raw bool, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool {
|
||||
return actool.Build(actool.Spec{
|
||||
Name: name, Description: desc, Schema: schema,
|
||||
Name: name, Description: desc, Schema: schema, RawInput: raw,
|
||||
Permissions: func(context.Context, json.RawMessage, acperm.Context) acperm.Decision { return acperm.Allowed() },
|
||||
Run: func(ctx context.Context, in json.RawMessage, _ *actool.ToolContext) (actool.Result, error) {
|
||||
return run(ctx, in)
|
||||
@@ -419,6 +432,12 @@ func (t *ToolSet) writeExpTool(name, desc string, schema map[string]any, run fun
|
||||
return writeTool(name, desc, schema, t.needExploration(name, run))
|
||||
}
|
||||
|
||||
// writeExpToolRaw is writeExpTool with the harness schema check skipped (see
|
||||
// writeRawTool); the handler validates the input itself.
|
||||
func (t *ToolSet) writeExpToolRaw(name, desc string, schema map[string]any, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool {
|
||||
return writeRawTool(name, desc, schema, t.needExploration(name, run))
|
||||
}
|
||||
|
||||
// needExploration wraps a handler so it only runs with an exploration store.
|
||||
// Tools that degrade more usefully than "unavailable" (report_finding points at
|
||||
// add_task_hint, set_goals/set_constraints at the task itself) keep their own
|
||||
@@ -1588,15 +1607,30 @@ func (t *ToolSet) recordOneFact(it factItem, defaultIntent int64) (int64, error)
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// runningIntent is the intent this ToolSet works for (the worker's claimed
|
||||
// intent, set by SetOwnerNode), or 0 when the owner node is not an intent
|
||||
// (planner: the task root) or unknown.
|
||||
func (t *ToolSet) runningIntent() int64 {
|
||||
if t.ts == nil || t.ownerNode <= 0 {
|
||||
return 0
|
||||
}
|
||||
node, err := t.ts.GetNode(t.ownerNode)
|
||||
if err != nil || node == nil || node.Kind != db.KindIntent {
|
||||
return 0
|
||||
}
|
||||
return node.ID
|
||||
}
|
||||
|
||||
func (t *ToolSet) recordFact() actool.CoreTool {
|
||||
return t.writeExpTool("record_fact", "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n"+
|
||||
return t.writeExpToolRaw("record_fact", "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n"+
|
||||
"⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n"+
|
||||
"★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n"+
|
||||
"★facts is the only parameter: facts:[{...}] with one item per [mutually distinct] conclusion, usually exactly one item. Returns an ids array, same length and order as facts. An item without intent_id is linked to the intent you are running.\n"+
|
||||
"⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n"+
|
||||
" · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n"+
|
||||
" · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n"+
|
||||
" · source = where the content came from: target_response (content the target itself returned: page bodies, headers, error messages, banners, even when a tool fetched it) | tool_output (a tool's own result about the target: port states, scanner verdicts, status codes, timings) | third_party_intel (OSINT, CVE databases, public reports) | operator_input (given by the human operator) | inference (your own reasoning; the default when omitted). Any other value is refused.\n"+
|
||||
" · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen.",
|
||||
" · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen."+
|
||||
recordFactContractNote,
|
||||
recordFactSchema(),
|
||||
func(_ context.Context, in json.RawMessage) (actool.Result, error) {
|
||||
a, batch, err := decodeRecordFactInput(in)
|
||||
@@ -1607,7 +1641,12 @@ func (t *ToolSet) recordFact() actool.CoreTool {
|
||||
if !batch {
|
||||
items = []factItem{a.factItem}
|
||||
}
|
||||
defaultIntent := pid(a.factItem.IntentID) // top-level intent_id = batch default
|
||||
// Default intent of the items: a legacy top-level intent_id, else the
|
||||
// intent this worker is running (the published schema has no top level).
|
||||
defaultIntent := pid(a.factItem.IntentID)
|
||||
if defaultIntent <= 0 {
|
||||
defaultIntent = t.runningIntent()
|
||||
}
|
||||
|
||||
ids := make([]int64, len(items))
|
||||
errs := map[string]string{}
|
||||
|
||||
@@ -118,7 +118,7 @@ else
|
||||
command -v rsync >/dev/null 2>&1 || die "rsync not detected"
|
||||
info "Building frontend static assets"
|
||||
if [ "${ARTEX_SKIP_NPM_CI:-0}" != "1" ]; then
|
||||
(cd web && npm ci)
|
||||
(cd web && npm ci --ignore-scripts)
|
||||
fi
|
||||
(cd web && npm run build:static)
|
||||
info "Syncing frontend assets to server/webui/dist"
|
||||
|
||||
@@ -1143,6 +1143,7 @@ func TestAssetIPRejectsHostname(t *testing.T) {
|
||||
t.Fatalf("empty ip rejected: %v", err)
|
||||
}
|
||||
deleteAsset(d, id)
|
||||
_, _ = d.Exec(`DELETE FROM assets WHERE domain IN ('ip-guard.example.com','example.com')`)
|
||||
id, err = av2.UpsertIP(UpsertIPReq{IP: "203.0.113.7"})
|
||||
if err != nil {
|
||||
t.Fatalf("valid ip rejected: %v", err)
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestDashboardUsageIndexesMigration0010(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
SkipPGUnavailable(t, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = d.Close() })
|
||||
|
||||
wantIndexes := []struct {
|
||||
name string
|
||||
table string
|
||||
pred string
|
||||
}{
|
||||
{"idx_act_result_created", "activity", "kind = 'result'"},
|
||||
{"idx_conv_act_result", "conversation_activities", "kind = 'result'"},
|
||||
{"idx_llm_usage_judge_ts", "llm_usage", "worker = 'judge'"},
|
||||
}
|
||||
|
||||
for _, want := range wantIndexes {
|
||||
var valid, ready bool
|
||||
var def string
|
||||
err := d.QueryRow(`
|
||||
SELECT i.indisvalid, i.indisready, pg_get_indexdef(i.indexrelid)
|
||||
FROM pg_index i
|
||||
JOIN pg_class c ON c.oid = i.indexrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = 'public' AND c.relname = $1`, want.name).Scan(&valid, &ready, &def)
|
||||
if err != nil {
|
||||
t.Fatalf("index public.%s missing after Open: %v", want.name, err)
|
||||
}
|
||||
if !valid || !ready {
|
||||
t.Fatalf("index %s indisvalid=%v indisready=%v, want both true (%s)", want.name, valid, ready, def)
|
||||
}
|
||||
if !strings.Contains(def, want.table) || !strings.Contains(def, want.pred) {
|
||||
t.Fatalf("index %s definition %q missing table %q or predicate %q", want.name, def, want.table, want.pred)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify sargable TokenByModel matches both specific task_id and empty task_id.
|
||||
taskID := fmt.Sprintf("idx-test-%d", time.Now().UnixNano())
|
||||
t.Cleanup(func() {
|
||||
_, _ = d.Exec(`DELETE FROM llm_usage WHERE task_id = $1 OR profile_name = $2`, taskID, taskID)
|
||||
})
|
||||
if err := d.InsertLLMUsage(&LLMUsage{
|
||||
TaskID: taskID,
|
||||
Worker: "worker",
|
||||
Model: "claude-sonnet-4",
|
||||
ProfileName: taskID,
|
||||
InputTokens: 1000,
|
||||
OutputTokens: 200,
|
||||
Status: "ok",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.InsertLLMUsage(&LLMUsage{
|
||||
TaskID: "",
|
||||
Worker: "chat",
|
||||
Model: "gpt-5",
|
||||
ProfileName: taskID,
|
||||
InputTokens: 500,
|
||||
OutputTokens: 100,
|
||||
Status: "ok",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := d.TokenByModel(taskID)
|
||||
if err != nil {
|
||||
t.Fatalf("TokenByModel(%q): %v", taskID, err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Model != "claude-sonnet-4" || got[0].InputTokens != 1000 || got[0].OutputTokens != 200 {
|
||||
t.Fatalf("TokenByModel(%q) = %+v, want 1 claude-sonnet-4 row", taskID, got)
|
||||
}
|
||||
|
||||
emptyGot, err := d.TokenByModel("")
|
||||
if err != nil {
|
||||
t.Fatalf("TokenByModel(\"\"): %v", err)
|
||||
}
|
||||
foundChat := false
|
||||
for _, m := range emptyGot {
|
||||
if m.Model == "gpt-5" && m.InputTokens >= 500 {
|
||||
foundChat = true
|
||||
}
|
||||
}
|
||||
if !foundChat {
|
||||
t.Fatalf("TokenByModel(\"\") did not include NULL/empty task_id row: %+v", emptyGot)
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,7 @@ func TestFindingQualityByNodeID(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.DeleteFinding(rec.FindingID)
|
||||
if err := d.SetFindingVerification(rec.FindingID, VerificationVerified, 3, "reproduced twice on staging"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -24,10 +24,12 @@ func TestFindingReasoningFieldsRoundTrip(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.DeleteFinding(with.FindingID)
|
||||
without, err := es.RecordFinding(t.Context(), RecordFindingInput{TaskID: tk.ID, VulnClass: "IDOR", Severity: "low", Summary: "without reasoning", Worker: "test"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.DeleteFinding(without.FindingID)
|
||||
want := map[int64][2]string{
|
||||
with.FindingID: {"UUID guessing returned 404", "two accounts, same response body"},
|
||||
without.FindingID: {"", ""},
|
||||
|
||||
@@ -32,7 +32,11 @@ func TestFindingTrafficLegacyArchiveDefaults(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { d.Exec(`DELETE FROM task_archives WHERE task_id=$1`, task.ID); d.DeleteTask(task.ID) }()
|
||||
defer func() {
|
||||
_, _ = d.DeleteFindingsByTask(task.ID)
|
||||
_, _ = d.Exec(`DELETE FROM task_archives WHERE task_id=$1`, task.ID)
|
||||
_ = d.DeleteTask(task.ID)
|
||||
}()
|
||||
f, err := d.Exploration(task.ExplorationID).RecordFinding(t.Context(), RecordFindingInput{TaskID: task.ID, ExplorationID: task.ExplorationID, Summary: "legacy", Severity: "low"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
+9
-3
@@ -100,6 +100,7 @@ func TestSetFindingStatusMirrorsNodePayload(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.DeleteFinding(fid)
|
||||
|
||||
n, err := d.SetFindingStatus(fid, FindingFalsePositive)
|
||||
if err != nil || n != 1 {
|
||||
@@ -146,13 +147,13 @@ func TestFindingReportState(t *testing.T) {
|
||||
if err != nil {
|
||||
SkipPGUnavailable(t, err)
|
||||
}
|
||||
defer d.Close()
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
tk, err := d.CreateTask("finding report state", "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.DeleteTask(tk.ID)
|
||||
t.Cleanup(func() { _ = d.DeleteTask(tk.ID) })
|
||||
|
||||
es := d.Exploration(tk.ExplorationID)
|
||||
nodeID, err := es.AddNode(KindFinding, map[string]any{"summary": "x", "severity": "high"}, 5, "confirmed", "worker", nil)
|
||||
@@ -167,7 +168,11 @@ func TestFindingReportState(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { d.Exec(`DELETE FROM notification_events WHERE finding_id IN ($1,$2)`, fid, second) })
|
||||
t.Cleanup(func() {
|
||||
_, _ = d.Exec(`DELETE FROM notification_events WHERE finding_id IN ($1,$2)`, fid, second)
|
||||
_, _ = d.DeleteFinding(fid)
|
||||
_, _ = d.DeleteFinding(second)
|
||||
})
|
||||
|
||||
one, err := d.GetFinding(fid)
|
||||
if err != nil || one == nil || one.ReportState != FindingReportNone {
|
||||
@@ -633,6 +638,7 @@ func TestAddFindingFollowUpIntent(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.DeleteFinding(findingID)
|
||||
|
||||
auditInput := Activity{Worker: "system", Kind: "text", Summary: "Manually submitted vulnerability deep-exploitation intent", Detail: "Verify exploitability and build an evidence chain"}
|
||||
intentID, audit, err := store.AddFindingFollowUpIntent(findingID, findingNodeID, "Verify exploitability and build an evidence chain", auditInput)
|
||||
|
||||
+12
-2
@@ -191,11 +191,14 @@ func TestExtractLessonsScopesSingleCompanyFromTaskScope(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "extract task", d.DeleteTask(task.ID)) })
|
||||
if _, err := d.Exec(`
|
||||
var extractedFID int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,'xss','Reflected XSS','reflected in search','high','confirmed')`, task.ID); err != nil {
|
||||
VALUES ($1,'xss','Reflected XSS','reflected in search','high','confirmed')
|
||||
RETURNING id`, task.ID).Scan(&extractedFID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, extractedFID) })
|
||||
n, err := d.ExtractLessonsForTask(t.Context(), task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -477,6 +480,7 @@ func TestExtractLessonsRetractsReversedDisposition(t *testing.T) {
|
||||
RETURNING id`, task.ID, cue, label).Scan(&fid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
|
||||
n, err := d.ExtractLessonsForTask(t.Context(), task.ID)
|
||||
if err != nil {
|
||||
@@ -589,12 +593,14 @@ func TestExtractLessonsRetireOnlyWhenLastEvidence(t *testing.T) {
|
||||
RETURNING id`, task.ID, cue, label).Scan(&aID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, aID) })
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS',$3,'high','confirmed')
|
||||
RETURNING id`, task.ID, cue, label).Scan(&bID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, bID) })
|
||||
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -681,6 +687,7 @@ func TestExtractLessonsRetractsAllRetests(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
|
||||
r1 := insertLessonRetest(t, d, fid, "reproduced")
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
@@ -752,6 +759,7 @@ func TestExtractLessonsDoesNotUnretireWithEvidence(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -798,6 +806,7 @@ func TestExtractLessonsLeavesAdopted(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -841,6 +850,7 @@ func TestExtractLessonsReopenRetractsNothing(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
+21
-2
@@ -1,6 +1,7 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
@@ -71,14 +72,32 @@ VALUES (NULLIF($1,''),$2,NULLIF($3,''),NULLIF($4,''),NULLIF($5,''),$6,$7,$8,$9,$
|
||||
// Accurate even with per-agent model bindings, pool rotation/failover, and
|
||||
// interrupted runs, since every call (success or error) is metered.
|
||||
func (d *DB) TokenByModel(taskID string) ([]ModelTokenStat, error) {
|
||||
rows, err := d.Query(`
|
||||
// Keep the predicate sargable for non-empty task IDs so PostgreSQL can use
|
||||
// idx_llm_usage_model(task_id, model) instead of sequentially scanning the
|
||||
// table via COALESCE(task_id, '').
|
||||
var (
|
||||
rows *sql.Rows
|
||||
err error
|
||||
)
|
||||
if taskID == "" {
|
||||
rows, err = d.Query(`
|
||||
SELECT COALESCE(NULLIF(model,''),'(unknown)') AS model, COUNT(*) AS calls,
|
||||
COALESCE(SUM(input_tokens),0), COALESCE(SUM(output_tokens),0),
|
||||
COALESCE(SUM(cache_read),0), COALESCE(SUM(cache_write),0)
|
||||
FROM llm_usage
|
||||
WHERE COALESCE(task_id,'') = $1
|
||||
WHERE task_id IS NULL OR task_id = ''
|
||||
GROUP BY model
|
||||
ORDER BY SUM(input_tokens) + SUM(output_tokens) DESC, model`)
|
||||
} else {
|
||||
rows, err = d.Query(`
|
||||
SELECT COALESCE(NULLIF(model,''),'(unknown)') AS model, COUNT(*) AS calls,
|
||||
COALESCE(SUM(input_tokens),0), COALESCE(SUM(output_tokens),0),
|
||||
COALESCE(SUM(cache_read),0), COALESCE(SUM(cache_write),0)
|
||||
FROM llm_usage
|
||||
WHERE task_id = $1
|
||||
GROUP BY model
|
||||
ORDER BY SUM(input_tokens) + SUM(output_tokens) DESC, model`, taskID)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -44,7 +44,7 @@ func TestCostSeparatesFreshReadAndWriteBuckets(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
t.Cleanup(func() { d.Close() })
|
||||
priced, unpriced := costProfiles(t, d)
|
||||
// Every case adds 100k output tokens: 0.1 × 4.00 = 0.40.
|
||||
for _, tc := range []struct {
|
||||
@@ -82,7 +82,7 @@ func TestUsageByProfileCostAggregatesRows(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
t.Cleanup(func() { d.Close() })
|
||||
priced, unpriced := costProfiles(t, d)
|
||||
t.Cleanup(func() { d.Exec(`DELETE FROM llm_usage WHERE profile_name IN ($1,$2)`, priced, unpriced) })
|
||||
rows := []LLMUsage{
|
||||
|
||||
+4
-3
@@ -123,7 +123,7 @@ func readApplied(t *testing.T, d *DB) []appliedMigration {
|
||||
// embedded migration is recorded with the SHA-256 of its file.
|
||||
func assertEmbeddedApplied(t *testing.T, d *DB) []appliedMigration {
|
||||
t.Helper()
|
||||
for _, table := range []string{"llm_records", "llm_usage", "llm_context"} {
|
||||
for _, table := range []string{"llm_records", "llm_usage", "llm_context", "traffic_tamper_hooks", "traffic_tamper_hook_versions"} {
|
||||
var exists bool
|
||||
if err := d.QueryRow(`SELECT to_regclass($1) IS NOT NULL`, table).Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -138,9 +138,10 @@ func assertEmbeddedApplied(t *testing.T, d *DB) []appliedMigration {
|
||||
name string
|
||||
}{{1, llmRecordsMigration}, {2, llmUsageMigration}, {3, "0003_task_scope_icp_index.sql"}, {4, "0004_llm_usage_ts_index.sql"},
|
||||
{5, "0005_traffic_rewrite_rules.sql"}, {6, "0006_activity_outbox.sql"}, {7, "0007_finding_report_state.sql"},
|
||||
{8, "0008_traffic_evidence_rewrite_flag.sql"}, {9, "0009_llm_context.sql"}}
|
||||
{8, "0008_traffic_evidence_rewrite_flag.sql"}, {9, "0009_llm_context.sql"}, {10, "0010_dashboard_usage_indexes.sql"},
|
||||
{11, "0011_traffic_tamper_hooks.sql"}}
|
||||
if len(applied) != len(want) {
|
||||
t.Fatalf("schema_migrations = %+v, want versions 1 to 9", applied)
|
||||
t.Fatalf("schema_migrations = %+v, want versions 1 to 11", applied)
|
||||
}
|
||||
for i, w := range want {
|
||||
content, err := fs.ReadFile(migrationsFS(), w.name)
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
-- Partial covering indexes proved via EXPLAIN (ANALYZE, BUFFERS) on seeded
|
||||
-- production-shaped tables (P3.6 dashboard & usage query hygiene):
|
||||
--
|
||||
-- 1. idx_act_result_created: TokenDailyAll (db/exploration.go) aggregates
|
||||
-- activity WHERE kind='result' AND created_at >= now() - interval. The
|
||||
-- existing idx_act_result_usage is keyed on exploration_id, so date-bounded
|
||||
-- global daily rollups previously fell back to a bitmap heap scan across all
|
||||
-- result rows (~100x fewer shared buffers with an index-only scan).
|
||||
-- 2. idx_conv_act_result: ConversationTokenSummaries (db/conversation.go,
|
||||
-- polled every 5s by the dashboard) left-joins conversation_activities on
|
||||
-- (conversation_id) WHERE kind='result'. Without a partial index on
|
||||
-- kind='result', every poll sequentially scans the entire
|
||||
-- conversation_activities table (~65x fewer shared buffers).
|
||||
-- 3. idx_llm_usage_judge_ts: JudgeUsageStats (db/llm_usage.go) filters
|
||||
-- llm_usage WHERE worker='judge' (all-time and recent window). Judge rows
|
||||
-- are a small fraction of the append-only ledger; a partial covering index
|
||||
-- turns both queries into index-only scans (~60-80x fewer shared buffers).
|
||||
--
|
||||
-- Plain CREATE INDEX (migrations run in a transaction at startup before the
|
||||
-- HTTP server listens). Very large installs can pre-build them online with
|
||||
-- CREATE INDEX CONCURRENTLY IF NOT EXISTS before upgrading.
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_act_result_created
|
||||
ON activity(created_at)
|
||||
INCLUDE (input_tokens, output_tokens, cache_read_tokens)
|
||||
WHERE kind = 'result';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_conv_act_result
|
||||
ON conversation_activities(conversation_id)
|
||||
INCLUDE (input_tokens, output_tokens, cache_read_tokens, cache_write_tokens)
|
||||
WHERE kind = 'result';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_llm_usage_judge_ts
|
||||
ON llm_usage(ts)
|
||||
INCLUDE (input_tokens, output_tokens, cache_read, cache_write)
|
||||
WHERE worker = 'judge';
|
||||
@@ -0,0 +1,42 @@
|
||||
-- Constrained Go tampering hooks of the capture proxy (P4.2). Global
|
||||
-- configuration: neither table is part of a task archive.
|
||||
--
|
||||
-- source (TEXT) and matchers (JSONB) on BOTH traffic_tamper_hooks and
|
||||
-- traffic_tamper_hook_versions are sealed at rest by db/secrets.go because a
|
||||
-- hook can carry a hardcoded secret (for example an HMAC signing key).
|
||||
--
|
||||
-- Execution order is priority ascending, then id ascending. A hook is always
|
||||
-- created disabled (enabled = false).
|
||||
CREATE TABLE IF NOT EXISTS traffic_tamper_hooks (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
enabled BOOLEAN NOT NULL DEFAULT false,
|
||||
phase TEXT NOT NULL CHECK (phase IN ('request', 'response')),
|
||||
priority INTEGER NOT NULL DEFAULT 0,
|
||||
matchers JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
source TEXT NOT NULL,
|
||||
source_sha256 TEXT NOT NULL,
|
||||
version INTEGER NOT NULL DEFAULT 1,
|
||||
note TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
DROP TRIGGER IF EXISTS trg_traffic_tamper_hooks_upd ON traffic_tamper_hooks;
|
||||
CREATE TRIGGER trg_traffic_tamper_hooks_upd BEFORE UPDATE ON traffic_tamper_hooks
|
||||
FOR EACH ROW EXECUTE FUNCTION set_updated_at();
|
||||
|
||||
CREATE TABLE IF NOT EXISTS traffic_tamper_hook_versions (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
hook_id BIGINT NOT NULL REFERENCES traffic_tamper_hooks(id) ON DELETE CASCADE,
|
||||
version INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
phase TEXT NOT NULL CHECK (phase IN ('request', 'response')),
|
||||
priority INTEGER NOT NULL DEFAULT 0,
|
||||
matchers JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
source TEXT NOT NULL,
|
||||
source_sha256 TEXT NOT NULL,
|
||||
note TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (hook_id, version)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_tamper_hook_versions_hook ON traffic_tamper_hook_versions(hook_id, version DESC);
|
||||
+17
-6
@@ -534,7 +534,7 @@ func TestSetFindingStatusWithNotifyOnlyEmitsOnRealChange(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.DeleteTask(tk.ID)
|
||||
t.Cleanup(func() { _ = d.DeleteTask(tk.ID) })
|
||||
es := d.Exploration(tk.ExplorationID)
|
||||
f, err := es.RecordFinding(ctx, RecordFindingInput{
|
||||
TaskID: tk.ID, Worker: "test", VulnClass: "SQL injection", Name: "status change case",
|
||||
@@ -543,7 +543,10 @@ func TestSetFindingStatusWithNotifyOnlyEmitsOnRealChange(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { d.Exec(`DELETE FROM notification_events WHERE finding_id=$1`, f.FindingID) })
|
||||
t.Cleanup(func() {
|
||||
_, _ = d.Exec(`DELETE FROM notification_events WHERE finding_id=$1`, f.FindingID)
|
||||
_, _ = d.DeleteFinding(f.FindingID)
|
||||
})
|
||||
|
||||
// Writing the finding already registered one finding_created event; count it as the baseline.
|
||||
var base int
|
||||
@@ -617,7 +620,7 @@ func TestSetFindingStatusesWithNotify(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.DeleteTask(tk.ID)
|
||||
t.Cleanup(func() { _ = d.DeleteTask(tk.ID) })
|
||||
es := d.Exploration(tk.ExplorationID)
|
||||
record := func(name string) int64 {
|
||||
fi, err := es.RecordFinding(ctx, RecordFindingInput{
|
||||
@@ -627,7 +630,10 @@ func TestSetFindingStatusesWithNotify(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { d.Exec(`DELETE FROM notification_events WHERE finding_id=$1`, fi.FindingID) })
|
||||
t.Cleanup(func() {
|
||||
_, _ = d.Exec(`DELETE FROM notification_events WHERE finding_id=$1`, fi.FindingID)
|
||||
_, _ = d.DeleteFinding(fi.FindingID)
|
||||
})
|
||||
return fi.FindingID
|
||||
}
|
||||
first, second := record("bulk one"), record("bulk two")
|
||||
@@ -891,7 +897,7 @@ func TestFinishFindingRetestEmitsStatusChange(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.DeleteTask(tk.ID)
|
||||
t.Cleanup(func() { _ = d.DeleteTask(tk.ID) })
|
||||
es := d.Exploration(tk.ExplorationID)
|
||||
f, err := es.RecordFinding(ctx, RecordFindingInput{
|
||||
TaskID: tk.ID, Worker: "test", VulnClass: "SQL injection", Name: "retest target",
|
||||
@@ -900,7 +906,10 @@ func TestFinishFindingRetestEmitsStatusChange(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { d.Exec(`DELETE FROM notification_events WHERE finding_id=$1`, f.FindingID) })
|
||||
t.Cleanup(func() {
|
||||
_, _ = d.Exec(`DELETE FROM notification_events WHERE finding_id=$1`, f.FindingID)
|
||||
_, _ = d.DeleteFinding(f.FindingID)
|
||||
})
|
||||
|
||||
// Create a retest record and drive it straight to completion.
|
||||
rt, _, _, err := d.CreateFindingRetest(ctx, f.FindingID, "review")
|
||||
@@ -910,6 +919,8 @@ func TestFinishFindingRetestEmitsStatusChange(t *testing.T) {
|
||||
if rt.ConversationID == nil {
|
||||
t.Fatal("a retest should be associated with a session")
|
||||
}
|
||||
retestConvID := *rt.ConversationID
|
||||
t.Cleanup(func() { _ = d.DeleteConversation(retestConvID) })
|
||||
// A retest must enter running before a verdict can land (consistent with the real flow).
|
||||
if ok, err := d.StartFindingRetest(ctx, rt.ID); err != nil || !ok {
|
||||
t.Fatalf("failed to start retest: ok=%v err=%v", ok, err)
|
||||
|
||||
+6
-1
@@ -90,7 +90,12 @@ var (
|
||||
// Capture-proxy rewrite rules: an action may carry a token, a matcher a header value.
|
||||
colRewriteMatchers = secretColumn{"traffic_rewrite_rules", "matchers", true, ""}
|
||||
colRewriteActions = secretColumn{"traffic_rewrite_rules", "actions", true, ""}
|
||||
secretColumns = append([]secretColumn{colLLMKey, colLLMProxy, colMCPEnv, colCredPass, colNotifConfig, colToolExec, colEvReqHead, colEvRespHead, colRewriteMatchers, colRewriteActions}, secretSettingColumns()...)
|
||||
// Capture-proxy tampering hooks and their version history: source may carry hardcoded secrets.
|
||||
colHookMatchers = secretColumn{"traffic_tamper_hooks", "matchers", true, ""}
|
||||
colHookSource = secretColumn{"traffic_tamper_hooks", "source", false, ""}
|
||||
colHookVerMatchers = secretColumn{"traffic_tamper_hook_versions", "matchers", true, ""}
|
||||
colHookVerSource = secretColumn{"traffic_tamper_hook_versions", "source", false, ""}
|
||||
secretColumns = append([]secretColumn{colLLMKey, colLLMProxy, colMCPEnv, colCredPass, colNotifConfig, colToolExec, colEvReqHead, colEvRespHead, colRewriteMatchers, colRewriteActions, colHookMatchers, colHookSource, colHookVerMatchers, colHookVerSource}, secretSettingColumns()...)
|
||||
)
|
||||
|
||||
// secretSettingKeys are the settings stored encrypted, AAD "settings.<key>".
|
||||
|
||||
@@ -154,19 +154,28 @@ func TestTaskArchiveDatabaseRoundTrip(t *testing.T) {
|
||||
VALUES($1,$2,0,'quota_exhausted','balance exhausted',$3,$4,$3)`, task.ID, llmProfileID, exhaustedAt, chainCreatedAt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var assetID, findingID int64
|
||||
defer func() {
|
||||
_, _ = d.Exec(`DELETE FROM llm_usage WHERE task_id=$1 OR exploration_id=$2`, fmt.Sprint(task.ID), task.ExplorationID)
|
||||
_, _ = d.Exec(`DELETE FROM llm_records WHERE task_id=$1`, fmt.Sprint(task.ID))
|
||||
_, _ = d.Exec(`DELETE FROM skill_usage WHERE task_id=$1 OR exploration_id=$2`, task.ID, task.ExplorationID)
|
||||
_, _ = d.Exec(`DELETE FROM tool_usage WHERE task_id=$1 OR exploration_id=$2`, task.ID, task.ExplorationID)
|
||||
_, _ = d.DeleteFindingsByTask(task.ID)
|
||||
if findingID != 0 {
|
||||
_, _ = d.DeleteFinding(findingID)
|
||||
}
|
||||
_, _ = d.Exec(`DELETE FROM task_archives WHERE task_id=$1`, task.ID)
|
||||
_ = d.DeleteTask(task.ID)
|
||||
if assetID != 0 {
|
||||
_, _ = d.Exec(`DELETE FROM assets WHERE id=$1`, assetID)
|
||||
}
|
||||
_, _ = d.Exec(`DELETE FROM llm_profiles WHERE id=$1`, llmProfileID)
|
||||
_, _ = d.Exec(`DELETE FROM companies WHERE id=$1`, companyID)
|
||||
}()
|
||||
if err := d.SetPaused(task.ID, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assetID, err := d.Assets().UpsertRootDomain(UpsertRootDomainReq{Domain: fmt.Sprintf("archive-%d.example", task.ID), TaskID: task.ID})
|
||||
assetID, err = d.Assets().UpsertRootDomain(UpsertRootDomainReq{Domain: fmt.Sprintf("archive-%d.example", task.ID), TaskID: task.ID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -197,7 +206,7 @@ VALUES($1,$2,0,'quota_exhausted','balance exhausted',$3,$4,$3)`, task.ID, llmPro
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.AddFinding(task.ID, nodeID, vulnclass, "archive finding", SeverityCritical, "summary", "evidence", "worker", []int64{assetID}); err != nil {
|
||||
if findingID, err = d.AddFinding(task.ID, nodeID, vulnclass, "archive finding", SeverityCritical, "summary", "evidence", "worker", []int64{assetID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
archive, err := d.QueueTaskArchive(task.ID)
|
||||
|
||||
@@ -168,8 +168,9 @@ func TestIntentAssetsIncludesDirectSourceProvenance(t *testing.T) {
|
||||
cleanupErr(t, "source task", d.DeleteTask(source.ID))
|
||||
})
|
||||
|
||||
intentHost := fmt.Sprintf("intent-%d.example.test", time.Now().UnixNano())
|
||||
assetID, err := d.Assets().UpsertHTTPService(UpsertHTTPServiceReq{
|
||||
URL: fmt.Sprintf("https://intent-%d.example.test", time.Now().UnixNano()), TaskID: source.ID,
|
||||
URL: "https://" + intentHost, TaskID: source.ID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -177,6 +178,9 @@ func TestIntentAssetsIncludesDirectSourceProvenance(t *testing.T) {
|
||||
t.Cleanup(func() {
|
||||
_, err := d.Assets().DeleteByIDs([]int64{assetID})
|
||||
cleanupErr(t, "asset", err)
|
||||
// UpsertHTTPService also creates the parent subdomain and root_domain rows.
|
||||
_, err = d.Exec(`DELETE FROM assets WHERE domain IN ($1, 'example.test')`, intentHost)
|
||||
cleanupErr(t, "parent assets", err)
|
||||
})
|
||||
intentID, err := d.Exploration(source.ExplorationID).AddIntent(map[string]any{"summary": "source worker"}, 5, []int64{assetID}, "planner")
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Limits on capture-proxy tampering hooks (P4.2).
|
||||
const (
|
||||
MaxTrafficTamperHooks = 50
|
||||
MaxEnabledTrafficTamperHooks = 20
|
||||
MaxTamperHookVersions = 20
|
||||
)
|
||||
|
||||
var (
|
||||
ErrTrafficTamperHookLimit = errors.New("the limit of 50 tampering hooks is reached")
|
||||
ErrTrafficTamperHookEnabledLimit = errors.New("the limit of 20 enabled tampering hooks is reached")
|
||||
)
|
||||
|
||||
// TrafficTamperHook is one row of traffic_tamper_hooks. Source and Matchers
|
||||
// are sealed at rest with db/secrets.go and decrypted here.
|
||||
type TrafficTamperHook struct {
|
||||
ID int64
|
||||
Name string
|
||||
Enabled bool
|
||||
Phase string // request | response
|
||||
Priority int
|
||||
Matchers json.RawMessage
|
||||
Source string
|
||||
SourceSHA256 string
|
||||
Version int
|
||||
Note string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
// TrafficTamperHookVersion is one immutable historical revision in
|
||||
// traffic_tamper_hook_versions. Source and Matchers are sealed at rest.
|
||||
type TrafficTamperHookVersion struct {
|
||||
ID int64
|
||||
HookID int64
|
||||
Version int
|
||||
Name string
|
||||
Phase string
|
||||
Priority int
|
||||
Matchers json.RawMessage
|
||||
Source string
|
||||
SourceSHA256 string
|
||||
Note string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
tamperHookCols = `id, name, enabled, phase, priority, matchers::text, source, source_sha256, version, note, created_at, updated_at`
|
||||
tamperHookVerCols = `id, hook_id, version, name, phase, priority, matchers::text, source, source_sha256, note, created_at`
|
||||
)
|
||||
|
||||
func hookSourceSHA256(src string) string {
|
||||
sum := sha256.Sum256([]byte(src))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func normalizeMatchersJSON(raw json.RawMessage) string {
|
||||
if len(raw) == 0 {
|
||||
return "{}"
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
func scanTamperHook(row interface{ Scan(...any) error }) (TrafficTamperHook, error) {
|
||||
var h TrafficTamperHook
|
||||
var matchers, source string
|
||||
err := row.Scan(&h.ID, &h.Name, &h.Enabled, &h.Phase, &h.Priority, &matchers, &source, &h.SourceSHA256, &h.Version, &h.Note, &h.CreatedAt, &h.UpdatedAt)
|
||||
if err == nil {
|
||||
matchers, err = openSecret(colHookMatchers, matchers)
|
||||
}
|
||||
if err == nil {
|
||||
source, err = openSecret(colHookSource, source)
|
||||
}
|
||||
h.Matchers, h.Source = json.RawMessage(matchers), source
|
||||
return h, err
|
||||
}
|
||||
|
||||
func scanTamperHookVersion(row interface{ Scan(...any) error }) (TrafficTamperHookVersion, error) {
|
||||
var v TrafficTamperHookVersion
|
||||
var matchers, source string
|
||||
err := row.Scan(&v.ID, &v.HookID, &v.Version, &v.Name, &v.Phase, &v.Priority, &matchers, &source, &v.SourceSHA256, &v.Note, &v.CreatedAt)
|
||||
if err == nil {
|
||||
matchers, err = openSecret(colHookVerMatchers, matchers)
|
||||
}
|
||||
if err == nil {
|
||||
source, err = openSecret(colHookVerSource, source)
|
||||
}
|
||||
v.Matchers, v.Source = json.RawMessage(matchers), source
|
||||
return v, err
|
||||
}
|
||||
|
||||
func sealTamperHook(h TrafficTamperHook) (matchers, source string, err error) {
|
||||
if matchers, err = sealSecret(colHookMatchers, normalizeMatchersJSON(h.Matchers)); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
source, err = sealSecret(colHookSource, h.Source)
|
||||
return matchers, source, err
|
||||
}
|
||||
|
||||
func sealTamperHookVersion(h TrafficTamperHook) (matchers, source string, err error) {
|
||||
if matchers, err = sealSecret(colHookVerMatchers, normalizeMatchersJSON(h.Matchers)); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
source, err = sealSecret(colHookVerSource, h.Source)
|
||||
return matchers, source, err
|
||||
}
|
||||
|
||||
func (d *DB) tamperHookTx(fn func(tx *sql.Tx) error) error {
|
||||
tx, err := d.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
if _, err := tx.Exec(`LOCK TABLE traffic_tamper_hooks IN SHARE ROW EXCLUSIVE MODE`); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := fn(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func tamperHookEnabledRoom(tx *sql.Tx, id int64) error {
|
||||
var n int
|
||||
if err := tx.QueryRow(`SELECT count(*) FROM traffic_tamper_hooks WHERE enabled AND id<>$1`, id).Scan(&n); err != nil {
|
||||
return err
|
||||
}
|
||||
if n >= MaxEnabledTrafficTamperHooks {
|
||||
return ErrTrafficTamperHookEnabledLimit
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func recordTamperHookVersionTx(tx *sql.Tx, h TrafficTamperHook) error {
|
||||
matchers, source, err := sealTamperHookVersion(h)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(`
|
||||
INSERT INTO traffic_tamper_hook_versions(hook_id, version, name, phase, priority, matchers, source, source_sha256, note)
|
||||
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8, $9)`,
|
||||
h.ID, h.Version, h.Name, h.Phase, h.Priority, matchers, source, h.SourceSHA256, h.Note); err != nil {
|
||||
return err
|
||||
}
|
||||
// Retain at most MaxTamperHookVersions newest revisions per hook.
|
||||
_, err = tx.Exec(`
|
||||
DELETE FROM traffic_tamper_hook_versions
|
||||
WHERE hook_id=$1 AND id NOT IN (
|
||||
SELECT id FROM traffic_tamper_hook_versions WHERE hook_id=$1 ORDER BY version DESC LIMIT $2
|
||||
)`, h.ID, MaxTamperHookVersions)
|
||||
return err
|
||||
}
|
||||
|
||||
// ListTrafficTamperHooks returns all hooks in execution order (priority ASC, id ASC).
|
||||
func (d *DB) ListTrafficTamperHooks() ([]TrafficTamperHook, error) {
|
||||
rows, err := d.Query(`SELECT ` + tamperHookCols + ` FROM traffic_tamper_hooks ORDER BY priority, id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []TrafficTamperHook
|
||||
for rows.Next() {
|
||||
h, err := scanTamperHook(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, h)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// GetTrafficTamperHook returns one hook, or sql.ErrNoRows when absent.
|
||||
func (d *DB) GetTrafficTamperHook(id int64) (TrafficTamperHook, error) {
|
||||
return scanTamperHook(d.QueryRow(`SELECT `+tamperHookCols+` FROM traffic_tamper_hooks WHERE id=$1`, id))
|
||||
}
|
||||
|
||||
// CreateTrafficTamperHook inserts a new hook (always disabled, version 1) and
|
||||
// records version 1 in traffic_tamper_hook_versions.
|
||||
func (d *DB) CreateTrafficTamperHook(h TrafficTamperHook) (out TrafficTamperHook, err error) {
|
||||
h.Enabled = false
|
||||
h.Version = 1
|
||||
h.SourceSHA256 = hookSourceSHA256(h.Source)
|
||||
matchers, source, err := sealTamperHook(h)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
err = d.tamperHookTx(func(tx *sql.Tx) error {
|
||||
var n int
|
||||
if err := tx.QueryRow(`SELECT count(*) FROM traffic_tamper_hooks`).Scan(&n); err != nil {
|
||||
return err
|
||||
}
|
||||
if n >= MaxTrafficTamperHooks {
|
||||
return ErrTrafficTamperHookLimit
|
||||
}
|
||||
out, err = scanTamperHook(tx.QueryRow(`
|
||||
INSERT INTO traffic_tamper_hooks(name, enabled, phase, priority, matchers, source, source_sha256, version, note)
|
||||
VALUES ($1, false, $2, $3, $4::jsonb, $5, $6, 1, $7)
|
||||
RETURNING `+tamperHookCols, h.Name, h.Phase, h.Priority, matchers, source, h.SourceSHA256, h.Note))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return recordTamperHookVersionTx(tx, out)
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
// UpdateTrafficTamperHook updates a hook, increments its version, and appends
|
||||
// the new revision to traffic_tamper_hook_versions.
|
||||
func (d *DB) UpdateTrafficTamperHook(id int64, edit func(stored TrafficTamperHook) (TrafficTamperHook, error)) (out TrafficTamperHook, err error) {
|
||||
err = d.tamperHookTx(func(tx *sql.Tx) error {
|
||||
stored, err := scanTamperHook(tx.QueryRow(`SELECT `+tamperHookCols+` FROM traffic_tamper_hooks WHERE id=$1`, id))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next, err := edit(stored)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if next.Enabled && !stored.Enabled {
|
||||
if err := tamperHookEnabledRoom(tx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
next.Version = stored.Version + 1
|
||||
next.SourceSHA256 = hookSourceSHA256(next.Source)
|
||||
matchers, source, err := sealTamperHook(next)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out, err = scanTamperHook(tx.QueryRow(`
|
||||
UPDATE traffic_tamper_hooks
|
||||
SET name=$2, enabled=$3, phase=$4, priority=$5, matchers=$6::jsonb, source=$7, source_sha256=$8, version=$9, note=$10
|
||||
WHERE id=$1
|
||||
RETURNING `+tamperHookCols, id, next.Name, next.Enabled, next.Phase, next.Priority, matchers, source, next.SourceSHA256, next.Version, next.Note))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return recordTamperHookVersionTx(tx, out)
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
// ToggleTrafficTamperHook sets the enabled state of a hook without creating a
|
||||
// new source version.
|
||||
func (d *DB) ToggleTrafficTamperHook(id int64, enabled bool) error {
|
||||
return d.tamperHookTx(func(tx *sql.Tx) error {
|
||||
if enabled {
|
||||
if err := tamperHookEnabledRoom(tx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return oneRow(tx.Exec(`UPDATE traffic_tamper_hooks SET enabled=$2 WHERE id=$1`, id, enabled))
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteTrafficTamperHook removes a hook and cascades its version history.
|
||||
func (d *DB) DeleteTrafficTamperHook(id int64) error {
|
||||
return oneRow(d.Exec(`DELETE FROM traffic_tamper_hooks WHERE id=$1`, id))
|
||||
}
|
||||
|
||||
// ListTrafficTamperHookVersions returns stored revisions of hookID newest-first.
|
||||
func (d *DB) ListTrafficTamperHookVersions(hookID int64) ([]TrafficTamperHookVersion, error) {
|
||||
rows, err := d.Query(`SELECT `+tamperHookVerCols+` FROM traffic_tamper_hook_versions WHERE hook_id=$1 ORDER BY version DESC`, hookID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []TrafficTamperHookVersion
|
||||
for rows.Next() {
|
||||
v, err := scanTamperHookVersion(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RollbackTrafficTamperHook restores hookID to targetVersion's code, matchers,
|
||||
// phase, priority, name, and note as a new monotonic version (preserving full
|
||||
// append-only version history and keeping the current enabled flag).
|
||||
func (d *DB) RollbackTrafficTamperHook(hookID int64, targetVersion int) (out TrafficTamperHook, err error) {
|
||||
err = d.tamperHookTx(func(tx *sql.Tx) error {
|
||||
stored, err := scanTamperHook(tx.QueryRow(`SELECT `+tamperHookCols+` FROM traffic_tamper_hooks WHERE id=$1`, hookID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rev, err := scanTamperHookVersion(tx.QueryRow(`SELECT `+tamperHookVerCols+` FROM traffic_tamper_hook_versions WHERE hook_id=$1 AND version=$2`, hookID, targetVersion))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
next := TrafficTamperHook{
|
||||
ID: hookID,
|
||||
Name: rev.Name,
|
||||
Enabled: stored.Enabled,
|
||||
Phase: rev.Phase,
|
||||
Priority: rev.Priority,
|
||||
Matchers: rev.Matchers,
|
||||
Source: rev.Source,
|
||||
SourceSHA256: hookSourceSHA256(rev.Source),
|
||||
Version: stored.Version + 1,
|
||||
Note: rev.Note,
|
||||
}
|
||||
matchers, source, err := sealTamperHook(next)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out, err = scanTamperHook(tx.QueryRow(`
|
||||
UPDATE traffic_tamper_hooks
|
||||
SET name=$2, phase=$3, priority=$4, matchers=$5::jsonb, source=$6, source_sha256=$7, version=$8, note=$9
|
||||
WHERE id=$1
|
||||
RETURNING `+tamperHookCols, hookID, next.Name, next.Phase, next.Priority, matchers, source, next.SourceSHA256, next.Version, next.Note))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return recordTamperHookVersionTx(tx, out)
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTrafficTamperHookCRUDSealedAndVersions(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
SkipPGUnavailable(t, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = d.Close() })
|
||||
|
||||
const srcV1 = `package hook
|
||||
const apiSecret = "super-secret-hmac-key-v1"
|
||||
func OnRequest(req *Request) error { req.SetHeader("X-V", "1"); return nil }`
|
||||
const srcV2 = `package hook
|
||||
const apiSecret = "super-secret-hmac-key-v2"
|
||||
func OnRequest(req *Request) error { req.SetHeader("X-V", "2"); return nil }`
|
||||
|
||||
h, err := d.CreateTrafficTamperHook(TrafficTamperHook{
|
||||
Name: "t-hook-sealed",
|
||||
Enabled: true, // Create always forces disabled
|
||||
Phase: "request",
|
||||
Priority: 5,
|
||||
Matchers: json.RawMessage(`{"host":"api.example.test"}`),
|
||||
Source: srcV1,
|
||||
Note: "v1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateTrafficTamperHook: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = d.DeleteTrafficTamperHook(h.ID) })
|
||||
|
||||
if h.Enabled || h.Version != 1 || h.Source != srcV1 {
|
||||
t.Fatalf("created hook = %+v", h)
|
||||
}
|
||||
|
||||
// Verify BOTH traffic_tamper_hooks AND traffic_tamper_hook_versions seal source & matchers at rest.
|
||||
var rawSrc, rawMatchers string
|
||||
if err := d.QueryRow(`SELECT source, matchers::text FROM traffic_tamper_hooks WHERE id=$1`, h.ID).Scan(&rawSrc, &rawMatchers); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(rawSrc, secretPrefix) || strings.Contains(rawSrc, "super-secret-hmac-key") {
|
||||
t.Fatalf("traffic_tamper_hooks.source is not sealed at rest: %q", rawSrc)
|
||||
}
|
||||
if !strings.Contains(rawMatchers, sealedJSONKey) || strings.Contains(rawMatchers, "api.example.test") {
|
||||
t.Fatalf("traffic_tamper_hooks.matchers is not sealed at rest: %q", rawMatchers)
|
||||
}
|
||||
|
||||
var verRawSrc, verRawMatchers string
|
||||
if err := d.QueryRow(`SELECT source, matchers::text FROM traffic_tamper_hook_versions WHERE hook_id=$1 AND version=1`, h.ID).Scan(&verRawSrc, &verRawMatchers); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(verRawSrc, secretPrefix) || strings.Contains(verRawSrc, "super-secret-hmac-key") {
|
||||
t.Fatalf("traffic_tamper_hook_versions.source is not sealed at rest: %q", verRawSrc)
|
||||
}
|
||||
if !strings.Contains(verRawMatchers, sealedJSONKey) || strings.Contains(verRawMatchers, "api.example.test") {
|
||||
t.Fatalf("traffic_tamper_hook_versions.matchers is not sealed at rest: %q", verRawMatchers)
|
||||
}
|
||||
|
||||
// Update -> version 2 + toggle enabled.
|
||||
v2, err := d.UpdateTrafficTamperHook(h.ID, func(stored TrafficTamperHook) (TrafficTamperHook, error) {
|
||||
stored.Source = srcV2
|
||||
stored.Note = "v2"
|
||||
return stored, nil
|
||||
})
|
||||
if err != nil || v2.Version != 2 || v2.Source != srcV2 {
|
||||
t.Fatalf("update = (%+v, %v)", v2, err)
|
||||
}
|
||||
if err := d.ToggleTrafficTamperHook(h.ID, true); err != nil {
|
||||
t.Fatalf("toggle: %v", err)
|
||||
}
|
||||
got, err := d.GetTrafficTamperHook(h.ID)
|
||||
if err != nil || !got.Enabled || got.Version != 2 {
|
||||
t.Fatalf("get after toggle = (%+v, %v)", got, err)
|
||||
}
|
||||
|
||||
// Rollback to version 1 -> creates version 3 with v1 source & keeps enabled=true.
|
||||
v3, err := d.RollbackTrafficTamperHook(h.ID, 1)
|
||||
if err != nil || v3.Version != 3 || v3.Source != srcV1 || !v3.Enabled {
|
||||
t.Fatalf("rollback = (%+v, %v)", v3, err)
|
||||
}
|
||||
vers, err := d.ListTrafficTamperHookVersions(h.ID)
|
||||
if err != nil || len(vers) != 3 || vers[0].Version != 3 || vers[2].Version != 1 {
|
||||
t.Fatalf("versions = (%+v, %v)", vers, err)
|
||||
}
|
||||
|
||||
// Delete cascades versions.
|
||||
if err := d.DeleteTrafficTamperHook(h.ID); err != nil {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
if _, err := d.GetTrafficTamperHook(h.ID); !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatalf("after delete err = %v, want sql.ErrNoRows", err)
|
||||
}
|
||||
}
|
||||
@@ -38,6 +38,8 @@ services:
|
||||
ARTEX_LLM_MODEL: ${ARTEX_LLM_MODEL:-}
|
||||
ARTEX_LLM_BASE_URL: ${ARTEX_LLM_BASE_URL:-}
|
||||
ARTEX_LLM_PROXY: ${ARTEX_LLM_PROXY:-}
|
||||
# Exact host:port list the model-list fetch may reach on private addresses (e.g. host.docker.internal:11434 for Ollama).
|
||||
ARTEX_LLM_PRIVATE_ENDPOINTS: ${ARTEX_LLM_PRIVATE_ENDPOINTS:-}
|
||||
ARTEX_JWT_KEY_DIR: /app/keys
|
||||
# Optional: key for secrets at rest; empty = keys/secret.key (generated at first start).
|
||||
ARTEX_SECRET_KEY: ${ARTEX_SECRET_KEY:-}
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# Opt-in SOCKS5 UDP ASSOCIATE & Raw UDP Recording (P4.1c)
|
||||
|
||||
ARTEX extends the opt-in SOCKS5 inbound listener with **RFC 1928 `UDP ASSOCIATE` (`0x03`)**, **IPv4 first**, enforcing per-datagram destination policy and recording forwarded datagrams separately from HTTP traffic in SQLite. Transparent UDP (`IP_RECVORIGDSTADDR`) is out of scope here and planned separately (C2).
|
||||
|
||||
## Opt-in Configuration (Off by Default)
|
||||
|
||||
| Environment Variable | Default | Behavior |
|
||||
|---|---|---|
|
||||
| `ARTEX_SOCKS5_UDP_ENABLED` | unset (off) | Set to `1` / `true` / `on` / `yes` to enable RFC 1928 `UDP ASSOCIATE` on the SOCKS5 listener (starting the SOCKS5 listener on `ARTEX_SOCKS5_ADDR` or `127.0.0.1:1080` unless `ARTEX_SOCKS5_ENABLED` is explicitly `0`/`false`/`off`/`no`). |
|
||||
| `ARTEX_SOCKS5_ENABLED` | unset (off) | Explicit `0` / `false` / `off` / `no` disables the entire SOCKS5 listener (TCP and UDP) even if `ARTEX_SOCKS5_UDP_ENABLED` is set. |
|
||||
| `ARTEX_SOCKS5_ADDR` | `127.0.0.1:1080` | Bind address for the SOCKS5 TCP control listener; per-association IPv4 UDP relay sockets bind on the same local IPv4 interface on an ephemeral port (`:0`). |
|
||||
|
||||
- **Flag off means no listener**:
|
||||
- With all SOCKS5 flags unset/off, no SOCKS5 listener and no UDP relay socket are started.
|
||||
- With SOCKS5 TCP enabled and `ARTEX_SOCKS5_UDP_ENABLED` off, `UDP ASSOCIATE` (`0x03`) is refused with RFC 1928 reply `0x07` (*Command not supported*) and no UDP socket is opened.
|
||||
- **Requires traffic recorder**: like P4.1a and P4.1b, the server only starts the SOCKS5 listener when the traffic recorder is enabled (inside the traffic proxy block in `server/manager.go`).
|
||||
- **Non-loopback warning**: the same authentication caveat from [docs/traffic-socks5.md](traffic-socks5.md) applies—RFC 1929 username/password accepts any credentials, so keep the listener bound to loopback (`127.0.0.1`) or restrict access at the host firewall.
|
||||
|
||||
## Protocol Scope & Lifetime
|
||||
|
||||
- **IPv4 first**:
|
||||
- `UDP ASSOCIATE` accepts IPv4 (`0x01`) and domain (`0x03`) requests (including `0.0.0.0:0` per RFC 1928 §4). IPv6 (`0x04`) `UDP ASSOCIATE` requests return `0x08` (*Address type not supported*); IPv6 UDP datagram destinations are dropped.
|
||||
- Each client UDP datagram header (`RSV=0x0000`, `FRAG=0x00`, `ATYP=0x01` IPv4 or `ATYP=0x03` domain resolved to IPv4) is parsed individually.
|
||||
- **No fragmentation reassembly**:
|
||||
- Per RFC 1928 §7, datagrams with `FRAG != 0` are dropped without reassembly.
|
||||
- Each forwarded `FRAG == 0` datagram (and each upstream reply datagram) is recorded as it arrives without cross-datagram reassembly.
|
||||
- **Client source pinning**:
|
||||
- Incoming client UDP datagrams on the ephemeral relay port are accepted only from the expected IPv4 client address (the `DST.ADDR`/`DST.PORT` given in `UDP ASSOCIATE`, or the TCP control connection's peer IP pinned on the first valid datagram).
|
||||
- Upstream reply datagrams are accepted only from destinations (`ip:port`) to which the association previously forwarded an allowed datagram.
|
||||
- **Tied to the TCP control connection**:
|
||||
- Closing the SOCKS5 TCP control connection immediately closes both UDP sockets and terminates the association.
|
||||
- Exceeding `UDPIdleTimeout`, `UDPMaxDatagrams`, or `UDPMaxBytes`, or calling `Traffic.Close()`, also closes the association and its TCP control connection cleanly.
|
||||
|
||||
## Policy Enforcement Per Datagram Destination
|
||||
|
||||
Before forwarding **every** client-to-server UDP datagram to `(host, port)`:
|
||||
|
||||
1. **Target gate (`intercept.Interceptor.ResolveUDP4` / `intercept.PinnedUDP4Resolver`)**: enforces global asset block rules (domain, IP, and CIDR, before and after fresh IPv4 DNS resolution) and any enabled `intercept.TargetScope` pins/exemptions, returning the verified IPv4 `*net.UDPAddr`.
|
||||
2. **Egress governor (`egress.Do(egress.Current(), ...)`)**: enforces the active noise profile (`PaceTimeout`, default `30s`) before sending.
|
||||
3. **Fail-closed drop (no silent passthrough)**: any datagram whose destination is denied by the target gate, fails DNS/IPv4 resolution, or is refused by the egress governor is **dropped** without sending bytes upstream and without recording a forwarded datagram. Subsequent allowed datagrams on the same association continue normally.
|
||||
|
||||
## Raw UDP Storage (SQLite)
|
||||
|
||||
Each forwarded datagram writes one row to the `raw_udp_datagrams` table in the SQLite traffic index (`traffic/traffic.go`, `indexSchema` — no PostgreSQL migration):
|
||||
|
||||
- Columns: `id` (`udp-<ms>-<seq>`), `association_id` (`udpa-<ms>-<seq>`), `protocol` (`'udp'`), `host`, `port`, `direction` (`'c2s'` | `'s2c'`), `ts` (unix milliseconds), `size`, `preview`, `blob`.
|
||||
- **No HTTP exchange row**: raw UDP datagrams never insert into `exchanges`.
|
||||
- **Preview & atomic blob spilling**:
|
||||
- Payloads up to `UDPPreviewBytes` (default `8 KiB`) stay inline in `preview`.
|
||||
- Payloads above `UDPPreviewBytes` keep the leading `UDPPreviewBytes` inline and spill the full datagram payload atomically (temp file + rename) into `_blobs/sha256/` with a reference registered in `blob_refs` under the datagram's `udp-*` ID.
|
||||
|
||||
## Operational Bounds
|
||||
|
||||
| Parameter | Default |
|
||||
|---|---|
|
||||
| `MaxConcurrent` | `64` active TCP streams + UDP associations combined |
|
||||
| `UDPMaxDatagrams` | `1000` forwarded datagrams (`c2s` + `s2c`) per association |
|
||||
| `UDPMaxBytes` | `4 MiB` forwarded payload bytes (`c2s` + `s2c`) per association |
|
||||
| `UDPPreviewBytes` | `8 KiB` inline preview per datagram before blob spill |
|
||||
| `UDPIdleTimeout` | `2m` inactivity timeout per association |
|
||||
| `HandshakeTimeout` | `10s` for greeting, auth, and `UDP ASSOCIATE` request |
|
||||
| `DialTimeout` | `30s` per-datagram target gate / DNS lookup bound |
|
||||
| `PaceTimeout` | `30s` per-datagram egress governor wait bound |
|
||||
@@ -0,0 +1,56 @@
|
||||
# Opt-in SOCKS5 Inbound Proxy & Raw TCP Recording (P4.1a)
|
||||
|
||||
ARTEX ships an opt-in **RFC 1928 SOCKS5 `CONNECT`** inbound listener in `traffic/` that records raw TCP connections separately from HTTP traffic.
|
||||
|
||||
## Opt-in Configuration (Off by Default)
|
||||
|
||||
| Environment Variable | Default | Behavior |
|
||||
|---|---|---|
|
||||
| `ARTEX_SOCKS5_ENABLED` | unset (off) | Set to `1` / `true` / `on` / `yes` to start the SOCKS5 listener on `127.0.0.1:1080` (loopback). Explicit `0` / `false` / `off` / `no` forces it off even if `ARTEX_SOCKS5_ADDR` is set. |
|
||||
| `ARTEX_SOCKS5_ADDR` | `127.0.0.1:1080` | Explicit TCP bind address; setting a non-empty value also enables the listener unless `ARTEX_SOCKS5_ENABLED` is explicitly false. |
|
||||
|
||||
- **Flag off**: no SOCKS5 listener is started and the HTTP MITM recording proxy is byte-identical.
|
||||
- **Requires traffic recorder**: the server only starts the SOCKS5 listener when the traffic recorder is enabled (inside the traffic proxy block in `server/manager.go`). If traffic capture is disabled, setting `ARTEX_SOCKS5_ENABLED` / `ARTEX_SOCKS5_ADDR` does not open a listener.
|
||||
- **Agents unchanged**: workers and tools keep `HTTP_PROXY`, `HTTPS_PROXY`, and `ALL_PROXY` pointing to the existing HTTP proxy; ARTEX does not silently switch agents to SOCKS5.
|
||||
- **Non-loopback bind warning**: RFC 1929 username/password (`0x02`) accepts **any** username and password without verifying credentials (so the pace marker `artex-pace` can ride in the username). Binding `ARTEX_SOCKS5_ADDR` to a non-loopback address therefore exposes an **unauthenticated proxy** to anyone who can reach that port. The target gate and egress governor still apply to every connection, but keep the listener on loopback (`127.0.0.1`) or restrict access at the host firewall.
|
||||
|
||||
## Protocol Scope (P4.1a)
|
||||
|
||||
- **Commands**: `CONNECT` (`0x01`) by default. `BIND` (`0x02`) and unknown commands return RFC 1928 reply `0x07` (*Command not supported*). `UDP ASSOCIATE` (`0x03`) also returns `0x07` unless `ARTEX_SOCKS5_UDP_ENABLED=1` is set (see [docs/traffic-socks5-udp.md](traffic-socks5-udp.md)).
|
||||
- **Addressing**: IPv4 (`0x01`), domain name (`0x03`), and IPv6 (`0x04`). Unknown address types return `0x08` (*Address type not supported*).
|
||||
- **Authentication**:
|
||||
- Loopback peers (`net.IP.IsLoopback()`) may connect with **No Auth** (`0x00`). Non-loopback peers offering only `0x00` are rejected with `0x05 0xFF`.
|
||||
- **Username/Password** (`0x02`, RFC 1929) is accepted without requiring credentials so clients can pass the pace marker (`artex-pace`, `egress.PaceUser`) in the username.
|
||||
- **Relay**: Full-duplex, half-close aware (`CloseWrite` on EOF in each direction).
|
||||
|
||||
## Policy Enforcement Before Dialing
|
||||
|
||||
Before opening any outbound TCP socket, the listener invokes the same controls used by the HTTP path:
|
||||
|
||||
1. **Egress governor (`egress.Do(egress.Current(), ...)`)**: enforces the active noise profile (`off` is a zero-delay passthrough). If waiting for a slot/token exceeds `PaceTimeout`, the connection is refused before dialing with SOCKS5 reply `0x02` (*Connection not allowed by ruleset*).
|
||||
2. **Target gate (`intercept.Interceptor.DialSOCKS5` / `intercept.PinnedDialer`)**: enforces global asset block rules (before and after fresh DNS resolution) and any enabled `intercept.TargetScope` pins/exemptions, dialing only verified addresses. Blocked targets return `0x02` before dialing; DNS lookup failures return `0x04` (*Host unreachable*).
|
||||
|
||||
## Raw TCP Storage (SQLite)
|
||||
|
||||
Each completed relayed connection writes one row to the `raw_tcp_streams` table in the SQLite traffic index (`traffic/traffic.go`, `indexSchema` — no PostgreSQL migration):
|
||||
|
||||
- Columns: `id` (`tcp-<ms>-<seq>`), `protocol` (`'tcp'`), `host`, `port`, `started_at`, `ended_at`, `duration_ms`, `c2s_bytes`, `s2c_bytes`, `truncated`, `c2s_preview`, `c2s_blob`, `s2c_preview`, `s2c_blob`.
|
||||
- **No HTTP exchange row**: raw TCP streams never insert into `exchanges`.
|
||||
- **Preview & blob spilling**:
|
||||
- Directional payloads up to `PreviewBytes` (default `8 KiB`) stay inline in `c2s_preview` / `s2c_preview`.
|
||||
- Payloads above `PreviewBytes` keep the leading `PreviewBytes` inline and spill the captured bytes (up to `MaxBytesPerDir`) atomically (temp file + rename) into `_blobs/sha256/` with references registered in `blob_refs` under the stream's `tcp-*` ID.
|
||||
- Exceeding `MaxBytesPerDir` sets `truncated = 1` while continuing to count exact total relayed `c2s_bytes` / `s2c_bytes`.
|
||||
|
||||
## Operational Bounds
|
||||
|
||||
| Parameter | Default |
|
||||
|---|---|
|
||||
| `MaxConcurrent` | `64` active streams (excess connections receive `0x01`) |
|
||||
| `MaxBytesPerDir` | `4 MiB` captured per direction |
|
||||
| `PreviewBytes` | `8 KiB` inline preview per direction |
|
||||
| `IdleTimeout` | `2m` per-direction read/write deadline |
|
||||
| `HandshakeTimeout` | `10s` for greeting, auth, and `CONNECT` request |
|
||||
| `DialTimeout` | `30s` outbound TCP connect timeout |
|
||||
| `PaceTimeout` | `30s` egress governor wait bound |
|
||||
|
||||
Closing `Traffic` stops accepting new SOCKS5 connections, closes active sockets, and waits for all handler goroutines to exit before closing SQLite.
|
||||
@@ -0,0 +1,110 @@
|
||||
# Constrained Go Tampering Hooks (P4.2 — PR1 Backend, API & Trace)
|
||||
|
||||
ARTEX supports opt-in, **operator-only** HTTP tampering hooks written in a constrained, deterministic subset of Go—with **no Python runtime** and **no external interpreter dependency**. Hooks run inside the capture proxy after declarative rewrite rules in each phase (`request` or `response`), fail open on any error, and record field-level diffs into SQLite `exchange_rewrites`.
|
||||
|
||||
## Kill Switch & Authority
|
||||
|
||||
- **Kill switch (`ARTEX_TAMPER_HOOKS_ENABLED=1`, off by default)**: when unset or false, **zero hooks are loaded** onto the capture proxy and the proxy follows the unchanged baseline path.
|
||||
- **Operator-only**: hooks are global admin configuration managed only via `/api/traffic/hooks*` and are **never exposed as an agent tool**.
|
||||
- **Created disabled**: every hook is created with `enabled: false` and must be explicitly enabled via toggle or update.
|
||||
- **Sealed at rest**: `source` and `matchers` on both `traffic_tamper_hooks` and `traffic_tamper_hook_versions` (migration `0011_traffic_tamper_hooks.sql`) are encrypted at rest with `db/secrets.go` (AES-256-GCM) because a hook may contain hardcoded signing keys or tokens.
|
||||
|
||||
## Execution Order, Failure Mode & Limits
|
||||
|
||||
- **Order**: per phase (`priority ASC, id ASC`), **declarative rewrite rules run first**, then **tampering hooks** see the post-rule message.
|
||||
- **Matchers prefilter**: hooks reuse the exact `traffic/rewrite` matchers (`methods`, `host`, `url_regex`, `headers`, `body_regex`, `status`, `resp_headers`, `resp_body_regex`). Non-matching exchanges never invoke the VM.
|
||||
- **Fail-open only (0.7)**: any returned error, panic, timeout, step-limit, memory-limit, or forbidden HTTP mutation **rolls back all mutations of that hook atomically**, passes the pre-hook message unchanged, records an `outcome='error'` trace row in `exchange_rewrites` with the reason (`panic`, `timeout`, `step_limit`, `memory_limit`, `forbidden_mutation`, `forbidden_method`, `invalid_url`, `invalid_status`, `forbidden_header`, `invalid_header`, `body_too_large`, `hook_error`), and updates in-memory `last_error` (`class`, `at`, `count`).
|
||||
- **Skipped messages**:
|
||||
- **Binary bodies are not modified in 0.7** (`not rewritten: binary`).
|
||||
- Bodies larger than **1 MiB** (`not rewritten: body too large`), compressed bodies (`Content-Encoding` other than `identity`), WebSocket upgrades, SSE response bodies, and responses ≥ 5 MiB (`StreamLargeBodies`) are skipped.
|
||||
- **Tainted fields stay masked**: if a `sensitive: true` rewrite rule wrote a field earlier in the phase, any subsequent hook trace line for that field emits only keyed HMAC fingerprints and never stores plaintext `after_inline`/`after_blob`.
|
||||
|
||||
### Resource & Storage Bounds
|
||||
|
||||
| Limit | Default |
|
||||
|---|---|
|
||||
| Per-hook wall-clock timeout | `25 ms` |
|
||||
| Per-phase cumulative hook timeout | `100 ms` |
|
||||
| CPU step budget per invocation | `100,000` steps |
|
||||
| Memory allocation budget per invocation | `4 MiB` |
|
||||
| Call depth | `32` |
|
||||
| Regexp pattern size | `1 KiB` (RE2) |
|
||||
| Source size | `64 KiB` |
|
||||
| Body size | `1 MiB` |
|
||||
| Stored / enabled hooks | `50` total / `20` enabled |
|
||||
| Retained version history | `20` newest versions per hook |
|
||||
|
||||
## Go Subset Specification
|
||||
|
||||
Hooks are parsed with Go's standard `go/parser` and must declare `package hook`. They are **not** full idiomatic Go; only the exact syntax, fields, and virtual stdlib functions below are supported.
|
||||
|
||||
### Entrypoints
|
||||
|
||||
- Request phase (`phase: "request"`):
|
||||
```go
|
||||
package hook
|
||||
func OnRequest(req *Request) error
|
||||
```
|
||||
- Response phase (`phase: "response"`):
|
||||
```go
|
||||
package hook
|
||||
func OnResponse(req *Request, resp *Response) error
|
||||
```
|
||||
In `OnResponse`, `req` holds an isolated copy of the request as sent to the target: field assignments (`req.Method`, `req.Path`, `req.RawQuery`, `req.Body`, `req.Headers`) and mutator methods (`req.SetHeader`, `req.AddHeader`, `req.DelHeader`) fail with `forbidden_mutation`, while direct map/slice element writes (such as `req.Headers[k] = ...`, `delete(req.Headers, k)`, or `req.Body[0] = ...`) mutate only the VM copy and are discarded without affecting the wire exchange or trace.
|
||||
|
||||
### Allowed Syntax
|
||||
|
||||
- Package-level `import`, `const` (basic literals / bool / unary `-`), and helper `func` definitions (no receivers, no `init()`, no package-level `var`, no `type` declarations).
|
||||
- Statements: `:=`, `=`, `+=`, `-=`, `++`, `--`, local `var`, `if` / `else if` / `else`, `for` (C-style, condition-only, infinite `for {}`, and `for k, v := range ...` with **lexicographically sorted map keys**), `break`, `continue`, `return`.
|
||||
- Expressions: literals (`int`, `float`, `string`, `rune`, `bool`, `nil`), unary (`!`, `-`, `+`), binary (`+`, `-`, `*`, `/`, `%`, `==`, `!=`, `<`, `<=`, `>`, `>=`, `&&`, `||`), indexing (`x[i]`, `m[k]`, comma-ok `v, ok := m[k]`), slicing (`x[lo:hi]`), composite literals (`[]byte{...}`, `[]any{...}`, `map[string]any{...}`), type conversions (`[]byte(s)`, `string(b)`, `int(x)`, `int64(x)`, `float64(x)`, `byte(x)`), type assertions (`v, ok := x.(T)` for `string`, `bool`, `int`/`int64`, `float64`, `[]byte`, `[]any`, `map[string]any`).
|
||||
- Collections: `[]byte`, `[]any`, `map[string]any` (plus `map[string][]string` for `Headers` and `url.ParseQuery`).
|
||||
- Allowed builtins: `len`, `cap`, `append`, `make`, `delete`, `string`, `int`, `int64`, `float64`, `byte`, `panic`.
|
||||
|
||||
### Forbidden Constructs (Rejected with HTTP 400 at Validation)
|
||||
|
||||
- Forbidden imports: `os`, `io`, `io/fs`, `net`, `net/http`, `os/exec`, `syscall`, `unsafe`, `reflect`, `time`, `math/rand`, `crypto/rand`, `fmt`, `sort`, or any package outside the closed allowlist below; dot (`.`) and blank (`_`) imports.
|
||||
- Forbidden syntax: `go` (goroutines), `chan` / `select`, `defer`, `goto`, `switch`, `func(...)` anonymous literals, `&` (address-of), `init()`, package-level `var`, custom `type` / methods.
|
||||
- Forbidden builtins: `recover`, `print`, `println`, `close`, `complex`, `real`, `imag`, `new`.
|
||||
|
||||
### `Request` and `Response` API
|
||||
|
||||
| Type | Field / Method | Access | Notes |
|
||||
|---|---|---|---|
|
||||
| `*Request` | `Scheme`, `Host`, `Port` (`string`) | **Read-only** | Attempting to assign fails atomically (`forbidden_mutation`) |
|
||||
| `*Request` | `Method`, `Path`, `RawQuery` (`string`) | R/W (`OnRequest`), R/O (`OnResponse`) | `CONNECT` rejected; `Path` must start with `/` |
|
||||
| `*Request` | `Headers` (`map[string][]string`) | R/W (`OnRequest`), R/O (`OnResponse`) | Forbidden headers (`Host`, `Content-Length`, `Transfer-Encoding`, `Connection`, `Keep-Alive`, `TE`, `Trailer`, `Expect`, `Upgrade`, `Proxy-*`, `X-ARTEX-Replay-Nonce`) and CRLF/NUL values rejected atomically |
|
||||
| `*Request` | `Body` (`[]byte`) | R/W (`OnRequest`), R/O (`OnResponse`) | Capped at 1 MiB; mutating strips `Content-Length`, `ETag`, `Content-MD5`, `Content-Encoding` |
|
||||
| `*Request` / `*Response` | `GetHeader(name) string`, `SetHeader(name, val)`, `AddHeader(name, val)`, `DelHeader(name)` | Methods | Canonicalizes header keys |
|
||||
| `*Response` | `Status` (`int`) | R/W (`OnResponse`) | Must be `200..599` excluding `204` and `304` |
|
||||
| `*Response` | `Headers` (`map[string][]string`), `Body` (`[]byte`) | R/W (`OnResponse`) | Same header & 1 MiB body invariants |
|
||||
|
||||
### Allowed Virtual Stdlib Packages & Functions
|
||||
|
||||
- `strings`: `Contains`, `HasPrefix`, `HasSuffix`, `Index`, `TrimPrefix`, `TrimSuffix`, `ToLower`, `ToUpper`, `TrimSpace`, `ReplaceAll`, `Split` (returns `[]any` of strings), `Join`, `Repeat`.
|
||||
- `bytes`: `Contains`, `HasPrefix`, `HasSuffix`, `Equal`, `Index`, `TrimSpace`, `ToLower`, `ToUpper`, `ReplaceAll`, `Repeat`.
|
||||
- `encoding/json`: `Decode(b []byte) (any, error)`, `Marshal(v any) ([]byte, error)` (sorts object keys deterministically).
|
||||
- `encoding/base64`: `EncodeToString`, `DecodeString`, `URLEncodeToString`, `URLDecodeString`.
|
||||
- `encoding/hex`: `EncodeToString`, `DecodeString`.
|
||||
- `net/url`: `QueryEscape`, `QueryUnescape`, `ParseQuery`, `EncodeQuery`.
|
||||
- `regexp`: `MatchString`, `FindString`, `ReplaceAllString` (RE2, pattern ≤ 1 KiB).
|
||||
- `crypto/sha256`: `Sum256`, `Sum256Hex`.
|
||||
- `crypto/hmac`: `SHA256(key, data)`, `SHA256Hex(key, data)`.
|
||||
- `strconv`: `Atoi`, `Itoa`, `ParseBool`, `FormatBool`.
|
||||
- `errors`: `New`.
|
||||
|
||||
## Admin API & SQLite Trace
|
||||
|
||||
All routes require the administrator token:
|
||||
|
||||
| Route | Behavior |
|
||||
|---|---|
|
||||
| `GET /api/traffic/hooks` | Lists hooks in execution order (`priority ASC, id ASC`) with `version`, `load_error`, `last_error`, and `enabled_by_env`. |
|
||||
| `POST /api/traffic/hooks` | Validates AST + matchers (400 on error) and creates a disabled hook (`version: 1`). |
|
||||
| `PUT /api/traffic/hooks/{id}` | Validates and updates the hook, incrementing `version` and appending to `traffic_tamper_hook_versions`. |
|
||||
| `DELETE /api/traffic/hooks/{id}` | Deletes hook and cascades its version history. |
|
||||
| `POST /api/traffic/hooks/{id}/toggle` | Body `{"enabled": true|false}`. |
|
||||
| `GET /api/traffic/hooks/{id}/versions` | Lists stored revisions newest-first. |
|
||||
| `POST /api/traffic/hooks/{id}/rollback` | Body `{"version": N}`; restores revision `N` as a new monotonic version. |
|
||||
| `POST /api/traffic/hooks/dry-run` | Executes a stored (`hook_id`) or inline (`phase`, `matchers`, `source`) hook against synthetic `request`/`response` without touching live traffic. |
|
||||
|
||||
In SQLite, `exchange_rewrites` adds `source_kind` (`'rule'` | `'hook'`), `hook_id`, and `hook_version`. Hook trace rows store compact metadata JSON (`{"kind":"hook","id":...,"name":...,"version":...,"sha256":...}`) in `rule_snapshot` (never full source per exchange) and preserve original client/target data in `exchanges`/`exchange_bodies`.
|
||||
@@ -0,0 +1,81 @@
|
||||
# Opt-in Linux Transparent TCP Inbound & Raw TCP Recording (P4.1b)
|
||||
|
||||
ARTEX ships an opt-in **Linux-only IPv4 transparent TCP inbound listener** in `traffic/` that accepts connections redirected by `iptables` or `nftables` `REDIRECT` rules, recovers each socket's original IPv4 destination via `getsockopt(SOL_IP, SO_ORIGINAL_DST)`, enforces the existing target gate and egress governor before dialing, and records the connection in SQLite `raw_tcp_streams`.
|
||||
|
||||
## Opt-in Configuration (Off by Default)
|
||||
|
||||
| Environment Variable | Default | Behavior |
|
||||
|---|---|---|
|
||||
| `ARTEX_TRANSPARENT_TCP_ENABLED` | unset (off) | Set to `1` / `true` / `on` / `yes` to start the transparent TCP listener on `127.0.0.1:1081` (IPv4 loopback). Explicit `0` / `false` / `off` / `no` forces it off even if `ARTEX_TRANSPARENT_TCP_ADDR` is set. |
|
||||
| `ARTEX_TRANSPARENT_TCP_ADDR` | `127.0.0.1:1081` | Explicit IPv4 TCP bind address; setting a non-empty value also enables the listener unless `ARTEX_TRANSPARENT_TCP_ENABLED` is explicitly false. Use `0.0.0.0:1081` only when redirecting traffic from other interfaces (`PREROUTING`) and protect the port with a host firewall. |
|
||||
|
||||
- **Flag off**: no transparent TCP listener is started, no firewall rules are touched, and the HTTP MITM proxy is unchanged.
|
||||
- **Requires traffic recorder**: the server only starts the transparent TCP listener when the traffic recorder is enabled (inside the traffic proxy block in `server/manager.go`). If traffic capture is disabled, setting these environment variables does not open a listener.
|
||||
- **Linux only**: on non-Linux operating systems (`windows`, `darwin`), leaving the flag off is a no-op; explicitly enabling it logs `ErrTransparentTCPNotLinux` and leaves the listener disabled.
|
||||
- **IPv4 only (IPv6 not covered)**: P4.1b recovers `SO_ORIGINAL_DST` (`SOL_IP`, `AF_INET`) for IPv4 TCP sockets only. IPv6 (`IP6T_SO_ORIGINAL_DST`) is **not covered**; IPv6 bind addresses and IPv6 sockets are refused.
|
||||
- **Firewall never touched by ARTEX**: ARTEX never installs, modifies, or deletes `iptables`/`nftables` rules. Root or `CAP_NET_ADMIN` is required **only** by the operator when installing and cleaning up the host redirect rules—never to run ARTEX itself.
|
||||
|
||||
## Firewall Rules & Cleanup (Operator-Managed)
|
||||
|
||||
Choose **either** `iptables` **or** `nftables`. Always exclude the UID running ARTEX (example: `artex`) from local `OUTPUT` redirect rules so ARTEX's own outbound dials do not loop back into the transparent listener.
|
||||
|
||||
### Option 1: `iptables` (IPv4 local `OUTPUT` REDIRECT)
|
||||
|
||||
Install into a dedicated chain so cleanup is atomic:
|
||||
|
||||
```bash
|
||||
# Requires root or CAP_NET_ADMIN
|
||||
sudo iptables -t nat -N ARTEX_TRANSPARENT
|
||||
# Never redirect ARTEX's own outbound dials or loopback traffic
|
||||
sudo iptables -t nat -A ARTEX_TRANSPARENT -m owner --uid-owner artex -j RETURN
|
||||
sudo iptables -t nat -A ARTEX_TRANSPARENT -d 127.0.0.0/8 -j RETURN
|
||||
# Example: redirect outbound TCP to lab target 198.51.100.0/24 into ARTEX on 127.0.0.1:1081
|
||||
sudo iptables -t nat -A ARTEX_TRANSPARENT -p tcp -d 198.51.100.0/24 -j REDIRECT --to-ports 1081
|
||||
sudo iptables -t nat -A OUTPUT -p tcp -j ARTEX_TRANSPARENT
|
||||
```
|
||||
|
||||
Cleanup (exact removal):
|
||||
|
||||
```bash
|
||||
sudo iptables -t nat -D OUTPUT -p tcp -j ARTEX_TRANSPARENT
|
||||
sudo iptables -t nat -F ARTEX_TRANSPARENT
|
||||
sudo iptables -t nat -X ARTEX_TRANSPARENT
|
||||
```
|
||||
|
||||
*(For traffic arriving on another interface—for example a container or routed VM—bind `ARTEX_TRANSPARENT_TCP_ADDR=0.0.0.0:1081`, attach `ARTEX_TRANSPARENT` to `PREROUTING` instead of `OUTPUT`, and drop direct non-redirected packets to port 1081 at the host firewall.)*
|
||||
|
||||
### Option 2: `nftables` (IPv4 local `OUTPUT` REDIRECT)
|
||||
|
||||
Install into a dedicated table so cleanup is a single command:
|
||||
|
||||
```bash
|
||||
# Requires root or CAP_NET_ADMIN
|
||||
sudo nft -f - <<'NFT'
|
||||
table ip artex_transparent {
|
||||
chain output {
|
||||
type nat hook output priority -100; policy accept;
|
||||
meta skuid "artex" return
|
||||
ip daddr 127.0.0.0/8 return
|
||||
ip daddr 198.51.100.0/24 tcp redirect to :1081
|
||||
}
|
||||
}
|
||||
NFT
|
||||
```
|
||||
|
||||
Cleanup (exact removal):
|
||||
|
||||
```bash
|
||||
sudo nft delete table ip artex_transparent
|
||||
```
|
||||
|
||||
## Policy Enforcement, Loop Guard & Storage (Shared with P4.1a)
|
||||
|
||||
For every accepted TCP connection:
|
||||
|
||||
1. **Original destination recovery**: reads `getsockopt(SOL_IP, SO_ORIGINAL_DST)` under `HandshakeTimeout` (default `10s`), parses `struct sockaddr_in` (rejecting short buffers, non-`AF_INET` families including IPv6, port `0`, and `0.0.0.0`), and rejects **self-redirects** (direct non-redirected connections where `SO_ORIGINAL_DST` equals the listener's own address/port) before dialing.
|
||||
2. **Egress governor (`egress.Do(egress.Current(), ...)`)**: enforces the active noise profile (`PaceTimeout`, default `30s`) before dialing.
|
||||
3. **Target gate (`intercept.Interceptor.DialSOCKS5` / `intercept.PinnedDialer`)**: enforces global asset block rules and any enabled `intercept.TargetScope` pins/exemptions before opening any outbound socket. Because the transparent listener only knows the recovered destination IP (never a DNS hostname), hostname-based block or allow rules cannot match—IP and CIDR rules are the ones that apply. Refused connections are closed immediately without dialing and without writing a store row.
|
||||
4. **Relay & bounds**: reuses the P4.1a full-duplex half-close-aware relay (`CloseWrite`) and defaults (`MaxConcurrent=64`, `MaxBytesPerDir=4 MiB`, `PreviewBytes=8 KiB`, `IdleTimeout=2m`, `HandshakeTimeout=10s`, `DialTimeout=30s`, `PaceTimeout=30s`).
|
||||
5. **SQLite storage**: records one row per completed connection in `raw_tcp_streams` (`protocol='tcp'`, atomic temp+rename `_blobs/sha256/` spill + `blob_refs`), with **zero HTTP `exchanges` rows** and **no PostgreSQL migration**.
|
||||
|
||||
Closing `Traffic` stops accepting new transparent connections, closes active sockets, and waits for all handler goroutines to exit before closing SQLite.
|
||||
@@ -281,20 +281,6 @@ func hostOf(rawURL string) string {
|
||||
return u.Hostname()
|
||||
}
|
||||
|
||||
func isBlank(v any) bool {
|
||||
switch x := v.(type) {
|
||||
case nil:
|
||||
return true
|
||||
case string:
|
||||
return x == ""
|
||||
case float64:
|
||||
return x == 0
|
||||
case int:
|
||||
return x == 0
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func uniq(in []string) []string {
|
||||
seen := map[string]struct{}{}
|
||||
out := in[:0]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module github.com/Autumn-27/artex
|
||||
|
||||
go 1.26.10
|
||||
go 1.26.11
|
||||
|
||||
require (
|
||||
github.com/Autumn-27/norma v0.4.11
|
||||
@@ -13,19 +13,19 @@ require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/klauspost/compress v1.18.6
|
||||
github.com/klauspost/compress v1.20.2
|
||||
github.com/lqqyt2423/go-mitmproxy v1.9.2
|
||||
github.com/miekg/dns v1.1.72
|
||||
github.com/projectdiscovery/dnsx v1.2.3
|
||||
golang.org/x/crypto v0.58.0
|
||||
golang.org/x/crypto v0.59.0
|
||||
golang.org/x/net v0.62.0
|
||||
golang.org/x/text v0.43.0
|
||||
golang.org/x/text v0.44.0
|
||||
modernc.org/sqlite v1.52.0
|
||||
mvdan.cc/sh/v3 v3.14.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.2.1 // indirect
|
||||
github.com/andybalholm/brotli v1.2.7 // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect
|
||||
@@ -70,10 +70,10 @@ require (
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect
|
||||
golang.org/x/mod v0.41.0 // indirect
|
||||
golang.org/x/sync v0.24.0 // indirect
|
||||
golang.org/x/sys v0.49.0 // indirect
|
||||
golang.org/x/tools v0.51.0 // indirect
|
||||
golang.org/x/mod v0.42.0 // indirect
|
||||
golang.org/x/sync v0.25.0 // indirect
|
||||
golang.org/x/sys v0.50.0 // indirect
|
||||
golang.org/x/tools v0.52.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/libc v1.72.3 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
github.com/BaaRai/Norma v0.4.11 h1:Z5lSRFKBDI0EzAw1YDA5iQ5GXsa9e7J19YmIe0zaVcQ=
|
||||
github.com/BaaRai/Norma v0.4.11/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw=
|
||||
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
|
||||
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/brotli v1.2.7 h1:UgafRPGMam5ySHfRkhthMqOFA6IFoeF3D7mWIDLjG5A=
|
||||
github.com/andybalholm/brotli v1.2.7/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4=
|
||||
@@ -81,8 +81,8 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
|
||||
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo=
|
||||
github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
@@ -143,23 +143,23 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||
golang.org/x/crypto v0.59.0 h1:xS2jhqYbzNrpvkWP2YTDf4TJwBy3ykR03i8vFDxEc7c=
|
||||
golang.org/x/crypto v0.59.0/go.mod h1:veDyUwATGs3c+vN3VfTbzViMGXeTBk3WyhBOkn+F/4U=
|
||||
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA=
|
||||
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU=
|
||||
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
|
||||
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
|
||||
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
|
||||
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
|
||||
golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY=
|
||||
golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
|
||||
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
|
||||
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs=
|
||||
golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
|
||||
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
|
||||
golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI=
|
||||
golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA=
|
||||
golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs=
|
||||
golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo=
|
||||
golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY=
|
||||
golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
|
||||
golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
|
||||
+1
-1
@@ -106,7 +106,7 @@ JSON
|
||||
# embedding the frontend requires node to produce the static assets
|
||||
if command -v npm >/dev/null 2>&1; then
|
||||
info "building frontend static assets..."
|
||||
( cd web && npm ci && npm run build:static )
|
||||
( cd web && npm ci --ignore-scripts && npm run build:static )
|
||||
rm -rf server/webui/dist && cp -r web/out server/webui/dist
|
||||
info "compiling embedded single binary..."
|
||||
CGO_ENABLED=0 go build -tags embedui -trimpath -o artex ./cmd/artex
|
||||
|
||||
@@ -3,12 +3,15 @@ package intercept
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// The HTTP clients the platform owns (WebFetch and the custom http tool)
|
||||
@@ -114,9 +117,23 @@ func (i *Interceptor) dialTarget(ctx context.Context, scope TargetScope, toolNam
|
||||
func addrTarget(network, addr string) (Target, error) {
|
||||
switch network {
|
||||
case "tcp", "tcp4", "tcp6":
|
||||
return parseTargetAddr(addr)
|
||||
default:
|
||||
return Target{}, fmt.Errorf("network %q is not tcp", network)
|
||||
}
|
||||
}
|
||||
|
||||
// addrTargetUDP reads the address a UDP relay datagram names.
|
||||
func addrTargetUDP(network, addr string) (Target, error) {
|
||||
switch network {
|
||||
case "udp", "udp4", "udp6":
|
||||
return parseTargetAddr(addr)
|
||||
default:
|
||||
return Target{}, fmt.Errorf("network %q is not udp", network)
|
||||
}
|
||||
}
|
||||
|
||||
func parseTargetAddr(addr string) (Target, error) {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return Target{}, err
|
||||
@@ -178,3 +195,195 @@ func (e *dialError) Error() string {
|
||||
}
|
||||
return msg + " Do not fetch the target with another tool or by its address."
|
||||
}
|
||||
|
||||
// IsTargetBlocked reports whether err came from PinnedDialer / DialSOCKS5
|
||||
// refusing a target before dialing (rule block, unpinned target, or unavailable
|
||||
// target gate), returning its platform control code.
|
||||
func IsTargetBlocked(err error) (string, bool) {
|
||||
var de *dialError
|
||||
if errors.As(err, &de) {
|
||||
return de.code, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// DialSOCKS5 enforces the platform's target gate before dialing a SOCKS5
|
||||
// CONNECT destination, reusing the same policy functions as PinnedDialer:
|
||||
// - When ctx carries an enabled TargetScope, delegates directly to dialTarget
|
||||
// (asset block rules, exemptions, fresh DNS resolution, and pin checks).
|
||||
// - Otherwise enforces global asset block rules (domain, IP, and CIDR, before
|
||||
// and after fresh DNS resolution) and dials only the resolved addresses that
|
||||
// passed those rules.
|
||||
func (i *Interceptor) DialSOCKS5(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
if i == nil {
|
||||
return baseDial(ctx, network, addr)
|
||||
}
|
||||
if scope, ok := TargetScopeFromContext(ctx); ok && scope.Enabled != nil && scope.Enabled() {
|
||||
return i.dialTarget(ctx, scope, "socks5", network, addr)
|
||||
}
|
||||
t, err := addrTarget(network, addr)
|
||||
if err != nil {
|
||||
return nil, &dialError{code: TargetGateUnavailable, target: addr, reason: "the address of the connection could not be read (" + err.Error() + ")"}
|
||||
}
|
||||
if i.targets == nil {
|
||||
return i.baseDialer()(ctx, network, addr)
|
||||
}
|
||||
rules, err := i.targets.BlockRules(db.TargetScopeRef{})
|
||||
if err != nil {
|
||||
return nil, &dialError{code: TargetGateUnavailable, target: t.String(), reason: "the asset rules could not be read: " + err.Error()}
|
||||
}
|
||||
ct := &checkedTarget{Target: t}
|
||||
if t.Kind != TargetDomain {
|
||||
ct.ips, ct.resolver, ct.resolvedAt = []string{t.Host}, literalResolver, time.Now().UTC()
|
||||
}
|
||||
if rule, hit := blockRule(rules, ct); hit {
|
||||
return nil, &dialError{code: TargetRuleBlocked, target: t.String(), reason: fmt.Sprintf("%s is blocked by an asset rule. %s", t, rule.Reason())}
|
||||
}
|
||||
if t.Kind == TargetDomain {
|
||||
c := &targetCheck{i: i, ctx: ctx, tool: "socks5", fresh: true}
|
||||
c.resolve(ct, true)
|
||||
if ct.err != nil {
|
||||
return nil, ct.err
|
||||
}
|
||||
if rule, hit := blockRule(rules, ct); hit {
|
||||
return nil, &dialError{code: TargetRuleBlocked, target: t.String(), reason: fmt.Sprintf("%s is blocked by an asset rule. %s", t, rule.Reason())}
|
||||
}
|
||||
}
|
||||
return i.dialAddrs(ctx, network, ct.ips, t.Port)
|
||||
}
|
||||
|
||||
// PinnedUDP4Resolver returns a per-datagram IPv4 destination resolver that
|
||||
// enforces the platform's target gate identically to PinnedDialer. When scoped
|
||||
// is true, a missing or incomplete TargetScope fails closed with
|
||||
// TARGET_GATE_UNAVAILABLE.
|
||||
func PinnedUDP4Resolver(toolName string, scoped bool) func(ctx context.Context, addr string) (*net.UDPAddr, error) {
|
||||
return func(ctx context.Context, addr string) (*net.UDPAddr, error) {
|
||||
scope, ok := TargetScopeFromContext(ctx)
|
||||
switch {
|
||||
case !ok && scoped:
|
||||
return nil, &dialError{code: TargetGateUnavailable, target: addr, reason: "the target scope of the session did not reach the connection"}
|
||||
case !ok:
|
||||
return resolvePlainUDP4(ctx, addr)
|
||||
case scope.Enabled == nil || !scope.Enabled():
|
||||
if scope.Interceptor != nil {
|
||||
return scope.Interceptor.ResolveUDP4(ctx, addr)
|
||||
}
|
||||
return resolvePlainUDP4(ctx, addr)
|
||||
case scope.Interceptor == nil:
|
||||
return nil, &dialError{code: TargetGateUnavailable, target: addr, reason: "the session's target scope carries no gate"}
|
||||
}
|
||||
return scope.Interceptor.resolveTargetUDP4(ctx, scope, toolName, addr)
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveUDP4 enforces the platform's target gate before forwarding a SOCKS5
|
||||
// UDP ASSOCIATE datagram, reusing the exact same asset block rules, fresh DNS
|
||||
// resolution, and TargetScope pin/exemption logic as DialSOCKS5 / PinnedDialer.
|
||||
// Returns the first verified IPv4 *net.UDPAddr.
|
||||
func (i *Interceptor) ResolveUDP4(ctx context.Context, addr string) (*net.UDPAddr, error) {
|
||||
if i == nil {
|
||||
return resolvePlainUDP4(ctx, addr)
|
||||
}
|
||||
if scope, ok := TargetScopeFromContext(ctx); ok && scope.Enabled != nil && scope.Enabled() {
|
||||
return i.resolveTargetUDP4(ctx, scope, "socks5-udp", addr)
|
||||
}
|
||||
t, err := addrTargetUDP("udp4", addr)
|
||||
if err != nil {
|
||||
return nil, &dialError{code: TargetGateUnavailable, target: addr, reason: "the address of the datagram could not be read (" + err.Error() + ")"}
|
||||
}
|
||||
if i.targets == nil {
|
||||
return resolvePlainUDP4(ctx, addr)
|
||||
}
|
||||
rules, err := i.targets.BlockRules(db.TargetScopeRef{})
|
||||
if err != nil {
|
||||
return nil, &dialError{code: TargetGateUnavailable, target: t.String(), reason: "the asset rules could not be read: " + err.Error()}
|
||||
}
|
||||
ct := &checkedTarget{Target: t}
|
||||
if t.Kind != TargetDomain {
|
||||
ct.ips, ct.resolver, ct.resolvedAt = []string{t.Host}, literalResolver, time.Now().UTC()
|
||||
}
|
||||
if rule, hit := blockRule(rules, ct); hit {
|
||||
return nil, &dialError{code: TargetRuleBlocked, target: t.String(), reason: fmt.Sprintf("%s is blocked by an asset rule. %s", t, rule.Reason())}
|
||||
}
|
||||
if t.Kind == TargetDomain {
|
||||
c := &targetCheck{i: i, ctx: ctx, tool: "socks5-udp", fresh: true}
|
||||
c.resolve(ct, true)
|
||||
if ct.err != nil {
|
||||
return nil, ct.err
|
||||
}
|
||||
if rule, hit := blockRule(rules, ct); hit {
|
||||
return nil, &dialError{code: TargetRuleBlocked, target: t.String(), reason: fmt.Sprintf("%s is blocked by an asset rule. %s", t, rule.Reason())}
|
||||
}
|
||||
}
|
||||
return firstIPv4UDPAddr(ct.ips, t.Port)
|
||||
}
|
||||
|
||||
func (i *Interceptor) resolveTargetUDP4(ctx context.Context, scope TargetScope, toolName, addr string) (*net.UDPAddr, error) {
|
||||
input, _ := json.Marshal(map[string]string{"udp": addr})
|
||||
ctx = WithCall(ctx, toolName, input)
|
||||
t, err := addrTargetUDP("udp4", addr)
|
||||
if err != nil {
|
||||
res := undeterminedTarget(ctx, scope, toolName, input, "the address of the datagram could not be read ("+err.Error()+")")
|
||||
return nil, &dialError{code: res.Code, target: addr, reason: res.Reason}
|
||||
}
|
||||
blocked := func(res TargetResult) error {
|
||||
return &dialError{code: res.Code, target: t.String(), reason: res.Reason}
|
||||
}
|
||||
c := &targetCheck{i: i, ctx: ctx, scope: scope, tool: toolName, input: input, fresh: true}
|
||||
unpinned, res := c.run([]Target{t})
|
||||
if res.Code != "" {
|
||||
return nil, blocked(res)
|
||||
}
|
||||
if len(unpinned) > 0 {
|
||||
scope.WaitBound = 0
|
||||
res = i.RequestTargets(ctx, scope, toolName, input, unpinned, nil)
|
||||
if res.Code == TargetApprovalPending {
|
||||
res.Reason = notCovered(unpinned[0], scope)
|
||||
}
|
||||
return nil, blocked(res)
|
||||
}
|
||||
switch ct := c.checked[0]; ct.cover {
|
||||
case coverDerived, coverNameOnly:
|
||||
return resolvePlainUDP4(ctx, addr)
|
||||
default:
|
||||
return firstIPv4UDPAddr(ct.ips, t.Port)
|
||||
}
|
||||
}
|
||||
|
||||
func firstIPv4UDPAddr(ips []string, port int) (*net.UDPAddr, error) {
|
||||
for _, raw := range ips {
|
||||
a, err := netip.ParseAddr(raw)
|
||||
if err == nil && a.Is4() {
|
||||
return &net.UDPAddr{IP: net.IP(a.AsSlice()), Port: port}, nil
|
||||
}
|
||||
}
|
||||
return nil, &net.AddrError{Err: "no IPv4 address found", Addr: strings.Join(ips, ", ")}
|
||||
}
|
||||
|
||||
func resolvePlainUDP4(ctx context.Context, addr string) (*net.UDPAddr, error) {
|
||||
host, portStr, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
port, err := strconv.Atoi(portStr)
|
||||
if err != nil || port <= 0 || port > 65535 {
|
||||
return nil, fmt.Errorf("invalid udp port %q", portStr)
|
||||
}
|
||||
if ip := net.ParseIP(strings.Trim(host, "[]")); ip != nil {
|
||||
ip4 := ip.To4()
|
||||
if ip4 == nil {
|
||||
return nil, &net.AddrError{Err: "IPv6 UDP destination not supported", Addr: addr}
|
||||
}
|
||||
return &net.UDPAddr{IP: ip4, Port: port}, nil
|
||||
}
|
||||
ips, err := net.DefaultResolver.LookupIP(ctx, "ip4", host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
return &net.UDPAddr{IP: ip4, Port: port}, nil
|
||||
}
|
||||
}
|
||||
return nil, &net.AddrError{Err: "no IPv4 address found", Addr: addr}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ package llmrec
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"iter"
|
||||
"log"
|
||||
"strings"
|
||||
@@ -16,6 +17,11 @@ import (
|
||||
"github.com/Autumn-27/norma/transcript"
|
||||
)
|
||||
|
||||
// errStreamAbandoned marks a stream whose consumer stopped reading before
|
||||
// message_stop while its context was still live. The call did not complete,
|
||||
// so it is metered as an error rather than inferred from ctx.Err().
|
||||
var errStreamAbandoned = errors.New("llmrec: consumer stopped reading before message_stop")
|
||||
|
||||
type taskIDContextKey struct{}
|
||||
type workerContextKey struct{}
|
||||
|
||||
@@ -190,6 +196,7 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S
|
||||
usage llm.Usage
|
||||
stopReason string
|
||||
streamErr error
|
||||
complete bool // message_stop was seen
|
||||
)
|
||||
|
||||
finished := false
|
||||
@@ -246,8 +253,21 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S
|
||||
stopReason = ev.StopReason
|
||||
}
|
||||
usage.MergeEcho(ev.Usage)
|
||||
case llm.SEMessageStop:
|
||||
complete = true
|
||||
}
|
||||
if !yield(ev, err) {
|
||||
// The consumer stopped early. Decide the status here from what
|
||||
// is known, not later in the deferred finish: the call only
|
||||
// counts as "ok" if message_stop arrived and ctx is live.
|
||||
switch {
|
||||
case ctx.Err() != nil:
|
||||
finish(ctx.Err())
|
||||
case !complete:
|
||||
finish(errStreamAbandoned)
|
||||
default:
|
||||
finish(nil)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
+12
-14
@@ -151,15 +151,9 @@ func (meteredStreamProvider) Stream(context.Context, llm.CompletionRequest) iter
|
||||
}
|
||||
|
||||
func TestSideUsageRecordedOnceOnConsumerCancellation(t *testing.T) {
|
||||
// Timing-sensitive on 2-core CI runners (BaaRai/ARTEX#9). Repro:
|
||||
// ARTEX_PG_DSN=postgres://artex:testpass@localhost:5432/artextest_race \
|
||||
// go test ./llmrec -run TestSideUsageRecordedOnceOnConsumerCancellation -count=20 -race -v
|
||||
// The `early=` t.Fatalf assertion below compares the single llm_usage row's
|
||||
// count/input/output/worker/status; in the `early` branch the status
|
||||
// occasionally comes back "ok" instead of "error" because the consumer's
|
||||
// cancellation and the recorder's deferred finish race on a loaded runner.
|
||||
// Keep the skip until the metering status is made cancellation-explicit.
|
||||
t.Skip("timing-sensitive on CI runners; repro and failing assertion documented above; BaaRai/ARTEX#9")
|
||||
// Previously skipped as timing-sensitive (BaaRai/ARTEX#9). The recorder now
|
||||
// decides the status at the point the consumer stops, so the result no
|
||||
// longer depends on when the deferred finish runs.
|
||||
dsn := os.Getenv("ARTEX_PG_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("requires isolated ARTEX_PG_DSN")
|
||||
@@ -169,7 +163,9 @@ func TestSideUsageRecordedOnceOnConsumerCancellation(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer pg.Close()
|
||||
for _, early := range []bool{false, true} {
|
||||
// full: read to the end. cancel: cancel ctx, then stop reading.
|
||||
// abandon: stop reading before message_stop with ctx still live.
|
||||
for _, mode := range []string{"full", "cancel", "abandon"} {
|
||||
profile := "btw-metering-" + uuid.NewString()
|
||||
ctx, cancel := context.WithCancel(transcript.WithSessionID(t.Context(), "exp0-btw-test"))
|
||||
recorder := Wrap(meteredStreamProvider{}, pg, "fixture", profile, "", "", func() bool { return false })
|
||||
@@ -177,8 +173,10 @@ func TestSideUsageRecordedOnceOnConsumerCancellation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if early && event.Type == llm.SEMessageDelta {
|
||||
cancel()
|
||||
if mode != "full" && event.Type == llm.SEMessageDelta {
|
||||
if mode == "cancel" {
|
||||
cancel()
|
||||
}
|
||||
break // Consumer exits before the provider can return a cancellation event.
|
||||
}
|
||||
}
|
||||
@@ -192,11 +190,11 @@ FROM llm_usage WHERE profile_name=$1`, profile).Scan(&count, &input, &output, &w
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantStatus := "ok"
|
||||
if early {
|
||||
if mode != "full" {
|
||||
wantStatus = "error"
|
||||
}
|
||||
if count != 1 || input != 23 || output != 5 || worker != "btw" || status != wantStatus {
|
||||
t.Fatalf("early=%v: count=%d usage=%d/%d worker=%q status=%q", early, count, input, output, worker, status)
|
||||
t.Fatalf("mode=%s: count=%d usage=%d/%d worker=%q status=%q", mode, count, input, output, worker, status)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ func TestChatUnavailableReasonDistinguishesStates(t *testing.T) {
|
||||
if err != nil {
|
||||
db.SkipPGUnavailable(t, err)
|
||||
}
|
||||
defer m.Close()
|
||||
t.Cleanup(func() { _ = m.Close() })
|
||||
td := t.TempDir()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s := New(ctx, m, td, td, td)
|
||||
@@ -68,7 +68,7 @@ func TestResolveChatAgentHonoursConversationProfile(t *testing.T) {
|
||||
if err != nil {
|
||||
db.SkipPGUnavailable(t, err)
|
||||
}
|
||||
defer m.Close()
|
||||
t.Cleanup(func() { _ = m.Close() })
|
||||
td := t.TempDir()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s := New(ctx, m, td, td, td)
|
||||
|
||||
@@ -84,7 +84,7 @@ func TestResetSettledRunReseedsBudgetFromUsage(t *testing.T) {
|
||||
if err != nil {
|
||||
pgdb.SkipPGUnavailable(t, err)
|
||||
}
|
||||
defer m.Close()
|
||||
t.Cleanup(func() { _ = m.Close() })
|
||||
|
||||
taskID := strconv.FormatInt(time.Now().UnixNano(), 10)
|
||||
t.Cleanup(func() {
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The record_fact texts seeded by the releases v0.4.0 to v0.6.0, copied from
|
||||
// agent/tools.go at those tags. They are kept here, apart from the agent
|
||||
// package's list of known defaults, so the test fails when that list drifts
|
||||
// from the released bytes.
|
||||
const fix410ReleasedDescription = "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n" +
|
||||
"⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n" +
|
||||
"★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n" +
|
||||
"⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n" +
|
||||
" · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n" +
|
||||
" · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n" +
|
||||
" · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen."
|
||||
|
||||
const fix410ReleasedFactsDescription = "[use when there are several distinct conclusions] array of facts; element fields same as the top-level fields below (summary/detail/evidence/confidence/intent_id/asset_ids); omitting intent_id falls back to the top-level intent_id. The returned ids match this array in length and order."
|
||||
|
||||
// A database seeded by a release stores the released record_fact description,
|
||||
// the released facts property description and facts items without properties.
|
||||
// The one-time refresh replaces both texts with the current ones and sets its
|
||||
// flag; texts an operator edited match no known default and stay as stored.
|
||||
func TestFix410ReleasedDefaultsRefreshed(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, text, sum string
|
||||
size int
|
||||
}{
|
||||
{"description", fix410ReleasedDescription, "0da5cec5cde7f5e6f41343418fcc2a7d132d6520f336efb39fcbf1ab6ac51957", 2038},
|
||||
{"facts description", fix410ReleasedFactsDescription, "eb054843a6b33dd39b94ba0648f292babb9cf9bdd106ee4eff6bf593d3f3edf4", 286},
|
||||
} {
|
||||
sum := sha256.Sum256([]byte(c.text))
|
||||
if len(c.text) != c.size || hex.EncodeToString(sum[:]) != c.sum {
|
||||
t.Fatalf("the released %s in this test is not the released text: %d bytes, sha256 %x", c.name, len(c.text), sum)
|
||||
}
|
||||
}
|
||||
skipWithoutDSN(t)
|
||||
m, err := NewManager(t.TempDir(), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer m.Close()
|
||||
s := &Server{m: m}
|
||||
const flag = "tool_schema_refresh_record_fact_facts_only"
|
||||
seed := recordFactSeed(t)
|
||||
seedSchema, _ := json.Marshal(seed.Schema)
|
||||
seedAgents, _ := json.Marshal(seed.Agents)
|
||||
defer func() {
|
||||
_ = m.pg.UpsertToolForce("record_fact", seed.Desc, seedSchema, seedAgents)
|
||||
_, _ = m.pg.Exec(`DELETE FROM settings WHERE key=$1`, flag)
|
||||
}()
|
||||
currentFacts := seed.Schema["properties"].(map[string]any)["facts"].(map[string]any)["description"]
|
||||
|
||||
// start stores the row as a release seeded it, with the given two texts,
|
||||
// runs the startup steps and returns the two texts as stored afterwards.
|
||||
start := func(desc, factsDesc string) (string, any) {
|
||||
t.Helper()
|
||||
str := func(d string) map[string]any { return map[string]any{"type": "string", "description": d} }
|
||||
schema, _ := json.Marshal(map[string]any{"type": "object", "properties": map[string]any{
|
||||
"facts": map[string]any{"type": "array", "description": factsDesc, "items": map[string]any{"type": "object"}},
|
||||
"summary": str("a [one-sentence summary] of this exploration's conclusion (a condensation of detail)"),
|
||||
"intent_id": map[string]any{"type": "integer", "description": "the id of the intent that produced this fact (the intent you claimed; the default for each item in batch mode)"},
|
||||
"detail": str("details related to this fact: write all of this exploration's observations here"),
|
||||
"evidence": str("[one line] of key evidence: the command + the one or two most proving output lines. Keep it brief; don't paste large output blocks (details go in detail)."),
|
||||
"confidence": str("observed (seen directly in output) | inferred (inferred from phenomena). Always label negative conclusions honestly."),
|
||||
"asset_ids": map[string]any{"type": "array", "items": map[string]any{"type": "integer"}, "description": "related asset ids (optional, 0/1/many): which assets this fact involves"},
|
||||
}})
|
||||
if err := m.pg.UpsertToolForce("record_fact", desc, schema, seedAgents); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.pg.Exec(`DELETE FROM settings WHERE key=$1`, flag); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.seedFactInputSchema()
|
||||
s.seedFactSourceSchema()
|
||||
s.refreshRecordFactContract()
|
||||
if v, _, _ := m.pg.GetSetting(flag); v != "true" {
|
||||
t.Errorf("flag = %q after the refresh, want true", v)
|
||||
}
|
||||
row, err := m.pg.GetTool("record_fact")
|
||||
if err != nil || row == nil {
|
||||
t.Fatalf("row: %v", err)
|
||||
}
|
||||
var stored struct {
|
||||
Properties map[string]map[string]any `json:"properties"`
|
||||
}
|
||||
if err := json.Unmarshal(row.Schema, &stored); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(stored.Properties) != 1 || stored.Properties["facts"] == nil {
|
||||
t.Fatalf("top level still holds fact fields: %s", row.Schema)
|
||||
}
|
||||
return row.Description, stored.Properties["facts"]["description"]
|
||||
}
|
||||
|
||||
desc, factsDesc := start(fix410ReleasedDescription, fix410ReleasedFactsDescription)
|
||||
if desc != seed.Desc {
|
||||
t.Errorf("released description not replaced by the current one: %.60q", desc)
|
||||
}
|
||||
if factsDesc != currentFacts {
|
||||
t.Errorf("released facts description not replaced by the current one: %v", factsDesc)
|
||||
}
|
||||
|
||||
editedDesc := fix410ReleasedDescription + "\nOperator note: record every negative result."
|
||||
editedFacts := fix410ReleasedFactsDescription + " Operator note: one fact per host."
|
||||
desc, factsDesc = start(editedDesc, editedFacts)
|
||||
if desc != editedDesc {
|
||||
t.Errorf("operator description changed: %.60q", desc)
|
||||
}
|
||||
if factsDesc != editedFacts {
|
||||
t.Errorf("operator facts description changed: %v", factsDesc)
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,99 @@ import (
|
||||
|
||||
// SeedTool only inserts new rows. Repair existing built-ins too, while keeping
|
||||
// descriptions, bindings, enabled/deferred flags and parameter annotations.
|
||||
|
||||
// refreshRecordFactContract brings an already-seeded record_fact row to the
|
||||
// facts-only contract (one-time, settings flag). SeedTool is first-insert-only
|
||||
// and ToolResolve serves the stored text, so without this an old database keeps
|
||||
// describing the earlier contract (fact fields at the top level beside facts[])
|
||||
// to the model. Only text equal to the previous code default is replaced:
|
||||
// - the tool description;
|
||||
// - the facts property and item descriptions that referred to the top level.
|
||||
//
|
||||
// Operator edits stay; a description that matches no known default is logged
|
||||
// as such and left in place. The schema structure itself is repaired by
|
||||
// seedFactInputSchema on every start (RepairRecordFactSchema).
|
||||
//
|
||||
// The flag is written only once the outcome is known: the UPDATE succeeded
|
||||
// (also when it changed no row) or the row is not the built-in one. A failed
|
||||
// read, decode or UPDATE leaves it unset, so the next start retries.
|
||||
func (s *Server) refreshRecordFactContract() {
|
||||
const flag = "tool_schema_refresh_record_fact_facts_only"
|
||||
if v, _, _ := s.m.pg.GetSetting(flag); v == "true" {
|
||||
return
|
||||
}
|
||||
done := func() { _ = s.m.pg.SetSetting(flag, "true") }
|
||||
var seed *agent.ToolSeed
|
||||
for _, sd := range agent.BuiltinToolSeeds() {
|
||||
if sd.Key == "record_fact" {
|
||||
seed = &sd
|
||||
break
|
||||
}
|
||||
}
|
||||
row, err := s.m.pg.GetTool("record_fact")
|
||||
if err != nil {
|
||||
log.Printf("[facts] read record_fact for the contract refresh: %v", err)
|
||||
return
|
||||
}
|
||||
if row == nil || seed == nil {
|
||||
return
|
||||
}
|
||||
if !row.System || row.Kind != "builtin" {
|
||||
done()
|
||||
return
|
||||
}
|
||||
legacyTool, legacyFacts, legacyItems := agent.RecordFactLegacyDescriptions()
|
||||
desc := row.Description
|
||||
switch {
|
||||
case desc == seed.Desc:
|
||||
case sliceHas(legacyTool, desc):
|
||||
desc = seed.Desc
|
||||
default:
|
||||
log.Printf("[facts] record_fact description does not match a known default (operator edit or an older release); the facts-only schema is published anyway, review the text in the tool catalog")
|
||||
}
|
||||
var schema map[string]any
|
||||
if err := json.Unmarshal(row.Schema, &schema); err != nil {
|
||||
log.Printf("[facts] decode record_fact schema: %v", err)
|
||||
return
|
||||
}
|
||||
repaired := agent.RepairRecordFactSchema(schema)
|
||||
facts := repaired["properties"].(map[string]any)["facts"].(map[string]any)
|
||||
canonFacts := seed.Schema["properties"].(map[string]any)["facts"].(map[string]any)
|
||||
if d, ok := facts["description"].(string); ok && sliceHas(legacyFacts, d) {
|
||||
facts["description"] = canonFacts["description"]
|
||||
}
|
||||
items := facts["items"].(map[string]any)["properties"].(map[string]any)
|
||||
canonItems := canonFacts["items"].(map[string]any)["properties"].(map[string]any)
|
||||
for name, old := range legacyItems {
|
||||
prop, _ := items[name].(map[string]any)
|
||||
want, _ := canonItems[name].(map[string]any)
|
||||
if prop != nil && want != nil && prop["description"] == old {
|
||||
prop["description"] = want["description"]
|
||||
}
|
||||
}
|
||||
raw, err := json.Marshal(repaired)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := s.m.pg.Exec(`UPDATE tools SET description=$1, schema=$2::jsonb, updated_at=now()
|
||||
WHERE key='record_fact' AND system AND kind='builtin'
|
||||
AND (description IS DISTINCT FROM $1 OR schema IS DISTINCT FROM $2::jsonb)`, desc, string(raw)); err != nil {
|
||||
log.Printf("[facts] refresh record_fact contract: %v", err)
|
||||
return
|
||||
}
|
||||
done()
|
||||
log.Printf("[facts] record_fact now publishes the facts-only contract (one-time)")
|
||||
}
|
||||
|
||||
func sliceHas(list []string, s string) bool {
|
||||
for _, v := range list {
|
||||
if v == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *Server) seedFactInputSchema() {
|
||||
row, err := s.m.pg.GetTool("record_fact")
|
||||
if err != nil {
|
||||
|
||||
@@ -3,6 +3,7 @@ package server
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -42,10 +43,16 @@ func TestRecordFactInputSchemaUpgrade(t *testing.T) {
|
||||
if err := pg.UpdateTool("record_fact", "operator tool", raw, bindings, old.Enabled); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Facts-only contract: the top level holds facts and
|
||||
// nothing else; the earlier top-level fact fields and their operator
|
||||
// annotations are dropped (the handler still accepts the legacy shapes),
|
||||
// item annotations survive.
|
||||
check := func(schema map[string]any) {
|
||||
t.Helper()
|
||||
if _, ok := schema["required"]; ok {
|
||||
t.Fatal("top-level summary still required for batches")
|
||||
if req, _ := schema["required"].([]any); len(req) != 1 || req[0] != "facts" {
|
||||
if reqS, _ := schema["required"].([]string); len(reqS) != 1 || reqS[0] != "facts" {
|
||||
t.Fatalf("facts must be the only required property: %v", schema["required"])
|
||||
}
|
||||
}
|
||||
if _, ok := schema["anyOf"]; ok {
|
||||
t.Fatal("root anyOf still present")
|
||||
@@ -54,14 +61,13 @@ func TestRecordFactInputSchemaUpgrade(t *testing.T) {
|
||||
t.Fatal("unknown fields allowed")
|
||||
}
|
||||
props := schema["properties"].(map[string]any)
|
||||
if len(props) != 1 || props["facts"] == nil {
|
||||
t.Fatalf("top level must hold facts only: %v", props)
|
||||
}
|
||||
items := props["facts"].(map[string]any)["items"].(map[string]any)
|
||||
itemProps := items["properties"].(map[string]any)
|
||||
if props["tested_asset_ids"] == nil || itemProps["tested_asset_ids"] == nil || itemProps["summary"] == nil {
|
||||
t.Fatal("repaired schema missing alias or batch summary")
|
||||
}
|
||||
if props["summary"].(map[string]any)["description"] != "operator summary" ||
|
||||
props["confidence"].(map[string]any)["default"] != "inferred" || props["custom"] != nil {
|
||||
t.Fatal("operator annotations lost")
|
||||
if itemProps["tested_asset_ids"] == nil || itemProps["summary"] == nil || itemProps["source"] == nil {
|
||||
t.Fatal("repaired schema missing alias, batch summary or source")
|
||||
}
|
||||
if err := actool.ValidateInput(schema, json.RawMessage(`{"facts":[{"summary":"complete batch item"}]}`)); err != nil {
|
||||
t.Fatal("valid batch rejected", err)
|
||||
@@ -104,3 +110,217 @@ func TestRecordFactInputSchemaUpgrade(t *testing.T) {
|
||||
t.Fatal("second migration changed the row")
|
||||
}
|
||||
}
|
||||
|
||||
// recordFactLegacySchema is the stored record_fact schema of the earlier
|
||||
// contract: fact fields at the top level beside facts[], with the texts that
|
||||
// referred to the top level, plus an operator annotation on an item field.
|
||||
func recordFactLegacySchema(t *testing.T) (string, map[string]string) {
|
||||
t.Helper()
|
||||
_, legacyFacts, legacyItems := agent.RecordFactLegacyDescriptions()
|
||||
item := map[string]any{"type": "object", "required": []string{"summary"}, "additionalProperties": false, "properties": map[string]any{
|
||||
"summary": map[string]any{"type": "string", "minLength": 1, "description": legacyItems["summary"]},
|
||||
"intent_id": map[string]any{"type": "integer", "description": legacyItems["intent_id"]},
|
||||
"detail": map[string]any{"type": "string", "description": "Details of this fact, including all related observations."},
|
||||
"confidence": map[string]any{"type": "string", "description": "operator confidence note", "default": "inferred"},
|
||||
"source": map[string]any{"type": "string", "enum": []string{"target_response", "tool_output", "third_party_intel", "operator_input", "inference"}, "description": "where the content came from"},
|
||||
}}
|
||||
schema := map[string]any{"type": "object", "additionalProperties": false, "properties": map[string]any{
|
||||
"summary": map[string]any{"type": "string", "minLength": 1, "description": legacyItems["summary"]},
|
||||
"intent_id": map[string]any{"type": "integer", "description": legacyItems["intent_id"]},
|
||||
"detail": map[string]any{"type": "string", "description": "Details of this fact, including all related observations."},
|
||||
"source": map[string]any{"type": "string", "enum": []string{"inference"}},
|
||||
"facts": map[string]any{"type": "array", "minItems": 1, "description": legacyFacts[0], "items": item},
|
||||
}}
|
||||
raw, _ := json.Marshal(schema)
|
||||
return string(raw), legacyItems
|
||||
}
|
||||
|
||||
func recordFactSeed(t *testing.T) agent.ToolSeed {
|
||||
t.Helper()
|
||||
for _, sd := range agent.BuiltinToolSeeds() {
|
||||
if sd.Key == "record_fact" {
|
||||
return sd
|
||||
}
|
||||
}
|
||||
t.Fatal("record_fact seed missing")
|
||||
return agent.ToolSeed{}
|
||||
}
|
||||
|
||||
// A database that still stores the earlier record_fact contract (text and
|
||||
// schema) is brought to the facts-only contract once: description replaced
|
||||
// when it is the previous default, facts/item descriptions replaced when they
|
||||
// are the earlier texts, operator annotations kept, top-level fact fields
|
||||
// gone, source enum present on the items. A second run changes nothing, and
|
||||
// an operator-edited description is left alone.
|
||||
func TestRefreshRecordFactContract(t *testing.T) {
|
||||
skipWithoutDSN(t)
|
||||
m, err := NewManager(t.TempDir(), "")
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v)", err)
|
||||
}
|
||||
defer m.Close()
|
||||
s := &Server{m: m}
|
||||
seed := recordFactSeed(t)
|
||||
seedSchema, _ := json.Marshal(seed.Schema)
|
||||
seedAgents, _ := json.Marshal(seed.Agents)
|
||||
restore := func() {
|
||||
_ = m.pg.UpsertToolForce("record_fact", seed.Desc, seedSchema, seedAgents)
|
||||
_, _ = m.pg.Exec(`DELETE FROM settings WHERE key='tool_schema_refresh_record_fact_facts_only'`)
|
||||
}
|
||||
defer restore()
|
||||
|
||||
legacyTool, _, legacyItems := agent.RecordFactLegacyDescriptions()
|
||||
legacySchema, _ := recordFactLegacySchema(t)
|
||||
run := func(desc string) {
|
||||
t.Helper()
|
||||
if err := m.pg.UpsertToolForce("record_fact", desc, json.RawMessage(legacySchema), seedAgents); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.pg.Exec(`DELETE FROM settings WHERE key='tool_schema_refresh_record_fact_facts_only'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.seedFactInputSchema()
|
||||
s.seedFactSourceSchema()
|
||||
s.refreshRecordFactContract()
|
||||
}
|
||||
for _, desc := range legacyTool {
|
||||
run(desc)
|
||||
row, err := m.pg.GetTool("record_fact")
|
||||
if err != nil || row == nil {
|
||||
t.Fatalf("row: %v", err)
|
||||
}
|
||||
if row.Description != seed.Desc {
|
||||
t.Fatalf("description not refreshed from a known default: %q", row.Description[:60])
|
||||
}
|
||||
var schema map[string]any
|
||||
if err := json.Unmarshal(row.Schema, &schema); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
props := schema["properties"].(map[string]any)
|
||||
if len(props) != 1 || props["facts"] == nil {
|
||||
t.Fatalf("top level still holds fact fields: %v", props)
|
||||
}
|
||||
facts := props["facts"].(map[string]any)
|
||||
items := facts["items"].(map[string]any)["properties"].(map[string]any)
|
||||
canonItems := seed.Schema["properties"].(map[string]any)["facts"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any)
|
||||
if facts["description"] != seed.Schema["properties"].(map[string]any)["facts"].(map[string]any)["description"] {
|
||||
t.Fatalf("facts description not refreshed: %v", facts["description"])
|
||||
}
|
||||
for name := range legacyItems {
|
||||
if items[name].(map[string]any)["description"] != canonItems[name].(map[string]any)["description"] {
|
||||
t.Fatalf("item %s description not refreshed: %v", name, items[name])
|
||||
}
|
||||
}
|
||||
if items["confidence"].(map[string]any)["description"] != "operator confidence note" || items["confidence"].(map[string]any)["default"] != "inferred" {
|
||||
t.Fatalf("operator annotation lost: %v", items["confidence"])
|
||||
}
|
||||
if enum, _ := items["source"].(map[string]any)["enum"].([]any); len(enum) != 5 {
|
||||
t.Fatalf("source enum missing on the items: %v", items["source"])
|
||||
}
|
||||
if v, _, _ := m.pg.GetSetting("tool_schema_refresh_record_fact_facts_only"); v != "true" {
|
||||
t.Fatal("flag not set")
|
||||
}
|
||||
// Second start: nothing changes.
|
||||
var updated1, updated2 string
|
||||
_ = m.pg.QueryRow(`SELECT updated_at::text FROM tools WHERE key='record_fact'`).Scan(&updated1)
|
||||
s.seedFactInputSchema()
|
||||
s.seedFactSourceSchema()
|
||||
s.refreshRecordFactContract()
|
||||
_ = m.pg.QueryRow(`SELECT updated_at::text FROM tools WHERE key='record_fact'`).Scan(&updated2)
|
||||
if updated1 != updated2 {
|
||||
t.Fatal("second start rewrote the row")
|
||||
}
|
||||
}
|
||||
// An operator-edited description is kept; the schema is still repaired.
|
||||
run("my own record_fact text")
|
||||
row, _ := m.pg.GetTool("record_fact")
|
||||
if row == nil || row.Description != "my own record_fact text" {
|
||||
t.Fatalf("operator description replaced: %v", row)
|
||||
}
|
||||
if !strings.Contains(string(row.Schema), `"facts"`) || strings.Contains(string(row.Schema), legacyItems["intent_id"]) {
|
||||
t.Fatalf("schema not repaired beside an operator description: %s", row.Schema)
|
||||
}
|
||||
}
|
||||
|
||||
// A start on which the refresh fails must not spend the one-time flag: the
|
||||
// next start retries and the stored default text is refreshed then.
|
||||
func TestFix410RefreshFlagNotSetOnFailure(t *testing.T) {
|
||||
skipWithoutDSN(t)
|
||||
m, err := NewManager(t.TempDir(), "")
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v)", err)
|
||||
}
|
||||
defer m.Close()
|
||||
s := &Server{m: m}
|
||||
const flag = "tool_schema_refresh_record_fact_facts_only"
|
||||
seed := recordFactSeed(t)
|
||||
seedSchema, _ := json.Marshal(seed.Schema)
|
||||
seedAgents, _ := json.Marshal(seed.Agents)
|
||||
legacyTool, _, _ := agent.RecordFactLegacyDescriptions()
|
||||
legacySchema, _ := recordFactLegacySchema(t)
|
||||
exec := func(query string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := m.pg.Exec(query, args...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const dropTrigger = `DROP TRIGGER IF EXISTS fix410_fail_update ON tools`
|
||||
defer func() {
|
||||
_, _ = m.pg.Exec(dropTrigger)
|
||||
_, _ = m.pg.Exec(`DROP FUNCTION IF EXISTS fix410_fail_update()`)
|
||||
_ = m.pg.UpsertToolForce("record_fact", seed.Desc, seedSchema, seedAgents)
|
||||
_, _ = m.pg.Exec(`DELETE FROM settings WHERE key=$1`, flag)
|
||||
}()
|
||||
exec(`CREATE OR REPLACE FUNCTION fix410_fail_update() RETURNS trigger LANGUAGE plpgsql AS
|
||||
'BEGIN RAISE EXCEPTION ''fix410: simulated transient failure''; END'`)
|
||||
|
||||
start := func() {
|
||||
s.seedFactInputSchema()
|
||||
s.seedFactSourceSchema()
|
||||
s.refreshRecordFactContract()
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
fail, clear func()
|
||||
}{
|
||||
{"update fails", func() {
|
||||
exec(`CREATE TRIGGER fix410_fail_update BEFORE UPDATE ON tools
|
||||
FOR EACH ROW WHEN (OLD.key = 'record_fact') EXECUTE FUNCTION fix410_fail_update()`)
|
||||
}, func() { exec(dropTrigger) }},
|
||||
{"schema does not decode", func() {
|
||||
exec(`UPDATE tools SET schema='[]'::jsonb WHERE key='record_fact'`)
|
||||
}, func() {
|
||||
exec(`UPDATE tools SET schema=$1::jsonb WHERE key='record_fact'`, legacySchema)
|
||||
}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// A database that stores the earlier default text and has not run the refresh.
|
||||
if err := m.pg.UpsertToolForce("record_fact", legacyTool[0], json.RawMessage(legacySchema), seedAgents); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exec(`DELETE FROM settings WHERE key=$1`, flag)
|
||||
|
||||
tc.fail()
|
||||
start()
|
||||
if v, ok, _ := m.pg.GetSetting(flag); ok {
|
||||
t.Errorf("flag is %q after a failed refresh: later starts skip it", v)
|
||||
}
|
||||
if row, _ := m.pg.GetTool("record_fact"); row == nil || row.Description != legacyTool[0] {
|
||||
t.Fatal("the simulated failure did not leave the earlier description in place")
|
||||
}
|
||||
|
||||
tc.clear()
|
||||
start()
|
||||
row, err := m.pg.GetTool("record_fact")
|
||||
if err != nil || row == nil {
|
||||
t.Fatalf("row: %v", err)
|
||||
}
|
||||
if row.Description != seed.Desc {
|
||||
t.Errorf("the next start did not refresh the description: %.60q", row.Description)
|
||||
}
|
||||
if v, _, _ := m.pg.GetSetting(flag); v != "true" {
|
||||
t.Errorf("flag = %q after a successful refresh, want true", v)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,12 +35,13 @@ func TestRecordFactSourceSchemaUpgrade(t *testing.T) {
|
||||
t.Cleanup(func() { pg.UpdateTool(old.Key, old.Description, old.Schema, bindings, old.Enabled) })
|
||||
|
||||
// Simulate a pre-provenance row customised by the operator: no source
|
||||
// property, an edited description and an edited parameter description.
|
||||
// property on the facts items, an edited description and an edited
|
||||
// parameter description (facts-only contract: the items carry the fields).
|
||||
var schema map[string]any
|
||||
if err := json.Unmarshal(old.Schema, &schema); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
props := schema["properties"].(map[string]any)
|
||||
props := schema["properties"].(map[string]any)["facts"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any)
|
||||
delete(props, "source")
|
||||
props["confidence"].(map[string]any)["description"] = "operator confidence text"
|
||||
custom, _ := json.Marshal(schema)
|
||||
@@ -72,16 +73,23 @@ func TestRecordFactSourceSchemaUpgrade(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
Properties map[string]map[string]any `json:"properties"`
|
||||
Properties struct {
|
||||
Facts struct {
|
||||
Items struct {
|
||||
Properties map[string]map[string]any `json:"properties"`
|
||||
} `json:"items"`
|
||||
} `json:"facts"`
|
||||
} `json:"properties"`
|
||||
}
|
||||
if err := json.Unmarshal(tool.Schema, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
itemProps := got.Properties.Facts.Items.Properties
|
||||
wantEnum := "[target_response tool_output third_party_intel operator_input inference]"
|
||||
if src := got.Properties["source"]; src == nil || fmt.Sprint(src["enum"]) != wantEnum {
|
||||
if src := itemProps["source"]; src == nil || fmt.Sprint(src["enum"]) != wantEnum {
|
||||
t.Fatalf("upgrade did not add the source enum: %s", tool.Schema)
|
||||
}
|
||||
if tool.Description != "operator description" || got.Properties["confidence"]["description"] != "operator confidence text" || len(got.Properties) != len(props)+1 {
|
||||
if tool.Description != "operator description" || itemProps["confidence"]["description"] != "operator confidence text" || len(itemProps) != len(props)+1 {
|
||||
t.Fatalf("operator configuration overwritten: %q %s", tool.Description, tool.Schema)
|
||||
}
|
||||
|
||||
@@ -92,15 +100,18 @@ func TestRecordFactSourceSchemaUpgrade(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// resolvedRecordFactProps returns the properties of record_fact as ToolResolve
|
||||
// serves it to the worker.
|
||||
// resolvedRecordFactProps returns the facts[] item properties of record_fact as
|
||||
// ToolResolve serves it to the worker.
|
||||
func resolvedRecordFactProps(t *testing.T) map[string]any {
|
||||
t.Helper()
|
||||
tools := agent.ToolResolve()(context.Background(), "worker", agent.NewToolSet(nil, "").WorkerTools())
|
||||
for _, tl := range tools {
|
||||
if tl.Name() == "record_fact" {
|
||||
props, _ := tl.InputSchema()["properties"].(map[string]any)
|
||||
return props
|
||||
facts, _ := props["facts"].(map[string]any)
|
||||
items, _ := facts["items"].(map[string]any)
|
||||
itemProps, _ := items["properties"].(map[string]any)
|
||||
return itemProps
|
||||
}
|
||||
}
|
||||
t.Fatal("worker lost record_fact")
|
||||
|
||||
@@ -38,7 +38,11 @@ func trafficEvidenceServer(t *testing.T) (*Server, *db.RecordedFinding, func(str
|
||||
t.Fatal(err)
|
||||
}
|
||||
tid, _ := strconv.ParseInt(task.ID, 10, 64)
|
||||
t.Cleanup(func() { m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, tid); m.pg.DeleteTask(tid) })
|
||||
t.Cleanup(func() {
|
||||
_, _ = m.pg.DeleteFindingsByTask(tid)
|
||||
_, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, tid)
|
||||
_ = m.pg.DeleteTask(tid)
|
||||
})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
|
||||
|
||||
@@ -36,7 +36,10 @@ func TestTaskLessonsExtractionAndIdempotency(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
|
||||
defer func() { _ = m.pg.DeleteTask(taskID) }()
|
||||
defer func() {
|
||||
_, _ = m.pg.DeleteFindingsByTask(taskID)
|
||||
_ = m.pg.DeleteTask(taskID)
|
||||
}()
|
||||
defer func() { _, _ = m.pg.Exec(`DELETE FROM rsi_lessons`) }()
|
||||
|
||||
if _, err := m.pg.Exec(`
|
||||
@@ -138,7 +141,10 @@ func TestTerminalStatusExtractsLessons(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
|
||||
defer func() { _ = m.pg.DeleteTask(taskID) }()
|
||||
defer func() {
|
||||
_, _ = m.pg.DeleteFindingsByTask(taskID)
|
||||
_ = m.pg.DeleteTask(taskID)
|
||||
}()
|
||||
if _, err := m.pg.Exec(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,'xss','Reflected XSS','reflected in search','high','confirmed')`, taskID); err != nil {
|
||||
|
||||
@@ -266,6 +266,10 @@ type Manager struct {
|
||||
rewriteMu sync.Mutex
|
||||
rewriteSet atomic.Pointer[rewrite.Set]
|
||||
rewriteErrors rewrite.ErrorLog
|
||||
// Capture-proxy constrained Go tampering hooks (P4.2), behind ARTEX_TAMPER_HOOKS_ENABLED.
|
||||
hookMu sync.Mutex
|
||||
hookSet atomic.Pointer[traffic.HookSet]
|
||||
hookLoadErrs map[int64]string
|
||||
|
||||
companyMu sync.Mutex // serializes task/company-scope commits with live handle registration
|
||||
// taskStateMu preserves commit order between PostgreSQL lifecycle writes and
|
||||
@@ -463,6 +467,26 @@ func NewManager(dir, proxyAddr string) (*Manager, error) {
|
||||
if tr != nil {
|
||||
m.traffic = tr
|
||||
m.reloadRewriteRules() // in place before the proxy accepts a connection
|
||||
m.reloadTamperHooks() // opt-in behind ARTEX_TAMPER_HOOKS_ENABLED=1
|
||||
if scfg, ok := traffic.SOCKS5ConfigFromEnv(); ok {
|
||||
scfg.DialContext = m.interceptor.DialSOCKS5
|
||||
scfg.ResolveUDP4 = m.interceptor.ResolveUDP4
|
||||
if bound, serr := tr.StartSOCKS5(scfg); serr != nil {
|
||||
log.Printf("[traffic/socks5] disabled: %v", serr)
|
||||
} else if scfg.EnableUDP {
|
||||
log.Printf("[traffic/socks5] opt-in SOCKS5 CONNECT + UDP ASSOCIATE listener on %s", bound)
|
||||
} else {
|
||||
log.Printf("[traffic/socks5] opt-in SOCKS5 CONNECT listener on %s", bound)
|
||||
}
|
||||
}
|
||||
if tcfg, ok := traffic.TransparentTCPConfigFromEnv(); ok {
|
||||
tcfg.DialContext = m.interceptor.DialSOCKS5
|
||||
if bound, terr := tr.StartTransparentTCP(tcfg); terr != nil {
|
||||
log.Printf("[traffic/transparent] disabled: %v", terr)
|
||||
} else {
|
||||
log.Printf("[traffic/transparent] opt-in transparent TCP listener on %s (Linux IPv4 SO_ORIGINAL_DST)", bound)
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
log.Printf("[traffic] recording proxy on %s (set HTTP_PROXY=%s + trust _ca CA)", proxyAddr, tr.ProxyAddr())
|
||||
if err := tr.Start(); err != nil {
|
||||
|
||||
@@ -18,7 +18,7 @@ func TestSeedAssociatesTargetAssetWithTask(t *testing.T) {
|
||||
if err != nil {
|
||||
pgdb.SkipPGUnavailable(t, err)
|
||||
}
|
||||
defer m.Close()
|
||||
t.Cleanup(func() { _ = m.Close() })
|
||||
task, err := m.CreateTask("seed ownership", "seed ownership", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -72,7 +72,10 @@ func newNotifyFixture(t *testing.T) *notifyFixture {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { pg.Exec(`DELETE FROM tasks WHERE id=$1`, taskID) })
|
||||
t.Cleanup(func() {
|
||||
_, _ = pg.DeleteFindingsByTask(taskID)
|
||||
_ = pg.DeleteTask(taskID)
|
||||
})
|
||||
|
||||
var mark int64
|
||||
if err := pg.QueryRow(`SELECT COALESCE(max(id),0) FROM notification_events`).Scan(&mark); err != nil {
|
||||
|
||||
+16
-3
@@ -573,6 +573,7 @@ func (s *Server) seedOrchestrationTools() {
|
||||
s.seedFindingEvidenceSchema() // optional counterevidence / confidence_rationale on report_finding, added once per DB
|
||||
s.seedFactInputSchema() // restore record_fact single/batch contracts
|
||||
s.seedFactSourceSchema() // source enum on record_fact, added once per DB
|
||||
s.refreshRecordFactContract() // record_fact publishes the facts-only contract (one-time)
|
||||
s.seedTaskScopeDescription() // add_task_scope: "coverage scope, not enforced" replaces the authorization-boundary text
|
||||
// Note: pentest's default tool bindings need no migration — BuiltinToolSeeds already seeds
|
||||
// list_assets/insert_assets/report_finding/list_findings/list_companies together with
|
||||
@@ -602,11 +603,23 @@ func (s *Server) seedFactSourceSchema() {
|
||||
if err := json.Unmarshal(obj["properties"], &properties); err != nil {
|
||||
return
|
||||
}
|
||||
if len(properties["source"]) == 0 {
|
||||
// The source enum lives on the facts[] items (facts-only contract); the
|
||||
// top level has no fact fields any more.
|
||||
var facts map[string]json.RawMessage
|
||||
var items map[string]json.RawMessage
|
||||
var itemProps map[string]json.RawMessage
|
||||
if err := json.Unmarshal(properties["facts"], &facts); err != nil {
|
||||
return
|
||||
}
|
||||
_, err = s.m.pg.Exec(`UPDATE tools SET schema=jsonb_set(schema,ARRAY['properties','source'],$1::jsonb,true),updated_at=now()
|
||||
WHERE key='record_fact' AND system AND NOT(COALESCE(schema->'properties','{}'::jsonb) ? 'source')`, string(properties["source"]))
|
||||
if err := json.Unmarshal(facts["items"], &items); err != nil {
|
||||
return
|
||||
}
|
||||
if err := json.Unmarshal(items["properties"], &itemProps); err != nil || len(itemProps["source"]) == 0 {
|
||||
return
|
||||
}
|
||||
_, err = s.m.pg.Exec(`UPDATE tools SET schema=jsonb_set(schema,ARRAY['properties','facts','items','properties','source'],$1::jsonb,true),updated_at=now()
|
||||
WHERE key='record_fact' AND system AND (schema->'properties'->'facts'->'items') ? 'properties'
|
||||
AND NOT(COALESCE(schema->'properties'->'facts'->'items'->'properties','{}'::jsonb) ? 'source')`, string(itemProps["source"]))
|
||||
if err != nil {
|
||||
log.Printf("[facts] upgrade record_fact.source: %v", err)
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ func TestPlannerQuietSkipEdge(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = m.DeleteTask(created.ID, DeleteTaskOptions{}) })
|
||||
tt, ok := m.Task(created.ID)
|
||||
if !ok {
|
||||
t.Fatal("task not found after creation")
|
||||
@@ -93,6 +94,7 @@ func TestPlannerQuietModeToggleMidRun(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = m.DeleteTask(created.ID, DeleteTaskOptions{}) })
|
||||
tt, ok := m.Task(created.ID)
|
||||
if !ok {
|
||||
t.Fatal("task not found")
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/norma/harness"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
actool "github.com/Autumn-27/norma/tool"
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// acceptsRawInput is the assertion norma's harness makes before its schema
|
||||
// check: only a tool that reports AcceptsRawInput skips it.
|
||||
func acceptsRawInput(t actool.CoreTool) bool {
|
||||
r, ok := t.(interface{ AcceptsRawInput() bool })
|
||||
return ok && r.AcceptsRawInput()
|
||||
}
|
||||
|
||||
// record_fact opts out of the harness schema check and validates its own
|
||||
// input. Every wrapper between the handler and the harness has to forward
|
||||
// that opt-out: an embedded CoreTool alone hides it from the harness's type
|
||||
// assertion, and the stricter published schema then rejects the calls the
|
||||
// handler exists to answer.
|
||||
func TestFix410AcceptsRawInputSurvivesWrappers(t *testing.T) {
|
||||
oldAugment, oldResolve := agent.ToolAugment(), agent.ToolResolve()
|
||||
t.Cleanup(func() {
|
||||
agent.SetToolAugment(oldAugment)
|
||||
agent.SetToolResolve(oldResolve)
|
||||
})
|
||||
ctx := context.Background()
|
||||
probe := func(raw bool) actool.CoreTool {
|
||||
return actool.Build(actool.Spec{Name: "fix410_probe", Schema: map[string]any{"type": "object"}, RawInput: raw})
|
||||
}
|
||||
|
||||
// AugmentTools without hooks applies guardPanic and nothing else.
|
||||
t.Run("guardedTool", func(t *testing.T) {
|
||||
agent.SetToolAugment(nil)
|
||||
agent.SetToolResolve(nil)
|
||||
for _, raw := range []bool{true, false} {
|
||||
out, _, cleanup := agent.AugmentTools(ctx, "worker", []actool.CoreTool{probe(raw)})
|
||||
cleanup()
|
||||
if len(out) != 1 {
|
||||
t.Fatalf("AugmentTools returned %d tools, want 1", len(out))
|
||||
}
|
||||
if got := acceptsRawInput(out[0]); got != raw {
|
||||
t.Errorf("%T: AcceptsRawInput = %v, wrapped tool reports %v", out[0], got, raw)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("meteredTool", func(t *testing.T) {
|
||||
for _, raw := range []bool{true, false} {
|
||||
wrapped := meterTool(probe(raw), &recordingToolUsage{}, "fix410_probe", "worker", agent.RunInfo{})
|
||||
if got := acceptsRawInput(wrapped); got != raw {
|
||||
t.Errorf("%T: AcceptsRawInput = %v, wrapped tool reports %v", wrapped, got, raw)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// The chain a worker's record_fact goes through on the server: the catalog
|
||||
// decorator and the usage meter (ToolResolve, wireTools), then guardPanic
|
||||
// (AugmentTools), then the harness itself.
|
||||
t.Run("server chain", func(t *testing.T) {
|
||||
dsn, _, err := db.DSN()
|
||||
if err != nil {
|
||||
db.SkipPGUnavailable(t, err)
|
||||
}
|
||||
pg, err := db.Open(dsn)
|
||||
if err != nil {
|
||||
db.SkipPGUnavailable(t, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = pg.Close() })
|
||||
agent.SetToolAugment(nil)
|
||||
wireTools(pg, nil)
|
||||
task, err := pg.CreateTaskWithOptions("fix410", "record_fact through the wrapper chain", db.TaskCreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = pg.DeleteTask(task.ID) })
|
||||
ts := agent.NewToolSet(pg.Exploration(task.ExplorationID), "worker")
|
||||
tools, _, cleanup := agent.AugmentTools(ctx, "worker", ts.WorkerTools())
|
||||
defer cleanup()
|
||||
recordFact := names(tools)["record_fact"]
|
||||
if recordFact == nil {
|
||||
t.Fatal("worker has no record_fact")
|
||||
}
|
||||
if !acceptsRawInput(recordFact) {
|
||||
t.Errorf("record_fact lost the harness opt-out behind %T", recordFact)
|
||||
}
|
||||
|
||||
// The first call is malformed (facts is not an array): the harness
|
||||
// schema check answers it with its generic error unless the tool opted
|
||||
// out. The second is the legacy single fact, which the published
|
||||
// schema does not describe and the handler still accepts.
|
||||
p := &toolScript{results: make(chan string, 4), calls: []scriptedCall{
|
||||
{"record_fact", `{"facts":{"summary":"not an array"}}`},
|
||||
{"record_fact", `{"summary":"legacy single fact"}`},
|
||||
}}
|
||||
for _, err := range harness.Query(ctx, harness.QueryInput{
|
||||
Provider: p,
|
||||
Tools: actool.NewRegistry(recordFact),
|
||||
Messages: []llm.Message{llm.UserText("record the facts")},
|
||||
}, harness.QueryDeps{}) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if refusal := p.next(t); !strings.Contains(refusal, "invalid record_fact input: facts must be an array") {
|
||||
t.Errorf("malformed call: want the handler's own refusal, not the SDK schema error, got: %s", refusal)
|
||||
}
|
||||
if legacy := p.next(t); strings.HasPrefix(legacy, "[error]") || ts.Writes().Facts != 1 {
|
||||
t.Errorf("legacy single fact not written (facts=%d): %s", ts.Writes().Facts, legacy)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1033,6 +1033,16 @@ func (s *Server) Handler() http.Handler {
|
||||
mux.HandleFunc("DELETE /api/traffic/rewrite-rules/{id}", s.rewriteDeleteRule)
|
||||
mux.HandleFunc("POST /api/traffic/rewrite-rules/{id}/toggle", s.rewriteToggleRule)
|
||||
|
||||
// capture-proxy constrained Go tampering hooks (P4.2, operator-only, opt-in via ARTEX_TAMPER_HOOKS_ENABLED)
|
||||
mux.HandleFunc("GET /api/traffic/hooks", s.tamperListHooks)
|
||||
mux.HandleFunc("POST /api/traffic/hooks", s.tamperCreateHook)
|
||||
mux.HandleFunc("PUT /api/traffic/hooks/{id}", s.tamperUpdateHook)
|
||||
mux.HandleFunc("DELETE /api/traffic/hooks/{id}", s.tamperDeleteHook)
|
||||
mux.HandleFunc("POST /api/traffic/hooks/{id}/toggle", s.tamperToggleHook)
|
||||
mux.HandleFunc("GET /api/traffic/hooks/{id}/versions", s.tamperListHookVersions)
|
||||
mux.HandleFunc("POST /api/traffic/hooks/{id}/rollback", s.tamperRollbackHook)
|
||||
mux.HandleFunc("POST /api/traffic/hooks/dry-run", s.tamperDryRunHook)
|
||||
|
||||
// asset interception rule management (global blocklist: domains/IPs/URLs/CIDRs)
|
||||
mux.HandleFunc("GET /api/asset-intercept/rules", s.assetInterceptListRules)
|
||||
mux.HandleFunc("POST /api/asset-intercept/rules", s.assetInterceptCreateRule)
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/traffic"
|
||||
)
|
||||
|
||||
func TestManagerSOCKS5OptInFlagAndAssetBlockRuleRefusal(t *testing.T) {
|
||||
// 1. Flag off (default): Manager starts NO SOCKS5 listener.
|
||||
t.Setenv(traffic.SOCKS5EnabledEnv, "")
|
||||
t.Setenv(traffic.SOCKS5AddrEnv, "")
|
||||
mOff, err := NewManager(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
db.SkipPGUnavailable(t, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = mOff.Close() })
|
||||
if mOff.traffic == nil {
|
||||
t.Fatal("expected HTTP recording proxy to start")
|
||||
}
|
||||
if got := mOff.traffic.SOCKS5Addr(); got != "" {
|
||||
t.Fatalf("SOCKS5Addr with flag off = %q, want empty", got)
|
||||
}
|
||||
|
||||
// 2. Flag on (ARTEX_SOCKS5_ADDR=127.0.0.1:0): listener starts and enforces
|
||||
// global asset block rules via m.interceptor.DialSOCKS5 BEFORE dialing.
|
||||
t.Setenv(traffic.SOCKS5AddrEnv, "127.0.0.1:0")
|
||||
mOn, err := NewManager(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = mOn.Close() })
|
||||
socksAddr := mOn.traffic.SOCKS5Addr()
|
||||
if socksAddr == "" {
|
||||
t.Fatal("expected SOCKS5 listener when ARTEX_SOCKS5_ADDR is set")
|
||||
}
|
||||
|
||||
// Add a global asset block rule for 127.0.0.1 and ensure cleanup.
|
||||
rule, err := mOn.pg.CreateAssetInterceptRule(
|
||||
"exact_ip",
|
||||
"127.0.0.1",
|
||||
fmt.Sprintf("p41a-block-test-%d", time.Now().UnixNano()),
|
||||
true,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = mOn.pg.DeleteAssetInterceptRule(rule.ID) })
|
||||
|
||||
// Target TCP server on 127.0.0.1: must NEVER receive a connection.
|
||||
targetLn, err := net.Listen("tcp4", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer targetLn.Close()
|
||||
var accepted atomic.Int32
|
||||
go func() {
|
||||
for {
|
||||
c, err := targetLn.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
accepted.Add(1)
|
||||
_ = c.Close()
|
||||
}
|
||||
}()
|
||||
_, portStr, _ := net.SplitHostPort(targetLn.Addr().String())
|
||||
port, _ := strconv.Atoi(portStr)
|
||||
|
||||
c, err := net.DialTimeout("tcp", socksAddr, 3*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
_ = c.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
// Greeting: VER=5, NMETHODS=1, NO AUTH
|
||||
if _, err := c.Write([]byte{0x05, 0x01, 0x00}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var sel [2]byte
|
||||
if _, err := io.ReadFull(c, sel[:]); err != nil || sel != [2]byte{0x05, 0x00} {
|
||||
t.Fatalf("method selection = %v err=%v", sel, err)
|
||||
}
|
||||
// CONNECT 127.0.0.1:<port>
|
||||
req := []byte{0x05, 0x01, 0x00, 0x01, 127, 0, 0, 1, 0, 0}
|
||||
binary.BigEndian.PutUint16(req[8:10], uint16(port))
|
||||
if _, err := c.Write(req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var rep [10]byte
|
||||
if _, err := io.ReadFull(c, rep[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rep[1] != 0x02 {
|
||||
t.Fatalf("blocked target SOCKS5 reply = 0x%02x, want 0x02 (not allowed by ruleset)", rep[1])
|
||||
}
|
||||
if accepted.Load() != 0 {
|
||||
t.Fatalf("blocked target accepted %d connections, want 0", accepted.Load())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/traffic"
|
||||
)
|
||||
|
||||
func TestManagerSOCKS5UDPOptInAndAssetBlockRuleDropsDatagram(t *testing.T) {
|
||||
// 1. Both flags off: NO SOCKS5 listener and UDP disabled.
|
||||
t.Setenv(traffic.SOCKS5EnabledEnv, "")
|
||||
t.Setenv(traffic.SOCKS5AddrEnv, "")
|
||||
t.Setenv(traffic.SOCKS5UDPEnabledEnv, "")
|
||||
mOff, err := NewManager(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
db.SkipPGUnavailable(t, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = mOff.Close() })
|
||||
if mOff.traffic.SOCKS5Addr() != "" || mOff.traffic.SOCKS5UDPEnabled() {
|
||||
t.Fatalf("flags off: addr=%q udp=%v, want empty/false", mOff.traffic.SOCKS5Addr(), mOff.traffic.SOCKS5UDPEnabled())
|
||||
}
|
||||
|
||||
// 2. SOCKS5 on with UDP flag off: UDP ASSOCIATE refused with 0x07.
|
||||
t.Setenv(traffic.SOCKS5AddrEnv, "127.0.0.1:0")
|
||||
t.Setenv(traffic.SOCKS5UDPEnabledEnv, "")
|
||||
mTCPOnly, err := NewManager(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = mTCPOnly.Close() })
|
||||
if mTCPOnly.traffic.SOCKS5UDPEnabled() {
|
||||
t.Fatal("expected UDP disabled when ARTEX_SOCKS5_UDP_ENABLED is unset")
|
||||
}
|
||||
|
||||
// 3. UDP flag on: listener accepts UDP ASSOCIATE and enforces global asset
|
||||
// block rules per datagram destination (drop denied, forward allowed).
|
||||
t.Setenv(traffic.SOCKS5AddrEnv, "127.0.0.1:0")
|
||||
t.Setenv(traffic.SOCKS5UDPEnabledEnv, "1")
|
||||
mOn, err := NewManager(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = mOn.Close() })
|
||||
socksAddr := mOn.traffic.SOCKS5Addr()
|
||||
if socksAddr == "" || !mOn.traffic.SOCKS5UDPEnabled() {
|
||||
t.Fatalf("expected SOCKS5+UDP enabled, got addr=%q udp=%v", socksAddr, mOn.traffic.SOCKS5UDPEnabled())
|
||||
}
|
||||
|
||||
// Block 127.0.0.2 via global asset intercept rule; leave 127.0.0.1 allowed.
|
||||
rule, err := mOn.pg.CreateAssetInterceptRule(
|
||||
"exact_ip",
|
||||
"127.0.0.2",
|
||||
fmt.Sprintf("p41c-udp-block-%d", time.Now().UnixNano()),
|
||||
true,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = mOn.pg.DeleteAssetInterceptRule(rule.ID) })
|
||||
|
||||
startEcho := func(bind string) (*net.UDPConn, *atomic.Int32) {
|
||||
t.Helper()
|
||||
ua, err := net.ResolveUDPAddr("udp4", bind)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ln, err := net.ListenUDP("udp4", ua)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var n atomic.Int32
|
||||
go func() {
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
rn, from, rerr := ln.ReadFromUDP(buf)
|
||||
if rerr != nil {
|
||||
return
|
||||
}
|
||||
n.Add(1)
|
||||
_, _ = ln.WriteToUDP(append([]byte("OK:"), buf[:rn]...), from)
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() { _ = ln.Close() })
|
||||
return ln, &n
|
||||
}
|
||||
|
||||
blockedLn, blockedCount := startEcho("127.0.0.2:0")
|
||||
allowedLn, allowedCount := startEcho("127.0.0.1:0")
|
||||
blockedAddr := blockedLn.LocalAddr().(*net.UDPAddr)
|
||||
allowedAddr := allowedLn.LocalAddr().(*net.UDPAddr)
|
||||
|
||||
// Open SOCKS5 UDP ASSOCIATE control connection.
|
||||
ctrl, err := net.DialTimeout("tcp4", socksAddr, 3*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ctrl.Close()
|
||||
_ = ctrl.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
if _, err := ctrl.Write([]byte{0x05, 0x01, 0x00}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var sel [2]byte
|
||||
if _, err := io.ReadFull(ctrl, sel[:]); err != nil || sel != [2]byte{0x05, 0x00} {
|
||||
t.Fatalf("method selection = %v err=%v", sel, err)
|
||||
}
|
||||
// CMD=0x03 (UDP ASSOCIATE), ATYP=0x01, 0.0.0.0:0
|
||||
if _, err := ctrl.Write([]byte{0x05, 0x03, 0x00, 0x01, 0, 0, 0, 0, 0, 0}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var rep [10]byte
|
||||
if _, err := io.ReadFull(ctrl, rep[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rep[1] != 0x00 {
|
||||
t.Fatalf("UDP ASSOCIATE rep = 0x%02x, want 0x00", rep[1])
|
||||
}
|
||||
relayAddr := &net.UDPAddr{
|
||||
IP: net.IPv4(rep[4], rep[5], rep[6], rep[7]),
|
||||
Port: int(binary.BigEndian.Uint16(rep[8:10])),
|
||||
}
|
||||
_ = ctrl.SetDeadline(time.Time{})
|
||||
|
||||
cliUDP, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cliUDP.Close()
|
||||
|
||||
buildPkt := func(dst *net.UDPAddr, msg string) []byte {
|
||||
p := make([]byte, 10+len(msg))
|
||||
p[3] = 0x01
|
||||
copy(p[4:8], dst.IP.To4())
|
||||
binary.BigEndian.PutUint16(p[8:10], uint16(dst.Port))
|
||||
copy(p[10:], msg)
|
||||
return p
|
||||
}
|
||||
|
||||
// Send datagram to blocked 127.0.0.2 (must be dropped by asset rule), then to allowed 127.0.0.1.
|
||||
_, _ = cliUDP.WriteToUDP(buildPkt(blockedAddr, "blocked-udp"), relayAddr)
|
||||
_, _ = cliUDP.WriteToUDP(buildPkt(allowedAddr, "allowed-udp"), relayAddr)
|
||||
|
||||
_ = cliUDP.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
buf := make([]byte, 2048)
|
||||
rn, _, err := cliUDP.ReadFromUDP(buf)
|
||||
if err != nil {
|
||||
t.Fatalf("read allowed udp reply: %v", err)
|
||||
}
|
||||
if got := string(buf[10:rn]); got != "OK:allowed-udp" {
|
||||
t.Fatalf("reply = %q, want OK:allowed-udp", got)
|
||||
}
|
||||
if blockedCount.Load() != 0 {
|
||||
t.Fatalf("blocked 127.0.0.2 received %d datagrams, want 0", blockedCount.Load())
|
||||
}
|
||||
if allowedCount.Load() != 1 {
|
||||
t.Fatalf("allowed 127.0.0.1 received %d datagrams, want 1", allowedCount.Load())
|
||||
}
|
||||
|
||||
// Wait briefly for SQLite writes and verify raw_udp_datagrams has 127.0.0.1 only, 0 HTTP exchanges.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
n, err := mOn.traffic.CountRawUDPDatagrams("", "127.0.0.1")
|
||||
if err == nil && n >= 2 {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
if nBlocked, err := mOn.traffic.CountRawUDPDatagrams("", "127.0.0.2"); err != nil || nBlocked != 0 {
|
||||
t.Fatalf("blocked host recorded %d rows (err=%v), want 0", nBlocked, err)
|
||||
}
|
||||
if nAllowed, err := mOn.traffic.CountRawUDPDatagrams("", "127.0.0.1"); err != nil || nAllowed != 2 {
|
||||
t.Fatalf("allowed host recorded %d rows (err=%v), want 2", nAllowed, err)
|
||||
}
|
||||
}
|
||||
@@ -174,6 +174,7 @@ func (f *targetScopeFixture) newTask(t *testing.T) *Task {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = f.m.DeleteTask(created.ID, DeleteTaskOptions{}) })
|
||||
task, ok := f.m.Task(created.ID)
|
||||
if !ok {
|
||||
t.Fatal("task not found after creation")
|
||||
|
||||
@@ -650,6 +650,15 @@ func TestTaskLLMResolutionReportsDatabaseFailure(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taskID := mustTaskID(t, task.ID)
|
||||
t.Cleanup(func() {
|
||||
if dsn, _, err := db.DSN(); err == nil {
|
||||
if pg, err := db.Open(dsn); err == nil {
|
||||
_ = pg.DeleteTask(taskID)
|
||||
_ = pg.Close()
|
||||
}
|
||||
}
|
||||
})
|
||||
if err := m.pg.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -49,6 +49,7 @@ func TestTaskCredentialsFlow(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.m.pg.DeleteTask(id) })
|
||||
t.Cleanup(func() { s.m.PG().Exec(`DELETE FROM tasks WHERE id=$1`, id) })
|
||||
|
||||
// List: both rows, user source, order stable.
|
||||
|
||||
+9
-11
@@ -193,20 +193,18 @@ func healthIDFor(prov llm.Provider, profileID int64) int64 {
|
||||
return profileID
|
||||
}
|
||||
|
||||
// current resolves the provider this role runs on, by precedence:
|
||||
// Agent binding → task LLM profile chain → global/environment configuration.
|
||||
// currentFor resolves the provider this role runs on, by precedence:
|
||||
// Agent binding → task LLM profile chain → global/environment configuration,
|
||||
// honoring a per-run profile pin set by the engine for per-intent rotation (#85).
|
||||
// Binding wins over the task chain: a role explicitly assigned a model keeps running
|
||||
// on that model; only when the binding is missing or fails to build does it fall back
|
||||
// to the task chain, and to the global configuration when the chain is empty.
|
||||
// The returned profile id is non-zero only on the task-chain path — streamTaskLLM uses it to decide whether a quota error should advance the task's failover state (binding/global paths leave the chain state untouched, preserving existing semantics).
|
||||
func (r *taskLLMRuntime) current() (taskLLMSelection, error) {
|
||||
return r.currentFor(context.Background())
|
||||
}
|
||||
|
||||
// currentFor resolves like current, honoring a per-run profile pin set by the
|
||||
// engine for per-intent rotation (#85). A pin is only a dispatch preference:
|
||||
// the chain must still be in a ready state (an exhausted chain with no active
|
||||
// head stays exhausted) and the pinned id must belong to the task's chain.
|
||||
// A pin is only a dispatch preference: the chain must still be in a ready state
|
||||
// (an exhausted chain with no active head stays exhausted) and the pinned id must
|
||||
// belong to the task's chain. The returned profile id is non-zero only on the
|
||||
// task-chain path — streamTaskLLM uses it to decide whether a quota error should
|
||||
// advance the task's failover state (binding/global paths leave the chain state
|
||||
// untouched, preserving existing semantics).
|
||||
func (r *taskLLMRuntime) currentFor(ctx context.Context) (taskLLMSelection, error) {
|
||||
taskNum, err := parseTaskID(r.taskID)
|
||||
if err != nil {
|
||||
|
||||
@@ -656,7 +656,7 @@ func TestProfileDeleteRestoresQuotaBlockedIntentWhenFallbackAvailable(t *testing
|
||||
if err != nil {
|
||||
db.SkipPGUnavailable(t, err)
|
||||
}
|
||||
defer m.Close()
|
||||
t.Cleanup(func() { _ = m.Close() })
|
||||
|
||||
profileID, err := m.pg.SaveProfile(&db.LLMProfile{
|
||||
Name: fmt.Sprintf("delete-quota-fallback-%d", time.Now().UnixNano()),
|
||||
|
||||
@@ -22,7 +22,10 @@ func TestTaskOutcomeCard(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
|
||||
defer func() { _ = m.pg.DeleteTask(taskID) }()
|
||||
defer func() {
|
||||
_, _ = m.pg.DeleteFindingsByTask(taskID)
|
||||
_ = m.pg.DeleteTask(taskID)
|
||||
}()
|
||||
// skill_usage has no foreign keys, so DeleteTask leaves these rows behind,
|
||||
// and their misses would outrank TestSkillGapsReport's fixture in later runs.
|
||||
defer func() { _, _ = m.pg.Exec(`DELETE FROM skill_usage WHERE task_id=$1`, taskID) }()
|
||||
|
||||
@@ -42,7 +42,7 @@ func (s *Server) resolutionFromProfile(p *db.LLMProfile, source string) taskLLMR
|
||||
return result
|
||||
}
|
||||
|
||||
// resolveTaskRoleLLM mirrors taskLLMRuntime.current without creating a provider:
|
||||
// resolveTaskRoleLLM mirrors taskLLMRuntime.currentFor without creating a provider:
|
||||
// role Agent binding -> explicit task chain -> active/global environment config.
|
||||
// Database failures are returned rather than represented as an unavailable model.
|
||||
func (s *Server) resolveTaskRoleLLM(t *Task, agentKey string) (taskLLMResolution, error) {
|
||||
|
||||
@@ -91,7 +91,7 @@ func TestConversationPatchReturnsPinState(t *testing.T) {
|
||||
if err != nil {
|
||||
db.SkipPGUnavailable(t, err)
|
||||
}
|
||||
defer m.Close()
|
||||
t.Cleanup(func() { _ = m.Close() })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
|
||||
defer stopTestServer(t, s, cancel)
|
||||
|
||||
@@ -33,6 +33,14 @@ func meterTool(t actool.CoreTool, recorder toolUsageRecorder, toolKey, agentKey
|
||||
return &meteredTool{CoreTool: t, recorder: recorder, toolKey: toolKey, agentKey: agentKey, ri: ri}
|
||||
}
|
||||
|
||||
// AcceptsRawInput forwards the resolved tool's opt-out of the harness schema
|
||||
// check; the embedded interface alone would hide it from the harness's type
|
||||
// assertion.
|
||||
func (m *meteredTool) AcceptsRawInput() bool {
|
||||
r, ok := m.CoreTool.(interface{ AcceptsRawInput() bool })
|
||||
return ok && r.AcceptsRawInput()
|
||||
}
|
||||
|
||||
func (m *meteredTool) Call(ctx context.Context, input json.RawMessage, tc *actool.ToolContext) (actool.Result, error) {
|
||||
if err := m.recorder.InsertToolUsage(&db.ToolUsage{
|
||||
ToolKey: m.toolKey,
|
||||
|
||||
@@ -73,12 +73,19 @@ func TestWireTools(t *testing.T) {
|
||||
// Edit description + add a default to a scalar param, then confirm the resolved
|
||||
// worker tool reflects both. Restore from code default on cleanup.
|
||||
t.Cleanup(func() { _ = pg.UpsertToolForce(rf.Key, rf.Description, rf.Schema, mustJSON(rf.Agents)) })
|
||||
// Facts-only contract: the fact fields, and so the
|
||||
// editable scalar annotations, live on the facts[] items.
|
||||
var schema map[string]any
|
||||
_ = json.Unmarshal(rf.Schema, &schema)
|
||||
if props, ok := schema["properties"].(map[string]any); ok {
|
||||
if conf, ok := props["confidence"].(map[string]any); ok {
|
||||
conf["default"] = "inferred"
|
||||
}
|
||||
itemProps := func(sc map[string]any) map[string]any {
|
||||
props, _ := sc["properties"].(map[string]any)
|
||||
facts, _ := props["facts"].(map[string]any)
|
||||
items, _ := facts["items"].(map[string]any)
|
||||
ip, _ := items["properties"].(map[string]any)
|
||||
return ip
|
||||
}
|
||||
if conf, ok := itemProps(schema)["confidence"].(map[string]any); ok {
|
||||
conf["default"] = "inferred"
|
||||
}
|
||||
edited := mustJSON(schema)
|
||||
if err := pg.UpdateTool(rf.Key, "EDITED DESC", edited, mustJSON([]string{"worker"}), true); err != nil {
|
||||
@@ -96,9 +103,8 @@ func TestWireTools(t *testing.T) {
|
||||
// Default injection: omit confidence → handler input should carry the default.
|
||||
// We can't run the real handler (nil store), but InputSchema must show the default.
|
||||
sc := got.InputSchema()
|
||||
props := sc["properties"].(map[string]any)
|
||||
conf := props["confidence"].(map[string]any)
|
||||
if conf["default"] != "inferred" {
|
||||
conf, _ := itemProps(sc)["confidence"].(map[string]any)
|
||||
if conf == nil || conf["default"] != "inferred" {
|
||||
t.Errorf("confidence.default = %v, want inferred", conf["default"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/traffic"
|
||||
"github.com/Autumn-27/artex/traffic/rewrite"
|
||||
)
|
||||
|
||||
// --- capture-proxy constrained Go tampering hooks (P4.2) ---
|
||||
// Global configuration, operator/admin only, never exposed as an agent tool.
|
||||
// Execution is opt-in behind ARTEX_TAMPER_HOOKS_ENABLED=1 (off by default).
|
||||
|
||||
type tamperHookReq struct {
|
||||
Name string `json:"name"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Phase rewrite.Phase `json:"phase"`
|
||||
Priority int `json:"priority"`
|
||||
Matchers rewrite.Matchers `json:"matchers"`
|
||||
Source string `json:"source"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
type tamperHookDTO struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Phase string `json:"phase"`
|
||||
Priority int `json:"priority"`
|
||||
Matchers rewrite.Matchers `json:"matchers"`
|
||||
Source string `json:"source"`
|
||||
SourceSHA256 string `json:"source_sha256"`
|
||||
Version int `json:"version"`
|
||||
Note string `json:"note"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
LoadError string `json:"load_error,omitempty"`
|
||||
LastError *rewrite.LastError `json:"last_error,omitempty"`
|
||||
}
|
||||
|
||||
type tamperHookVersionDTO struct {
|
||||
ID int64 `json:"id"`
|
||||
HookID int64 `json:"hook_id"`
|
||||
Version int `json:"version"`
|
||||
Name string `json:"name"`
|
||||
Phase string `json:"phase"`
|
||||
Priority int `json:"priority"`
|
||||
Matchers rewrite.Matchers `json:"matchers"`
|
||||
Source string `json:"source"`
|
||||
SourceSHA256 string `json:"source_sha256"`
|
||||
Note string `json:"note"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type tamperDryRunReq struct {
|
||||
HookID int64 `json:"hook_id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Phase rewrite.Phase `json:"phase,omitempty"`
|
||||
Matchers rewrite.Matchers `json:"matchers"`
|
||||
Source string `json:"source,omitempty"`
|
||||
Request struct {
|
||||
Method string `json:"method"`
|
||||
URL string `json:"url"`
|
||||
Headers map[string][]string `json:"headers"`
|
||||
Body string `json:"body"`
|
||||
} `json:"request"`
|
||||
Response struct {
|
||||
Status int `json:"status"`
|
||||
Headers map[string][]string `json:"headers"`
|
||||
Body string `json:"body"`
|
||||
} `json:"response"`
|
||||
}
|
||||
|
||||
var errTamperHookInvalid = errors.New("invalid tampering hook")
|
||||
|
||||
func tamperHookInvalid(err error) error { return fmt.Errorf("%w: %v", errTamperHookInvalid, err) }
|
||||
|
||||
func tamperHookWriteErr(w http.ResponseWriter, err error) {
|
||||
switch {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
writeErr(w, 404, "tampering hook not found")
|
||||
case errors.Is(err, errTamperHookInvalid), errors.Is(err, db.ErrTrafficTamperHookLimit), errors.Is(err, db.ErrTrafficTamperHookEnabledLimit):
|
||||
writeErr(w, 400, err.Error())
|
||||
default:
|
||||
writeErr(w, 500, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func decodeTamperHookRow(row db.TrafficTamperHook) (traffic.HookDef, error) {
|
||||
var m rewrite.Matchers
|
||||
if len(row.Matchers) > 0 {
|
||||
if err := json.Unmarshal(row.Matchers, &m); err != nil {
|
||||
return traffic.HookDef{}, err
|
||||
}
|
||||
}
|
||||
return traffic.HookDef{
|
||||
ID: row.ID, Name: row.Name, Enabled: row.Enabled, Phase: rewrite.Phase(row.Phase),
|
||||
Priority: row.Priority, Matchers: m, Source: row.Source, SourceSHA256: row.SourceSHA256,
|
||||
Version: row.Version, Note: row.Note,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Manager) tamperHookDTO(row db.TrafficTamperHook) tamperHookDTO {
|
||||
var matchers rewrite.Matchers
|
||||
_ = json.Unmarshal(row.Matchers, &matchers)
|
||||
out := tamperHookDTO{
|
||||
ID: row.ID, Name: row.Name, Enabled: row.Enabled, Phase: row.Phase,
|
||||
Priority: row.Priority, Matchers: matchers, Source: row.Source,
|
||||
SourceSHA256: row.SourceSHA256, Version: row.Version, Note: row.Note,
|
||||
CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt,
|
||||
}
|
||||
m.hookMu.Lock()
|
||||
if m.hookLoadErrs != nil {
|
||||
out.LoadError = m.hookLoadErrs[row.ID]
|
||||
}
|
||||
m.hookMu.Unlock()
|
||||
if hs := m.hookSet.Load(); hs != nil {
|
||||
out.LastError = hs.LastError(row.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reloadTamperHooks compiles enabled hooks into an immutable snapshot and swaps
|
||||
// it onto the capture proxy. When ARTEX_TAMPER_HOOKS_ENABLED is off (default),
|
||||
// zero hooks are loaded and the proxy receives nil (baseline fast path).
|
||||
func (m *Manager) reloadTamperHooks() {
|
||||
m.hookMu.Lock()
|
||||
defer m.hookMu.Unlock()
|
||||
if !traffic.TamperHooksEnabledFromEnv() {
|
||||
m.hookLoadErrs = nil
|
||||
m.hookSet.Store(nil)
|
||||
if m.traffic != nil {
|
||||
m.traffic.SetTamperHooks(nil)
|
||||
}
|
||||
return
|
||||
}
|
||||
rows, err := m.pg.ListTrafficTamperHooks()
|
||||
if err != nil {
|
||||
log.Printf("[traffic] tampering hooks not reloaded: %v", err)
|
||||
return
|
||||
}
|
||||
defs := make([]traffic.HookDef, 0, len(rows))
|
||||
decodeErrs := map[int64]string{}
|
||||
for _, row := range rows {
|
||||
d, err := decodeTamperHookRow(row)
|
||||
if err != nil {
|
||||
if row.Enabled {
|
||||
decodeErrs[row.ID] = "the stored hook cannot be decoded"
|
||||
}
|
||||
continue
|
||||
}
|
||||
defs = append(defs, d)
|
||||
}
|
||||
hs, loadErrs := traffic.CompileHooks(defs, db.TrafficTraceKey())
|
||||
for id, msg := range decodeErrs {
|
||||
loadErrs[id] = msg
|
||||
}
|
||||
for id, msg := range loadErrs {
|
||||
log.Printf("[traffic] tampering hook %d not loaded: %s", id, msg)
|
||||
}
|
||||
m.hookLoadErrs = loadErrs
|
||||
m.hookSet.Store(hs)
|
||||
if m.traffic != nil {
|
||||
m.traffic.SetTamperHooks(hs)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) tamperListHooks(w http.ResponseWriter, r *http.Request) {
|
||||
pg := s.pg(w)
|
||||
if pg == nil {
|
||||
return
|
||||
}
|
||||
rows, err := pg.ListTrafficTamperHooks()
|
||||
if err != nil {
|
||||
writeErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
hooks := make([]tamperHookDTO, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
hooks = append(hooks, s.m.tamperHookDTO(row))
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{
|
||||
"hooks": hooks,
|
||||
"enabled_by_env": traffic.TamperHooksEnabledFromEnv(),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) tamperCreateHook(w http.ResponseWriter, r *http.Request) {
|
||||
pg := s.pg(w)
|
||||
if pg == nil {
|
||||
return
|
||||
}
|
||||
var req tamperHookReq
|
||||
if err := decode(w, r, &req); err != nil {
|
||||
writeErr(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
def := traffic.HookDef{
|
||||
Name: req.Name, Enabled: false, Phase: req.Phase, Priority: req.Priority,
|
||||
Matchers: req.Matchers, Source: req.Source, Note: req.Note,
|
||||
}
|
||||
if err := traffic.ValidateHookDef(def); err != nil {
|
||||
tamperHookWriteErr(w, tamperHookInvalid(err))
|
||||
return
|
||||
}
|
||||
matchersJSON, _ := json.Marshal(req.Matchers)
|
||||
row, err := pg.CreateTrafficTamperHook(db.TrafficTamperHook{
|
||||
Name: req.Name, Enabled: false, Phase: string(req.Phase), Priority: req.Priority,
|
||||
Matchers: matchersJSON, Source: req.Source, Note: req.Note,
|
||||
})
|
||||
if err != nil {
|
||||
tamperHookWriteErr(w, err)
|
||||
return
|
||||
}
|
||||
s.m.reloadTamperHooks()
|
||||
writeJSON(w, 200, s.m.tamperHookDTO(row))
|
||||
}
|
||||
|
||||
func (s *Server) tamperUpdateHook(w http.ResponseWriter, r *http.Request) {
|
||||
pg := s.pg(w)
|
||||
if pg == nil {
|
||||
return
|
||||
}
|
||||
id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
|
||||
if err != nil || id <= 0 {
|
||||
writeErr(w, 400, "invalid hook id")
|
||||
return
|
||||
}
|
||||
var req tamperHookReq
|
||||
if err := decode(w, r, &req); err != nil {
|
||||
writeErr(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
def := traffic.HookDef{
|
||||
ID: id, Name: req.Name, Enabled: req.Enabled, Phase: req.Phase, Priority: req.Priority,
|
||||
Matchers: req.Matchers, Source: req.Source, Note: req.Note,
|
||||
}
|
||||
if err := traffic.ValidateHookDef(def); err != nil {
|
||||
tamperHookWriteErr(w, tamperHookInvalid(err))
|
||||
return
|
||||
}
|
||||
matchersJSON, _ := json.Marshal(req.Matchers)
|
||||
row, err := pg.UpdateTrafficTamperHook(id, func(stored db.TrafficTamperHook) (db.TrafficTamperHook, error) {
|
||||
return db.TrafficTamperHook{
|
||||
Name: req.Name, Enabled: req.Enabled, Phase: string(req.Phase), Priority: req.Priority,
|
||||
Matchers: matchersJSON, Source: req.Source, Note: req.Note,
|
||||
}, nil
|
||||
})
|
||||
if err != nil {
|
||||
tamperHookWriteErr(w, err)
|
||||
return
|
||||
}
|
||||
s.m.reloadTamperHooks()
|
||||
writeJSON(w, 200, s.m.tamperHookDTO(row))
|
||||
}
|
||||
|
||||
func (s *Server) tamperDeleteHook(w http.ResponseWriter, r *http.Request) {
|
||||
pg := s.pg(w)
|
||||
if pg == nil {
|
||||
return
|
||||
}
|
||||
id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
|
||||
if err != nil || id <= 0 {
|
||||
writeErr(w, 400, "invalid hook id")
|
||||
return
|
||||
}
|
||||
if err := pg.DeleteTrafficTamperHook(id); err != nil {
|
||||
tamperHookWriteErr(w, err)
|
||||
return
|
||||
}
|
||||
s.m.reloadTamperHooks()
|
||||
writeJSON(w, 200, map[string]any{"deleted": true})
|
||||
}
|
||||
|
||||
func (s *Server) tamperToggleHook(w http.ResponseWriter, r *http.Request) {
|
||||
pg := s.pg(w)
|
||||
if pg == nil {
|
||||
return
|
||||
}
|
||||
id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
|
||||
if err != nil || id <= 0 {
|
||||
writeErr(w, 400, "invalid hook id")
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
if err := decode(w, r, &body); err != nil {
|
||||
writeErr(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
if err := pg.ToggleTrafficTamperHook(id, body.Enabled); err != nil {
|
||||
tamperHookWriteErr(w, err)
|
||||
return
|
||||
}
|
||||
row, err := pg.GetTrafficTamperHook(id)
|
||||
if err != nil {
|
||||
tamperHookWriteErr(w, err)
|
||||
return
|
||||
}
|
||||
s.m.reloadTamperHooks()
|
||||
writeJSON(w, 200, s.m.tamperHookDTO(row))
|
||||
}
|
||||
|
||||
func (s *Server) tamperListHookVersions(w http.ResponseWriter, r *http.Request) {
|
||||
pg := s.pg(w)
|
||||
if pg == nil {
|
||||
return
|
||||
}
|
||||
id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
|
||||
if err != nil || id <= 0 {
|
||||
writeErr(w, 400, "invalid hook id")
|
||||
return
|
||||
}
|
||||
if _, err := pg.GetTrafficTamperHook(id); err != nil {
|
||||
tamperHookWriteErr(w, err)
|
||||
return
|
||||
}
|
||||
vers, err := pg.ListTrafficTamperHookVersions(id)
|
||||
if err != nil {
|
||||
writeErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
out := make([]tamperHookVersionDTO, 0, len(vers))
|
||||
for _, v := range vers {
|
||||
var m rewrite.Matchers
|
||||
_ = json.Unmarshal(v.Matchers, &m)
|
||||
out = append(out, tamperHookVersionDTO{
|
||||
ID: v.ID, HookID: v.HookID, Version: v.Version, Name: v.Name,
|
||||
Phase: v.Phase, Priority: v.Priority, Matchers: m, Source: v.Source,
|
||||
SourceSHA256: v.SourceSHA256, Note: v.Note, CreatedAt: v.CreatedAt,
|
||||
})
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"versions": out})
|
||||
}
|
||||
|
||||
func (s *Server) tamperRollbackHook(w http.ResponseWriter, r *http.Request) {
|
||||
pg := s.pg(w)
|
||||
if pg == nil {
|
||||
return
|
||||
}
|
||||
id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
|
||||
if err != nil || id <= 0 {
|
||||
writeErr(w, 400, "invalid hook id")
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Version int `json:"version"`
|
||||
}
|
||||
if err := decode(w, r, &body); err != nil {
|
||||
writeErr(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
if body.Version <= 0 {
|
||||
writeErr(w, 400, "version must be > 0")
|
||||
return
|
||||
}
|
||||
row, err := pg.RollbackTrafficTamperHook(id, body.Version)
|
||||
if err != nil {
|
||||
tamperHookWriteErr(w, err)
|
||||
return
|
||||
}
|
||||
s.m.reloadTamperHooks()
|
||||
writeJSON(w, 200, s.m.tamperHookDTO(row))
|
||||
}
|
||||
|
||||
func (s *Server) tamperDryRunHook(w http.ResponseWriter, r *http.Request) {
|
||||
pg := s.pg(w)
|
||||
if pg == nil {
|
||||
return
|
||||
}
|
||||
var req tamperDryRunReq
|
||||
if err := decode(w, r, &req); err != nil {
|
||||
writeErr(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
var def traffic.HookDef
|
||||
if req.HookID > 0 {
|
||||
row, err := pg.GetTrafficTamperHook(req.HookID)
|
||||
if err != nil {
|
||||
tamperHookWriteErr(w, err)
|
||||
return
|
||||
}
|
||||
def, err = decodeTamperHookRow(row)
|
||||
if err != nil {
|
||||
writeErr(w, 400, "stored hook cannot be decoded")
|
||||
return
|
||||
}
|
||||
} else {
|
||||
name := req.Name
|
||||
if name == "" {
|
||||
name = "dry-run"
|
||||
}
|
||||
def = traffic.HookDef{
|
||||
Name: name, Phase: req.Phase, Matchers: req.Matchers, Source: req.Source, Version: 1,
|
||||
}
|
||||
}
|
||||
|
||||
rawURL := req.Request.URL
|
||||
if rawURL == "" {
|
||||
rawURL = "https://example.test/"
|
||||
}
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
writeErr(w, 400, "invalid request.url: "+err.Error())
|
||||
return
|
||||
}
|
||||
method := req.Request.Method
|
||||
if method == "" {
|
||||
method = http.MethodGet
|
||||
}
|
||||
status := req.Response.Status
|
||||
if status == 0 {
|
||||
status = http.StatusOK
|
||||
}
|
||||
reqHdr := http.Header{}
|
||||
for k, vs := range req.Request.Headers {
|
||||
reqHdr[http.CanonicalHeaderKey(k)] = append([]string(nil), vs...)
|
||||
}
|
||||
respHdr := http.Header{}
|
||||
for k, vs := range req.Response.Headers {
|
||||
respHdr[http.CanonicalHeaderKey(k)] = append([]string(nil), vs...)
|
||||
}
|
||||
|
||||
res, err := traffic.DryRunHook(def, rewrite.Input{
|
||||
Request: rewrite.Request{Method: method, URL: u, Header: reqHdr, Body: []byte(req.Request.Body)},
|
||||
Response: rewrite.Response{Status: status, Header: respHdr, Body: []byte(req.Response.Body)},
|
||||
}, db.TrafficTraceKey())
|
||||
if err != nil {
|
||||
tamperHookWriteErr(w, tamperHookInvalid(err))
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{
|
||||
"changed": res.Changed,
|
||||
"request": map[string]any{
|
||||
"method": res.Request.Method,
|
||||
"url": res.Request.URL.String(),
|
||||
"headers": res.Request.Header,
|
||||
"body": string(res.Request.Body),
|
||||
},
|
||||
"response": map[string]any{
|
||||
"status": res.Response.Status,
|
||||
"headers": res.Response.Header,
|
||||
"body": string(res.Response.Body),
|
||||
},
|
||||
"lines": res.Lines,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/traffic"
|
||||
)
|
||||
|
||||
func TestTamperHooksAPIValidationKillSwitchCRUDVersionsRollbackDryRun(t *testing.T) {
|
||||
t.Setenv(traffic.TamperHooksEnabledEnv, "")
|
||||
m, err := NewManager(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
db.SkipPGUnavailable(t, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = m.Close() })
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
|
||||
token, err := signJWT(s.jwtKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
do := func(method, path string, body any) (int, map[string]any) {
|
||||
t.Helper()
|
||||
var rdr *bytes.Reader
|
||||
if body != nil {
|
||||
b, _ := json.Marshal(body)
|
||||
rdr = bytes.NewReader(b)
|
||||
} else {
|
||||
rdr = bytes.NewReader(nil)
|
||||
}
|
||||
req := httptest.NewRequest(method, path, rdr)
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
rec := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rec, req)
|
||||
var out map[string]any
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &out)
|
||||
return rec.Code, out
|
||||
}
|
||||
|
||||
// 1. Forbidden constructs/imports rejected at AST validation with HTTP 400.
|
||||
forbidden := []string{
|
||||
`package hook; import "os"; func OnRequest(req *Request) error { _ = os.Getenv("X"); return nil }`,
|
||||
`package hook; import "io/fs"; func OnRequest(req *Request) error { return nil }`,
|
||||
`package hook; import "net"; func OnRequest(req *Request) error { return nil }`,
|
||||
`package hook; import "os/exec"; func OnRequest(req *Request) error { return nil }`,
|
||||
`package hook; import "syscall"; func OnRequest(req *Request) error { return nil }`,
|
||||
`package hook; import "unsafe"; func OnRequest(req *Request) error { return nil }`,
|
||||
`package hook; import "reflect"; func OnRequest(req *Request) error { return nil }`,
|
||||
`package hook; import "time"; func OnRequest(req *Request) error { _ = time.Now(); return nil }`,
|
||||
`package hook; import "math/rand"; func OnRequest(req *Request) error { _ = rand.Int(); return nil }`,
|
||||
`package hook; func f(){}; func OnRequest(req *Request) error { go f(); return nil }`,
|
||||
`package hook; func OnRequest(req *Request) error { var c chan int; _ = c; return nil }`,
|
||||
`package hook; func OnRequest(req *Request) error { select {}; return nil }`,
|
||||
`package hook; func f(){}; func OnRequest(req *Request) error { defer f(); return nil }`,
|
||||
`package hook; func OnRequest(req *Request) error { goto end; end: return nil }`,
|
||||
`package hook; func init() {}; func OnRequest(req *Request) error { return nil }`,
|
||||
`package hook; var g = 1; func OnRequest(req *Request) error { g++; return nil }`,
|
||||
}
|
||||
for i, src := range forbidden {
|
||||
code, _ := do("POST", "/api/traffic/hooks", map[string]any{
|
||||
"name": fmt.Sprintf("bad-%d", i), "phase": "request", "source": src,
|
||||
})
|
||||
if code != 400 {
|
||||
t.Fatalf("forbidden hook %d returned HTTP %d, want 400", i, code)
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Create valid hook (always created disabled, version 1).
|
||||
const srcV1 = `package hook
|
||||
func OnRequest(req *Request) error {
|
||||
req.SetHeader("X-Ver", "1")
|
||||
return nil
|
||||
}`
|
||||
const srcV2 = `package hook
|
||||
func OnRequest(req *Request) error {
|
||||
req.SetHeader("X-Ver", "2")
|
||||
return nil
|
||||
}`
|
||||
code, created := do("POST", "/api/traffic/hooks", map[string]any{
|
||||
"name": "p42-api-hook", "enabled": true, "phase": "request", "priority": 5,
|
||||
"matchers": map[string]any{"host": "api.example.test"},
|
||||
"source": srcV1, "note": "v1",
|
||||
})
|
||||
if code != 200 {
|
||||
t.Fatalf("create hook HTTP %d: %v", code, created)
|
||||
}
|
||||
id := int64(created["id"].(float64))
|
||||
t.Cleanup(func() { _ = m.pg.DeleteTrafficTamperHook(id) })
|
||||
if created["enabled"] != false || created["version"].(float64) != 1 {
|
||||
t.Fatalf("expected created disabled v1, got %v", created)
|
||||
}
|
||||
|
||||
// 3. Toggle enabled while kill switch is OFF: zero hooks loaded onto proxy.
|
||||
code, toggled := do("POST", fmt.Sprintf("/api/traffic/hooks/%d/toggle", id), map[string]any{"enabled": true})
|
||||
if code != 200 || toggled["enabled"] != true {
|
||||
t.Fatalf("toggle HTTP %d: %v", code, toggled)
|
||||
}
|
||||
if m.traffic.TamperHooks() != nil {
|
||||
t.Fatal("expected nil TamperHooks on proxy when ARTEX_TAMPER_HOOKS_ENABLED is off")
|
||||
}
|
||||
|
||||
// 4. Enable kill switch and reload: snapshot now holds 1 request hook.
|
||||
t.Setenv(traffic.TamperHooksEnabledEnv, "1")
|
||||
m.reloadTamperHooks()
|
||||
if m.traffic.TamperHooks().Len("request") != 1 {
|
||||
t.Fatalf("expected 1 loaded request hook when kill switch on, got %d", m.traffic.TamperHooks().Len("request"))
|
||||
}
|
||||
|
||||
// 5. Update -> v2, list versions, rollback to v1 -> v3.
|
||||
code, upd := do("PUT", fmt.Sprintf("/api/traffic/hooks/%d", id), map[string]any{
|
||||
"name": "p42-api-hook", "enabled": true, "phase": "request", "priority": 5,
|
||||
"matchers": map[string]any{"host": "api.example.test"},
|
||||
"source": srcV2, "note": "v2",
|
||||
})
|
||||
if code != 200 || upd["version"].(float64) != 2 {
|
||||
t.Fatalf("update HTTP %d: %v", code, upd)
|
||||
}
|
||||
code, vers := do("GET", fmt.Sprintf("/api/traffic/hooks/%d/versions", id), nil)
|
||||
if code != 200 || len(vers["versions"].([]any)) != 2 {
|
||||
t.Fatalf("versions HTTP %d: %v", code, vers)
|
||||
}
|
||||
code, rb := do("POST", fmt.Sprintf("/api/traffic/hooks/%d/rollback", id), map[string]any{"version": 1})
|
||||
if code != 200 || rb["version"].(float64) != 3 || rb["source"] != srcV1 {
|
||||
t.Fatalf("rollback HTTP %d: %v", code, rb)
|
||||
}
|
||||
|
||||
// 6. Dry-run endpoint against stored hook_id.
|
||||
code, dry := do("POST", "/api/traffic/hooks/dry-run", map[string]any{
|
||||
"hook_id": id,
|
||||
"request": map[string]any{"method": "GET", "url": "https://api.example.test/v1/ping"},
|
||||
})
|
||||
if code != 200 || dry["changed"] != true {
|
||||
t.Fatalf("dry-run HTTP %d: %v", code, dry)
|
||||
}
|
||||
reqMap := dry["request"].(map[string]any)
|
||||
hdrs := reqMap["headers"].(map[string]any)
|
||||
if xver := hdrs["X-Ver"].([]any); len(xver) != 1 || xver[0] != "1" {
|
||||
t.Fatalf("dry-run X-Ver = %v, want [1]", xver)
|
||||
}
|
||||
|
||||
// 7. Delete cleans up and updates proxy snapshot.
|
||||
code, _ = do("DELETE", fmt.Sprintf("/api/traffic/hooks/%d", id), nil)
|
||||
if code != 200 {
|
||||
t.Fatalf("delete HTTP %d", code)
|
||||
}
|
||||
if m.traffic.TamperHooks().Len("request") != 0 {
|
||||
t.Fatalf("expected 0 loaded hooks after delete, got %d", m.traffic.TamperHooks().Len("request"))
|
||||
}
|
||||
}
|
||||
@@ -269,6 +269,7 @@ VALUES(?,?, 'HTTP/1.1 200 OK', ?, ?)`, id, "GET "+path+" HTTP/1.1\nHost: "+host,
|
||||
}
|
||||
tidV1, _ := strconv.ParseInt(taskV1.ID, 10, 64)
|
||||
t.Cleanup(func() {
|
||||
_, _ = s.m.pg.DeleteFindingsByTask(tidV1)
|
||||
_, _ = s.m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, tidV1)
|
||||
_ = s.m.pg.DeleteTask(tidV1)
|
||||
})
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/traffic"
|
||||
)
|
||||
|
||||
func TestManagerTransparentTCPOptInFlag(t *testing.T) {
|
||||
// 1. Flag off (default): Manager starts NO transparent TCP listener.
|
||||
t.Setenv(traffic.TransparentTCPEnabledEnv, "")
|
||||
t.Setenv(traffic.TransparentTCPAddrEnv, "")
|
||||
|
||||
mOff, err := NewManager(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
db.SkipPGUnavailable(t, err)
|
||||
}
|
||||
defer mOff.Close()
|
||||
if mOff.traffic == nil {
|
||||
t.Fatal("expected traffic recorder to be initialized")
|
||||
}
|
||||
if got := mOff.traffic.TransparentTCPAddr(); got != "" {
|
||||
t.Fatalf("TransparentTCPAddr with flag off = %q, want empty", got)
|
||||
}
|
||||
|
||||
// 2. Flag on (ARTEX_TRANSPARENT_TCP_ADDR=127.0.0.1:0): starts on Linux,
|
||||
// stays cleanly disabled on non-Linux without crashing NewManager.
|
||||
t.Setenv(traffic.TransparentTCPAddrEnv, "127.0.0.1:0")
|
||||
|
||||
mOn, err := NewManager(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
db.SkipPGUnavailable(t, err)
|
||||
}
|
||||
defer mOn.Close()
|
||||
|
||||
gotAddr := mOn.traffic.TransparentTCPAddr()
|
||||
if runtime.GOOS == "linux" {
|
||||
if gotAddr == "" {
|
||||
t.Fatal("expected transparent TCP listener on Linux when ARTEX_TRANSPARENT_TCP_ADDR is set")
|
||||
}
|
||||
} else {
|
||||
if gotAddr != "" {
|
||||
t.Fatalf("expected transparent TCP listener disabled on %s, got %q", runtime.GOOS, gotAddr)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -69,16 +69,6 @@ func workspacePathDenied(realRoot, realPath string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// wsResolve maps a user-supplied relative path to an absolute path INSIDE the work
|
||||
// dir. It returns ok=false if the path would escape the root (including via symlinks).
|
||||
func (s *Server) wsResolve(rel string) (string, bool) {
|
||||
cp, ok := s.wsResolveConfined(rel)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
return cp.realPath, true
|
||||
}
|
||||
|
||||
// wsRel renders an absolute path back as a workspace-relative path (forward slashes).
|
||||
func (s *Server) wsRel(abs string) string {
|
||||
base, err := resolveRealRoot(s.m.dir)
|
||||
|
||||
+85
-85
@@ -13,7 +13,7 @@
|
||||
},
|
||||
"node_modules/@puppeteer/browsers": {
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmmirror.com/@puppeteer/browsers/-/browsers-2.6.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.6.1.tgz",
|
||||
"integrity": "sha512-aBSREisdsGH890S2rQqK82qmQYU3uFpSH8wcZWHgHzl3LfzsxAKbLNiAG9mO8v1Y0UICBeClICxPJvyr0rcuxg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
@@ -35,13 +35,13 @@
|
||||
},
|
||||
"node_modules/@tootallnate/quickjs-emscripten": {
|
||||
"version": "0.23.0",
|
||||
"resolved": "https://registry.npmmirror.com/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz",
|
||||
"integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "26.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/@types/node/-/node-26.0.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-26.0.1.tgz",
|
||||
"integrity": "sha512-fc3KiUoBt6kie0N9bIW3E47vZsuaMf0PM2AaUpLCLT0s/LvX1nxAim6Fc049cNxODPpGm6qRAuUOB86SkRuPQw==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
@@ -51,7 +51,7 @@
|
||||
},
|
||||
"node_modules/@types/yauzl": {
|
||||
"version": "2.10.3",
|
||||
"resolved": "https://registry.npmmirror.com/@types/yauzl/-/yauzl-2.10.3.tgz",
|
||||
"resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz",
|
||||
"integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
@@ -61,7 +61,7 @@
|
||||
},
|
||||
"node_modules/agent-base": {
|
||||
"version": "7.1.4",
|
||||
"resolved": "https://registry.npmmirror.com/agent-base/-/agent-base-7.1.4.tgz",
|
||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz",
|
||||
"integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -70,7 +70,7 @@
|
||||
},
|
||||
"node_modules/ansi-regex": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-5.0.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
|
||||
"integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -79,7 +79,7 @@
|
||||
},
|
||||
"node_modules/ansi-styles": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-4.3.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
|
||||
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -94,7 +94,7 @@
|
||||
},
|
||||
"node_modules/ast-types": {
|
||||
"version": "0.13.4",
|
||||
"resolved": "https://registry.npmmirror.com/ast-types/-/ast-types-0.13.4.tgz",
|
||||
"resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz",
|
||||
"integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -106,7 +106,7 @@
|
||||
},
|
||||
"node_modules/b4a": {
|
||||
"version": "1.8.1",
|
||||
"resolved": "https://registry.npmmirror.com/b4a/-/b4a-1.8.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.1.tgz",
|
||||
"integrity": "sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==",
|
||||
"license": "Apache-2.0",
|
||||
"peerDependencies": {
|
||||
@@ -120,7 +120,7 @@
|
||||
},
|
||||
"node_modules/bare-events": {
|
||||
"version": "2.9.1",
|
||||
"resolved": "https://registry.npmmirror.com/bare-events/-/bare-events-2.9.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.9.1.tgz",
|
||||
"integrity": "sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==",
|
||||
"license": "Apache-2.0",
|
||||
"peerDependencies": {
|
||||
@@ -134,7 +134,7 @@
|
||||
},
|
||||
"node_modules/bare-fs": {
|
||||
"version": "4.7.2",
|
||||
"resolved": "https://registry.npmmirror.com/bare-fs/-/bare-fs-4.7.2.tgz",
|
||||
"resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.7.2.tgz",
|
||||
"integrity": "sha512-aTvMFUWkBmjzKtEQMDGGDNF8bkfpD5N1b/FCwt7A3wrU4t1o/e/85Wzkluh6JlODCjqVESYCkQCdTXqZ9G7VFg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
@@ -158,7 +158,7 @@
|
||||
},
|
||||
"node_modules/bare-os": {
|
||||
"version": "3.9.3",
|
||||
"resolved": "https://registry.npmmirror.com/bare-os/-/bare-os-3.9.3.tgz",
|
||||
"resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.9.3.tgz",
|
||||
"integrity": "sha512-fF4Q7QsyKVF5Rj0qvI8BgUNjqzC2JvQlpTaPLjVJVxYVUX5Zr9un+y3w1HmA4nNKdFmRBT8z/WmrjvXzXVerKQ==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
@@ -167,7 +167,7 @@
|
||||
},
|
||||
"node_modules/bare-path": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/bare-path/-/bare-path-3.0.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.1.tgz",
|
||||
"integrity": "sha512-ghj2DSK/2e99a1anTVPCV4m4YIYtrbXhfM7V3D7XZLOTsybnYyaJloymGqssQc8l/or0UoDyRtNQkmkEF/ysgQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
@@ -176,7 +176,7 @@
|
||||
},
|
||||
"node_modules/bare-stream": {
|
||||
"version": "2.13.3",
|
||||
"resolved": "https://registry.npmmirror.com/bare-stream/-/bare-stream-2.13.3.tgz",
|
||||
"resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.13.3.tgz",
|
||||
"integrity": "sha512-Kc+brLqvEqGkjyfiwJmImAOqLZL7OsoLKuavx+hJjgVV3nLTOjloJyPMFxjUPerGGHrNH0fLU06jjykMLWrERQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
@@ -203,7 +203,7 @@
|
||||
},
|
||||
"node_modules/bare-url": {
|
||||
"version": "2.4.5",
|
||||
"resolved": "https://registry.npmmirror.com/bare-url/-/bare-url-2.4.5.tgz",
|
||||
"resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.4.5.tgz",
|
||||
"integrity": "sha512-K+y9xF1tN+CdPu4qWwr0QiK1Al07eFPGYK5M2pDXcmHdMdgC/tT/bpmMe1hrmRHaidKLkXrC+cRNYf3XVDUhSQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
@@ -212,7 +212,7 @@
|
||||
},
|
||||
"node_modules/base64-js": {
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmmirror.com/base64-js/-/base64-js-1.5.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
|
||||
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
|
||||
"funding": [
|
||||
{
|
||||
@@ -232,7 +232,7 @@
|
||||
},
|
||||
"node_modules/basic-ftp": {
|
||||
"version": "5.3.1",
|
||||
"resolved": "https://registry.npmmirror.com/basic-ftp/-/basic-ftp-5.3.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.3.1.tgz",
|
||||
"integrity": "sha512-bopVNp6ugyA150DDuZfPFdt1KZ5a94ZDiwX4hMgZDzF+GttD80lEy8kj98kbyhLXnPvhtIo93mdnLIjpCAeeOw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -241,7 +241,7 @@
|
||||
},
|
||||
"node_modules/buffer": {
|
||||
"version": "5.7.1",
|
||||
"resolved": "https://registry.npmmirror.com/buffer/-/buffer-5.7.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
|
||||
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
|
||||
"funding": [
|
||||
{
|
||||
@@ -265,7 +265,7 @@
|
||||
},
|
||||
"node_modules/buffer-crc32": {
|
||||
"version": "0.2.13",
|
||||
"resolved": "https://registry.npmmirror.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
|
||||
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
|
||||
"integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -274,7 +274,7 @@
|
||||
},
|
||||
"node_modules/chromium-bidi": {
|
||||
"version": "0.11.0",
|
||||
"resolved": "https://registry.npmmirror.com/chromium-bidi/-/chromium-bidi-0.11.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.11.0.tgz",
|
||||
"integrity": "sha512-6CJWHkNRoyZyjV9Rwv2lYONZf1Xm0IuDyNq97nwSsxxP3wf5Bwy15K5rOvVKMtJ127jJBmxFUanSAOjgFRxgrA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
@@ -287,7 +287,7 @@
|
||||
},
|
||||
"node_modules/cliui": {
|
||||
"version": "8.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/cliui/-/cliui-8.0.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
|
||||
"integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
@@ -301,7 +301,7 @@
|
||||
},
|
||||
"node_modules/color-convert": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-2.0.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
|
||||
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -313,13 +313,13 @@
|
||||
},
|
||||
"node_modules/color-name": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.4.tgz",
|
||||
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/data-uri-to-buffer": {
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmmirror.com/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz",
|
||||
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz",
|
||||
"integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -328,7 +328,7 @@
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "4.4.3",
|
||||
"resolved": "https://registry.npmmirror.com/debug/-/debug-4.4.3.tgz",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
|
||||
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -345,7 +345,7 @@
|
||||
},
|
||||
"node_modules/degenerator": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/degenerator/-/degenerator-5.0.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz",
|
||||
"integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -359,19 +359,19 @@
|
||||
},
|
||||
"node_modules/devtools-protocol": {
|
||||
"version": "0.0.1367902",
|
||||
"resolved": "https://registry.npmmirror.com/devtools-protocol/-/devtools-protocol-0.0.1367902.tgz",
|
||||
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1367902.tgz",
|
||||
"integrity": "sha512-XxtPuC3PGakY6PD7dG66/o8KwJ/LkH2/EKe19Dcw58w53dv4/vSQEkn/SzuyhHE2q4zPgCkxQBxus3VV4ql+Pg==",
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/emoji-regex": {
|
||||
"version": "8.0.0",
|
||||
"resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-8.0.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
|
||||
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/end-of-stream": {
|
||||
"version": "1.4.5",
|
||||
"resolved": "https://registry.npmmirror.com/end-of-stream/-/end-of-stream-1.4.5.tgz",
|
||||
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz",
|
||||
"integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -380,7 +380,7 @@
|
||||
},
|
||||
"node_modules/escalade": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmmirror.com/escalade/-/escalade-3.2.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
|
||||
"integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -389,7 +389,7 @@
|
||||
},
|
||||
"node_modules/escodegen": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmmirror.com/escodegen/-/escodegen-2.1.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz",
|
||||
"integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
@@ -410,7 +410,7 @@
|
||||
},
|
||||
"node_modules/esprima": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/esprima/-/esprima-4.0.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
|
||||
"integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
|
||||
"license": "BSD-2-Clause",
|
||||
"bin": {
|
||||
@@ -423,7 +423,7 @@
|
||||
},
|
||||
"node_modules/estraverse": {
|
||||
"version": "5.3.0",
|
||||
"resolved": "https://registry.npmmirror.com/estraverse/-/estraverse-5.3.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
|
||||
"integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
@@ -432,7 +432,7 @@
|
||||
},
|
||||
"node_modules/esutils": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmmirror.com/esutils/-/esutils-2.0.3.tgz",
|
||||
"resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
|
||||
"integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
@@ -441,7 +441,7 @@
|
||||
},
|
||||
"node_modules/events-universal": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/events-universal/-/events-universal-1.0.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz",
|
||||
"integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
@@ -450,7 +450,7 @@
|
||||
},
|
||||
"node_modules/extract-zip": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/extract-zip/-/extract-zip-2.0.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz",
|
||||
"integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
@@ -470,13 +470,13 @@
|
||||
},
|
||||
"node_modules/fast-fifo": {
|
||||
"version": "1.3.2",
|
||||
"resolved": "https://registry.npmmirror.com/fast-fifo/-/fast-fifo-1.3.2.tgz",
|
||||
"resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz",
|
||||
"integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fd-slicer": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmmirror.com/fd-slicer/-/fd-slicer-1.1.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz",
|
||||
"integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -485,7 +485,7 @@
|
||||
},
|
||||
"node_modules/get-caller-file": {
|
||||
"version": "2.0.5",
|
||||
"resolved": "https://registry.npmmirror.com/get-caller-file/-/get-caller-file-2.0.5.tgz",
|
||||
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
|
||||
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
@@ -494,7 +494,7 @@
|
||||
},
|
||||
"node_modules/get-stream": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmmirror.com/get-stream/-/get-stream-5.2.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
|
||||
"integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -509,7 +509,7 @@
|
||||
},
|
||||
"node_modules/get-uri": {
|
||||
"version": "6.0.5",
|
||||
"resolved": "https://registry.npmmirror.com/get-uri/-/get-uri-6.0.5.tgz",
|
||||
"resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz",
|
||||
"integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -523,7 +523,7 @@
|
||||
},
|
||||
"node_modules/http-proxy-agent": {
|
||||
"version": "7.0.2",
|
||||
"resolved": "https://registry.npmmirror.com/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz",
|
||||
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz",
|
||||
"integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -536,7 +536,7 @@
|
||||
},
|
||||
"node_modules/https-proxy-agent": {
|
||||
"version": "7.0.6",
|
||||
"resolved": "https://registry.npmmirror.com/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz",
|
||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz",
|
||||
"integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -549,7 +549,7 @@
|
||||
},
|
||||
"node_modules/ieee754": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmmirror.com/ieee754/-/ieee754-1.2.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
|
||||
"integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
|
||||
"funding": [
|
||||
{
|
||||
@@ -569,7 +569,7 @@
|
||||
},
|
||||
"node_modules/ip-address": {
|
||||
"version": "10.2.0",
|
||||
"resolved": "https://registry.npmmirror.com/ip-address/-/ip-address-10.2.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz",
|
||||
"integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -578,7 +578,7 @@
|
||||
},
|
||||
"node_modules/is-fullwidth-code-point": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
|
||||
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -587,7 +587,7 @@
|
||||
},
|
||||
"node_modules/lru-cache": {
|
||||
"version": "7.18.3",
|
||||
"resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-7.18.3.tgz",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
|
||||
"integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
@@ -596,19 +596,19 @@
|
||||
},
|
||||
"node_modules/mitt": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/mitt/-/mitt-3.0.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz",
|
||||
"integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/ms": {
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
||||
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/netmask": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmmirror.com/netmask/-/netmask-2.1.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/netmask/-/netmask-2.1.1.tgz",
|
||||
"integrity": "sha512-eonl3sLUha+S1GzTPxychyhnUzKyeQkZ7jLjKrBagJgPla13F+uQ71HgpFefyHgqrjEbCPkDArxYsjY8/+gLKA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -617,7 +617,7 @@
|
||||
},
|
||||
"node_modules/once": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmmirror.com/once/-/once-1.4.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
||||
"integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
@@ -626,7 +626,7 @@
|
||||
},
|
||||
"node_modules/pac-proxy-agent": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmmirror.com/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz",
|
||||
"integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -645,7 +645,7 @@
|
||||
},
|
||||
"node_modules/pac-resolver": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/pac-resolver/-/pac-resolver-7.0.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz",
|
||||
"integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -658,13 +658,13 @@
|
||||
},
|
||||
"node_modules/pend": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmmirror.com/pend/-/pend-1.2.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
|
||||
"integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/progress": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmmirror.com/progress/-/progress-2.0.3.tgz",
|
||||
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
|
||||
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -673,7 +673,7 @@
|
||||
},
|
||||
"node_modules/proxy-agent": {
|
||||
"version": "6.5.0",
|
||||
"resolved": "https://registry.npmmirror.com/proxy-agent/-/proxy-agent-6.5.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz",
|
||||
"integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -692,13 +692,13 @@
|
||||
},
|
||||
"node_modules/proxy-from-env": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmmirror.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
|
||||
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/pump": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmmirror.com/pump/-/pump-3.0.4.tgz",
|
||||
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz",
|
||||
"integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -708,7 +708,7 @@
|
||||
},
|
||||
"node_modules/puppeteer-core": {
|
||||
"version": "23.11.1",
|
||||
"resolved": "https://registry.npmmirror.com/puppeteer-core/-/puppeteer-core-23.11.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-23.11.1.tgz",
|
||||
"integrity": "sha512-3HZ2/7hdDKZvZQ7dhhITOUg4/wOrDRjyK2ZBllRB0ZCOi9u0cwq1ACHDjBB+nX+7+kltHjQvBRdeY7+W0T+7Gg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
@@ -725,7 +725,7 @@
|
||||
},
|
||||
"node_modules/require-directory": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmmirror.com/require-directory/-/require-directory-2.1.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
||||
"integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -734,7 +734,7 @@
|
||||
},
|
||||
"node_modules/semver": {
|
||||
"version": "7.8.5",
|
||||
"resolved": "https://registry.npmmirror.com/semver/-/semver-7.8.5.tgz",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz",
|
||||
"integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==",
|
||||
"license": "ISC",
|
||||
"bin": {
|
||||
@@ -746,7 +746,7 @@
|
||||
},
|
||||
"node_modules/smart-buffer": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmmirror.com/smart-buffer/-/smart-buffer-4.2.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
|
||||
"integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -756,7 +756,7 @@
|
||||
},
|
||||
"node_modules/socks": {
|
||||
"version": "2.8.9",
|
||||
"resolved": "https://registry.npmmirror.com/socks/-/socks-2.8.9.tgz",
|
||||
"resolved": "https://registry.npmjs.org/socks/-/socks-2.8.9.tgz",
|
||||
"integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -770,7 +770,7 @@
|
||||
},
|
||||
"node_modules/socks-proxy-agent": {
|
||||
"version": "8.0.5",
|
||||
"resolved": "https://registry.npmmirror.com/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz",
|
||||
"resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz",
|
||||
"integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -784,7 +784,7 @@
|
||||
},
|
||||
"node_modules/source-map": {
|
||||
"version": "0.6.1",
|
||||
"resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
|
||||
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
|
||||
"license": "BSD-3-Clause",
|
||||
"optional": true,
|
||||
@@ -794,7 +794,7 @@
|
||||
},
|
||||
"node_modules/streamx": {
|
||||
"version": "2.28.0",
|
||||
"resolved": "https://registry.npmmirror.com/streamx/-/streamx-2.28.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/streamx/-/streamx-2.28.0.tgz",
|
||||
"integrity": "sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -805,7 +805,7 @@
|
||||
},
|
||||
"node_modules/string-width": {
|
||||
"version": "4.2.3",
|
||||
"resolved": "https://registry.npmmirror.com/string-width/-/string-width-4.2.3.tgz",
|
||||
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
|
||||
"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -819,7 +819,7 @@
|
||||
},
|
||||
"node_modules/strip-ansi": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-6.0.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
|
||||
"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -831,7 +831,7 @@
|
||||
},
|
||||
"node_modules/tar-fs": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmmirror.com/tar-fs/-/tar-fs-3.1.3.tgz",
|
||||
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.3.tgz",
|
||||
"integrity": "sha512-/hU4AXnIdZu+Gvl1pk0oI5f5HxWsCJRtY2aFaJdk9VvyL48DWU6iU5WAIPG+wIi1YvWA6eTJvIviP/tMAZZNwQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -845,7 +845,7 @@
|
||||
},
|
||||
"node_modules/tar-stream": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmmirror.com/tar-stream/-/tar-stream-3.2.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.2.0.tgz",
|
||||
"integrity": "sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -857,7 +857,7 @@
|
||||
},
|
||||
"node_modules/teex": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/teex/-/teex-1.0.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz",
|
||||
"integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -866,7 +866,7 @@
|
||||
},
|
||||
"node_modules/text-decoder": {
|
||||
"version": "1.2.7",
|
||||
"resolved": "https://registry.npmmirror.com/text-decoder/-/text-decoder-1.2.7.tgz",
|
||||
"resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz",
|
||||
"integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
@@ -875,25 +875,25 @@
|
||||
},
|
||||
"node_modules/through": {
|
||||
"version": "2.3.8",
|
||||
"resolved": "https://registry.npmmirror.com/through/-/through-2.3.8.tgz",
|
||||
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
|
||||
"integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tslib": {
|
||||
"version": "2.8.1",
|
||||
"resolved": "https://registry.npmmirror.com/tslib/-/tslib-2.8.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
|
||||
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
|
||||
"license": "0BSD"
|
||||
},
|
||||
"node_modules/typed-query-selector": {
|
||||
"version": "2.12.2",
|
||||
"resolved": "https://registry.npmmirror.com/typed-query-selector/-/typed-query-selector-2.12.2.tgz",
|
||||
"resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.2.tgz",
|
||||
"integrity": "sha512-EOPFbyIub4ngnEdqi2yOcNeDLaX/0jcE1JoAXQDDMIthap7FoN795lc/SHfIq2d416VufXpM8z/lD+WRm2gfOQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/unbzip2-stream": {
|
||||
"version": "1.4.3",
|
||||
"resolved": "https://registry.npmmirror.com/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz",
|
||||
"resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz",
|
||||
"integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -903,14 +903,14 @@
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "8.3.0",
|
||||
"resolved": "https://registry.npmmirror.com/undici-types/-/undici-types-8.3.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz",
|
||||
"integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==",
|
||||
"license": "MIT",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/wrap-ansi": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmmirror.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
|
||||
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -927,13 +927,13 @@
|
||||
},
|
||||
"node_modules/wrappy": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmmirror.com/wrappy/-/wrappy-1.0.2.tgz",
|
||||
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
|
||||
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.21.0",
|
||||
"resolved": "https://registry.npmmirror.com/ws/-/ws-8.21.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz",
|
||||
"integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -954,7 +954,7 @@
|
||||
},
|
||||
"node_modules/y18n": {
|
||||
"version": "5.0.8",
|
||||
"resolved": "https://registry.npmmirror.com/y18n/-/y18n-5.0.8.tgz",
|
||||
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
|
||||
"integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
@@ -963,7 +963,7 @@
|
||||
},
|
||||
"node_modules/yargs": {
|
||||
"version": "17.7.3",
|
||||
"resolved": "https://registry.npmmirror.com/yargs/-/yargs-17.7.3.tgz",
|
||||
"resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz",
|
||||
"integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -981,7 +981,7 @@
|
||||
},
|
||||
"node_modules/yargs-parser": {
|
||||
"version": "21.1.1",
|
||||
"resolved": "https://registry.npmmirror.com/yargs-parser/-/yargs-parser-21.1.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
|
||||
"integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
@@ -990,7 +990,7 @@
|
||||
},
|
||||
"node_modules/yauzl": {
|
||||
"version": "2.10.0",
|
||||
"resolved": "https://registry.npmmirror.com/yauzl/-/yauzl-2.10.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz",
|
||||
"integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -1000,7 +1000,7 @@
|
||||
},
|
||||
"node_modules/zod": {
|
||||
"version": "3.23.8",
|
||||
"resolved": "https://registry.npmmirror.com/zod/-/zod-3.23.8.tgz",
|
||||
"resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz",
|
||||
"integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==",
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,268 @@
|
||||
package hookvm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const regexpCompileCost = 32 << 10
|
||||
|
||||
func (e *vm) checkRepeat(elemLen, cnt int64) int {
|
||||
if cnt < 0 {
|
||||
e.failf("Repeat: negative count")
|
||||
}
|
||||
if cnt > 0 && elemLen > e.memLimit/cnt {
|
||||
e.fail(ErrMemoryLimit)
|
||||
}
|
||||
e.alloc(elemLen * cnt)
|
||||
return int(cnt)
|
||||
}
|
||||
|
||||
func (e *vm) callPkg(pkg, fn string, args []any) any {
|
||||
need := func(n int) {
|
||||
if len(args) != n {
|
||||
e.failf("%s.%s expects %d arg(s)", pkg, fn, n)
|
||||
}
|
||||
}
|
||||
switch pkg {
|
||||
case "strings":
|
||||
switch fn {
|
||||
case "Contains", "HasPrefix", "HasSuffix", "Index", "TrimPrefix", "TrimSuffix", "Split", "Join", "Repeat":
|
||||
need(2)
|
||||
switch fn {
|
||||
case "Contains":
|
||||
return strings.Contains(e.str(args[0]), e.str(args[1]))
|
||||
case "HasPrefix":
|
||||
return strings.HasPrefix(e.str(args[0]), e.str(args[1]))
|
||||
case "HasSuffix":
|
||||
return strings.HasSuffix(e.str(args[0]), e.str(args[1]))
|
||||
case "Index":
|
||||
return int64(strings.Index(e.str(args[0]), e.str(args[1])))
|
||||
case "TrimPrefix":
|
||||
return strings.TrimPrefix(e.str(args[0]), e.str(args[1]))
|
||||
case "TrimSuffix":
|
||||
return strings.TrimSuffix(e.str(args[0]), e.str(args[1]))
|
||||
case "Split":
|
||||
a, sep := e.str(args[0]), e.str(args[1])
|
||||
parts := strings.Split(a, sep)
|
||||
e.alloc(int64(len(a)) + int64(len(parts))*16)
|
||||
out := make([]any, len(parts))
|
||||
for i, p := range parts {
|
||||
out[i] = p
|
||||
}
|
||||
return out
|
||||
case "Join":
|
||||
ss, sep := e.strSlice(args[0]), e.str(args[1])
|
||||
var est int64
|
||||
for _, p := range ss {
|
||||
est += int64(len(p) + len(sep))
|
||||
}
|
||||
e.alloc(est)
|
||||
return strings.Join(ss, sep)
|
||||
case "Repeat":
|
||||
s := e.str(args[0])
|
||||
return strings.Repeat(s, e.checkRepeat(int64(len(s)), e.int(args[1])))
|
||||
}
|
||||
case "ToLower", "ToUpper", "TrimSpace":
|
||||
need(1)
|
||||
s := e.str(args[0])
|
||||
if fn == "TrimSpace" {
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
e.alloc(int64(len(s)))
|
||||
if fn == "ToLower" {
|
||||
return strings.ToLower(s)
|
||||
}
|
||||
return strings.ToUpper(s)
|
||||
case "ReplaceAll":
|
||||
need(3)
|
||||
out := strings.ReplaceAll(e.str(args[0]), e.str(args[1]), e.str(args[2]))
|
||||
e.alloc(int64(len(out)))
|
||||
return out
|
||||
}
|
||||
case "bytes":
|
||||
switch fn {
|
||||
case "Contains", "HasPrefix", "HasSuffix", "Equal", "Index", "Repeat":
|
||||
need(2)
|
||||
switch fn {
|
||||
case "Contains":
|
||||
return bytes.Contains(e.bytes(args[0]), e.bytes(args[1]))
|
||||
case "HasPrefix":
|
||||
return bytes.HasPrefix(e.bytes(args[0]), e.bytes(args[1]))
|
||||
case "HasSuffix":
|
||||
return bytes.HasSuffix(e.bytes(args[0]), e.bytes(args[1]))
|
||||
case "Equal":
|
||||
return bytes.Equal(e.bytes(args[0]), e.bytes(args[1]))
|
||||
case "Index":
|
||||
return int64(bytes.Index(e.bytes(args[0]), e.bytes(args[1])))
|
||||
case "Repeat":
|
||||
b := e.bytes(args[0])
|
||||
return bytes.Repeat(b, e.checkRepeat(int64(len(b)), e.int(args[1])))
|
||||
}
|
||||
case "TrimSpace", "ToLower", "ToUpper":
|
||||
need(1)
|
||||
b := e.bytes(args[0])
|
||||
e.alloc(int64(len(b)))
|
||||
if fn == "TrimSpace" {
|
||||
return append([]byte(nil), bytes.TrimSpace(b)...)
|
||||
}
|
||||
if fn == "ToLower" {
|
||||
return bytes.ToLower(b)
|
||||
}
|
||||
return bytes.ToUpper(b)
|
||||
case "ReplaceAll":
|
||||
need(3)
|
||||
out := bytes.ReplaceAll(e.bytes(args[0]), e.bytes(args[1]), e.bytes(args[2]))
|
||||
e.alloc(int64(len(out)))
|
||||
return out
|
||||
}
|
||||
case "encoding/json":
|
||||
need(1)
|
||||
if fn == "Marshal" {
|
||||
raw, err := json.Marshal(args[0])
|
||||
if err == nil {
|
||||
e.alloc(int64(len(raw)))
|
||||
}
|
||||
return multiVal{raw, err}
|
||||
}
|
||||
if fn == "Decode" {
|
||||
b := e.bytes(args[0])
|
||||
e.alloc(int64(len(b)) * 4)
|
||||
var dst any
|
||||
return multiVal{dst, json.Unmarshal(b, &dst)}
|
||||
}
|
||||
case "encoding/base64":
|
||||
need(1)
|
||||
enc := base64.StdEncoding
|
||||
if strings.HasPrefix(fn, "URL") {
|
||||
enc = base64.RawURLEncoding
|
||||
}
|
||||
if strings.HasSuffix(fn, "EncodeToString") {
|
||||
b := e.bytes(args[0])
|
||||
e.alloc(int64(enc.EncodedLen(len(b))))
|
||||
return enc.EncodeToString(b)
|
||||
}
|
||||
if strings.HasSuffix(fn, "DecodeString") {
|
||||
s := e.str(args[0])
|
||||
e.alloc(int64(len(s)))
|
||||
b, err := enc.DecodeString(s)
|
||||
return multiVal{b, err}
|
||||
}
|
||||
case "encoding/hex":
|
||||
need(1)
|
||||
if fn == "EncodeToString" {
|
||||
b := e.bytes(args[0])
|
||||
e.alloc(int64(len(b) * 2))
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
if fn == "DecodeString" {
|
||||
s := e.str(args[0])
|
||||
e.alloc(int64(len(s)))
|
||||
b, err := hex.DecodeString(s)
|
||||
return multiVal{b, err}
|
||||
}
|
||||
case "net/url":
|
||||
need(1)
|
||||
switch fn {
|
||||
case "QueryEscape":
|
||||
out := url.QueryEscape(e.str(args[0]))
|
||||
e.alloc(int64(len(out)))
|
||||
return out
|
||||
case "QueryUnescape":
|
||||
out, err := url.QueryUnescape(e.str(args[0]))
|
||||
if err == nil {
|
||||
e.alloc(int64(len(out)))
|
||||
}
|
||||
return multiVal{out, err}
|
||||
case "ParseQuery":
|
||||
s := e.str(args[0])
|
||||
e.alloc(int64(len(s)) * 2)
|
||||
v, err := url.ParseQuery(s)
|
||||
return multiVal{map[string][]string(v), err}
|
||||
case "EncodeQuery":
|
||||
m, ok := args[0].(map[string][]string)
|
||||
if !ok {
|
||||
e.failf("url.EncodeQuery expects map[string][]string")
|
||||
}
|
||||
out := url.Values(m).Encode()
|
||||
e.alloc(int64(len(out)))
|
||||
return out
|
||||
}
|
||||
case "regexp":
|
||||
if len(args) < 2 {
|
||||
e.failf("regexp.%s expects at least 2 args", fn)
|
||||
}
|
||||
pat, s := e.str(args[0]), e.str(args[1])
|
||||
if len(pat) > DefaultMaxRegexBytes {
|
||||
e.failf("regexp pattern exceeds %d bytes", DefaultMaxRegexBytes)
|
||||
}
|
||||
e.alloc(regexpCompileCost + int64(len(pat))*32)
|
||||
re, err := regexp.Compile(pat)
|
||||
if err != nil {
|
||||
e.fail(err)
|
||||
}
|
||||
switch fn {
|
||||
case "MatchString":
|
||||
need(2)
|
||||
return multiVal{re.MatchString(s), nil}
|
||||
case "FindString":
|
||||
need(2)
|
||||
return re.FindString(s)
|
||||
case "ReplaceAllString":
|
||||
need(3)
|
||||
out := re.ReplaceAllString(s, e.str(args[2]))
|
||||
e.alloc(int64(len(out)))
|
||||
return out
|
||||
}
|
||||
case "crypto/sha256":
|
||||
need(1)
|
||||
sum := sha256.Sum256(e.bytes(args[0]))
|
||||
if fn == "Sum256" {
|
||||
return append([]byte(nil), sum[:]...)
|
||||
}
|
||||
if fn == "Sum256Hex" {
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
case "crypto/hmac":
|
||||
need(2)
|
||||
m := hmac.New(sha256.New, e.bytes(args[0]))
|
||||
m.Write(e.bytes(args[1]))
|
||||
sum := m.Sum(nil)
|
||||
if fn == "SHA256" {
|
||||
return sum
|
||||
}
|
||||
if fn == "SHA256Hex" {
|
||||
return hex.EncodeToString(sum)
|
||||
}
|
||||
case "strconv":
|
||||
need(1)
|
||||
switch fn {
|
||||
case "Atoi":
|
||||
n, err := strconv.Atoi(e.str(args[0]))
|
||||
return multiVal{int64(n), err}
|
||||
case "Itoa":
|
||||
return strconv.FormatInt(e.int(args[0]), 10)
|
||||
case "ParseBool":
|
||||
b, err := strconv.ParseBool(e.str(args[0]))
|
||||
return multiVal{b, err}
|
||||
case "FormatBool":
|
||||
return strconv.FormatBool(e.bool(args[0]))
|
||||
}
|
||||
case "errors":
|
||||
if fn == "New" {
|
||||
need(1)
|
||||
return errors.New(e.str(args[0]))
|
||||
}
|
||||
}
|
||||
e.failf("unsupported stdlib call %s.%s", pkg, fn)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
// Package hookvm implements a small, deterministic, resource-bounded Go-subset
|
||||
// evaluator built on Go's standard library go/parser and go/ast packages.
|
||||
// Zero external dependencies; no filesystem, network, subprocess, environment,
|
||||
// wall-clock, randomness, goroutine, channel, pointer, or global-state access.
|
||||
package hookvm
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxSourceBytes = 64 * 1024
|
||||
DefaultMaxSteps = 100_000
|
||||
DefaultMaxMemory = 4 * 1024 * 1024
|
||||
DefaultMaxRegexBytes = 1024
|
||||
MaxCallDepth = 32
|
||||
)
|
||||
|
||||
var (
|
||||
ErrStepLimit = errors.New("hook exceeded step limit")
|
||||
ErrTimeout = errors.New("hook timed out")
|
||||
ErrMemoryLimit = errors.New("hook exceeded memory limit")
|
||||
ErrForbiddenMutation = errors.New("hook attempted forbidden mutation")
|
||||
ErrCallDepth = errors.New("hook exceeded call depth limit")
|
||||
)
|
||||
|
||||
// AllowedImports is the closed set of pure, deterministic virtual packages.
|
||||
var AllowedImports = map[string]bool{
|
||||
"strings": true, "bytes": true, "encoding/json": true, "encoding/base64": true,
|
||||
"encoding/hex": true, "net/url": true, "regexp": true, "crypto/sha256": true,
|
||||
"crypto/hmac": true, "strconv": true, "errors": true,
|
||||
}
|
||||
|
||||
var allowedBuiltins = map[string]bool{
|
||||
"len": true, "cap": true, "append": true, "make": true, "delete": true,
|
||||
"string": true, "int": true, "int64": true, "float64": true, "byte": true, "panic": true,
|
||||
}
|
||||
|
||||
// Program is a validated AST ready for concurrent read-only execution.
|
||||
type Program struct {
|
||||
phase string
|
||||
imports map[string]string
|
||||
funcs map[string]*ast.FuncDecl
|
||||
consts map[string]any
|
||||
entry *ast.FuncDecl
|
||||
}
|
||||
|
||||
// Compile parses and statically validates a Go-subset tampering hook for phase
|
||||
// ("request" or "response").
|
||||
func Compile(phase, source string) (*Program, error) {
|
||||
if phase != "request" && phase != "response" {
|
||||
return nil, fmt.Errorf("invalid hook phase %q", phase)
|
||||
}
|
||||
if len(source) == 0 || len(source) > MaxSourceBytes {
|
||||
return nil, fmt.Errorf("hook source length must be 1..%d bytes", MaxSourceBytes)
|
||||
}
|
||||
file, err := parser.ParseFile(token.NewFileSet(), "hook.go", source, 0)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse hook: %w", err)
|
||||
}
|
||||
if file.Name == nil || file.Name.Name != "hook" {
|
||||
return nil, errors.New("hook must declare 'package hook'")
|
||||
}
|
||||
|
||||
prog := &Program{
|
||||
phase: phase, imports: map[string]string{},
|
||||
funcs: map[string]*ast.FuncDecl{}, consts: map[string]any{},
|
||||
}
|
||||
for _, imp := range file.Imports {
|
||||
path := strings.Trim(imp.Path.Value, `"`)
|
||||
if !AllowedImports[path] {
|
||||
return nil, fmt.Errorf("forbidden import %q", path)
|
||||
}
|
||||
alias := path[strings.LastIndexByte(path, '/')+1:]
|
||||
if imp.Name != nil {
|
||||
if imp.Name.Name == "." || imp.Name.Name == "_" {
|
||||
return nil, fmt.Errorf("dot/blank import %q is not allowed", path)
|
||||
}
|
||||
alias = imp.Name.Name
|
||||
}
|
||||
prog.imports[alias] = path
|
||||
}
|
||||
|
||||
for _, decl := range file.Decls {
|
||||
switch d := decl.(type) {
|
||||
case *ast.GenDecl:
|
||||
switch d.Tok {
|
||||
case token.IMPORT:
|
||||
case token.CONST:
|
||||
if err := collectConsts(d, prog.consts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case token.VAR:
|
||||
return nil, errors.New("package-level variables are forbidden (hooks must be stateless)")
|
||||
case token.TYPE:
|
||||
return nil, errors.New("custom package-level types are not supported in hooks")
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported declaration %s", d.Tok)
|
||||
}
|
||||
case *ast.FuncDecl:
|
||||
if d.Recv != nil {
|
||||
return nil, errors.New("custom receiver methods are not supported in hooks")
|
||||
}
|
||||
name := d.Name.Name
|
||||
if name == "init" {
|
||||
return nil, errors.New("init() is forbidden in hooks")
|
||||
}
|
||||
if _, dup := prog.funcs[name]; dup || d.Body == nil {
|
||||
return nil, fmt.Errorf("invalid or duplicate function %q", name)
|
||||
}
|
||||
if err := validateBlock(d.Body); err != nil {
|
||||
return nil, fmt.Errorf("function %s: %w", name, err)
|
||||
}
|
||||
prog.funcs[name] = d
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported top-level declaration %T", decl)
|
||||
}
|
||||
}
|
||||
|
||||
entryName, wantParams := "OnRequest", 1
|
||||
if phase == "response" {
|
||||
entryName, wantParams = "OnResponse", 2
|
||||
}
|
||||
entry := prog.funcs[entryName]
|
||||
if entry == nil {
|
||||
return nil, fmt.Errorf("phase %q requires function %s", phase, entryName)
|
||||
}
|
||||
if countFields(entry.Type.Params) != wantParams || countFields(entry.Type.Results) != 1 {
|
||||
return nil, fmt.Errorf("%s must accept %d parameter(s) and return error", entryName, wantParams)
|
||||
}
|
||||
prog.entry = entry
|
||||
return prog, nil
|
||||
}
|
||||
|
||||
func countFields(fl *ast.FieldList) int {
|
||||
if fl == nil {
|
||||
return 0
|
||||
}
|
||||
n := 0
|
||||
for _, f := range fl.List {
|
||||
n += max(1, len(f.Names))
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func collectConsts(d *ast.GenDecl, out map[string]any) error {
|
||||
for _, spec := range d.Specs {
|
||||
vs, ok := spec.(*ast.ValueSpec)
|
||||
if !ok || len(vs.Names) != len(vs.Values) {
|
||||
return errors.New("const declarations must have explicit values")
|
||||
}
|
||||
for i, name := range vs.Names {
|
||||
v, err := evalConstExpr(vs.Values[i])
|
||||
if err != nil {
|
||||
return fmt.Errorf("const %s: %w", name.Name, err)
|
||||
}
|
||||
out[name.Name] = v
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func evalConstExpr(e ast.Expr) (any, error) {
|
||||
switch x := e.(type) {
|
||||
case *ast.BasicLit:
|
||||
return parseBasicLit(x)
|
||||
case *ast.Ident:
|
||||
if x.Name == "true" || x.Name == "false" {
|
||||
return x.Name == "true", nil
|
||||
}
|
||||
case *ast.UnaryExpr:
|
||||
if x.Op == token.SUB {
|
||||
if v, err := evalConstExpr(x.X); err == nil {
|
||||
if n, ok := v.(int64); ok {
|
||||
return -n, nil
|
||||
}
|
||||
if f, ok := v.(float64); ok {
|
||||
return -f, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported const expression %T", e)
|
||||
}
|
||||
|
||||
func validateBlock(body *ast.BlockStmt) error {
|
||||
var firstErr error
|
||||
ast.Inspect(body, func(n ast.Node) bool {
|
||||
if firstErr != nil || n == nil {
|
||||
return false
|
||||
}
|
||||
switch x := n.(type) {
|
||||
case *ast.GoStmt:
|
||||
firstErr = errors.New("goroutines ('go') are forbidden in hooks")
|
||||
case *ast.SendStmt, *ast.ChanType, *ast.SelectStmt:
|
||||
firstErr = errors.New("channels and select are forbidden in hooks")
|
||||
case *ast.DeferStmt:
|
||||
firstErr = errors.New("defer is forbidden in hooks")
|
||||
case *ast.SwitchStmt, *ast.TypeSwitchStmt:
|
||||
firstErr = errors.New("switch is not supported in hooks; use if / else if")
|
||||
case *ast.FuncLit:
|
||||
firstErr = errors.New("anonymous function literals are not supported in hooks")
|
||||
case *ast.UnaryExpr:
|
||||
if x.Op == token.AND {
|
||||
firstErr = errors.New("address-of (&) is not supported in hooks")
|
||||
}
|
||||
case *ast.BranchStmt:
|
||||
if x.Tok == token.GOTO {
|
||||
firstErr = errors.New("goto is forbidden in hooks")
|
||||
}
|
||||
case *ast.CallExpr:
|
||||
if id, ok := x.Fun.(*ast.Ident); ok {
|
||||
switch id.Name {
|
||||
case "recover", "print", "println", "close", "complex", "real", "imag", "new":
|
||||
firstErr = fmt.Errorf("builtin %q is forbidden in hooks", id.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return firstErr == nil
|
||||
})
|
||||
return firstErr
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
package hookvm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCompileRejectsForbiddenConstructsAndImports(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
phase string
|
||||
src string
|
||||
}{
|
||||
{"import_os", "request", `package hook; import "os"; func OnRequest(req *Request) error { _ = os.Getenv("HOME"); return nil }`},
|
||||
{"import_io_fs", "request", `package hook; import "io/fs"; func OnRequest(req *Request) error { return nil }`},
|
||||
{"import_net", "request", `package hook; import "net"; func OnRequest(req *Request) error { return nil }`},
|
||||
{"import_net_http", "request", `package hook; import "net/http"; func OnRequest(req *Request) error { return nil }`},
|
||||
{"import_os_exec", "request", `package hook; import "os/exec"; func OnRequest(req *Request) error { return nil }`},
|
||||
{"import_syscall", "request", `package hook; import "syscall"; func OnRequest(req *Request) error { return nil }`},
|
||||
{"import_unsafe", "request", `package hook; import "unsafe"; func OnRequest(req *Request) error { return nil }`},
|
||||
{"import_reflect", "request", `package hook; import "reflect"; func OnRequest(req *Request) error { return nil }`},
|
||||
{"import_time", "request", `package hook; import "time"; func OnRequest(req *Request) error { _ = time.Now(); return nil }`},
|
||||
{"import_math_rand", "request", `package hook; import "math/rand"; func OnRequest(req *Request) error { _ = rand.Int(); return nil }`},
|
||||
{"import_crypto_rand", "request", `package hook; import "crypto/rand"; func OnRequest(req *Request) error { return nil }`},
|
||||
{"import_fmt_disallowed", "request", `package hook; import "fmt"; func OnRequest(req *Request) error { return nil }`},
|
||||
{"import_sort_disallowed", "request", `package hook; import "sort"; func OnRequest(req *Request) error { return nil }`},
|
||||
{"dot_import", "request", `package hook; import . "strings"; func OnRequest(req *Request) error { return nil }`},
|
||||
{"blank_import", "request", `package hook; import _ "strings"; func OnRequest(req *Request) error { return nil }`},
|
||||
{"goroutine", "request", `package hook; func f(){}; func OnRequest(req *Request) error { go f(); return nil }`},
|
||||
{"channel_type", "request", `package hook; func OnRequest(req *Request) error { var c chan int; _ = c; return nil }`},
|
||||
{"select_stmt", "request", `package hook; func OnRequest(req *Request) error { select {}; return nil }`},
|
||||
{"defer_stmt", "request", `package hook; func f(){}; func OnRequest(req *Request) error { defer f(); return nil }`},
|
||||
{"goto_stmt", "request", `package hook; func OnRequest(req *Request) error { goto end; end: return nil }`},
|
||||
{"init_func", "request", `package hook; func init() {}; func OnRequest(req *Request) error { return nil }`},
|
||||
{"mutable_global", "request", `package hook; var counter = 0; func OnRequest(req *Request) error { counter++; return nil }`},
|
||||
{"switch_stmt", "request", `package hook; func OnRequest(req *Request) error { switch req.Method { case "GET": }; return nil }`},
|
||||
{"address_of", "request", `package hook; func OnRequest(req *Request) error { x := 1; _ = &x; return nil }`},
|
||||
{"func_lit", "request", `package hook; func OnRequest(req *Request) error { f := func() {}; _ = f; return nil }`},
|
||||
{"builtin_recover", "request", `package hook; func OnRequest(req *Request) error { _ = recover(); return nil }`},
|
||||
{"builtin_print", "request", `package hook; func OnRequest(req *Request) error { print("x"); return nil }`},
|
||||
{"builtin_println", "request", `package hook; func OnRequest(req *Request) error { println("x"); return nil }`},
|
||||
{"wrong_package", "request", `package main; func OnRequest(req *Request) error { return nil }`},
|
||||
{"missing_entry", "request", `package hook; func Helper() error { return nil }`},
|
||||
{"wrong_arity_response", "response", `package hook; func OnResponse(resp *Response) error { return nil }`},
|
||||
{"oversized_source", "request", "package hook\nfunc OnRequest(req *Request) error { return nil }\n//" + strings.Repeat("a", MaxSourceBytes)},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if _, err := Compile(tc.phase, tc.src); err == nil {
|
||||
t.Fatalf("expected Compile(%s) to reject %s", tc.phase, tc.name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestAndResponseTransformationsAndHelpers(t *testing.T) {
|
||||
src := `package hook
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const secretKey = "k-123"
|
||||
|
||||
func sign(data []byte) string {
|
||||
return hmac.SHA256Hex([]byte(secretKey), data)
|
||||
}
|
||||
|
||||
func OnRequest(req *Request) error {
|
||||
req.SetHeader("X-Tampered", "yes")
|
||||
req.DelHeader("X-Remove")
|
||||
if strings.HasPrefix(req.Path, "/v1/") {
|
||||
req.Path = strings.ReplaceAll(req.Path, "/v1/", "/v2/")
|
||||
}
|
||||
q, err := url.ParseQuery(req.RawQuery)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q["debug"] = []any{"1"}
|
||||
req.RawQuery = url.EncodeQuery(q)
|
||||
|
||||
val, err := json.Decode(req.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
obj, ok := val.(map[string]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
obj["role"] = "admin"
|
||||
out, err := json.Marshal(obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Body = out
|
||||
req.SetHeader("X-Sig", sign(out))
|
||||
req.SetHeader("X-B64", base64.EncodeToString([]byte(req.Host)))
|
||||
return nil
|
||||
}
|
||||
`
|
||||
prog, err := Compile("request", src)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile: %v", err)
|
||||
}
|
||||
req := &Request{
|
||||
Scheme: "https",
|
||||
Host: "api.example.test",
|
||||
Port: "443",
|
||||
Method: "POST",
|
||||
Path: "/v1/users",
|
||||
RawQuery: "a=2",
|
||||
Headers: map[string][]string{"X-Remove": {"bye"}},
|
||||
Body: []byte(`{"role":"user","id":7}`),
|
||||
}
|
||||
if err := prog.RunRequest(context.Background(), req, RunOptions{}); err != nil {
|
||||
t.Fatalf("RunRequest: %v", err)
|
||||
}
|
||||
if req.Path != "/v2/users" {
|
||||
t.Fatalf("Path = %q, want /v2/users", req.Path)
|
||||
}
|
||||
if req.RawQuery != "a=2&debug=1" {
|
||||
t.Fatalf("RawQuery = %q, want a=2&debug=1", req.RawQuery)
|
||||
}
|
||||
if got := req.Headers["X-Tampered"]; len(got) != 1 || got[0] != "yes" {
|
||||
t.Fatalf("X-Tampered = %v", got)
|
||||
}
|
||||
if _, exists := req.Headers["X-Remove"]; exists {
|
||||
t.Fatal("X-Remove was not deleted")
|
||||
}
|
||||
if !bytes.Contains(req.Body, []byte(`"role":"admin"`)) {
|
||||
t.Fatalf("Body = %s", req.Body)
|
||||
}
|
||||
if len(req.Headers["X-Sig"]) != 1 || len(req.Headers["X-Sig"][0]) != 64 {
|
||||
t.Fatalf("X-Sig = %v", req.Headers["X-Sig"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestForbiddenHostSchemePortAndReadOnlyRequest(t *testing.T) {
|
||||
for _, field := range []string{"Host", "Scheme", "Port"} {
|
||||
t.Run("mutate_"+field, func(t *testing.T) {
|
||||
src := `package hook
|
||||
func OnRequest(req *Request) error {
|
||||
req.` + field + ` = "evil.example.test"
|
||||
return nil
|
||||
}`
|
||||
prog, err := Compile("request", src)
|
||||
if err != nil {
|
||||
t.Fatalf("Compile: %v", err)
|
||||
}
|
||||
req := &Request{Scheme: "https", Host: "good.example.test", Port: "443"}
|
||||
err = prog.RunRequest(context.Background(), req, RunOptions{})
|
||||
if !errors.Is(err, ErrForbiddenMutation) {
|
||||
t.Fatalf("err = %v, want ErrForbiddenMutation", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("mutate_request_in_response_phase", func(t *testing.T) {
|
||||
src := `package hook
|
||||
func OnResponse(req *Request, resp *Response) error {
|
||||
req.SetHeader("X-Bad", "1")
|
||||
return nil
|
||||
}`
|
||||
prog, err := Compile("response", src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = prog.RunResponse(context.Background(), &Request{}, &Response{Status: 200}, RunOptions{})
|
||||
if !errors.Is(err, ErrForbiddenMutation) {
|
||||
t.Fatalf("err = %v, want ErrForbiddenMutation", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("response_phase_req_headers_index_only_mutates_vm_copy", func(t *testing.T) {
|
||||
src := `package hook
|
||||
func OnResponse(req *Request, resp *Response) error {
|
||||
req.Headers["X-Vm-Only"] = []any{"1"}
|
||||
resp.SetHeader("X-Seen", req.GetHeader("X-Vm-Only"))
|
||||
return nil
|
||||
}`
|
||||
prog, err := Compile("response", src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vmReq := &Request{Headers: map[string][]string{"X-Orig": {"keep"}}}
|
||||
resp := &Response{Status: 200, Headers: map[string][]string{}}
|
||||
if err := prog.RunResponse(context.Background(), vmReq, resp, RunOptions{}); err != nil {
|
||||
t.Fatalf("RunResponse: %v", err)
|
||||
}
|
||||
if len(resp.Headers["X-Seen"]) != 1 || resp.Headers["X-Seen"][0] != "1" {
|
||||
t.Fatalf("expected VM-local mutation visible inside hook, got %v", resp.Headers)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestResourceBoundsMemoryStepTimeoutPanicAndRegexp(t *testing.T) {
|
||||
t.Run("make_100MiB_stopped_by_memory_budget", func(t *testing.T) {
|
||||
src := `package hook
|
||||
func OnRequest(req *Request) error {
|
||||
b := make([]byte, 100*1024*1024)
|
||||
_ = b
|
||||
return nil
|
||||
}`
|
||||
prog, err := Compile("request", src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = prog.RunRequest(context.Background(), &Request{}, RunOptions{MaxMemory: 1 << 20})
|
||||
if !errors.Is(err, ErrMemoryLimit) {
|
||||
t.Fatalf("err = %v, want ErrMemoryLimit", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("strings_repeat_stopped_by_memory_budget", func(t *testing.T) {
|
||||
src := `package hook
|
||||
import "strings"
|
||||
func OnRequest(req *Request) error {
|
||||
s := strings.Repeat("A", 10*1024*1024)
|
||||
_ = s
|
||||
return nil
|
||||
}`
|
||||
prog, err := Compile("request", src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = prog.RunRequest(context.Background(), &Request{}, RunOptions{MaxMemory: 1 << 20})
|
||||
if !errors.Is(err, ErrMemoryLimit) {
|
||||
t.Fatalf("err = %v, want ErrMemoryLimit", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("bytes_contains_string_loop_stopped_by_memory_budget", func(t *testing.T) {
|
||||
src := `package hook
|
||||
import "bytes"
|
||||
func OnRequest(req *Request) error {
|
||||
s := string(req.Body)
|
||||
for i := 0; i < 1000; i++ {
|
||||
if bytes.Contains(s, "needle") {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}`
|
||||
prog, err := Compile("request", src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := &Request{Body: bytes.Repeat([]byte("A"), 1<<20)}
|
||||
err = prog.RunRequest(context.Background(), req, RunOptions{})
|
||||
if !errors.Is(err, ErrMemoryLimit) {
|
||||
t.Fatalf("err = %v, want ErrMemoryLimit", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("regexp_compile_loop_stopped_by_memory_budget", func(t *testing.T) {
|
||||
src := `package hook
|
||||
import "regexp"
|
||||
func OnRequest(req *Request) error {
|
||||
for i := 0; i < 500; i++ {
|
||||
_, err := regexp.MatchString("foo[0-9]+bar", "hello")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}`
|
||||
prog, err := Compile("request", src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = prog.RunRequest(context.Background(), &Request{}, RunOptions{})
|
||||
if !errors.Is(err, ErrMemoryLimit) {
|
||||
t.Fatalf("err = %v, want ErrMemoryLimit", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("strslice_join_loop_stopped_by_memory_budget", func(t *testing.T) {
|
||||
src := `package hook
|
||||
import "strings"
|
||||
func OnRequest(req *Request) error {
|
||||
items := make([]any, 1000)
|
||||
for i := 0; i < 1000; i++ {
|
||||
items[i] = ""
|
||||
}
|
||||
for i := 0; i < 1000; i++ {
|
||||
_ = strings.Join(items, "")
|
||||
}
|
||||
return nil
|
||||
}`
|
||||
prog, err := Compile("request", src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = prog.RunRequest(context.Background(), &Request{}, RunOptions{MaxMemory: 256 << 10})
|
||||
if !errors.Is(err, ErrMemoryLimit) {
|
||||
t.Fatalf("err = %v, want ErrMemoryLimit", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("infinite_loop_stopped_by_step_limit", func(t *testing.T) {
|
||||
src := `package hook
|
||||
func OnRequest(req *Request) error {
|
||||
for {
|
||||
}
|
||||
return nil
|
||||
}`
|
||||
prog, err := Compile("request", src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = prog.RunRequest(context.Background(), &Request{}, RunOptions{MaxSteps: 500})
|
||||
if !errors.Is(err, ErrStepLimit) {
|
||||
t.Fatalf("err = %v, want ErrStepLimit", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("infinite_loop_stopped_by_context_timeout", func(t *testing.T) {
|
||||
src := `package hook
|
||||
func OnRequest(req *Request) error {
|
||||
i := 0
|
||||
for {
|
||||
i++
|
||||
}
|
||||
return nil
|
||||
}`
|
||||
prog, err := Compile("request", src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
|
||||
defer cancel()
|
||||
err = prog.RunRequest(ctx, &Request{}, RunOptions{MaxSteps: 100_000_000})
|
||||
if !errors.Is(err, ErrTimeout) {
|
||||
t.Fatalf("err = %v, want ErrTimeout", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("panic_recovered_as_panic_error", func(t *testing.T) {
|
||||
src := `package hook
|
||||
func OnRequest(req *Request) error {
|
||||
panic("boom")
|
||||
}`
|
||||
prog, err := Compile("request", src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = prog.RunRequest(context.Background(), &Request{}, RunOptions{})
|
||||
var pe *PanicError
|
||||
if !errors.As(err, &pe) || pe.Value != "boom" {
|
||||
t.Fatalf("err = %v, want PanicError(boom)", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("regexp_pattern_bounded", func(t *testing.T) {
|
||||
src := `package hook
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
func OnRequest(req *Request) error {
|
||||
pat := strings.Repeat("a", 2048)
|
||||
_, err := regexp.MatchString(pat, "aaa")
|
||||
return err
|
||||
}`
|
||||
prog, err := Compile("request", src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := prog.RunRequest(context.Background(), &Request{}, RunOptions{}); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("expected regexp size limit error, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDeterminismAndSortedMapIteration(t *testing.T) {
|
||||
src := `package hook
|
||||
import "encoding/json"
|
||||
func OnRequest(req *Request) error {
|
||||
v, err := json.Decode(req.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m, _ := v.(map[string]any)
|
||||
joined := ""
|
||||
for k, val := range m {
|
||||
s, _ := val.(string)
|
||||
joined += k + "=" + s + ";"
|
||||
}
|
||||
req.SetHeader("X-Order", joined)
|
||||
out, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Body = out
|
||||
return nil
|
||||
}`
|
||||
prog, err := Compile("request", src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inputBody := []byte(`{"z":"last","a":"first","m":"mid","b":"second","k":"fifth"}`)
|
||||
var firstHeader, firstHash string
|
||||
for run := 0; run < 20; run++ {
|
||||
req := &Request{
|
||||
Scheme: "https", Host: "example.test", Port: "443",
|
||||
Method: "POST", Path: "/api",
|
||||
Body: append([]byte(nil), inputBody...),
|
||||
}
|
||||
if err := prog.RunRequest(context.Background(), req, RunOptions{}); err != nil {
|
||||
t.Fatalf("run %d: %v", run, err)
|
||||
}
|
||||
h := req.Headers["X-Order"][0]
|
||||
sum := sha256.Sum256(req.Body)
|
||||
bodyHash := hex.EncodeToString(sum[:])
|
||||
if run == 0 {
|
||||
firstHeader, firstHash = h, bodyHash
|
||||
if firstHeader != "a=first;b=second;k=fifth;m=mid;z=last;" {
|
||||
t.Fatalf("unsorted map iteration: %q", firstHeader)
|
||||
}
|
||||
} else if h != firstHeader || bodyHash != firstHash {
|
||||
t.Fatalf("run %d diverged: (%q, %s) != (%q, %s)", run, h, bodyHash, firstHeader, firstHash)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RawTCPStream is one recorded raw TCP connection relayed by the opt-in SOCKS5
|
||||
// listener. It never creates an HTTP exchanges row.
|
||||
type RawTCPStream struct {
|
||||
ID string `json:"id"`
|
||||
Protocol string `json:"protocol"`
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
StartedAt int64 `json:"started_at"` // unix milliseconds
|
||||
EndedAt int64 `json:"ended_at"` // unix milliseconds
|
||||
DurationMs int64 `json:"duration_ms"` // non-negative duration in milliseconds
|
||||
C2SBytes int64 `json:"c2s_bytes"`
|
||||
S2CBytes int64 `json:"s2c_bytes"`
|
||||
Truncated bool `json:"truncated"`
|
||||
C2SPreview []byte `json:"c2s_preview,omitempty"`
|
||||
C2SBlob string `json:"c2s_blob,omitempty"`
|
||||
S2CPreview []byte `json:"s2c_preview,omitempty"`
|
||||
S2CBlob string `json:"s2c_blob,omitempty"`
|
||||
}
|
||||
|
||||
// RawTCPFilter narrows ListRawTCPStreams queries.
|
||||
type RawTCPFilter struct {
|
||||
Host string
|
||||
Port int
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
func (t *Traffic) newRawTCPStreamID(started time.Time) string {
|
||||
n := t.seq.Add(1)
|
||||
return fmt.Sprintf("tcp-%d-%04d", started.UnixMilli(), n%10000)
|
||||
}
|
||||
|
||||
// spillRawDir stores one direction's captured bytes: up to previewCap stays
|
||||
// inline in preview with no blob; beyond previewCap, the full captured bytes
|
||||
// are written to the content-addressed _blobs store and preview holds the
|
||||
// leading previewCap bytes.
|
||||
func (t *Traffic) spillRawDir(data []byte, previewCap int) (preview []byte, blobHash string) {
|
||||
if len(data) == 0 {
|
||||
return nil, ""
|
||||
}
|
||||
if previewCap <= 0 {
|
||||
previewCap = DefaultSOCKS5PreviewBytes
|
||||
}
|
||||
if len(data) <= previewCap {
|
||||
cp := make([]byte, len(data))
|
||||
copy(cp, data)
|
||||
return cp, ""
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
h := hex.EncodeToString(sum[:])
|
||||
blobDir := filepath.Join(t.dir, "_blobs", "sha256", h[:2])
|
||||
if err := os.MkdirAll(blobDir, 0o755); err != nil {
|
||||
log.Printf("[traffic/socks5] failed to create blob directory: %v", err)
|
||||
return append([]byte(nil), data[:previewCap]...), ""
|
||||
}
|
||||
blobPath := filepath.Join(blobDir, h+".bin")
|
||||
if _, err := os.Stat(blobPath); os.IsNotExist(err) {
|
||||
if err := writeBlobAtomic(blobDir, blobPath, h, data); err != nil {
|
||||
log.Printf("[traffic/socks5] failed to write blob %s: %v", h, err)
|
||||
return append([]byte(nil), data[:previewCap]...), ""
|
||||
}
|
||||
}
|
||||
return append([]byte(nil), data[:previewCap]...), h
|
||||
}
|
||||
|
||||
// writeBlobAtomic writes data to a temporary file inside blobDir and renames it
|
||||
// into place at blobPath so a crash mid-write cannot leave a partial file under
|
||||
// a content-addressed SHA-256 hash.
|
||||
func writeBlobAtomic(blobDir, blobPath, hash string, data []byte) error {
|
||||
tmp, err := os.CreateTemp(blobDir, hash+".*.tmp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
cleanup := true
|
||||
defer func() {
|
||||
if cleanup {
|
||||
_ = os.Remove(tmpPath)
|
||||
}
|
||||
}()
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
_ = tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Chmod(0o644); err != nil {
|
||||
_ = tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmpPath, blobPath); err != nil {
|
||||
// Another goroutine may have won the race writing the same
|
||||
// content-addressed hash (notably on Windows where Rename can fail
|
||||
// if the destination already exists).
|
||||
if _, statErr := os.Stat(blobPath); statErr == nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
cleanup = false
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecordRawTCPStream persists one raw TCP connection into raw_tcp_streams and
|
||||
// registers any spilled directional blobs in blob_refs. It never writes to the
|
||||
// HTTP exchanges table.
|
||||
func (t *Traffic) RecordRawTCPStream(rec RawTCPCapture, previewBytes int) (string, error) {
|
||||
if t == nil || t.db == nil {
|
||||
return "", fmt.Errorf("traffic store unavailable")
|
||||
}
|
||||
started := rec.StartedAt.UTC()
|
||||
if started.IsZero() {
|
||||
started = time.Now().UTC()
|
||||
}
|
||||
ended := rec.EndedAt.UTC()
|
||||
if ended.IsZero() || ended.Before(started) {
|
||||
ended = started
|
||||
}
|
||||
durMs := ended.Sub(started).Milliseconds()
|
||||
if durMs < 0 {
|
||||
durMs = 0
|
||||
}
|
||||
|
||||
c2sPrev, c2sBlob := t.spillRawDir(rec.C2SData, previewBytes)
|
||||
s2cPrev, s2cBlob := t.spillRawDir(rec.S2CData, previewBytes)
|
||||
|
||||
t.wmu.Lock()
|
||||
defer t.wmu.Unlock()
|
||||
|
||||
id := t.newRawTCPStreamID(started)
|
||||
trunc := 0
|
||||
if rec.Truncated {
|
||||
trunc = 1
|
||||
}
|
||||
|
||||
tx, err := t.db.Begin()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("begin raw tcp tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
|
||||
if _, err := tx.Exec(`
|
||||
INSERT INTO raw_tcp_streams(
|
||||
id, protocol, host, port, started_at, ended_at, duration_ms,
|
||||
c2s_bytes, s2c_bytes, truncated, c2s_preview, c2s_blob, s2c_preview, s2c_blob
|
||||
) VALUES(?, 'tcp', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
id, strings.ToLower(strings.TrimSpace(rec.Host)), rec.Port,
|
||||
started.UnixMilli(), ended.UnixMilli(), durMs,
|
||||
rec.C2SBytes, rec.S2CBytes, trunc,
|
||||
c2sPrev, nullIfEmpty(c2sBlob), s2cPrev, nullIfEmpty(s2cBlob),
|
||||
); err != nil {
|
||||
return "", fmt.Errorf("insert raw_tcp_streams: %w", err)
|
||||
}
|
||||
|
||||
for _, h := range []string{c2sBlob, s2cBlob} {
|
||||
if h == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash, exchange_id) VALUES(?, ?)`, h, id); err != nil {
|
||||
return "", fmt.Errorf("register raw tcp blob ref: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return "", fmt.Errorf("commit raw tcp tx: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// GetRawTCPStream fetches a single recorded raw TCP stream by ID. Returns nil, nil
|
||||
// when not found.
|
||||
func (t *Traffic) GetRawTCPStream(id string) (*RawTCPStream, error) {
|
||||
if t == nil || t.db == nil {
|
||||
return nil, fmt.Errorf("traffic store unavailable")
|
||||
}
|
||||
var s RawTCPStream
|
||||
var trunc int
|
||||
var c2sBlob, s2cBlob sql.NullString
|
||||
err := t.db.QueryRow(`
|
||||
SELECT id, protocol, host, port, started_at, ended_at, duration_ms,
|
||||
c2s_bytes, s2c_bytes, truncated, c2s_preview, c2s_blob, s2c_preview, s2c_blob
|
||||
FROM raw_tcp_streams WHERE id = ?`, id).Scan(
|
||||
&s.ID, &s.Protocol, &s.Host, &s.Port, &s.StartedAt, &s.EndedAt, &s.DurationMs,
|
||||
&s.C2SBytes, &s.S2CBytes, &trunc, &s.C2SPreview, &c2sBlob, &s.S2CPreview, &s2cBlob,
|
||||
)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.Truncated = trunc != 0
|
||||
s.C2SBlob = c2sBlob.String
|
||||
s.S2CBlob = s2cBlob.String
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
// ListRawTCPStreams returns raw TCP streams newest-first along with the total
|
||||
// count matching the filter.
|
||||
func (t *Traffic) ListRawTCPStreams(f RawTCPFilter) ([]RawTCPStream, int, error) {
|
||||
if t == nil || t.db == nil {
|
||||
return nil, 0, fmt.Errorf("traffic store unavailable")
|
||||
}
|
||||
var conds []string
|
||||
var args []any
|
||||
if h := strings.ToLower(strings.TrimSpace(f.Host)); h != "" {
|
||||
conds = append(conds, "host = ?")
|
||||
args = append(args, h)
|
||||
}
|
||||
if f.Port > 0 {
|
||||
conds = append(conds, "port = ?")
|
||||
args = append(args, f.Port)
|
||||
}
|
||||
where := ""
|
||||
if len(conds) > 0 {
|
||||
where = " WHERE " + strings.Join(conds, " AND ")
|
||||
}
|
||||
|
||||
var total int
|
||||
if err := t.db.QueryRow(`SELECT COUNT(*) FROM raw_tcp_streams`+where, args...).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
limit := f.Limit
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
if limit > 500 {
|
||||
limit = 500
|
||||
}
|
||||
offset := f.Offset
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
|
||||
qArgs := append(append([]any{}, args...), limit, offset)
|
||||
rows, err := t.db.Query(`
|
||||
SELECT id, protocol, host, port, started_at, ended_at, duration_ms,
|
||||
c2s_bytes, s2c_bytes, truncated, c2s_preview, c2s_blob, s2c_preview, s2c_blob
|
||||
FROM raw_tcp_streams`+where+`
|
||||
ORDER BY started_at DESC, id DESC
|
||||
LIMIT ? OFFSET ?`, qArgs...)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := []RawTCPStream{}
|
||||
for rows.Next() {
|
||||
var s RawTCPStream
|
||||
var trunc int
|
||||
var c2sBlob, s2cBlob sql.NullString
|
||||
if err := rows.Scan(
|
||||
&s.ID, &s.Protocol, &s.Host, &s.Port, &s.StartedAt, &s.EndedAt, &s.DurationMs,
|
||||
&s.C2SBytes, &s.S2CBytes, &trunc, &s.C2SPreview, &c2sBlob, &s.S2CPreview, &s2cBlob,
|
||||
); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
s.Truncated = trunc != 0
|
||||
s.C2SBlob = c2sBlob.String
|
||||
s.S2CBlob = s2cBlob.String
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, total, rows.Err()
|
||||
}
|
||||
|
||||
// CountRawTCPStreams returns the number of recorded raw TCP streams (all hosts
|
||||
// when host is empty, or filtered to one exact host).
|
||||
func (t *Traffic) CountRawTCPStreams(host string) (int, error) {
|
||||
if t == nil || t.db == nil {
|
||||
return 0, fmt.Errorf("traffic store unavailable")
|
||||
}
|
||||
var n int
|
||||
h := strings.ToLower(strings.TrimSpace(host))
|
||||
if h == "" {
|
||||
err := t.db.QueryRow(`SELECT COUNT(*) FROM raw_tcp_streams`).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
err := t.db.QueryRow(`SELECT COUNT(*) FROM raw_tcp_streams WHERE host = ?`, h).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RawUDPDatagram is one recorded UDP datagram relayed by opt-in SOCKS5 UDP
|
||||
// ASSOCIATE (P4.1c). It never creates an HTTP exchanges row.
|
||||
type RawUDPDatagram struct {
|
||||
ID string `json:"id"`
|
||||
AssociationID string `json:"association_id"`
|
||||
Protocol string `json:"protocol"`
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
Direction string `json:"direction"` // "c2s" | "s2c"
|
||||
Timestamp int64 `json:"ts"` // unix milliseconds
|
||||
Size int64 `json:"size"`
|
||||
Preview []byte `json:"preview,omitempty"`
|
||||
Blob string `json:"blob,omitempty"`
|
||||
}
|
||||
|
||||
// RawUDPFilter narrows ListRawUDPDatagrams queries.
|
||||
type RawUDPFilter struct {
|
||||
AssociationID string
|
||||
Host string
|
||||
Port int
|
||||
Direction string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
func (t *Traffic) newRawUDPDatagramID(ts time.Time) string {
|
||||
n := t.seq.Add(1)
|
||||
return fmt.Sprintf("udp-%d-%04d", ts.UnixMilli(), n%10000)
|
||||
}
|
||||
|
||||
// RecordRawUDPDatagram persists one forwarded UDP datagram into
|
||||
// raw_udp_datagrams and registers any spilled payload blob in blob_refs.
|
||||
// Payloads above previewBytes are spilled atomically via spillRawDir.
|
||||
// It never writes to the HTTP exchanges table.
|
||||
func (t *Traffic) RecordRawUDPDatagram(rec RawUDPCapture, previewBytes int) (string, error) {
|
||||
if t == nil || t.db == nil {
|
||||
return "", fmt.Errorf("traffic store unavailable")
|
||||
}
|
||||
dir := strings.ToLower(strings.TrimSpace(rec.Direction))
|
||||
if dir != "c2s" && dir != "s2c" {
|
||||
return "", fmt.Errorf("invalid raw udp direction %q", rec.Direction)
|
||||
}
|
||||
ts := rec.Timestamp.UTC()
|
||||
if ts.IsZero() {
|
||||
ts = time.Now().UTC()
|
||||
}
|
||||
|
||||
preview, blobHash := t.spillRawDir(rec.Payload, previewBytes)
|
||||
|
||||
t.wmu.Lock()
|
||||
defer t.wmu.Unlock()
|
||||
|
||||
id := t.newRawUDPDatagramID(ts)
|
||||
|
||||
tx, err := t.db.Begin()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("begin raw udp tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
|
||||
if _, err := tx.Exec(`
|
||||
INSERT INTO raw_udp_datagrams(
|
||||
id, association_id, protocol, host, port, direction, ts, size, preview, blob
|
||||
) VALUES(?, ?, 'udp', ?, ?, ?, ?, ?, ?, ?)`,
|
||||
id, strings.TrimSpace(rec.AssociationID),
|
||||
strings.ToLower(strings.TrimSpace(rec.Host)), rec.Port,
|
||||
dir, ts.UnixMilli(), int64(len(rec.Payload)),
|
||||
preview, nullIfEmpty(blobHash),
|
||||
); err != nil {
|
||||
return "", fmt.Errorf("insert raw_udp_datagrams: %w", err)
|
||||
}
|
||||
|
||||
if blobHash != "" {
|
||||
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash, exchange_id) VALUES(?, ?)`, blobHash, id); err != nil {
|
||||
return "", fmt.Errorf("register raw udp blob ref: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return "", fmt.Errorf("commit raw udp tx: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// GetRawUDPDatagram fetches one recorded UDP datagram by ID. Returns nil, nil
|
||||
// when not found.
|
||||
func (t *Traffic) GetRawUDPDatagram(id string) (*RawUDPDatagram, error) {
|
||||
if t == nil || t.db == nil {
|
||||
return nil, fmt.Errorf("traffic store unavailable")
|
||||
}
|
||||
var d RawUDPDatagram
|
||||
var blob sql.NullString
|
||||
err := t.db.QueryRow(`
|
||||
SELECT id, association_id, protocol, host, port, direction, ts, size, preview, blob
|
||||
FROM raw_udp_datagrams WHERE id = ?`, id).Scan(
|
||||
&d.ID, &d.AssociationID, &d.Protocol, &d.Host, &d.Port,
|
||||
&d.Direction, &d.Timestamp, &d.Size, &d.Preview, &blob,
|
||||
)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.Blob = blob.String
|
||||
return &d, nil
|
||||
}
|
||||
|
||||
// ListRawUDPDatagrams returns recorded UDP datagrams newest-first along with
|
||||
// the total count matching the filter.
|
||||
func (t *Traffic) ListRawUDPDatagrams(f RawUDPFilter) ([]RawUDPDatagram, int, error) {
|
||||
if t == nil || t.db == nil {
|
||||
return nil, 0, fmt.Errorf("traffic store unavailable")
|
||||
}
|
||||
var conds []string
|
||||
var args []any
|
||||
if a := strings.TrimSpace(f.AssociationID); a != "" {
|
||||
conds = append(conds, "association_id = ?")
|
||||
args = append(args, a)
|
||||
}
|
||||
if h := strings.ToLower(strings.TrimSpace(f.Host)); h != "" {
|
||||
conds = append(conds, "host = ?")
|
||||
args = append(args, h)
|
||||
}
|
||||
if f.Port > 0 {
|
||||
conds = append(conds, "port = ?")
|
||||
args = append(args, f.Port)
|
||||
}
|
||||
if dir := strings.ToLower(strings.TrimSpace(f.Direction)); dir != "" {
|
||||
conds = append(conds, "direction = ?")
|
||||
args = append(args, dir)
|
||||
}
|
||||
where := ""
|
||||
if len(conds) > 0 {
|
||||
where = " WHERE " + strings.Join(conds, " AND ")
|
||||
}
|
||||
|
||||
var total int
|
||||
if err := t.db.QueryRow(`SELECT COUNT(*) FROM raw_udp_datagrams`+where, args...).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
limit := f.Limit
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
if limit > 500 {
|
||||
limit = 500
|
||||
}
|
||||
offset := f.Offset
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
|
||||
qArgs := append(append([]any{}, args...), limit, offset)
|
||||
rows, err := t.db.Query(`
|
||||
SELECT id, association_id, protocol, host, port, direction, ts, size, preview, blob
|
||||
FROM raw_udp_datagrams`+where+`
|
||||
ORDER BY ts DESC, id DESC
|
||||
LIMIT ? OFFSET ?`, qArgs...)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := []RawUDPDatagram{}
|
||||
for rows.Next() {
|
||||
var d RawUDPDatagram
|
||||
var blob sql.NullString
|
||||
if err := rows.Scan(
|
||||
&d.ID, &d.AssociationID, &d.Protocol, &d.Host, &d.Port,
|
||||
&d.Direction, &d.Timestamp, &d.Size, &d.Preview, &blob,
|
||||
); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
d.Blob = blob.String
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, total, rows.Err()
|
||||
}
|
||||
|
||||
// CountRawUDPDatagrams returns the number of recorded raw UDP datagrams
|
||||
// optionally filtered by association_id and/or exact host.
|
||||
func (t *Traffic) CountRawUDPDatagrams(assocID, host string) (int, error) {
|
||||
if t == nil || t.db == nil {
|
||||
return 0, fmt.Errorf("traffic store unavailable")
|
||||
}
|
||||
var conds []string
|
||||
var args []any
|
||||
if a := strings.TrimSpace(assocID); a != "" {
|
||||
conds = append(conds, "association_id = ?")
|
||||
args = append(args, a)
|
||||
}
|
||||
if h := strings.ToLower(strings.TrimSpace(host)); h != "" {
|
||||
conds = append(conds, "host = ?")
|
||||
args = append(args, h)
|
||||
}
|
||||
q := `SELECT COUNT(*) FROM raw_udp_datagrams`
|
||||
if len(conds) > 0 {
|
||||
q += " WHERE " + strings.Join(conds, " AND ")
|
||||
}
|
||||
var n int
|
||||
err := t.db.QueryRow(q, args...).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
@@ -48,6 +48,9 @@ type Result struct {
|
||||
Response Response
|
||||
Changed bool
|
||||
Lines []TraceLine
|
||||
// Tainted lists fields written by a sensitive action in this phase so
|
||||
// downstream tampering hooks keep those fields HMAC-masked in the trace.
|
||||
Tainted map[string]bool
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -106,6 +109,9 @@ func (s *Set) Apply(phase Phase, in Input) Result {
|
||||
}
|
||||
}
|
||||
res.Lines = x.lines
|
||||
if len(x.tainted) > 0 {
|
||||
res.Tainted = x.tainted
|
||||
}
|
||||
if x.changed {
|
||||
res.Changed, res.Request, res.Response = true, x.st.req, x.st.resp
|
||||
}
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
package rewrite
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
// CompiledMatchers wraps the compiled matcher state of a rewrite rule so
|
||||
// tampering hooks reuse the exact same matcher validation and evaluation.
|
||||
type CompiledMatchers struct {
|
||||
r *compiledRule
|
||||
}
|
||||
|
||||
// CompileMatchers validates and compiles m for phase ("request" or "response").
|
||||
func CompileMatchers(phase Phase, m Matchers) (*CompiledMatchers, error) {
|
||||
if phase != PhaseRequest && phase != PhaseResponse {
|
||||
return nil, validatePhaseErr(phase)
|
||||
}
|
||||
r := &compiledRule{phase: phase}
|
||||
if err := r.compileMatchers(m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &CompiledMatchers{r: r}, nil
|
||||
}
|
||||
|
||||
func validatePhaseErr(phase Phase) error {
|
||||
_, err := compileRule(Rule{Name: "m", Phase: phase})
|
||||
return err
|
||||
}
|
||||
|
||||
// MatchHead evaluates head-only matchers (method, host, url, headers, status).
|
||||
func (cm *CompiledMatchers) MatchHead(req Request, resp Response) bool {
|
||||
if cm == nil || cm.r == nil {
|
||||
return true
|
||||
}
|
||||
return cm.r.matchHead(&state{req: req, resp: resp})
|
||||
}
|
||||
|
||||
// MatchBody evaluates body regex matchers.
|
||||
func (cm *CompiledMatchers) MatchBody(req Request, resp Response) bool {
|
||||
if cm == nil || cm.r == nil {
|
||||
return true
|
||||
}
|
||||
return cm.r.matchBody(&state{req: req, resp: resp})
|
||||
}
|
||||
|
||||
// ValidateHeaderTarget validates that name is a legal, non-forbidden header name
|
||||
// and returns its canonical form.
|
||||
func ValidateHeaderTarget(name string) (string, error) { return headerTarget(name) }
|
||||
|
||||
// ValidateHeaderValue rejects CRLF and NUL bytes in a header value.
|
||||
func ValidateHeaderValue(v string) error { return headerValue(v) }
|
||||
|
||||
// ValidToken reports whether s is an RFC 9110 token.
|
||||
func ValidToken(s string) bool { return validToken(s) }
|
||||
|
||||
// ParsePathQuery parses a path and query starting with '/'.
|
||||
func ParsePathQuery(s string) (*url.URL, bool) { return parsePathQuery(s) }
|
||||
|
||||
// BodyAllowedForStatus reports whether HTTP status allows a body (not 1xx/204/304).
|
||||
func BodyAllowedForStatus(status int) bool { return bodyAllowedForStatus(status) }
|
||||
|
||||
// BodyDependentHeaders returns headers invalidated when a body is rewritten.
|
||||
func BodyDependentHeaders() []string { return bodyDependentHeaders }
|
||||
|
||||
// SHA256Hex returns the lowercase hex SHA-256 digest of b.
|
||||
func SHA256Hex(b []byte) string { return sha256Hex(b) }
|
||||
|
||||
// Fingerprint returns the 128-bit keyed HMAC-SHA256 hex digest of b, or ""
|
||||
// when key is empty (matching sensitive rewrite-rule trace fingerprints).
|
||||
func Fingerprint(key, b []byte) string {
|
||||
if len(key) == 0 {
|
||||
return ""
|
||||
}
|
||||
m := hmac.New(sha256.New, key)
|
||||
m.Write(b)
|
||||
return hex.EncodeToString(m.Sum(nil)[:16])
|
||||
}
|
||||
|
||||
// HasConnectionHeader reports whether name is listed in the Connection header.
|
||||
func HasConnectionHeader(h http.Header, canonicalName string) bool {
|
||||
for _, v := range h.Values("Connection") {
|
||||
for _, part := range splitComma(v) {
|
||||
if http.CanonicalHeaderKey(part) == canonicalName {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func splitComma(s string) []string {
|
||||
var out []string
|
||||
start := 0
|
||||
for i := 0; i <= len(s); i++ {
|
||||
if i == len(s) || s[i] == ',' {
|
||||
part := trimOWS(s[start:i])
|
||||
if part != "" {
|
||||
out = append(out, part)
|
||||
}
|
||||
start = i + 1
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func trimOWS(s string) string {
|
||||
for len(s) > 0 && (s[0] == ' ' || s[0] == '\t') {
|
||||
s = s[1:]
|
||||
}
|
||||
for len(s) > 0 && (s[len(s)-1] == ' ' || s[len(s)-1] == '\t') {
|
||||
s = s[:len(s)-1]
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -35,13 +35,21 @@ const (
|
||||
ErrClassTransport = "transport_error"
|
||||
)
|
||||
|
||||
// Source kinds of a trace line in exchange_rewrites.
|
||||
const (
|
||||
SourceKindRule = "rule"
|
||||
SourceKindHook = "hook"
|
||||
)
|
||||
|
||||
// TraceLine is one row of an exchange's rewrite trace, in order. An exchange is
|
||||
// rewritten when it has at least one applied line.
|
||||
type TraceLine struct {
|
||||
Phase Phase
|
||||
RuleID int64 // 0 on a global line
|
||||
// RuleSnapshot is the rule as JSON, on its first line in the exchange only:
|
||||
// matchers in clear, sensitive action values left out, longer values summarized.
|
||||
Phase Phase
|
||||
SourceKind string // "rule" (default when empty) | "hook"
|
||||
RuleID int64 // 0 on a global or hook line
|
||||
HookID int64 // 0 on a rule or global line
|
||||
HookVersion int // 0 on a rule or global line
|
||||
// RuleSnapshot is the rule or hook metadata as JSON, on its first line in the exchange only.
|
||||
RuleSnapshot string
|
||||
Action string
|
||||
Field string // "method", "url", "header:<Name>", "body" or "status"
|
||||
|
||||
+63
-22
@@ -76,6 +76,7 @@ type reqSnap struct {
|
||||
// slot any more. The flow itself is not referenced.
|
||||
done <-chan struct{}
|
||||
rules *rewrite.Set // the snapshot of rules the flow runs with, both phases
|
||||
hooks *HookSet // the snapshot of tampering hooks the flow runs with, both phases
|
||||
host string // host name: the only part of the request a log shows
|
||||
tomb bool
|
||||
|
||||
@@ -360,10 +361,10 @@ func (t *Traffic) RewriteStats() RewriteStats {
|
||||
// newEntry builds the snapshot of a request before the rules' result is put on
|
||||
// the flow: URL and headers cloned, the body referenced, and only when a body
|
||||
// action applied (otherwise the flow's body is still the client's).
|
||||
func newEntry(f *mproxy.Flow, set *rewrite.Set, res rewrite.Result) *reqSnap {
|
||||
func newEntry(f *mproxy.Flow, set *rewrite.Set, hooks *HookSet, res rewrite.Result) *reqSnap {
|
||||
r := f.Request
|
||||
u := *r.URL
|
||||
e := &reqSnap{done: f.Done(), rules: set, host: r.URL.Hostname(), rewritten: res.Changed,
|
||||
e := &reqSnap{done: f.Done(), rules: set, hooks: hooks, host: r.URL.Hostname(), rewritten: res.Changed,
|
||||
method: r.Method, url: &u, header: r.Header.Clone(), lines: res.Lines}
|
||||
if raw := r.Raw(); raw != nil {
|
||||
e.rawHost = raw.Host
|
||||
@@ -461,7 +462,8 @@ func (s *sink) Request(f *mproxy.Flow) {
|
||||
return
|
||||
}
|
||||
set := s.t.rewriteRules.Load()
|
||||
if set.Len(rewrite.PhaseRequest) == 0 {
|
||||
hooks := s.t.tamperHooks.Load()
|
||||
if set.Len(rewrite.PhaseRequest) == 0 && hooks.Len(rewrite.PhaseRequest) == 0 {
|
||||
return
|
||||
}
|
||||
host := f.Request.URL.Hostname()
|
||||
@@ -470,12 +472,22 @@ func (s *sink) Request(f *mproxy.Flow) {
|
||||
log.Printf("[traffic] rewrite rules stopped on a request to %s (class %s): it is sent as the client sent it", host, rewrite.ErrClassPanic)
|
||||
}
|
||||
}()
|
||||
res := set.Apply(rewrite.PhaseRequest, rewrite.Input{Request: engineRequest(f.Request), IsBinary: rulesIsBinary})
|
||||
in := rewrite.Input{Request: engineRequest(f.Request), IsBinary: rulesIsBinary}
|
||||
res := set.Apply(rewrite.PhaseRequest, in)
|
||||
if hooks.Len(rewrite.PhaseRequest) > 0 {
|
||||
hIn := in
|
||||
hIn.Request = res.Request
|
||||
hRes := hooks.Apply(rewrite.PhaseRequest, hIn, res.Tainted)
|
||||
if hRes.Changed {
|
||||
res.Changed, res.Request = true, hRes.Request
|
||||
}
|
||||
res.Lines = append(res.Lines, hRes.Lines...)
|
||||
}
|
||||
runTestHook("request")
|
||||
if len(res.Lines) == 0 {
|
||||
return
|
||||
}
|
||||
if !s.t.snaps.reserve(keyOf(f), newEntry(f, set, res)) {
|
||||
if !s.t.snaps.reserve(keyOf(f), newEntry(f, set, hooks, res)) {
|
||||
return
|
||||
}
|
||||
if res.Changed {
|
||||
@@ -486,8 +498,8 @@ func (s *sink) Request(f *mproxy.Flow) {
|
||||
|
||||
// Response records the exchange and runs the response phase: the exchange
|
||||
// keeps the client's request and the target's response, the client gets the
|
||||
// rewritten response. Without a slot and without response rules this is the
|
||||
// plain record.
|
||||
// rewritten response. Without a slot and without response rules/hooks this is
|
||||
// the plain record.
|
||||
func (s *sink) Response(f *mproxy.Flow) {
|
||||
if f.Request == nil || f.Response == nil {
|
||||
return
|
||||
@@ -498,16 +510,18 @@ func (s *sink) Response(f *mproxy.Flow) {
|
||||
}
|
||||
snap := s.t.snaps.take(keyOf(f))
|
||||
set := s.t.rewriteRules.Load()
|
||||
hooks := s.t.tamperHooks.Load()
|
||||
if snap != nil {
|
||||
set = snap.rules
|
||||
hooks = snap.hooks
|
||||
}
|
||||
if snap == nil && set.Len(rewrite.PhaseResponse) == 0 {
|
||||
if snap == nil && set.Len(rewrite.PhaseResponse) == 0 && hooks.Len(rewrite.PhaseResponse) == 0 {
|
||||
s.t.record(f)
|
||||
return
|
||||
}
|
||||
rq, lines := originalRequest(f, snap)
|
||||
rs := flowResponseView(f.Response)
|
||||
res := s.applyResponse(f, set, rewrite.Input{
|
||||
res := s.applyResponse(f, set, hooks, rewrite.Input{
|
||||
Request: engineRequest(f.Request), // as sent to the target
|
||||
Response: rewrite.Response{Status: rs.status, Header: rs.header, Body: rs.body},
|
||||
IsBinary: rulesIsBinary,
|
||||
@@ -518,9 +532,10 @@ func (s *sink) Response(f *mproxy.Flow) {
|
||||
}
|
||||
}
|
||||
|
||||
// applyResponse runs the response phase; a panic yields no change and no line.
|
||||
func (s *sink) applyResponse(f *mproxy.Flow, set *rewrite.Set, in rewrite.Input) (res rewrite.Result) {
|
||||
if set.Len(rewrite.PhaseResponse) == 0 {
|
||||
// applyResponse runs the response phase (rules first, then hooks); a panic
|
||||
// yields no change and no line.
|
||||
func (s *sink) applyResponse(f *mproxy.Flow, set *rewrite.Set, hooks *HookSet, in rewrite.Input) (res rewrite.Result) {
|
||||
if set.Len(rewrite.PhaseResponse) == 0 && hooks.Len(rewrite.PhaseResponse) == 0 {
|
||||
return rewrite.Result{}
|
||||
}
|
||||
defer func() {
|
||||
@@ -530,6 +545,15 @@ func (s *sink) applyResponse(f *mproxy.Flow, set *rewrite.Set, in rewrite.Input)
|
||||
}
|
||||
}()
|
||||
r := set.Apply(rewrite.PhaseResponse, in)
|
||||
if hooks.Len(rewrite.PhaseResponse) > 0 {
|
||||
hIn := in
|
||||
hIn.Response = r.Response
|
||||
hRes := hooks.Apply(rewrite.PhaseResponse, hIn, r.Tainted)
|
||||
if hRes.Changed {
|
||||
r.Changed, r.Response = true, hRes.Response
|
||||
}
|
||||
r.Lines = append(r.Lines, hRes.Lines...)
|
||||
}
|
||||
runTestHook("response")
|
||||
return r
|
||||
}
|
||||
@@ -552,8 +576,10 @@ func (s *sink) rewriteStream(f *mproxy.Flow, kind string) (rq requestView, rs re
|
||||
}
|
||||
snap := s.t.snaps.take(keyOf(f))
|
||||
set := s.t.rewriteRules.Load()
|
||||
hooks := s.t.tamperHooks.Load()
|
||||
if snap != nil {
|
||||
set = snap.rules
|
||||
hooks = snap.hooks
|
||||
}
|
||||
rq, lines = originalRequest(f, snap)
|
||||
if kind == "sse" {
|
||||
@@ -561,10 +587,10 @@ func (s *sink) rewriteStream(f *mproxy.Flow, kind string) (rq requestView, rs re
|
||||
}
|
||||
cur := engineRequest(f.Request)
|
||||
if kind != "websocket" {
|
||||
res = s.applyResponse(f, set, rewrite.Input{Request: cur, Response: rewrite.Response{Status: rs.status, Header: rs.header}, Stream: true, IsBinary: rulesIsBinary})
|
||||
res = s.applyResponse(f, set, hooks, rewrite.Input{Request: cur, Response: rewrite.Response{Status: rs.status, Header: rs.header}, Stream: true, IsBinary: rulesIsBinary})
|
||||
return rq, rs, append(lines, res.Lines...), res
|
||||
}
|
||||
if set.Len(rewrite.PhaseRequest)+set.Len(rewrite.PhaseResponse) == 0 {
|
||||
if set.Len(rewrite.PhaseRequest)+set.Len(rewrite.PhaseResponse)+hooks.Len(rewrite.PhaseRequest)+hooks.Len(rewrite.PhaseResponse) == 0 {
|
||||
return rq, rs, lines, rewrite.Result{}
|
||||
}
|
||||
// The engine never changes a request with Upgrade; a ws:// stream arrives
|
||||
@@ -577,21 +603,26 @@ func (s *sink) rewriteStream(f *mproxy.Flow, kind string) (rq requestView, rs re
|
||||
cur.Header.Set("Upgrade", "websocket")
|
||||
}
|
||||
if snap == nil {
|
||||
lines = append(lines, s.applyRequestTrace(f, set, cur)...)
|
||||
lines = append(lines, s.applyRequestTrace(f, set, hooks, cur)...)
|
||||
}
|
||||
r := s.applyResponse(f, set, rewrite.Input{Request: cur, Response: rewrite.Response{Status: http.StatusSwitchingProtocols}, Stream: true, IsBinary: rulesIsBinary})
|
||||
r := s.applyResponse(f, set, hooks, rewrite.Input{Request: cur, Response: rewrite.Response{Status: http.StatusSwitchingProtocols}, Stream: true, IsBinary: rulesIsBinary})
|
||||
return rq, rs, append(lines, r.Lines...), rewrite.Result{}
|
||||
}
|
||||
|
||||
// applyRequestTrace runs the request phase for its trace only.
|
||||
func (s *sink) applyRequestTrace(f *mproxy.Flow, set *rewrite.Set, req rewrite.Request) (lines []rewrite.TraceLine) {
|
||||
func (s *sink) applyRequestTrace(f *mproxy.Flow, set *rewrite.Set, hooks *HookSet, req rewrite.Request) (lines []rewrite.TraceLine) {
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
lines = nil
|
||||
log.Printf("[traffic] rewrite rules stopped on a request to %s (class %s)", f.Request.URL.Hostname(), rewrite.ErrClassPanic)
|
||||
}
|
||||
}()
|
||||
return set.Apply(rewrite.PhaseRequest, rewrite.Input{Request: req, IsBinary: rulesIsBinary}).Lines
|
||||
in := rewrite.Input{Request: req, IsBinary: rulesIsBinary}
|
||||
out := set.Apply(rewrite.PhaseRequest, in).Lines
|
||||
if hooks.Len(rewrite.PhaseRequest) > 0 {
|
||||
out = append(out, hooks.Apply(rewrite.PhaseRequest, in, nil).Lines...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// applyStreamResponse puts the rewritten status and headers of an SSE
|
||||
@@ -670,16 +701,26 @@ func writeTrace(tx *sql.Tx, id string, lines []rewrite.TraceLine, trace []stored
|
||||
}
|
||||
}
|
||||
for i, l := range lines {
|
||||
var ruleID, after any
|
||||
var ruleID, hookID, hookVer, after any
|
||||
if l.RuleID != 0 {
|
||||
ruleID = l.RuleID
|
||||
}
|
||||
if l.HookID != 0 {
|
||||
hookID = l.HookID
|
||||
}
|
||||
if l.HookVersion != 0 {
|
||||
hookVer = l.HookVersion
|
||||
}
|
||||
sourceKind := l.SourceKind
|
||||
if sourceKind == "" {
|
||||
sourceKind = rewrite.SourceKindRule
|
||||
}
|
||||
if trace[i].inline != nil {
|
||||
after = trace[i].inline
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO exchange_rewrites(exchange_id,seq,phase,rule_id,rule_snapshot,action,field,outcome,reason,before_hash,after_hash,after_inline,after_blob)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
id, i, string(l.Phase), ruleID, nullIfEmpty(l.RuleSnapshot), nullIfEmpty(l.Action), nullIfEmpty(l.Field), l.Outcome, l.Reason,
|
||||
if _, err := tx.Exec(`INSERT INTO exchange_rewrites(exchange_id,seq,phase,source_kind,rule_id,hook_id,hook_version,rule_snapshot,action,field,outcome,reason,before_hash,after_hash,after_inline,after_blob)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
id, i, string(l.Phase), sourceKind, ruleID, hookID, hookVer, nullIfEmpty(l.RuleSnapshot), nullIfEmpty(l.Action), nullIfEmpty(l.Field), l.Outcome, l.Reason,
|
||||
nullIfEmpty(l.BeforeHash), nullIfEmpty(l.AfterHash), after, nullIfEmpty(trace[i].hash)); err != nil {
|
||||
return &recordError{"writing rewrite trace", err}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,603 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/traffic/hookvm"
|
||||
"github.com/Autumn-27/artex/traffic/rewrite"
|
||||
)
|
||||
|
||||
// Opt-in kill switch for constrained Go tampering hooks (P4.2).
|
||||
// Off by default: when unset/false, zero hooks are loaded and the proxy
|
||||
// follows the unchanged baseline path.
|
||||
const (
|
||||
TamperHooksEnabledEnv = "ARTEX_TAMPER_HOOKS_ENABLED"
|
||||
DefaultHookTimeout = 25 * time.Millisecond
|
||||
DefaultPhaseHookTimeout = 100 * time.Millisecond
|
||||
)
|
||||
|
||||
// TamperHooksEnabledFromEnv reports whether ARTEX_TAMPER_HOOKS_ENABLED is truthy.
|
||||
func TamperHooksEnabledFromEnv() bool {
|
||||
switch strings.ToLower(strings.TrimSpace(os.Getenv(TamperHooksEnabledEnv))) {
|
||||
case "1", "true", "yes", "on":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// HookDef is the input model compiled into a HookSet.
|
||||
type HookDef struct {
|
||||
ID int64
|
||||
Name string
|
||||
Enabled bool
|
||||
Phase rewrite.Phase
|
||||
Priority int
|
||||
Matchers rewrite.Matchers
|
||||
Source string
|
||||
SourceSHA256 string
|
||||
Version int
|
||||
Note string
|
||||
}
|
||||
|
||||
type compiledHook struct {
|
||||
id int64
|
||||
name string
|
||||
phase rewrite.Phase
|
||||
priority int
|
||||
version int
|
||||
snapshot string
|
||||
matchers *rewrite.CompiledMatchers
|
||||
prog *hookvm.Program
|
||||
}
|
||||
|
||||
type hookErrorTable struct {
|
||||
mu sync.Mutex
|
||||
errs map[int64]rewrite.LastError
|
||||
}
|
||||
|
||||
func (t *hookErrorTable) record(id int64, class string) {
|
||||
if t == nil || id == 0 {
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
if t.errs == nil {
|
||||
t.errs = map[int64]rewrite.LastError{}
|
||||
}
|
||||
cur := t.errs[id]
|
||||
cur.Class = class
|
||||
cur.At = time.Now().UTC()
|
||||
cur.Count++
|
||||
t.errs[id] = cur
|
||||
t.mu.Unlock()
|
||||
}
|
||||
|
||||
func (t *hookErrorTable) get(id int64) *rewrite.LastError {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if cur, ok := t.errs[id]; ok {
|
||||
cp := cur
|
||||
return &cp
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HookSet is an immutable compiled snapshot of enabled tampering hooks.
|
||||
type HookSet struct {
|
||||
req []*compiledHook
|
||||
resp []*compiledHook
|
||||
traceKey []byte
|
||||
errors *hookErrorTable
|
||||
// hookTimeout and phaseTimeout are overridable in tests.
|
||||
hookTimeout time.Duration
|
||||
phaseTimeout time.Duration
|
||||
maxSteps int
|
||||
maxMemory int64
|
||||
}
|
||||
|
||||
// ValidateHookDef validates metadata, matchers, and Go-subset AST for one hook.
|
||||
func ValidateHookDef(def HookDef) error {
|
||||
_, err := compileHookDef(def)
|
||||
return err
|
||||
}
|
||||
|
||||
func compileHookDef(def HookDef) (*compiledHook, error) {
|
||||
if strings.TrimSpace(def.Name) == "" {
|
||||
return nil, errors.New("name cannot be empty")
|
||||
}
|
||||
if def.Phase != rewrite.PhaseRequest && def.Phase != rewrite.PhaseResponse {
|
||||
return nil, errors.New("phase must be request or response")
|
||||
}
|
||||
cm, err := rewrite.CompileMatchers(def.Phase, def.Matchers)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("matchers: %w", err)
|
||||
}
|
||||
prog, err := hookvm.Compile(string(def.Phase), def.Source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sha := def.SourceSHA256
|
||||
if sha == "" {
|
||||
sha = rewrite.SHA256Hex([]byte(def.Source))
|
||||
}
|
||||
ver := def.Version
|
||||
if ver <= 0 {
|
||||
ver = 1
|
||||
}
|
||||
snap, _ := json.Marshal(map[string]any{
|
||||
"kind": rewrite.SourceKindHook,
|
||||
"id": def.ID,
|
||||
"name": def.Name,
|
||||
"version": ver,
|
||||
"sha256": sha,
|
||||
})
|
||||
return &compiledHook{
|
||||
id: def.ID, name: def.Name, phase: def.Phase, priority: def.Priority,
|
||||
version: ver, snapshot: string(snap), matchers: cm, prog: prog,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CompileHooks compiles enabled hooks in (priority ASC, id ASC) order. Invalid
|
||||
// enabled hooks are omitted and reported in loadErrors.
|
||||
func CompileHooks(defs []HookDef, traceKey []byte) (*HookSet, map[int64]string) {
|
||||
sorted := append([]HookDef(nil), defs...)
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
if sorted[i].Priority != sorted[j].Priority {
|
||||
return sorted[i].Priority < sorted[j].Priority
|
||||
}
|
||||
return sorted[i].ID < sorted[j].ID
|
||||
})
|
||||
hs := &HookSet{
|
||||
traceKey: append([]byte(nil), traceKey...),
|
||||
errors: &hookErrorTable{},
|
||||
hookTimeout: DefaultHookTimeout,
|
||||
phaseTimeout: DefaultPhaseHookTimeout,
|
||||
maxSteps: hookvm.DefaultMaxSteps,
|
||||
maxMemory: hookvm.DefaultMaxMemory,
|
||||
}
|
||||
loadErrors := map[int64]string{}
|
||||
for _, d := range sorted {
|
||||
if !d.Enabled {
|
||||
continue
|
||||
}
|
||||
ch, err := compileHookDef(d)
|
||||
if err != nil {
|
||||
loadErrors[d.ID] = err.Error()
|
||||
continue
|
||||
}
|
||||
if ch.phase == rewrite.PhaseRequest {
|
||||
hs.req = append(hs.req, ch)
|
||||
} else {
|
||||
hs.resp = append(hs.resp, ch)
|
||||
}
|
||||
}
|
||||
return hs, loadErrors
|
||||
}
|
||||
|
||||
// Len returns the number of compiled hooks for phase. Safe on nil receiver.
|
||||
func (s *HookSet) Len(phase rewrite.Phase) int {
|
||||
if s == nil {
|
||||
return 0
|
||||
}
|
||||
if phase == rewrite.PhaseRequest {
|
||||
return len(s.req)
|
||||
}
|
||||
return len(s.resp)
|
||||
}
|
||||
|
||||
// LastError returns the most recent runtime error recorded for hook id.
|
||||
func (s *HookSet) LastError(id int64) *rewrite.LastError {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
return s.errors.get(id)
|
||||
}
|
||||
|
||||
// DryRunHook compiles and executes a single hook (regardless of Enabled) on a
|
||||
// copy of in without affecting live traffic.
|
||||
func DryRunHook(def HookDef, in rewrite.Input, traceKey []byte) (rewrite.Result, error) {
|
||||
ch, err := compileHookDef(def)
|
||||
if err != nil {
|
||||
return rewrite.Result{}, err
|
||||
}
|
||||
hs := &HookSet{
|
||||
traceKey: append([]byte(nil), traceKey...),
|
||||
errors: &hookErrorTable{},
|
||||
hookTimeout: DefaultHookTimeout,
|
||||
phaseTimeout: DefaultPhaseHookTimeout,
|
||||
maxSteps: hookvm.DefaultMaxSteps,
|
||||
maxMemory: hookvm.DefaultMaxMemory,
|
||||
}
|
||||
if ch.phase == rewrite.PhaseRequest {
|
||||
hs.req = []*compiledHook{ch}
|
||||
} else {
|
||||
hs.resp = []*compiledHook{ch}
|
||||
}
|
||||
return hs.Apply(ch.phase, in, nil), nil
|
||||
}
|
||||
|
||||
// Apply runs all compiled hooks for phase in order, failing open per hook and
|
||||
// preserving tainted-field HMAC masking from earlier rewrite rules.
|
||||
func (s *HookSet) Apply(phase rewrite.Phase, in rewrite.Input, tainted map[string]bool) rewrite.Result {
|
||||
res := rewrite.Result{Request: in.Request, Response: in.Response}
|
||||
if s.Len(phase) == 0 {
|
||||
return res
|
||||
}
|
||||
hooks := s.req
|
||||
if phase == rewrite.PhaseResponse {
|
||||
hooks = s.resp
|
||||
}
|
||||
curReq, curResp := in.Request, in.Response
|
||||
if curReq.URL == nil {
|
||||
curReq.URL = &url.URL{}
|
||||
}
|
||||
upgrade := len(in.Request.Header.Values("Upgrade")) > 0
|
||||
|
||||
phaseBudget := s.phaseTimeout
|
||||
if phaseBudget <= 0 {
|
||||
phaseBudget = DefaultPhaseHookTimeout
|
||||
}
|
||||
phaseDeadline := time.Now().Add(phaseBudget)
|
||||
|
||||
var lines []rewrite.TraceLine
|
||||
changed := false
|
||||
for _, h := range hooks {
|
||||
if !h.matchers.MatchHead(curReq, curResp) {
|
||||
continue
|
||||
}
|
||||
if upgrade {
|
||||
lines = append(lines, hookTraceSkip(h, phase, rewrite.ReasonStream))
|
||||
continue
|
||||
}
|
||||
if reason := hookBodyIneligible(phase, in, curReq, curResp); reason != "" {
|
||||
lines = append(lines, hookTraceSkip(h, phase, reason))
|
||||
continue
|
||||
}
|
||||
if !h.matchers.MatchBody(curReq, curResp) {
|
||||
continue
|
||||
}
|
||||
|
||||
remaining := time.Until(phaseDeadline)
|
||||
if remaining <= 0 {
|
||||
s.errors.record(h.id, "timeout")
|
||||
lines = append(lines, hookTraceError(h, phase, "timeout"))
|
||||
continue
|
||||
}
|
||||
timeout := s.hookTimeout
|
||||
if timeout <= 0 {
|
||||
timeout = DefaultHookTimeout
|
||||
}
|
||||
if remaining < timeout {
|
||||
timeout = remaining
|
||||
}
|
||||
|
||||
nextReq, nextResp, hLines, errClass := s.runOne(h, phase, curReq, curResp, tainted, timeout)
|
||||
if errClass != "" {
|
||||
s.errors.record(h.id, errClass)
|
||||
lines = append(lines, hookTraceError(h, phase, errClass))
|
||||
continue
|
||||
}
|
||||
if len(hLines) > 0 {
|
||||
curReq, curResp, changed = nextReq, nextResp, true
|
||||
lines = append(lines, hLines...)
|
||||
}
|
||||
}
|
||||
|
||||
res.Lines = lines
|
||||
if changed {
|
||||
res.Changed, res.Request, res.Response = true, curReq, curResp
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func hookBodyIneligible(phase rewrite.Phase, in rewrite.Input, req rewrite.Request, resp rewrite.Response) string {
|
||||
isBin := in.IsBinary
|
||||
if isBin == nil {
|
||||
isBin = func(_ string, b []byte) bool { return bytes.IndexByte(b[:min(len(b), 512)], 0) >= 0 }
|
||||
}
|
||||
check := func(h http.Header, body []byte) string {
|
||||
if len(body) > rewrite.MaxBodySize {
|
||||
return rewrite.ReasonBodyTooLarge
|
||||
}
|
||||
if ce := strings.TrimSpace(h.Get("Content-Encoding")); ce != "" && !strings.EqualFold(ce, "identity") {
|
||||
return rewrite.ReasonContentEncoding
|
||||
}
|
||||
if isBin(h.Get("Content-Type"), body) {
|
||||
return rewrite.ReasonBinary
|
||||
}
|
||||
return ""
|
||||
}
|
||||
if phase == rewrite.PhaseRequest {
|
||||
return check(req.Header, req.Body)
|
||||
}
|
||||
if in.Stream {
|
||||
return rewrite.ReasonStream
|
||||
}
|
||||
if req.Method == http.MethodHead || !rewrite.BodyAllowedForStatus(resp.Status) {
|
||||
return rewrite.ReasonNoBodyAllowed
|
||||
}
|
||||
return check(resp.Header, resp.Body)
|
||||
}
|
||||
|
||||
func (s *HookSet) runOne(h *compiledHook, phase rewrite.Phase, req rewrite.Request, resp rewrite.Response, tainted map[string]bool, timeout time.Duration) (rewrite.Request, rewrite.Response, []rewrite.TraceLine, string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
vmReq := &hookvm.Request{
|
||||
Scheme: req.URL.Scheme,
|
||||
Host: req.URL.Hostname(),
|
||||
Port: req.URL.Port(),
|
||||
Method: req.Method,
|
||||
Path: req.URL.Path,
|
||||
RawQuery: req.URL.RawQuery,
|
||||
Headers: cloneHeaderMap(req.Header),
|
||||
Body: append([]byte(nil), req.Body...),
|
||||
}
|
||||
if vmReq.Path == "" {
|
||||
vmReq.Path = "/"
|
||||
}
|
||||
opts := hookvm.RunOptions{MaxSteps: s.maxSteps, MaxMemory: s.maxMemory}
|
||||
|
||||
if phase == rewrite.PhaseRequest {
|
||||
if err := h.prog.RunRequest(ctx, vmReq, opts); err != nil {
|
||||
return req, resp, nil, classifyHookErr(err)
|
||||
}
|
||||
return s.commitRequest(h, req, resp, vmReq, tainted)
|
||||
}
|
||||
|
||||
vmResp := &hookvm.Response{
|
||||
Status: resp.Status,
|
||||
Headers: cloneHeaderMap(resp.Header),
|
||||
Body: append([]byte(nil), resp.Body...),
|
||||
}
|
||||
if err := h.prog.RunResponse(ctx, vmReq, vmResp, opts); err != nil {
|
||||
return req, resp, nil, classifyHookErr(err)
|
||||
}
|
||||
return s.commitResponse(h, req, resp, vmResp, tainted)
|
||||
}
|
||||
|
||||
func (s *HookSet) commitRequest(h *compiledHook, origReq rewrite.Request, origResp rewrite.Response, vmReq *hookvm.Request, tainted map[string]bool) (rewrite.Request, rewrite.Response, []rewrite.TraceLine, string) {
|
||||
nextReq := origReq
|
||||
nextReq.Header = origReq.Header.Clone()
|
||||
if nextReq.Header == nil {
|
||||
nextReq.Header = http.Header{}
|
||||
}
|
||||
var lines []rewrite.TraceLine
|
||||
emit := func(field string, before, after []byte) {
|
||||
lines = append(lines, s.makeAppliedLine(h, rewrite.PhaseRequest, len(lines) == 0, field, before, after, tainted[field]))
|
||||
}
|
||||
|
||||
// 1. Method
|
||||
if vmReq.Method != origReq.Method {
|
||||
if !rewrite.ValidToken(vmReq.Method) || strings.EqualFold(vmReq.Method, http.MethodConnect) {
|
||||
return origReq, origResp, nil, "forbidden_method"
|
||||
}
|
||||
nextReq.Method = vmReq.Method
|
||||
emit("method", []byte(origReq.Method), []byte(vmReq.Method))
|
||||
}
|
||||
|
||||
// 2. Path & RawQuery (never Scheme/Host/Port)
|
||||
origURI := origReq.URL.RequestURI()
|
||||
wantURI := vmReq.Path
|
||||
if vmReq.RawQuery != "" {
|
||||
wantURI += "?" + vmReq.RawQuery
|
||||
}
|
||||
if wantURI != origURI {
|
||||
u, ok := rewrite.ParsePathQuery(wantURI)
|
||||
if !ok {
|
||||
return origReq, origResp, nil, rewrite.ErrClassInvalidURL
|
||||
}
|
||||
cp := *origReq.URL
|
||||
cp.Path, cp.RawPath, cp.RawQuery = u.Path, u.RawPath, u.RawQuery
|
||||
nextReq.URL = &cp
|
||||
emit("url", []byte(origURI), []byte(cp.RequestURI()))
|
||||
}
|
||||
|
||||
// 3. Headers (diffed & validated atomically)
|
||||
hdrLines, errClass := s.diffAndValidateHeaders(h, rewrite.PhaseRequest, origReq.Header, vmReq.Headers, nextReq.Header, tainted, len(lines) == 0)
|
||||
if errClass != "" {
|
||||
return origReq, origResp, nil, errClass
|
||||
}
|
||||
lines = append(lines, hdrLines...)
|
||||
|
||||
// 4. Body
|
||||
if !bytes.Equal(vmReq.Body, origReq.Body) {
|
||||
if len(vmReq.Body) > rewrite.MaxBodySize {
|
||||
return origReq, origResp, nil, "body_too_large"
|
||||
}
|
||||
nextReq.Body = append([]byte(nil), vmReq.Body...)
|
||||
for _, hdr := range rewrite.BodyDependentHeaders() {
|
||||
nextReq.Header.Del(hdr)
|
||||
}
|
||||
emit("body", origReq.Body, nextReq.Body)
|
||||
}
|
||||
|
||||
return nextReq, origResp, lines, ""
|
||||
}
|
||||
|
||||
func (s *HookSet) commitResponse(h *compiledHook, origReq rewrite.Request, origResp rewrite.Response, vmResp *hookvm.Response, tainted map[string]bool) (rewrite.Request, rewrite.Response, []rewrite.TraceLine, string) {
|
||||
nextResp := origResp
|
||||
nextResp.Header = origResp.Header.Clone()
|
||||
if nextResp.Header == nil {
|
||||
nextResp.Header = http.Header{}
|
||||
}
|
||||
var lines []rewrite.TraceLine
|
||||
emit := func(field string, before, after []byte) {
|
||||
lines = append(lines, s.makeAppliedLine(h, rewrite.PhaseResponse, len(lines) == 0, field, before, after, tainted[field]))
|
||||
}
|
||||
|
||||
// 1. Headers
|
||||
hdrLines, errClass := s.diffAndValidateHeaders(h, rewrite.PhaseResponse, origResp.Header, vmResp.Headers, nextResp.Header, tainted, len(lines) == 0)
|
||||
if errClass != "" {
|
||||
return origReq, origResp, nil, errClass
|
||||
}
|
||||
lines = append(lines, hdrLines...)
|
||||
|
||||
// 2. Body
|
||||
if !bytes.Equal(vmResp.Body, origResp.Body) {
|
||||
if len(vmResp.Body) > rewrite.MaxBodySize {
|
||||
return origReq, origResp, nil, "body_too_large"
|
||||
}
|
||||
nextResp.Body = append([]byte(nil), vmResp.Body...)
|
||||
for _, hdr := range rewrite.BodyDependentHeaders() {
|
||||
nextResp.Header.Del(hdr)
|
||||
}
|
||||
emit("body", origResp.Body, nextResp.Body)
|
||||
}
|
||||
|
||||
// 3. Status
|
||||
if vmResp.Status != origResp.Status {
|
||||
if vmResp.Status < 200 || vmResp.Status > 599 || !rewrite.BodyAllowedForStatus(vmResp.Status) {
|
||||
return origReq, origResp, nil, "invalid_status"
|
||||
}
|
||||
nextResp.Status = vmResp.Status
|
||||
emit("status", []byte(strconv.Itoa(origResp.Status)), []byte(strconv.Itoa(vmResp.Status)))
|
||||
}
|
||||
|
||||
return origReq, nextResp, lines, ""
|
||||
}
|
||||
|
||||
func (s *HookSet) diffAndValidateHeaders(h *compiledHook, phase rewrite.Phase, orig http.Header, mutated map[string][]string, dst http.Header, tainted map[string]bool, firstLine bool) ([]rewrite.TraceLine, string) {
|
||||
// Canonicalize mutated headers so case variants merge cleanly.
|
||||
canonMut := http.Header{}
|
||||
for k, vs := range mutated {
|
||||
ck := http.CanonicalHeaderKey(k)
|
||||
canonMut[ck] = append(canonMut[ck], vs...)
|
||||
}
|
||||
allKeys := map[string]bool{}
|
||||
for k := range orig {
|
||||
allKeys[http.CanonicalHeaderKey(k)] = true
|
||||
}
|
||||
for k := range canonMut {
|
||||
allKeys[k] = true
|
||||
}
|
||||
sortedKeys := make([]string, 0, len(allKeys))
|
||||
for k := range allKeys {
|
||||
sortedKeys = append(sortedKeys, k)
|
||||
}
|
||||
sort.Strings(sortedKeys)
|
||||
|
||||
var lines []rewrite.TraceLine
|
||||
for _, k := range sortedKeys {
|
||||
oldVals := orig.Values(k)
|
||||
newVals := canonMut.Values(k)
|
||||
if slices.Equal(oldVals, newVals) {
|
||||
continue
|
||||
}
|
||||
// Forbidden header / replay nonce / Connection-listed header check.
|
||||
ck, err := rewrite.ValidateHeaderTarget(k)
|
||||
if err != nil || strings.EqualFold(ck, replayNonceHeader) || rewrite.HasConnectionHeader(orig, ck) {
|
||||
return nil, "forbidden_header"
|
||||
}
|
||||
for _, v := range newVals {
|
||||
if err := rewrite.ValidateHeaderValue(v); err != nil {
|
||||
return nil, "invalid_header"
|
||||
}
|
||||
}
|
||||
if len(newVals) == 0 {
|
||||
dst.Del(ck)
|
||||
} else {
|
||||
dst[ck] = append([]string(nil), newVals...)
|
||||
}
|
||||
field := "header:" + ck
|
||||
before := []byte(strings.Join(oldVals, "\n"))
|
||||
after := []byte(strings.Join(newVals, "\n"))
|
||||
lines = append(lines, s.makeAppliedLine(h, phase, firstLine && len(lines) == 0, field, before, after, tainted[field]))
|
||||
}
|
||||
return lines, ""
|
||||
}
|
||||
|
||||
func (s *HookSet) makeAppliedLine(h *compiledHook, phase rewrite.Phase, includeSnap bool, field string, before, after []byte, masked bool) rewrite.TraceLine {
|
||||
l := rewrite.TraceLine{
|
||||
Phase: phase,
|
||||
SourceKind: rewrite.SourceKindHook,
|
||||
HookID: h.id,
|
||||
HookVersion: h.version,
|
||||
Action: "hook",
|
||||
Field: field,
|
||||
Outcome: rewrite.OutcomeApplied,
|
||||
}
|
||||
if includeSnap {
|
||||
l.RuleSnapshot = h.snapshot
|
||||
}
|
||||
if masked {
|
||||
l.BeforeHash = rewrite.Fingerprint(s.traceKey, before)
|
||||
l.AfterHash = rewrite.Fingerprint(s.traceKey, after)
|
||||
l.After = nil
|
||||
} else {
|
||||
l.BeforeHash = rewrite.SHA256Hex(before)
|
||||
l.AfterHash = rewrite.SHA256Hex(after)
|
||||
l.After = append([]byte(nil), after...)
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func hookTraceSkip(h *compiledHook, phase rewrite.Phase, reason string) rewrite.TraceLine {
|
||||
return rewrite.TraceLine{
|
||||
Phase: phase,
|
||||
SourceKind: rewrite.SourceKindHook,
|
||||
HookID: h.id,
|
||||
HookVersion: h.version,
|
||||
RuleSnapshot: h.snapshot,
|
||||
Action: "hook",
|
||||
Outcome: rewrite.OutcomeSkipped,
|
||||
Reason: reason,
|
||||
}
|
||||
}
|
||||
|
||||
func hookTraceError(h *compiledHook, phase rewrite.Phase, class string) rewrite.TraceLine {
|
||||
return rewrite.TraceLine{
|
||||
Phase: phase,
|
||||
SourceKind: rewrite.SourceKindHook,
|
||||
HookID: h.id,
|
||||
HookVersion: h.version,
|
||||
RuleSnapshot: h.snapshot,
|
||||
Action: "hook",
|
||||
Outcome: rewrite.OutcomeError,
|
||||
Reason: class,
|
||||
}
|
||||
}
|
||||
|
||||
func classifyHookErr(err error) string {
|
||||
var pe *hookvm.PanicError
|
||||
switch {
|
||||
case errors.As(err, &pe):
|
||||
return rewrite.ErrClassPanic
|
||||
case errors.Is(err, hookvm.ErrTimeout):
|
||||
return "timeout"
|
||||
case errors.Is(err, hookvm.ErrStepLimit):
|
||||
return "step_limit"
|
||||
case errors.Is(err, hookvm.ErrMemoryLimit):
|
||||
return "memory_limit"
|
||||
case errors.Is(err, hookvm.ErrForbiddenMutation):
|
||||
return "forbidden_mutation"
|
||||
case errors.Is(err, hookvm.ErrCallDepth):
|
||||
return "call_depth"
|
||||
default:
|
||||
return "hook_error"
|
||||
}
|
||||
}
|
||||
|
||||
func cloneHeaderMap(h http.Header) map[string][]string {
|
||||
out := make(map[string][]string, len(h))
|
||||
for k, vs := range h {
|
||||
out[http.CanonicalHeaderKey(k)] = append([]string(nil), vs...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/traffic/rewrite"
|
||||
)
|
||||
|
||||
func TestTamperHookHeaderAndBodyMutationAndTrace(t *testing.T) {
|
||||
var seenHeader, seenBody string
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
seenHeader = r.Header.Get("X-Hook")
|
||||
seenBody = string(b)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
defer target.Close()
|
||||
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
proxyAddr := ln.Addr().String()
|
||||
_ = ln.Close()
|
||||
|
||||
tr, err := Open(t.TempDir(), proxyAddr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
go func() { _ = tr.Start() }()
|
||||
waitProxyReady(t, proxyAddr)
|
||||
|
||||
reqSrc := `package hook
|
||||
import "encoding/json"
|
||||
func OnRequest(req *Request) error {
|
||||
req.SetHeader("X-Hook", "v1")
|
||||
v, err := json.Decode(req.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m, _ := v.(map[string]any)
|
||||
m["tampered"] = true
|
||||
out, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Body = out
|
||||
return nil
|
||||
}`
|
||||
respSrc := `package hook
|
||||
func OnResponse(req *Request, resp *Response) error {
|
||||
resp.SetHeader("X-Resp-Hook", "done")
|
||||
resp.Status = 201
|
||||
return nil
|
||||
}`
|
||||
hs, loadErrs := CompileHooks([]HookDef{
|
||||
{ID: 11, Name: "req-hook", Enabled: true, Phase: rewrite.PhaseRequest, Priority: 1, Source: reqSrc, Version: 2},
|
||||
{ID: 12, Name: "resp-hook", Enabled: true, Phase: rewrite.PhaseResponse, Priority: 1, Source: respSrc, Version: 3},
|
||||
}, []byte("test-trace-key-32-bytes-long-000"))
|
||||
if len(loadErrs) != 0 {
|
||||
t.Fatalf("unexpected load errors: %v", loadErrs)
|
||||
}
|
||||
tr.SetTamperHooks(hs)
|
||||
|
||||
pURL, _ := url.Parse(tr.ProxyAddr())
|
||||
client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(pURL)}}
|
||||
req, _ := http.NewRequest(http.MethodPost, target.URL+"/api/item", strings.NewReader(`{"tampered":false}`))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("proxy do: %v", err)
|
||||
}
|
||||
_, _ = io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
|
||||
if seenHeader != "v1" || !strings.Contains(seenBody, `"tampered":true`) {
|
||||
t.Fatalf("target saw header=%q body=%q", seenHeader, seenBody)
|
||||
}
|
||||
if resp.StatusCode != 201 || resp.Header.Get("X-Resp-Hook") != "done" {
|
||||
t.Fatalf("client saw status=%d header=%q", resp.StatusCode, resp.Header.Get("X-Resp-Hook"))
|
||||
}
|
||||
|
||||
// Verify original request preserved in exchange_bodies and hook trace rows in exchange_rewrites.
|
||||
var origBody []byte
|
||||
if err := tr.DB().QueryRow(`SELECT req_body FROM exchange_bodies LIMIT 1`).Scan(&origBody); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(origBody) != `{"tampered":false}` {
|
||||
t.Fatalf("stored original req_body = %q, want original client body", origBody)
|
||||
}
|
||||
|
||||
var hookRows int
|
||||
if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchange_rewrites WHERE source_kind='hook' AND outcome='applied' AND hook_id IN (11,12)`).Scan(&hookRows); err != nil || hookRows < 4 {
|
||||
t.Fatalf("hook applied trace rows = (%d, %v), want >= 4", hookRows, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTamperHookFailOpenRollbackForbiddenMutationsAndSkips(t *testing.T) {
|
||||
makeReqInput := func(body []byte, hdr http.Header) rewrite.Input {
|
||||
u, _ := url.Parse("https://api.example.test/v1/test")
|
||||
if hdr == nil {
|
||||
hdr = http.Header{"Content-Type": {"application/json"}}
|
||||
}
|
||||
return rewrite.Input{
|
||||
Request: rewrite.Request{Method: "POST", URL: u, Header: hdr, Body: body},
|
||||
Response: rewrite.Response{
|
||||
Status: 200,
|
||||
Header: http.Header{"Content-Type": {"application/json"}},
|
||||
Body: []byte(`{"ok":true}`),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("error_rolls_back_all_mutations", func(t *testing.T) {
|
||||
src := `package hook
|
||||
import "errors"
|
||||
func OnRequest(req *Request) error {
|
||||
req.SetHeader("X-Partial", "leaked")
|
||||
req.Body = []byte("mutated")
|
||||
return errors.New("abort")
|
||||
}`
|
||||
hs, _ := CompileHooks([]HookDef{{ID: 1, Name: "err", Enabled: true, Phase: rewrite.PhaseRequest, Source: src}}, nil)
|
||||
in := makeReqInput([]byte("orig"), nil)
|
||||
res := hs.Apply(rewrite.PhaseRequest, in, nil)
|
||||
if res.Changed || string(res.Request.Body) != "orig" || res.Request.Header.Get("X-Partial") != "" {
|
||||
t.Fatalf("partial mutation leaked on error: %+v", res)
|
||||
}
|
||||
if len(res.Lines) != 1 || res.Lines[0].Outcome != rewrite.OutcomeError || hs.LastError(1) == nil {
|
||||
t.Fatalf("expected error trace + LastError, got %+v / %+v", res.Lines, hs.LastError(1))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("panic_and_timeout_fail_open", func(t *testing.T) {
|
||||
panicSrc := `package hook
|
||||
func OnRequest(req *Request) error {
|
||||
req.SetHeader("X-Partial", "leaked")
|
||||
panic("kaboom")
|
||||
}`
|
||||
loopSrc := `package hook
|
||||
func OnRequest(req *Request) error {
|
||||
for {}
|
||||
return nil
|
||||
}`
|
||||
hs, _ := CompileHooks([]HookDef{
|
||||
{ID: 2, Name: "panic", Enabled: true, Phase: rewrite.PhaseRequest, Priority: 1, Source: panicSrc},
|
||||
{ID: 3, Name: "loop", Enabled: true, Phase: rewrite.PhaseRequest, Priority: 2, Source: loopSrc},
|
||||
}, nil)
|
||||
hs.hookTimeout = 15 * time.Millisecond
|
||||
hs.maxSteps = 1000
|
||||
res := hs.Apply(rewrite.PhaseRequest, makeReqInput([]byte("orig"), nil), nil)
|
||||
if res.Changed || len(res.Lines) != 2 {
|
||||
t.Fatalf("expected fail-open with 2 error lines, got changed=%v lines=%+v", res.Changed, res.Lines)
|
||||
}
|
||||
if res.Lines[0].Reason != rewrite.ErrClassPanic || (res.Lines[1].Reason != "step_limit" && res.Lines[1].Reason != "timeout") {
|
||||
t.Fatalf("unexpected error reasons: %+v", res.Lines)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("forbidden_http_mutations_rejected_atomically", func(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
phase rewrite.Phase
|
||||
src string
|
||||
want string
|
||||
}{
|
||||
{"host_change", rewrite.PhaseRequest, `package hook; func OnRequest(req *Request) error { req.SetHeader("X-Leak","1"); req.Host = "evil.test"; return nil }`, "forbidden_mutation"},
|
||||
{"scheme_change", rewrite.PhaseRequest, `package hook; func OnRequest(req *Request) error { req.SetHeader("X-Leak","1"); req.Scheme = "http"; return nil }`, "forbidden_mutation"},
|
||||
{"port_change", rewrite.PhaseRequest, `package hook; func OnRequest(req *Request) error { req.SetHeader("X-Leak","1"); req.Port = "8080"; return nil }`, "forbidden_mutation"},
|
||||
{"crlf_header", rewrite.PhaseRequest, `package hook; func OnRequest(req *Request) error { req.SetHeader("X-Leak","1"); req.SetHeader("X-Bad", "a\r\nInjected: 1"); return nil }`, "invalid_header"},
|
||||
{"connect_method", rewrite.PhaseRequest, `package hook; func OnRequest(req *Request) error { req.SetHeader("X-Leak","1"); req.Method = "CONNECT"; return nil }`, "forbidden_method"},
|
||||
{"forbidden_te_header", rewrite.PhaseRequest, `package hook; func OnRequest(req *Request) error { req.SetHeader("Transfer-Encoding", "chunked"); return nil }`, "forbidden_header"},
|
||||
{"status_204", rewrite.PhaseResponse, `package hook; func OnResponse(req *Request, resp *Response) error { resp.SetHeader("X-Leak","1"); resp.Status = 204; return nil }`, "invalid_status"},
|
||||
{"status_304", rewrite.PhaseResponse, `package hook; func OnResponse(req *Request, resp *Response) error { resp.SetHeader("X-Leak","1"); resp.Status = 304; return nil }`, "invalid_status"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
hs, errs := CompileHooks([]HookDef{{ID: 10, Name: tc.name, Enabled: true, Phase: tc.phase, Source: tc.src}}, nil)
|
||||
if len(errs) != 0 {
|
||||
t.Fatalf("compile errs: %v", errs)
|
||||
}
|
||||
res := hs.Apply(tc.phase, makeReqInput([]byte("orig"), nil), nil)
|
||||
if res.Changed {
|
||||
t.Fatalf("%s must not apply any mutation", tc.name)
|
||||
}
|
||||
if len(res.Lines) != 1 || res.Lines[0].Outcome != rewrite.OutcomeError || res.Lines[0].Reason != tc.want {
|
||||
t.Fatalf("%s lines = %+v, want error/%s", tc.name, res.Lines, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("oversized_compressed_and_binary_bodies_skipped", func(t *testing.T) {
|
||||
src := `package hook; func OnRequest(req *Request) error { req.SetHeader("X-Should-Skip","1"); return nil }`
|
||||
hs, _ := CompileHooks([]HookDef{{ID: 20, Name: "skip", Enabled: true, Phase: rewrite.PhaseRequest, Source: src}}, nil)
|
||||
|
||||
// > 1 MiB
|
||||
resLarge := hs.Apply(rewrite.PhaseRequest, makeReqInput(bytes.Repeat([]byte("a"), rewrite.MaxBodySize+1), nil), nil)
|
||||
if resLarge.Changed || len(resLarge.Lines) != 1 || resLarge.Lines[0].Reason != rewrite.ReasonBodyTooLarge {
|
||||
t.Fatalf("large body = %+v", resLarge.Lines)
|
||||
}
|
||||
// Compressed
|
||||
resGzip := hs.Apply(rewrite.PhaseRequest, makeReqInput([]byte("abc"), http.Header{"Content-Encoding": {"gzip"}}), nil)
|
||||
if resGzip.Changed || len(resGzip.Lines) != 1 || resGzip.Lines[0].Reason != rewrite.ReasonContentEncoding {
|
||||
t.Fatalf("compressed body = %+v", resGzip.Lines)
|
||||
}
|
||||
// Binary
|
||||
resBin := hs.Apply(rewrite.PhaseRequest, makeReqInput([]byte{0x89, 'P', 'N', 'G', 0x00, 0x01}, http.Header{"Content-Type": {"image/png"}}), nil)
|
||||
if resBin.Changed || len(resBin.Lines) != 1 || resBin.Lines[0].Reason != rewrite.ReasonBinary {
|
||||
t.Fatalf("binary body = %+v", resBin.Lines)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestTamperHookRulesFirstTaintedMaskingDeterminismAndNoHookBaseline(t *testing.T) {
|
||||
traceKey := []byte("0123456789abcdef0123456789abcdef")
|
||||
|
||||
// 1. Rules run first (setting a sensitive Authorization header -> taints header:Authorization),
|
||||
// then hook reads & transforms Authorization: trace must stay HMAC-masked (no plaintext secret!).
|
||||
ruleSet := rewrite.Compile([]rewrite.Rule{{
|
||||
ID: 1, Name: "auth", Enabled: true, Phase: rewrite.PhaseRequest,
|
||||
Actions: []rewrite.Action{{Type: rewrite.ActionSetHeader, Name: "Authorization", Value: "Bearer super-secret-tok", Sensitive: true}},
|
||||
}}, rewrite.Options{TraceKey: traceKey})
|
||||
|
||||
hookSrc := `package hook
|
||||
func OnRequest(req *Request) error {
|
||||
tok := req.GetHeader("Authorization")
|
||||
req.SetHeader("Authorization", tok + "-signed")
|
||||
return nil
|
||||
}`
|
||||
hs, _ := CompileHooks([]HookDef{{ID: 5, Name: "sign-auth", Enabled: true, Phase: rewrite.PhaseRequest, Source: hookSrc, Version: 1}}, traceKey)
|
||||
|
||||
u, _ := url.Parse("https://api.example.test/v1/secret")
|
||||
in := rewrite.Input{Request: rewrite.Request{Method: "GET", URL: u, Header: http.Header{}}}
|
||||
|
||||
r1 := ruleSet.Apply(rewrite.PhaseRequest, in)
|
||||
hIn := in
|
||||
hIn.Request = r1.Request
|
||||
h1 := hs.Apply(rewrite.PhaseRequest, hIn, r1.Tainted)
|
||||
|
||||
if h1.Request.Header.Get("Authorization") != "Bearer super-secret-tok-signed" {
|
||||
t.Fatalf("rules-first-then-hooks failed: got %q", h1.Request.Header.Get("Authorization"))
|
||||
}
|
||||
if len(h1.Lines) != 1 || h1.Lines[0].After != nil || strings.Contains(h1.Lines[0].AfterHash, "super-secret") {
|
||||
t.Fatalf("tainted field was not masked in hook trace: %+v", h1.Lines)
|
||||
}
|
||||
if h1.Lines[0].AfterHash == rewrite.SHA256Hex([]byte("Bearer super-secret-tok-signed")) {
|
||||
t.Fatal("tainted field used plain SHA-256 instead of keyed HMAC fingerprint")
|
||||
}
|
||||
|
||||
// 2. Determinism: two runs on identical input produce identical output and trace hashes.
|
||||
h2 := hs.Apply(rewrite.PhaseRequest, hIn, r1.Tainted)
|
||||
if h1.Lines[0].BeforeHash != h2.Lines[0].BeforeHash || h1.Lines[0].AfterHash != h2.Lines[0].AfterHash {
|
||||
t.Fatalf("non-deterministic hashes: %+v vs %+v", h1.Lines[0], h2.Lines[0])
|
||||
}
|
||||
|
||||
// 3. No-hook baseline byte-identical (kill switch off / nil HookSet).
|
||||
t.Setenv(TamperHooksEnabledEnv, "")
|
||||
if TamperHooksEnabledFromEnv() {
|
||||
t.Fatal("ARTEX_TAMPER_HOOKS_ENABLED must default to false")
|
||||
}
|
||||
var nilHooks *HookSet
|
||||
base := nilHooks.Apply(rewrite.PhaseRequest, in, nil)
|
||||
if base.Changed || len(base.Lines) != 0 {
|
||||
t.Fatalf("nil HookSet must be a no-op, got %+v", base)
|
||||
}
|
||||
}
|
||||
|
||||
func waitProxyReady(t *testing.T, addr string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if c, err := net.DialTimeout("tcp", addr, 100*time.Millisecond); err == nil {
|
||||
_ = c.Close()
|
||||
return
|
||||
}
|
||||
time.Sleep(15 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("proxy at %s did not start", addr)
|
||||
}
|
||||
@@ -0,0 +1,783 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/egress"
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
)
|
||||
|
||||
// Opt-in environment flags for the SOCKS5 inbound listener (P4.1a).
|
||||
// Off by default: when neither is set, no SOCKS5 listener is started and the
|
||||
// HTTP recording proxy is unchanged.
|
||||
const (
|
||||
// SOCKS5EnabledEnv turns on the SOCKS5 CONNECT listener on DefaultSOCKS5Addr
|
||||
// (loopback) when set to 1/true/on/yes.
|
||||
SOCKS5EnabledEnv = "ARTEX_SOCKS5_ENABLED"
|
||||
// SOCKS5AddrEnv optionally overrides the bind address (default 127.0.0.1:1080).
|
||||
// Setting a non-empty address also enables the listener unless
|
||||
// ARTEX_SOCKS5_ENABLED is explicitly false/0/off/no.
|
||||
SOCKS5AddrEnv = "ARTEX_SOCKS5_ADDR"
|
||||
// SOCKS5UDPEnabledEnv turns on RFC 1928 UDP ASSOCIATE (IPv4 first) on the
|
||||
// SOCKS5 listener when set to 1/true/on/yes. Off by default: when off,
|
||||
// UDP ASSOCIATE is refused with reply 0x07 and no UDP relay socket is opened.
|
||||
// Setting ARTEX_SOCKS5_UDP_ENABLED=1 also enables the SOCKS5 listener on
|
||||
// ARTEX_SOCKS5_ADDR (or DefaultSOCKS5Addr) unless ARTEX_SOCKS5_ENABLED is
|
||||
// explicitly false/0/off/no.
|
||||
SOCKS5UDPEnabledEnv = "ARTEX_SOCKS5_UDP_ENABLED"
|
||||
|
||||
DefaultSOCKS5Addr = "127.0.0.1:1080"
|
||||
DefaultSOCKS5MaxConcurrent = 64
|
||||
DefaultSOCKS5MaxBytesPerDir = 4 * 1024 * 1024
|
||||
DefaultSOCKS5PreviewBytes = blobPreview
|
||||
DefaultSOCKS5IdleTimeout = 2 * time.Minute
|
||||
DefaultSOCKS5HandshakeTimeout = 10 * time.Second
|
||||
DefaultSOCKS5DialTimeout = 30 * time.Second
|
||||
DefaultSOCKS5PaceTimeout = paceWaitMax
|
||||
DefaultSOCKS5UDPMaxDatagrams = 1000
|
||||
DefaultSOCKS5UDPMaxBytes = 4 * 1024 * 1024
|
||||
DefaultSOCKS5UDPIdleTimeout = 2 * time.Minute
|
||||
DefaultSOCKS5UDPMaxDatagramSize = 65535
|
||||
)
|
||||
|
||||
// RFC 1928 / RFC 1929 protocol constants.
|
||||
const (
|
||||
socks5Version = 0x05
|
||||
socks5AuthNone = 0x00
|
||||
socks5AuthUserPass = 0x02
|
||||
socks5AuthNoAccept = 0xFF
|
||||
socks5UserPassVer = 0x01
|
||||
|
||||
socks5CmdConnect = 0x01
|
||||
socks5CmdBind = 0x02
|
||||
socks5CmdUDPAssociate = 0x03
|
||||
|
||||
socks5AtypIPv4 = 0x01
|
||||
socks5AtypDomain = 0x03
|
||||
socks5AtypIPv6 = 0x04
|
||||
|
||||
socks5RepSuccess = 0x00
|
||||
socks5RepGeneralFailure = 0x01
|
||||
socks5RepNotAllowed = 0x02
|
||||
socks5RepNetworkUnreachable = 0x03
|
||||
socks5RepHostUnreachable = 0x04
|
||||
socks5RepConnectionRefused = 0x05
|
||||
socks5RepTTLExpired = 0x06
|
||||
socks5RepCmdNotSupported = 0x07
|
||||
socks5RepAtypNotSupported = 0x08
|
||||
)
|
||||
|
||||
// SOCKS5Config configures the opt-in RFC 1928 SOCKS5 CONNECT listener.
|
||||
type SOCKS5Config struct {
|
||||
// Addr is the TCP bind address (default DefaultSOCKS5Addr, loopback-bound).
|
||||
Addr string
|
||||
// MaxConcurrent bounds simultaneous active SOCKS5 streams (default 64).
|
||||
MaxConcurrent int
|
||||
// MaxBytesPerDir caps bytes captured for storage per direction (default 4 MiB).
|
||||
// Relay continues counting total bytes; exceeding the cap sets Truncated=true.
|
||||
MaxBytesPerDir int64
|
||||
// PreviewBytes caps the inline preview per direction (default 8 KiB).
|
||||
PreviewBytes int
|
||||
// IdleTimeout closes a relayed stream after no bytes move in either direction.
|
||||
IdleTimeout time.Duration
|
||||
// HandshakeTimeout bounds the RFC 1928 greeting, auth, and CONNECT request.
|
||||
HandshakeTimeout time.Duration
|
||||
// DialTimeout bounds the outbound TCP dial.
|
||||
DialTimeout time.Duration
|
||||
// PaceTimeout bounds waiting on the egress governor before refusing the dial.
|
||||
PaceTimeout time.Duration
|
||||
// DialContext opens the outbound TCP connection after policy checks.
|
||||
// Defaults to intercept.PinnedDialer("socks5", false).
|
||||
DialContext func(ctx context.Context, network, addr string) (net.Conn, error)
|
||||
// BaseContext, when non-nil, derives the context passed to the egress
|
||||
// governor and DialContext (e.g. carrying an intercept.TargetScope).
|
||||
BaseContext func() context.Context
|
||||
// OnRecord, when non-nil, receives each completed raw TCP capture.
|
||||
// When nil and attached to a Traffic store, the store's raw-TCP recorder runs.
|
||||
OnRecord func(RawTCPCapture)
|
||||
|
||||
// EnableUDP opts into RFC 1928 UDP ASSOCIATE (IPv4 first, P4.1c).
|
||||
// Off by default: when false, UDP ASSOCIATE returns 0x07.
|
||||
EnableUDP bool
|
||||
// UDPMaxDatagrams caps total forwarded datagrams (c2s + s2c) per UDP association (default 1000).
|
||||
UDPMaxDatagrams int
|
||||
// UDPMaxBytes caps total forwarded payload bytes (c2s + s2c) per UDP association (default 4 MiB).
|
||||
UDPMaxBytes int64
|
||||
// UDPIdleTimeout closes a UDP association after no forwarded datagrams move (default 2m).
|
||||
UDPIdleTimeout time.Duration
|
||||
// UDPPreviewBytes caps inline preview per stored UDP datagram before blob spill (default PreviewBytes / 8 KiB).
|
||||
UDPPreviewBytes int
|
||||
// ResolveUDP4 enforces the target gate and resolves each UDP datagram destination
|
||||
// to a verified IPv4 *net.UDPAddr before forwarding. Defaults to intercept.PinnedUDP4Resolver("socks5-udp", false).
|
||||
ResolveUDP4 func(ctx context.Context, addr string) (*net.UDPAddr, error)
|
||||
// OnRecordUDP, when non-nil, receives each forwarded UDP datagram capture.
|
||||
// When nil and attached to a Traffic store, the store's raw-UDP recorder runs.
|
||||
OnRecordUDP func(RawUDPCapture)
|
||||
}
|
||||
|
||||
// RawTCPCapture is the completed metadata and bounded directional payloads of
|
||||
// one relayed SOCKS5 TCP connection.
|
||||
type RawTCPCapture struct {
|
||||
Host string
|
||||
Port int
|
||||
StartedAt time.Time
|
||||
EndedAt time.Time
|
||||
C2SBytes int64
|
||||
S2CBytes int64
|
||||
C2SData []byte
|
||||
S2CData []byte
|
||||
Truncated bool
|
||||
Paced bool
|
||||
}
|
||||
|
||||
// SOCKS5ConfigFromEnv parses ARTEX_SOCKS5_ENABLED, ARTEX_SOCKS5_ADDR, and
|
||||
// ARTEX_SOCKS5_UDP_ENABLED. Returns (cfg, false) when disabled (the default).
|
||||
func SOCKS5ConfigFromEnv() (SOCKS5Config, bool) {
|
||||
rawEnabled := strings.ToLower(strings.TrimSpace(os.Getenv(SOCKS5EnabledEnv)))
|
||||
rawAddr := strings.TrimSpace(os.Getenv(SOCKS5AddrEnv))
|
||||
rawUDP := strings.ToLower(strings.TrimSpace(os.Getenv(SOCKS5UDPEnabledEnv)))
|
||||
|
||||
enableUDP := false
|
||||
switch rawUDP {
|
||||
case "", "0", "false", "off", "no":
|
||||
enableUDP = false
|
||||
case "1", "true", "on", "yes":
|
||||
enableUDP = true
|
||||
default:
|
||||
log.Printf("[traffic] %s=%q is not a boolean; leaving SOCKS5 UDP disabled", SOCKS5UDPEnabledEnv, rawUDP)
|
||||
}
|
||||
|
||||
switch rawEnabled {
|
||||
case "0", "false", "off", "no":
|
||||
return SOCKS5Config{}, false
|
||||
case "1", "true", "on", "yes":
|
||||
addr := rawAddr
|
||||
if addr == "" {
|
||||
addr = DefaultSOCKS5Addr
|
||||
}
|
||||
return SOCKS5Config{Addr: addr, EnableUDP: enableUDP}, true
|
||||
case "":
|
||||
if rawAddr == "" && !enableUDP {
|
||||
return SOCKS5Config{}, false
|
||||
}
|
||||
addr := rawAddr
|
||||
if addr == "" {
|
||||
addr = DefaultSOCKS5Addr
|
||||
}
|
||||
return SOCKS5Config{Addr: addr, EnableUDP: enableUDP}, true
|
||||
default:
|
||||
log.Printf("[traffic] %s=%q is not a boolean; leaving SOCKS5 disabled", SOCKS5EnabledEnv, rawEnabled)
|
||||
return SOCKS5Config{}, false
|
||||
}
|
||||
}
|
||||
|
||||
func (c SOCKS5Config) withDefaults() SOCKS5Config {
|
||||
if strings.TrimSpace(c.Addr) == "" {
|
||||
c.Addr = DefaultSOCKS5Addr
|
||||
}
|
||||
if c.MaxConcurrent <= 0 {
|
||||
c.MaxConcurrent = DefaultSOCKS5MaxConcurrent
|
||||
}
|
||||
if c.MaxBytesPerDir <= 0 {
|
||||
c.MaxBytesPerDir = DefaultSOCKS5MaxBytesPerDir
|
||||
}
|
||||
if c.PreviewBytes <= 0 {
|
||||
c.PreviewBytes = DefaultSOCKS5PreviewBytes
|
||||
}
|
||||
if c.IdleTimeout <= 0 {
|
||||
c.IdleTimeout = DefaultSOCKS5IdleTimeout
|
||||
}
|
||||
if c.HandshakeTimeout <= 0 {
|
||||
c.HandshakeTimeout = DefaultSOCKS5HandshakeTimeout
|
||||
}
|
||||
if c.DialTimeout <= 0 {
|
||||
c.DialTimeout = DefaultSOCKS5DialTimeout
|
||||
}
|
||||
if c.PaceTimeout <= 0 {
|
||||
c.PaceTimeout = DefaultSOCKS5PaceTimeout
|
||||
}
|
||||
if c.DialContext == nil {
|
||||
c.DialContext = intercept.PinnedDialer("socks5", false)
|
||||
}
|
||||
if c.UDPMaxDatagrams <= 0 {
|
||||
c.UDPMaxDatagrams = DefaultSOCKS5UDPMaxDatagrams
|
||||
}
|
||||
if c.UDPMaxBytes <= 0 {
|
||||
c.UDPMaxBytes = DefaultSOCKS5UDPMaxBytes
|
||||
}
|
||||
if c.UDPIdleTimeout <= 0 {
|
||||
c.UDPIdleTimeout = DefaultSOCKS5UDPIdleTimeout
|
||||
}
|
||||
if c.UDPPreviewBytes <= 0 {
|
||||
c.UDPPreviewBytes = c.PreviewBytes
|
||||
}
|
||||
if c.ResolveUDP4 == nil {
|
||||
c.ResolveUDP4 = intercept.PinnedUDP4Resolver("socks5-udp", false)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
type socks5Server struct {
|
||||
t *Traffic
|
||||
cfg SOCKS5Config
|
||||
ln net.Listener
|
||||
sem chan struct{}
|
||||
stop chan struct{}
|
||||
once sync.Once
|
||||
wg sync.WaitGroup
|
||||
|
||||
mu sync.Mutex
|
||||
active map[io.Closer]struct{}
|
||||
}
|
||||
|
||||
// StartSOCKS5 binds and starts the opt-in RFC 1928 SOCKS5 listener on t.
|
||||
// Returns the actual bound address (useful when cfg.Addr uses port 0).
|
||||
func (t *Traffic) StartSOCKS5(cfg SOCKS5Config) (string, error) {
|
||||
cfg = cfg.withDefaults()
|
||||
ln, err := net.Listen("tcp", cfg.Addr)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("socks5 listen %s: %w", cfg.Addr, err)
|
||||
}
|
||||
srv := &socks5Server{
|
||||
t: t,
|
||||
cfg: cfg,
|
||||
ln: ln,
|
||||
sem: make(chan struct{}, cfg.MaxConcurrent),
|
||||
stop: make(chan struct{}),
|
||||
active: map[io.Closer]struct{}{},
|
||||
}
|
||||
t.socksMu.Lock()
|
||||
if t.socks != nil {
|
||||
t.socksMu.Unlock()
|
||||
_ = ln.Close()
|
||||
return "", errors.New("socks5 listener already running")
|
||||
}
|
||||
t.socks = srv
|
||||
t.socksMu.Unlock()
|
||||
|
||||
srv.wg.Add(1)
|
||||
go srv.serve()
|
||||
return ln.Addr().String(), nil
|
||||
}
|
||||
|
||||
// SOCKS5Addr returns the bound SOCKS5 listener address, or "" when disabled.
|
||||
func (t *Traffic) SOCKS5Addr() string {
|
||||
t.socksMu.Lock()
|
||||
defer t.socksMu.Unlock()
|
||||
if t.socks == nil || t.socks.ln == nil {
|
||||
return ""
|
||||
}
|
||||
return t.socks.ln.Addr().String()
|
||||
}
|
||||
|
||||
// SOCKS5UDPEnabled reports whether the running SOCKS5 listener accepts RFC 1928
|
||||
// UDP ASSOCIATE requests.
|
||||
func (t *Traffic) SOCKS5UDPEnabled() bool {
|
||||
t.socksMu.Lock()
|
||||
defer t.socksMu.Unlock()
|
||||
return t.socks != nil && t.socks.cfg.EnableUDP
|
||||
}
|
||||
|
||||
func (s *socks5Server) Close() error {
|
||||
var closeErr error
|
||||
s.once.Do(func() {
|
||||
close(s.stop)
|
||||
closeErr = s.ln.Close()
|
||||
s.mu.Lock()
|
||||
for c := range s.active {
|
||||
_ = c.Close()
|
||||
}
|
||||
s.mu.Unlock()
|
||||
})
|
||||
s.wg.Wait()
|
||||
return closeErr
|
||||
}
|
||||
|
||||
func (s *socks5Server) stopping() bool {
|
||||
select {
|
||||
case <-s.stop:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (s *socks5Server) trackConn(c io.Closer) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.stopping() {
|
||||
return false
|
||||
}
|
||||
s.active[c] = struct{}{}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *socks5Server) untrackConn(c io.Closer) {
|
||||
s.mu.Lock()
|
||||
delete(s.active, c)
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *socks5Server) serve() {
|
||||
defer s.wg.Done()
|
||||
for {
|
||||
conn, err := s.ln.Accept()
|
||||
if err != nil {
|
||||
if s.stopping() || errors.Is(err, net.ErrClosed) {
|
||||
return
|
||||
}
|
||||
var ne net.Error
|
||||
if errors.As(err, &ne) && ne.Timeout() {
|
||||
continue
|
||||
}
|
||||
log.Printf("[traffic/socks5] accept error: %v", err)
|
||||
return
|
||||
}
|
||||
if !s.trackConn(conn) {
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
s.wg.Add(1)
|
||||
go func(c net.Conn) {
|
||||
defer s.wg.Done()
|
||||
defer s.untrackConn(c)
|
||||
defer c.Close()
|
||||
s.handleConn(c)
|
||||
}(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *socks5Server) handleConn(client net.Conn) {
|
||||
// 1. Handshake (greeting + optional RFC 1929 auth + CONNECT request) under HandshakeTimeout.
|
||||
_ = client.SetDeadline(time.Now().Add(s.cfg.HandshakeTimeout))
|
||||
|
||||
paced, ok := s.negotiateAuth(client)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
cmd, atyp, host, port, rep := readSOCKS5Request(client, s.cfg.EnableUDP)
|
||||
if rep != socks5RepSuccess {
|
||||
_ = writeSOCKS5Reply(client, rep, nil)
|
||||
return
|
||||
}
|
||||
if cmd == socks5CmdUDPAssociate {
|
||||
s.handleUDPAssociate(client, atyp, host, port)
|
||||
return
|
||||
}
|
||||
|
||||
// 2. Enforce max concurrent streams before touching policy or dialing.
|
||||
select {
|
||||
case s.sem <- struct{}{}:
|
||||
defer func() { <-s.sem }()
|
||||
default:
|
||||
_ = writeSOCKS5Reply(client, socks5RepGeneralFailure, nil)
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Apply the egress governor and target gate before dialing.
|
||||
targetAddr := net.JoinHostPort(host, strconv.Itoa(port))
|
||||
upstream, rep := s.dialTarget(host, port, targetAddr)
|
||||
if rep != socks5RepSuccess {
|
||||
_ = writeSOCKS5Reply(client, rep, nil)
|
||||
return
|
||||
}
|
||||
if !s.trackConn(upstream) {
|
||||
_ = upstream.Close()
|
||||
_ = writeSOCKS5Reply(client, socks5RepGeneralFailure, nil)
|
||||
return
|
||||
}
|
||||
defer s.untrackConn(upstream)
|
||||
defer upstream.Close()
|
||||
|
||||
// Clear handshake deadline before replying and starting the relay.
|
||||
_ = client.SetDeadline(time.Time{})
|
||||
if err := writeSOCKS5Reply(client, socks5RepSuccess, upstream.LocalAddr()); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// 4. Bidirectional half-close-aware relay with idle timeout and bounded capture.
|
||||
startedAt := time.Now().UTC()
|
||||
c2sCap := &limitedCapture{max: s.cfg.MaxBytesPerDir}
|
||||
s2cCap := &limitedCapture{max: s.cfg.MaxBytesPerDir}
|
||||
|
||||
var relayWG sync.WaitGroup
|
||||
relayWG.Add(2)
|
||||
go func() {
|
||||
defer relayWG.Done()
|
||||
relayHalf(upstream, client, c2sCap, s.cfg.IdleTimeout)
|
||||
closeWrite(upstream)
|
||||
}()
|
||||
go func() {
|
||||
defer relayWG.Done()
|
||||
relayHalf(client, upstream, s2cCap, s.cfg.IdleTimeout)
|
||||
closeWrite(client)
|
||||
}()
|
||||
relayWG.Wait()
|
||||
endedAt := time.Now().UTC()
|
||||
|
||||
rec := RawTCPCapture{
|
||||
Host: host,
|
||||
Port: port,
|
||||
StartedAt: startedAt,
|
||||
EndedAt: endedAt,
|
||||
C2SBytes: c2sCap.total,
|
||||
S2CBytes: s2cCap.total,
|
||||
C2SData: c2sCap.buf,
|
||||
S2CData: s2cCap.buf,
|
||||
Truncated: c2sCap.truncated || s2cCap.truncated,
|
||||
Paced: paced,
|
||||
}
|
||||
s.recordCapture(rec)
|
||||
}
|
||||
|
||||
func (s *socks5Server) recordCapture(rec RawTCPCapture) {
|
||||
if s.t != nil && s.t.db != nil {
|
||||
if _, err := s.t.RecordRawTCPStream(rec, s.cfg.PreviewBytes); err != nil {
|
||||
log.Printf("[traffic/socks5] failed to record raw tcp stream %s:%d: %v", rec.Host, rec.Port, err)
|
||||
}
|
||||
}
|
||||
if s.cfg.OnRecord != nil {
|
||||
s.cfg.OnRecord(rec)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *socks5Server) negotiateAuth(conn net.Conn) (paced bool, ok bool) {
|
||||
var hdr [2]byte
|
||||
if _, err := io.ReadFull(conn, hdr[:]); err != nil {
|
||||
return false, false
|
||||
}
|
||||
if hdr[0] != socks5Version || hdr[1] == 0 {
|
||||
return false, false
|
||||
}
|
||||
methods := make([]byte, int(hdr[1]))
|
||||
if _, err := io.ReadFull(conn, methods); err != nil {
|
||||
return false, false
|
||||
}
|
||||
|
||||
var hasNone, hasUserPass bool
|
||||
for _, m := range methods {
|
||||
switch m {
|
||||
case socks5AuthNone:
|
||||
hasNone = true
|
||||
case socks5AuthUserPass:
|
||||
hasUserPass = true
|
||||
}
|
||||
}
|
||||
|
||||
// Prefer username/password when offered so clients carrying the pace marker
|
||||
// (egress.PaceUser) can pass it through without requiring real credentials.
|
||||
// Accept no-auth for loopback clients.
|
||||
if hasUserPass {
|
||||
if _, err := conn.Write([]byte{socks5Version, socks5AuthUserPass}); err != nil {
|
||||
return false, false
|
||||
}
|
||||
return readSOCKS5UserPass(conn)
|
||||
}
|
||||
if hasNone && isLoopbackRemote(conn.RemoteAddr()) {
|
||||
if _, err := conn.Write([]byte{socks5Version, socks5AuthNone}); err != nil {
|
||||
return false, false
|
||||
}
|
||||
return false, true
|
||||
}
|
||||
_, _ = conn.Write([]byte{socks5Version, socks5AuthNoAccept})
|
||||
return false, false
|
||||
}
|
||||
|
||||
// readSOCKS5UserPass parses an RFC 1929 username/password subnegotiation.
|
||||
// Credentials are never required (any username/password succeeds); a username
|
||||
// equal to egress.PaceUser marks the connection as paced.
|
||||
func readSOCKS5UserPass(conn net.Conn) (paced bool, ok bool) {
|
||||
var head [2]byte
|
||||
if _, err := io.ReadFull(conn, head[:]); err != nil {
|
||||
return false, false
|
||||
}
|
||||
if head[0] != socks5UserPassVer {
|
||||
_, _ = conn.Write([]byte{socks5UserPassVer, 0x01})
|
||||
return false, false
|
||||
}
|
||||
uname := make([]byte, int(head[1]))
|
||||
if _, err := io.ReadFull(conn, uname); err != nil {
|
||||
return false, false
|
||||
}
|
||||
var plen [1]byte
|
||||
if _, err := io.ReadFull(conn, plen[:]); err != nil {
|
||||
return false, false
|
||||
}
|
||||
if plen[0] > 0 {
|
||||
pass := make([]byte, int(plen[0]))
|
||||
if _, err := io.ReadFull(conn, pass); err != nil {
|
||||
return false, false
|
||||
}
|
||||
}
|
||||
if _, err := conn.Write([]byte{socks5UserPassVer, 0x00}); err != nil {
|
||||
return false, false
|
||||
}
|
||||
return string(uname) == egress.PaceUser, true
|
||||
}
|
||||
|
||||
func readSOCKS5Request(conn net.Conn, enableUDP bool) (cmd byte, atyp byte, host string, port int, rep byte) {
|
||||
var hdr [4]byte
|
||||
if _, err := io.ReadFull(conn, hdr[:]); err != nil {
|
||||
return 0, 0, "", 0, socks5RepGeneralFailure
|
||||
}
|
||||
if hdr[0] != socks5Version || hdr[2] != 0x00 {
|
||||
return 0, 0, "", 0, socks5RepGeneralFailure
|
||||
}
|
||||
cmd = hdr[1]
|
||||
atyp = hdr[3]
|
||||
|
||||
// Read the address + port before rejecting unsupported commands so the stream
|
||||
// stays cleanly framed if the peer reads our reply.
|
||||
switch atyp {
|
||||
case socks5AtypIPv4:
|
||||
var ip [4]byte
|
||||
if _, err := io.ReadFull(conn, ip[:]); err != nil {
|
||||
return cmd, atyp, "", 0, socks5RepGeneralFailure
|
||||
}
|
||||
host = net.IP(ip[:]).String()
|
||||
case socks5AtypDomain:
|
||||
var l [1]byte
|
||||
if _, err := io.ReadFull(conn, l[:]); err != nil {
|
||||
return cmd, atyp, "", 0, socks5RepGeneralFailure
|
||||
}
|
||||
if l[0] == 0 {
|
||||
return cmd, atyp, "", 0, socks5RepGeneralFailure
|
||||
}
|
||||
dom := make([]byte, int(l[0]))
|
||||
if _, err := io.ReadFull(conn, dom); err != nil {
|
||||
return cmd, atyp, "", 0, socks5RepGeneralFailure
|
||||
}
|
||||
host = strings.TrimSpace(string(dom))
|
||||
if host == "" {
|
||||
return cmd, atyp, "", 0, socks5RepGeneralFailure
|
||||
}
|
||||
case socks5AtypIPv6:
|
||||
var ip [16]byte
|
||||
if _, err := io.ReadFull(conn, ip[:]); err != nil {
|
||||
return cmd, atyp, "", 0, socks5RepGeneralFailure
|
||||
}
|
||||
host = net.IP(ip[:]).String()
|
||||
default:
|
||||
return cmd, atyp, "", 0, socks5RepAtypNotSupported
|
||||
}
|
||||
|
||||
var pbuf [2]byte
|
||||
if _, err := io.ReadFull(conn, pbuf[:]); err != nil {
|
||||
return cmd, atyp, "", 0, socks5RepGeneralFailure
|
||||
}
|
||||
port = int(binary.BigEndian.Uint16(pbuf[:]))
|
||||
|
||||
switch cmd {
|
||||
case socks5CmdConnect:
|
||||
if port <= 0 || port > 65535 {
|
||||
return cmd, atyp, "", 0, socks5RepGeneralFailure
|
||||
}
|
||||
return cmd, atyp, host, port, socks5RepSuccess
|
||||
case socks5CmdUDPAssociate:
|
||||
if !enableUDP {
|
||||
return cmd, atyp, "", 0, socks5RepCmdNotSupported
|
||||
}
|
||||
// P4.1c covers IPv4 first; IPv6 UDP ASSOCIATE is rejected explicitly.
|
||||
if atyp == socks5AtypIPv6 {
|
||||
return cmd, atyp, "", 0, socks5RepAtypNotSupported
|
||||
}
|
||||
return cmd, atyp, host, port, socks5RepSuccess
|
||||
default:
|
||||
return cmd, atyp, "", 0, socks5RepCmdNotSupported
|
||||
}
|
||||
}
|
||||
|
||||
func writeSOCKS5Reply(conn net.Conn, rep byte, bindAddr net.Addr) error {
|
||||
var bndIP net.IP
|
||||
var bndPort int
|
||||
switch a := bindAddr.(type) {
|
||||
case *net.TCPAddr:
|
||||
if a != nil {
|
||||
bndIP = a.IP
|
||||
bndPort = a.Port
|
||||
}
|
||||
case *net.UDPAddr:
|
||||
if a != nil {
|
||||
bndIP = a.IP
|
||||
bndPort = a.Port
|
||||
}
|
||||
}
|
||||
var out []byte
|
||||
if ip4 := bndIP.To4(); ip4 != nil {
|
||||
out = make([]byte, 0, 10)
|
||||
out = append(out, socks5Version, rep, 0x00, socks5AtypIPv4)
|
||||
out = append(out, ip4...)
|
||||
} else if ip16 := bndIP.To16(); ip16 != nil {
|
||||
out = make([]byte, 0, 22)
|
||||
out = append(out, socks5Version, rep, 0x00, socks5AtypIPv6)
|
||||
out = append(out, ip16...)
|
||||
} else {
|
||||
out = make([]byte, 0, 10)
|
||||
out = append(out, socks5Version, rep, 0x00, socks5AtypIPv4, 0, 0, 0, 0)
|
||||
}
|
||||
var p [2]byte
|
||||
binary.BigEndian.PutUint16(p[:], uint16(bndPort))
|
||||
out = append(out, p[:]...)
|
||||
_, err := conn.Write(out)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *socks5Server) dialTarget(host string, port int, targetAddr string) (net.Conn, byte) {
|
||||
baseCtx := context.Background()
|
||||
if s.cfg.BaseContext != nil {
|
||||
if ctx := s.cfg.BaseContext(); ctx != nil {
|
||||
baseCtx = ctx
|
||||
}
|
||||
}
|
||||
stopCtx, stopCancel := context.WithCancel(baseCtx)
|
||||
defer stopCancel()
|
||||
go func() {
|
||||
select {
|
||||
case <-s.stop:
|
||||
stopCancel()
|
||||
case <-stopCtx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
// Run through the same egress governor (egress.Do) and target gate dialer
|
||||
// (intercept.PinnedDialer / Interceptor.DialSOCKS5) the HTTP path uses.
|
||||
govCtx, govCancel := context.WithTimeout(stopCtx, s.cfg.PaceTimeout)
|
||||
defer govCancel()
|
||||
|
||||
targetURL := &url.URL{Scheme: "tcp", Host: targetAddr}
|
||||
dialCalled := false
|
||||
conn, err := egress.Do(egress.Current(), govCtx, targetURL, func() (net.Conn, error) {
|
||||
dialCalled = true
|
||||
dialCtx, dialCancel := context.WithTimeout(stopCtx, s.cfg.DialTimeout)
|
||||
defer dialCancel()
|
||||
return s.cfg.DialContext(dialCtx, "tcp", targetAddr)
|
||||
})
|
||||
if err != nil {
|
||||
if !dialCalled {
|
||||
// Refused by the egress governor before dialing.
|
||||
return nil, socks5RepNotAllowed
|
||||
}
|
||||
return nil, mapDialErrorToSOCKS5Reply(err)
|
||||
}
|
||||
return conn, socks5RepSuccess
|
||||
}
|
||||
|
||||
func mapDialErrorToSOCKS5Reply(err error) byte {
|
||||
if _, blocked := intercept.IsTargetBlocked(err); blocked {
|
||||
return socks5RepNotAllowed
|
||||
}
|
||||
var dnsErr *net.DNSError
|
||||
if errors.As(err, &dnsErr) {
|
||||
return socks5RepHostUnreachable
|
||||
}
|
||||
if errors.Is(err, syscall.ECONNREFUSED) {
|
||||
return socks5RepConnectionRefused
|
||||
}
|
||||
if errors.Is(err, syscall.ENETUNREACH) {
|
||||
return socks5RepNetworkUnreachable
|
||||
}
|
||||
if errors.Is(err, syscall.EHOSTUNREACH) {
|
||||
return socks5RepHostUnreachable
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return socks5RepTTLExpired
|
||||
}
|
||||
low := strings.ToLower(err.Error())
|
||||
switch {
|
||||
case strings.Contains(low, "connection refused"):
|
||||
return socks5RepConnectionRefused
|
||||
case strings.Contains(low, "network is unreachable"):
|
||||
return socks5RepNetworkUnreachable
|
||||
case strings.Contains(low, "no route to host"), strings.Contains(low, "no such host"), strings.Contains(low, "host is unreachable"):
|
||||
return socks5RepHostUnreachable
|
||||
}
|
||||
return socks5RepGeneralFailure
|
||||
}
|
||||
|
||||
func isLoopbackRemote(addr net.Addr) bool {
|
||||
if addr == nil {
|
||||
return false
|
||||
}
|
||||
switch a := addr.(type) {
|
||||
case *net.TCPAddr:
|
||||
return a.IP != nil && a.IP.IsLoopback()
|
||||
default:
|
||||
h, _, err := net.SplitHostPort(addr.String())
|
||||
if err != nil {
|
||||
h = addr.String()
|
||||
}
|
||||
ip := net.ParseIP(strings.Trim(h, "[]"))
|
||||
return ip != nil && ip.IsLoopback()
|
||||
}
|
||||
}
|
||||
|
||||
type limitedCapture struct {
|
||||
max int64
|
||||
total int64
|
||||
buf []byte
|
||||
truncated bool
|
||||
}
|
||||
|
||||
func (c *limitedCapture) record(p []byte) {
|
||||
if len(p) == 0 {
|
||||
return
|
||||
}
|
||||
c.total += int64(len(p))
|
||||
rem := c.max - int64(len(c.buf))
|
||||
if rem <= 0 {
|
||||
c.truncated = true
|
||||
return
|
||||
}
|
||||
if int64(len(p)) > rem {
|
||||
c.buf = append(c.buf, p[:rem]...)
|
||||
c.truncated = true
|
||||
return
|
||||
}
|
||||
c.buf = append(c.buf, p...)
|
||||
}
|
||||
|
||||
func relayHalf(dst, src net.Conn, cap *limitedCapture, idle time.Duration) {
|
||||
buf := make([]byte, 32*1024)
|
||||
for {
|
||||
if idle > 0 {
|
||||
_ = src.SetReadDeadline(time.Now().Add(idle))
|
||||
}
|
||||
n, rerr := src.Read(buf)
|
||||
if n > 0 {
|
||||
chunk := buf[:n]
|
||||
cap.record(chunk)
|
||||
if idle > 0 {
|
||||
_ = dst.SetWriteDeadline(time.Now().Add(idle))
|
||||
}
|
||||
if _, werr := dst.Write(chunk); werr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
if rerr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func closeWrite(c net.Conn) {
|
||||
type halfCloser interface {
|
||||
CloseWrite() error
|
||||
}
|
||||
if hc, ok := c.(halfCloser); ok {
|
||||
_ = hc.CloseWrite()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,897 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/egress"
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
)
|
||||
|
||||
func startEchoServer(t *testing.T, network, addr string) (net.Listener, *atomic.Int32) {
|
||||
t.Helper()
|
||||
ln, err := net.Listen(network, addr)
|
||||
if err != nil {
|
||||
t.Fatalf("listen %s %s: %v", network, addr, err)
|
||||
}
|
||||
var accepted atomic.Int32
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
accepted.Add(1)
|
||||
wg.Add(1)
|
||||
go func(conn net.Conn) {
|
||||
defer wg.Done()
|
||||
defer conn.Close()
|
||||
_, _ = io.Copy(conn, conn)
|
||||
}(c)
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
_ = ln.Close()
|
||||
wg.Wait()
|
||||
})
|
||||
return ln, &accepted
|
||||
}
|
||||
|
||||
func socks5BuildRequest(cmd byte, host string, port int) []byte {
|
||||
req := []byte{socks5Version, cmd, 0x00}
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
req = append(req, socks5AtypIPv4)
|
||||
req = append(req, ip4...)
|
||||
} else {
|
||||
req = append(req, socks5AtypIPv6)
|
||||
req = append(req, ip.To16()...)
|
||||
}
|
||||
} else {
|
||||
req = append(req, socks5AtypDomain, byte(len(host)))
|
||||
req = append(req, []byte(host)...)
|
||||
}
|
||||
var p [2]byte
|
||||
binary.BigEndian.PutUint16(p[:], uint16(port))
|
||||
return append(req, p[:]...)
|
||||
}
|
||||
|
||||
func socks5ReadReply(t *testing.T, conn net.Conn) byte {
|
||||
t.Helper()
|
||||
var hdr [4]byte
|
||||
if _, err := io.ReadFull(conn, hdr[:]); err != nil {
|
||||
t.Fatalf("read socks5 reply header: %v", err)
|
||||
}
|
||||
if hdr[0] != socks5Version {
|
||||
t.Fatalf("reply version = 0x%02x, want 0x05", hdr[0])
|
||||
}
|
||||
switch hdr[3] {
|
||||
case socks5AtypIPv4:
|
||||
var buf [6]byte
|
||||
if _, err := io.ReadFull(conn, buf[:]); err != nil {
|
||||
t.Fatalf("read ipv4 reply tail: %v", err)
|
||||
}
|
||||
case socks5AtypIPv6:
|
||||
var buf [18]byte
|
||||
if _, err := io.ReadFull(conn, buf[:]); err != nil {
|
||||
t.Fatalf("read ipv6 reply tail: %v", err)
|
||||
}
|
||||
case socks5AtypDomain:
|
||||
var l [1]byte
|
||||
if _, err := io.ReadFull(conn, l[:]); err != nil {
|
||||
t.Fatalf("read domain len: %v", err)
|
||||
}
|
||||
buf := make([]byte, int(l[0])+2)
|
||||
if _, err := io.ReadFull(conn, buf); err != nil {
|
||||
t.Fatalf("read domain tail: %v", err)
|
||||
}
|
||||
default:
|
||||
t.Fatalf("unexpected reply atyp 0x%02x", hdr[3])
|
||||
}
|
||||
return hdr[1]
|
||||
}
|
||||
|
||||
func socks5DialNoAuth(t *testing.T, proxyAddr, host string, port int) (net.Conn, byte) {
|
||||
t.Helper()
|
||||
c, err := net.DialTimeout("tcp", proxyAddr, 3*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("dial proxy %s: %v", proxyAddr, err)
|
||||
}
|
||||
_ = c.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
if _, err := c.Write([]byte{socks5Version, 1, socks5AuthNone}); err != nil {
|
||||
_ = c.Close()
|
||||
t.Fatalf("write greeting: %v", err)
|
||||
}
|
||||
var sel [2]byte
|
||||
if _, err := io.ReadFull(c, sel[:]); err != nil {
|
||||
_ = c.Close()
|
||||
t.Fatalf("read method selection: %v", err)
|
||||
}
|
||||
if sel[0] != socks5Version || sel[1] != socks5AuthNone {
|
||||
_ = c.Close()
|
||||
t.Fatalf("method selection = %v, want [5 0]", sel)
|
||||
}
|
||||
if _, err := c.Write(socks5BuildRequest(socks5CmdConnect, host, port)); err != nil {
|
||||
_ = c.Close()
|
||||
t.Fatalf("write connect req: %v", err)
|
||||
}
|
||||
rep := socks5ReadReply(t, c)
|
||||
_ = c.SetDeadline(time.Time{})
|
||||
return c, rep
|
||||
}
|
||||
|
||||
func socks5DialUserPass(t *testing.T, proxyAddr, user, pass, host string, port int) (net.Conn, byte) {
|
||||
t.Helper()
|
||||
c, err := net.DialTimeout("tcp", proxyAddr, 3*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("dial proxy %s: %v", proxyAddr, err)
|
||||
}
|
||||
_ = c.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
if _, err := c.Write([]byte{socks5Version, 2, socks5AuthNone, socks5AuthUserPass}); err != nil {
|
||||
_ = c.Close()
|
||||
t.Fatalf("write greeting: %v", err)
|
||||
}
|
||||
var sel [2]byte
|
||||
if _, err := io.ReadFull(c, sel[:]); err != nil {
|
||||
_ = c.Close()
|
||||
t.Fatalf("read method selection: %v", err)
|
||||
}
|
||||
if sel[0] != socks5Version || sel[1] != socks5AuthUserPass {
|
||||
_ = c.Close()
|
||||
t.Fatalf("method selection = %v, want [5 2]", sel)
|
||||
}
|
||||
auth := []byte{socks5UserPassVer, byte(len(user))}
|
||||
auth = append(auth, []byte(user)...)
|
||||
auth = append(auth, byte(len(pass)))
|
||||
auth = append(auth, []byte(pass)...)
|
||||
if _, err := c.Write(auth); err != nil {
|
||||
_ = c.Close()
|
||||
t.Fatalf("write user/pass: %v", err)
|
||||
}
|
||||
var aresp [2]byte
|
||||
if _, err := io.ReadFull(c, aresp[:]); err != nil {
|
||||
_ = c.Close()
|
||||
t.Fatalf("read user/pass reply: %v", err)
|
||||
}
|
||||
if aresp[0] != socks5UserPassVer || aresp[1] != 0x00 {
|
||||
_ = c.Close()
|
||||
t.Fatalf("auth reply = %v, want [1 0]", aresp)
|
||||
}
|
||||
if _, err := c.Write(socks5BuildRequest(socks5CmdConnect, host, port)); err != nil {
|
||||
_ = c.Close()
|
||||
t.Fatalf("write connect req: %v", err)
|
||||
}
|
||||
rep := socks5ReadReply(t, c)
|
||||
_ = c.SetDeadline(time.Time{})
|
||||
return c, rep
|
||||
}
|
||||
|
||||
func waitForRawStreams(t *testing.T, tr *Traffic, want int) []RawTCPStream {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
items, total, err := tr.ListRawTCPStreams(RawTCPFilter{Limit: 100})
|
||||
if err != nil {
|
||||
t.Fatalf("ListRawTCPStreams: %v", err)
|
||||
}
|
||||
if total >= want {
|
||||
return items
|
||||
}
|
||||
time.Sleep(15 * time.Millisecond)
|
||||
}
|
||||
items, total, _ := tr.ListRawTCPStreams(RawTCPFilter{Limit: 100})
|
||||
t.Fatalf("timed out waiting for %d raw tcp streams; got total=%d (%+v)", want, total, items)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestSOCKS5FlagOffMeansNoListener(t *testing.T) {
|
||||
t.Setenv(SOCKS5EnabledEnv, "")
|
||||
t.Setenv(SOCKS5AddrEnv, "")
|
||||
if cfg, ok := SOCKS5ConfigFromEnv(); ok || cfg.Addr != "" {
|
||||
t.Fatalf("default SOCKS5ConfigFromEnv = (%+v, %v), want disabled", cfg, ok)
|
||||
}
|
||||
t.Setenv(SOCKS5EnabledEnv, "0")
|
||||
t.Setenv(SOCKS5AddrEnv, "127.0.0.1:1080")
|
||||
if _, ok := SOCKS5ConfigFromEnv(); ok {
|
||||
t.Fatalf("explicit ARTEX_SOCKS5_ENABLED=0 must stay disabled even when addr is set")
|
||||
}
|
||||
t.Setenv(SOCKS5EnabledEnv, "1")
|
||||
t.Setenv(SOCKS5AddrEnv, "")
|
||||
cfg, ok := SOCKS5ConfigFromEnv()
|
||||
if !ok || cfg.Addr != DefaultSOCKS5Addr {
|
||||
t.Fatalf("enabled default = (%+v, %v), want (%s, true)", cfg, ok, DefaultSOCKS5Addr)
|
||||
}
|
||||
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
if got := tr.SOCKS5Addr(); got != "" {
|
||||
t.Fatalf("SOCKS5Addr with flag off = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSOCKS5HandshakeSuccessAuthAndHalfCloseNoHTTPExchange(t *testing.T) {
|
||||
prev := egress.CurrentProfile()
|
||||
t.Cleanup(func() { egress.SetProfile(prev) })
|
||||
egress.SetProfile(egress.Profile{Name: "off"})
|
||||
|
||||
// Server that reads until client half-closes (CloseWrite), then replies.
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ln.Close()
|
||||
go func() {
|
||||
for {
|
||||
c, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func(conn net.Conn) {
|
||||
defer conn.Close()
|
||||
req, _ := io.ReadAll(conn)
|
||||
_, _ = conn.Write(append([]byte("ACK:"), req...))
|
||||
}(c)
|
||||
}
|
||||
}()
|
||||
targetHost, targetPortStr, _ := net.SplitHostPort(ln.Addr().String())
|
||||
targetPort, _ := strconv.Atoi(targetPortStr)
|
||||
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
var pacedSeen atomic.Bool
|
||||
var unpacedSeen atomic.Bool
|
||||
proxyAddr, err := tr.StartSOCKS5(SOCKS5Config{
|
||||
Addr: "127.0.0.1:0",
|
||||
OnRecord: func(rec RawTCPCapture) {
|
||||
if rec.Paced {
|
||||
pacedSeen.Store(true)
|
||||
} else {
|
||||
unpacedSeen.Store(true)
|
||||
}
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 1. Loopback no-auth + half-close relay.
|
||||
c1, rep := socks5DialNoAuth(t, proxyAddr, targetHost, targetPort)
|
||||
if rep != socks5RepSuccess {
|
||||
t.Fatalf("no-auth connect rep = 0x%02x, want 0x00", rep)
|
||||
}
|
||||
if _, err := c1.Write([]byte("hello-half-close")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c1.(*net.TCPConn).CloseWrite(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp1, err := io.ReadAll(c1)
|
||||
_ = c1.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(resp1) != "ACK:hello-half-close" {
|
||||
t.Fatalf("half-close reply = %q, want %q", resp1, "ACK:hello-half-close")
|
||||
}
|
||||
|
||||
// 2. Username/password with pace marker (egress.PaceUser).
|
||||
c2, rep := socks5DialUserPass(t, proxyAddr, egress.PaceUser, egress.PaceUser, targetHost, targetPort)
|
||||
if rep != socks5RepSuccess {
|
||||
t.Fatalf("user/pass pace connect rep = 0x%02x, want 0x00", rep)
|
||||
}
|
||||
_, _ = c2.Write([]byte("paced-payload"))
|
||||
_ = c2.(*net.TCPConn).CloseWrite()
|
||||
resp2, err := io.ReadAll(c2)
|
||||
_ = c2.Close()
|
||||
if err != nil || string(resp2) != "ACK:paced-payload" {
|
||||
t.Fatalf("paced reply = %q err=%v", resp2, err)
|
||||
}
|
||||
|
||||
// 3. Username/password with arbitrary/empty credentials still accepted.
|
||||
c3, rep := socks5DialUserPass(t, proxyAddr, "", "", targetHost, targetPort)
|
||||
if rep != socks5RepSuccess {
|
||||
t.Fatalf("empty user/pass connect rep = 0x%02x, want 0x00", rep)
|
||||
}
|
||||
_, _ = c3.Write([]byte("anon"))
|
||||
_ = c3.(*net.TCPConn).CloseWrite()
|
||||
_, _ = io.ReadAll(c3)
|
||||
_ = c3.Close()
|
||||
|
||||
streams := waitForRawStreams(t, tr, 3)
|
||||
if !pacedSeen.Load() || !unpacedSeen.Load() {
|
||||
t.Fatalf("paced=%v unpaced=%v, want both true", pacedSeen.Load(), unpacedSeen.Load())
|
||||
}
|
||||
|
||||
// Verify store fields, CountRawTCPStreams, GetRawTCPStream, and ZERO HTTP exchanges.
|
||||
cntAll, err := tr.CountRawTCPStreams("")
|
||||
if err != nil || cntAll != 3 {
|
||||
t.Fatalf("CountRawTCPStreams('') = %d err=%v, want 3", cntAll, err)
|
||||
}
|
||||
cntHost, err := tr.CountRawTCPStreams(targetHost)
|
||||
if err != nil || cntHost != 3 {
|
||||
t.Fatalf("CountRawTCPStreams(%q) = %d err=%v, want 3", targetHost, cntHost, err)
|
||||
}
|
||||
got, err := tr.GetRawTCPStream(streams[0].ID)
|
||||
if err != nil || got == nil {
|
||||
t.Fatalf("GetRawTCPStream(%q) = %+v err=%v", streams[0].ID, got, err)
|
||||
}
|
||||
if got.Protocol != "tcp" || got.Host != targetHost || got.Port != targetPort || got.Truncated {
|
||||
t.Fatalf("unexpected stream metadata: %+v", got)
|
||||
}
|
||||
if got.C2SBlob != "" || got.S2CBlob != "" {
|
||||
t.Fatalf("small payload unexpectedly spilled to blob: %+v", got)
|
||||
}
|
||||
|
||||
var httpRows int
|
||||
if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges`).Scan(&httpRows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if httpRows != 0 {
|
||||
t.Fatalf("raw TCP streams created %d HTTP exchanges rows, want 0", httpRows)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeRemoteAddrConn struct {
|
||||
net.Conn
|
||||
remote net.Addr
|
||||
}
|
||||
|
||||
func (f fakeRemoteAddrConn) RemoteAddr() net.Addr { return f.remote }
|
||||
|
||||
func TestSOCKS5HandshakeFailuresUnsupportedCommandsAndMalformedInput(t *testing.T) {
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
proxyAddr, err := tr.StartSOCKS5(SOCKS5Config{
|
||||
Addr: "127.0.0.1:0",
|
||||
HandshakeTimeout: 150 * time.Millisecond,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 1. BIND and UDP ASSOCIATE rejected with 0x07 (command not supported).
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
cmd byte
|
||||
}{
|
||||
{"bind", socks5CmdBind},
|
||||
{"udp_associate", socks5CmdUDPAssociate},
|
||||
{"unknown_cmd", 0x09},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
c, err := net.DialTimeout("tcp", proxyAddr, 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
_, _ = c.Write([]byte{socks5Version, 1, socks5AuthNone})
|
||||
var sel [2]byte
|
||||
if _, err := io.ReadFull(c, sel[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = c.Write(socks5BuildRequest(tc.cmd, "127.0.0.1", 8080))
|
||||
if rep := socks5ReadReply(t, c); rep != socks5RepCmdNotSupported {
|
||||
t.Fatalf("%s reply = 0x%02x, want 0x%02x", tc.name, rep, socks5RepCmdNotSupported)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// 2. Unsupported auth method -> 0x05 0xFF.
|
||||
t.Run("unsupported_auth_method", func(t *testing.T) {
|
||||
c, err := net.DialTimeout("tcp", proxyAddr, 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
_, _ = c.Write([]byte{socks5Version, 1, 0x03})
|
||||
var sel [2]byte
|
||||
if _, err := io.ReadFull(c, sel[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sel != [2]byte{socks5Version, socks5AuthNoAccept} {
|
||||
t.Fatalf("reply = %v, want [5 255]", sel)
|
||||
}
|
||||
})
|
||||
|
||||
// 3. Non-loopback client offering only no-auth -> rejected with 0x05 0xFF.
|
||||
t.Run("non_loopback_no_auth_rejected", func(t *testing.T) {
|
||||
srvSide, cliSide := net.Pipe()
|
||||
defer cliSide.Close()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
defer srvSide.Close()
|
||||
tr.socks.handleConn(fakeRemoteAddrConn{
|
||||
Conn: srvSide,
|
||||
remote: &net.TCPAddr{IP: net.ParseIP("198.51.100.10"), Port: 54321},
|
||||
})
|
||||
}()
|
||||
_, _ = cliSide.Write([]byte{socks5Version, 1, socks5AuthNone})
|
||||
var sel [2]byte
|
||||
if _, err := io.ReadFull(cliSide, sel[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sel != [2]byte{socks5Version, socks5AuthNoAccept} {
|
||||
t.Fatalf("non-loopback no-auth reply = %v, want [5 255]", sel)
|
||||
}
|
||||
<-done
|
||||
})
|
||||
|
||||
// 4. Unsupported ATYP -> 0x08.
|
||||
t.Run("unsupported_atyp", func(t *testing.T) {
|
||||
c, err := net.DialTimeout("tcp", proxyAddr, 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
_, _ = c.Write([]byte{socks5Version, 1, socks5AuthNone})
|
||||
var sel [2]byte
|
||||
_, _ = io.ReadFull(c, sel[:])
|
||||
_, _ = c.Write([]byte{socks5Version, socks5CmdConnect, 0x00, 0x09, 0x00, 0x50})
|
||||
if rep := socks5ReadReply(t, c); rep != socks5RepAtypNotSupported {
|
||||
t.Fatalf("atyp reply = 0x%02x, want 0x%02x", rep, socks5RepAtypNotSupported)
|
||||
}
|
||||
})
|
||||
|
||||
// 5. Malformed inputs & handshake deadline: must close cleanly without panic.
|
||||
malformed := [][]byte{
|
||||
{0x04, 0x01, 0x00}, // wrong version
|
||||
{0x05, 0x00}, // 0 methods
|
||||
{0x05}, // truncated greeting
|
||||
{0x05, 0x01, 0x02, 0x02, 0x01, 'a', 0x01, 'b'}, // bad RFC1929 version
|
||||
{0x05, 0x01, 0x00, 0x05, 0x01, 0xFF, 0x01, 127, 0, 0, 1, 0, 80}, // bad RSV
|
||||
{0x05, 0x01, 0x00, 0x05, 0x01, 0x00, 0x03, 0x00, 0, 80}, // empty domain
|
||||
{0x05, 0x01, 0x00, 0x05, 0x01, 0x00, 0x01, 127, 0, 0, 1, 0, 0}, // zero port
|
||||
nil, // silent client -> handshake timeout
|
||||
}
|
||||
baseG := runtime.NumGoroutine()
|
||||
for i, payload := range malformed {
|
||||
c, err := net.DialTimeout("tcp", proxyAddr, 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("malformed[%d] dial: %v", i, err)
|
||||
}
|
||||
if len(payload) > 0 {
|
||||
_, _ = c.Write(payload)
|
||||
_ = c.(*net.TCPConn).CloseWrite()
|
||||
}
|
||||
_ = c.SetReadDeadline(time.Now().Add(time.Second))
|
||||
_, _ = io.Copy(io.Discard, c)
|
||||
_ = c.Close()
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if afterG := runtime.NumGoroutine(); afterG > baseG+4 {
|
||||
t.Fatalf("possible goroutine leak after malformed handshakes: before=%d after=%d", baseG, afterG)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSOCKS5DeniedTargetRefusedBeforeDial(t *testing.T) {
|
||||
prev := egress.CurrentProfile()
|
||||
t.Cleanup(func() { egress.SetProfile(prev) })
|
||||
egress.SetProfile(egress.Profile{Name: "off"})
|
||||
|
||||
echoLn, accepted := startEchoServer(t, "tcp4", "127.0.0.1:0")
|
||||
host, portStr, _ := net.SplitHostPort(echoLn.Addr().String())
|
||||
port, _ := strconv.Atoi(portStr)
|
||||
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
// Use intercept.PinnedDialer with a gated TargetScope whose Interceptor is nil:
|
||||
// PinnedDialer returns TargetGateUnavailable (a platform dialError) BEFORE dialing.
|
||||
proxyAddr, err := tr.StartSOCKS5(SOCKS5Config{
|
||||
Addr: "127.0.0.1:0",
|
||||
BaseContext: func() context.Context {
|
||||
return intercept.WithTargetScope(context.Background(), intercept.TargetScope{
|
||||
TaskID: "99",
|
||||
Enabled: func() bool { return true },
|
||||
})
|
||||
},
|
||||
DialContext: intercept.PinnedDialer("socks5", true),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
c, rep := socks5DialNoAuth(t, proxyAddr, host, port)
|
||||
_ = c.Close()
|
||||
if rep != socks5RepNotAllowed {
|
||||
t.Fatalf("denied target reply = 0x%02x, want 0x%02x", rep, socks5RepNotAllowed)
|
||||
}
|
||||
if accepted.Load() != 0 {
|
||||
t.Fatalf("upstream accepted %d connections despite target gate refusal", accepted.Load())
|
||||
}
|
||||
if n, _ := tr.CountRawTCPStreams(""); n != 0 {
|
||||
t.Fatalf("refused connection recorded %d raw streams, want 0", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSOCKS5GovernorRefusalBeforeDial(t *testing.T) {
|
||||
prev := egress.CurrentProfile()
|
||||
t.Cleanup(func() { egress.SetProfile(prev) })
|
||||
// 1 token burst, very slow refill: 2nd request must wait far longer than PaceTimeout.
|
||||
egress.SetProfile(egress.Profile{Name: "tight", RPS: 0.01, Burst: 1, MaxPerHost: 1, MaxGlobal: 1})
|
||||
|
||||
echoLn, _ := startEchoServer(t, "tcp4", "127.0.0.1:0")
|
||||
host, portStr, _ := net.SplitHostPort(echoLn.Addr().String())
|
||||
port, _ := strconv.Atoi(portStr)
|
||||
|
||||
var dialCalls atomic.Int32
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
proxyAddr, err := tr.StartSOCKS5(SOCKS5Config{
|
||||
Addr: "127.0.0.1:0",
|
||||
PaceTimeout: 40 * time.Millisecond,
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
dialCalls.Add(1)
|
||||
return (&net.Dialer{Timeout: 2 * time.Second}).DialContext(ctx, network, addr)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// First call consumes the single burst token.
|
||||
c1, rep1 := socks5DialUserPass(t, proxyAddr, egress.PaceUser, egress.PaceUser, host, port)
|
||||
if rep1 != socks5RepSuccess {
|
||||
t.Fatalf("first connect rep = 0x%02x, want 0x00", rep1)
|
||||
}
|
||||
_ = c1.Close()
|
||||
|
||||
// Second call exceeds PaceTimeout inside egress.Do and is refused BEFORE dial.
|
||||
before := dialCalls.Load()
|
||||
c2, rep2 := socks5DialUserPass(t, proxyAddr, egress.PaceUser, egress.PaceUser, host, port)
|
||||
_ = c2.Close()
|
||||
if rep2 != socks5RepNotAllowed {
|
||||
t.Fatalf("governor refusal reply = 0x%02x, want 0x%02x", rep2, socks5RepNotAllowed)
|
||||
}
|
||||
if dialCalls.Load() != before {
|
||||
t.Fatalf("dialer was called despite egress governor refusal (before=%d after=%d)", before, dialCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSOCKS5ByteCapsTruncationAndBlobSpill(t *testing.T) {
|
||||
prev := egress.CurrentProfile()
|
||||
t.Cleanup(func() { egress.SetProfile(prev) })
|
||||
egress.SetProfile(egress.Profile{Name: "off"})
|
||||
|
||||
echoLn, _ := startEchoServer(t, "tcp4", "127.0.0.1:0")
|
||||
host, portStr, _ := net.SplitHostPort(echoLn.Addr().String())
|
||||
port, _ := strconv.Atoi(portStr)
|
||||
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
const (
|
||||
maxPerDir = 2048
|
||||
previewCap = 256
|
||||
sendTotal = 6144
|
||||
)
|
||||
proxyAddr, err := tr.StartSOCKS5(SOCKS5Config{
|
||||
Addr: "127.0.0.1:0",
|
||||
MaxBytesPerDir: maxPerDir,
|
||||
PreviewBytes: previewCap,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
payload := make([]byte, sendTotal)
|
||||
for i := range payload {
|
||||
payload[i] = byte(i % 251) // includes NUL bytes (binary stream)
|
||||
}
|
||||
|
||||
c, rep := socks5DialNoAuth(t, proxyAddr, host, port)
|
||||
if rep != socks5RepSuccess {
|
||||
t.Fatalf("connect rep = 0x%02x", rep)
|
||||
}
|
||||
go func() {
|
||||
_, _ = c.Write(payload)
|
||||
_ = c.(*net.TCPConn).CloseWrite()
|
||||
}()
|
||||
echoed, err := io.ReadAll(c)
|
||||
_ = c.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(echoed, payload) {
|
||||
t.Fatalf("relayed payload len=%d differed from sent len=%d", len(echoed), len(payload))
|
||||
}
|
||||
|
||||
streams := waitForRawStreams(t, tr, 1)
|
||||
s := streams[0]
|
||||
if !s.Truncated {
|
||||
t.Fatalf("expected Truncated=true when %d > %d", sendTotal, maxPerDir)
|
||||
}
|
||||
if s.C2SBytes != sendTotal || s.S2CBytes != sendTotal {
|
||||
t.Fatalf("byte counts = (%d, %d), want (%d, %d)", s.C2SBytes, s.S2CBytes, sendTotal, sendTotal)
|
||||
}
|
||||
if len(s.C2SPreview) != previewCap || len(s.S2CPreview) != previewCap {
|
||||
t.Fatalf("preview lengths = (%d, %d), want (%d, %d)", len(s.C2SPreview), len(s.S2CPreview), previewCap, previewCap)
|
||||
}
|
||||
if !bytes.Equal(s.C2SPreview, payload[:previewCap]) || !bytes.Equal(s.S2CPreview, payload[:previewCap]) {
|
||||
t.Fatalf("preview bytes mismatch")
|
||||
}
|
||||
if s.C2SBlob == "" || s.S2CBlob == "" {
|
||||
t.Fatalf("expected spilled blob hashes, got c2s=%q s2c=%q", s.C2SBlob, s.S2CBlob)
|
||||
}
|
||||
|
||||
// Blob must be readable via Traffic.Blob and registered in blob_refs.
|
||||
f, sz, err := tr.Blob(s.C2SBlob)
|
||||
if err != nil {
|
||||
t.Fatalf("tr.Blob(%q): %v", s.C2SBlob, err)
|
||||
}
|
||||
defer f.Close()
|
||||
blobData, _ := io.ReadAll(f)
|
||||
if sz != maxPerDir || !bytes.Equal(blobData, payload[:maxPerDir]) {
|
||||
t.Fatalf("spilled blob size=%d len=%d, want %d matching prefix", sz, len(blobData), maxPerDir)
|
||||
}
|
||||
var refs int
|
||||
if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM blob_refs WHERE hash=? AND exchange_id=?`, s.C2SBlob, s.ID).Scan(&refs); err != nil || refs != 1 {
|
||||
t.Fatalf("blob_refs count = %d err=%v, want 1", refs, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSOCKS5IPv4IPv6HostnameAndDNSFailure(t *testing.T) {
|
||||
prev := egress.CurrentProfile()
|
||||
t.Cleanup(func() { egress.SetProfile(prev) })
|
||||
egress.SetProfile(egress.Profile{Name: "off"})
|
||||
|
||||
echo4, _ := startEchoServer(t, "tcp4", "127.0.0.1:0")
|
||||
_, p4s, _ := net.SplitHostPort(echo4.Addr().String())
|
||||
p4, _ := strconv.Atoi(p4s)
|
||||
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
proxyAddr, err := tr.StartSOCKS5(SOCKS5Config{
|
||||
Addr: "127.0.0.1:0",
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
h, _, _ := net.SplitHostPort(addr)
|
||||
if h == "nonexistent.invalid" {
|
||||
return nil, &net.DNSError{Err: "no such host", Name: h, IsNotFound: true}
|
||||
}
|
||||
return (&net.Dialer{Timeout: 2 * time.Second}).DialContext(ctx, network, addr)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 1. IPv4 literal
|
||||
c4, rep := socks5DialNoAuth(t, proxyAddr, "127.0.0.1", p4)
|
||||
if rep != socks5RepSuccess {
|
||||
t.Fatalf("ipv4 rep = 0x%02x", rep)
|
||||
}
|
||||
_, _ = c4.Write([]byte("v4"))
|
||||
_ = c4.(*net.TCPConn).CloseWrite()
|
||||
got4, _ := io.ReadAll(c4)
|
||||
_ = c4.Close()
|
||||
if string(got4) != "v4" {
|
||||
t.Fatalf("ipv4 echo = %q", got4)
|
||||
}
|
||||
|
||||
// 2. Hostname (localhost)
|
||||
ch, rep := socks5DialNoAuth(t, proxyAddr, "localhost", p4)
|
||||
if rep != socks5RepSuccess {
|
||||
t.Fatalf("hostname rep = 0x%02x", rep)
|
||||
}
|
||||
_, _ = ch.Write([]byte("host"))
|
||||
_ = ch.(*net.TCPConn).CloseWrite()
|
||||
goth, _ := io.ReadAll(ch)
|
||||
_ = ch.Close()
|
||||
if string(goth) != "host" {
|
||||
t.Fatalf("hostname echo = %q", goth)
|
||||
}
|
||||
|
||||
// 3. DNS failure -> 0x04 Host unreachable
|
||||
cd, rep := socks5DialNoAuth(t, proxyAddr, "nonexistent.invalid", 443)
|
||||
_ = cd.Close()
|
||||
if rep != socks5RepHostUnreachable {
|
||||
t.Fatalf("dns failure rep = 0x%02x, want 0x%02x", rep, socks5RepHostUnreachable)
|
||||
}
|
||||
|
||||
// 4. IPv6 literal (::1) if available in this kernel/environment.
|
||||
wantStreams := 2
|
||||
if echo6, err := net.Listen("tcp6", "[::1]:0"); err == nil {
|
||||
t.Cleanup(func() { _ = echo6.Close() })
|
||||
go func() {
|
||||
for {
|
||||
c, err := echo6.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go func(conn net.Conn) {
|
||||
defer conn.Close()
|
||||
_, _ = io.Copy(conn, conn)
|
||||
}(c)
|
||||
}
|
||||
}()
|
||||
_, p6s, _ := net.SplitHostPort(echo6.Addr().String())
|
||||
p6, _ := strconv.Atoi(p6s)
|
||||
c6, rep6 := socks5DialNoAuth(t, proxyAddr, "::1", p6)
|
||||
if rep6 != socks5RepSuccess {
|
||||
t.Fatalf("ipv6 rep = 0x%02x", rep6)
|
||||
}
|
||||
_, _ = c6.Write([]byte("v6"))
|
||||
_ = c6.(*net.TCPConn).CloseWrite()
|
||||
got6, _ := io.ReadAll(c6)
|
||||
_ = c6.Close()
|
||||
if string(got6) != "v6" {
|
||||
t.Fatalf("ipv6 echo = %q", got6)
|
||||
}
|
||||
wantStreams++
|
||||
}
|
||||
|
||||
waitForRawStreams(t, tr, wantStreams)
|
||||
}
|
||||
|
||||
func TestSOCKS5ConcurrentStreamsMaxLimitAndIdleTimeout(t *testing.T) {
|
||||
prev := egress.CurrentProfile()
|
||||
t.Cleanup(func() { egress.SetProfile(prev) })
|
||||
egress.SetProfile(egress.Profile{Name: "off"})
|
||||
|
||||
echoLn, _ := startEchoServer(t, "tcp4", "127.0.0.1:0")
|
||||
host, portStr, _ := net.SplitHostPort(echoLn.Addr().String())
|
||||
port, _ := strconv.Atoi(portStr)
|
||||
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
proxyAddr, err := tr.StartSOCKS5(SOCKS5Config{
|
||||
Addr: "127.0.0.1:0",
|
||||
MaxConcurrent: 2,
|
||||
IdleTimeout: 200 * time.Millisecond,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Open 2 concurrent streams simultaneously.
|
||||
c1, rep1 := socks5DialNoAuth(t, proxyAddr, host, port)
|
||||
if rep1 != socks5RepSuccess {
|
||||
t.Fatalf("stream 1 rep = 0x%02x", rep1)
|
||||
}
|
||||
defer c1.Close()
|
||||
c2, rep2 := socks5DialNoAuth(t, proxyAddr, host, port)
|
||||
if rep2 != socks5RepSuccess {
|
||||
t.Fatalf("stream 2 rep = 0x%02x", rep2)
|
||||
}
|
||||
defer c2.Close()
|
||||
|
||||
// 3rd simultaneous stream must be rejected with 0x01 (MaxConcurrent=2).
|
||||
c3, rep3 := socks5DialNoAuth(t, proxyAddr, host, port)
|
||||
_ = c3.Close()
|
||||
if rep3 != socks5RepGeneralFailure {
|
||||
t.Fatalf("3rd concurrent stream rep = 0x%02x, want 0x%02x", rep3, socks5RepGeneralFailure)
|
||||
}
|
||||
|
||||
// Exchange bytes on both active streams while both are open.
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
checkEcho := func(c net.Conn, msg string) {
|
||||
defer wg.Done()
|
||||
if _, err := c.Write([]byte(msg)); err != nil {
|
||||
t.Errorf("write %s: %v", msg, err)
|
||||
return
|
||||
}
|
||||
buf := make([]byte, len(msg))
|
||||
if _, err := io.ReadFull(c, buf); err != nil || string(buf) != msg {
|
||||
t.Errorf("echo %s: got %q err=%v", msg, buf, err)
|
||||
}
|
||||
}
|
||||
go checkEcho(c1, "stream-one")
|
||||
go checkEcho(c2, "stream-two")
|
||||
wg.Wait()
|
||||
|
||||
// Leave both idle past IdleTimeout (200ms): relay must close them cleanly and record both.
|
||||
_ = c1.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
buf := make([]byte, 1)
|
||||
_, rerr := c1.Read(buf)
|
||||
if rerr == nil || !errors.Is(rerr, io.EOF) {
|
||||
// Peer close on idle may return EOF or connection reset depending on OS timing.
|
||||
if rerr == nil {
|
||||
t.Fatalf("expected idle stream 1 to close after IdleTimeout")
|
||||
}
|
||||
}
|
||||
_ = c1.Close()
|
||||
_ = c2.Close()
|
||||
|
||||
streams := waitForRawStreams(t, tr, 2)
|
||||
if len(streams) < 2 {
|
||||
t.Fatalf("want at least 2 recorded concurrent streams, got %d", len(streams))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawTCPAtomicBlobSpillConcurrent(t *testing.T) {
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
payload := bytes.Repeat([]byte("ATOMIC-BLOB-"), 128) // 1536 bytes > 64 preview
|
||||
const workers = 8
|
||||
var wg sync.WaitGroup
|
||||
hashes := make([]string, workers)
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
prev, h := tr.spillRawDir(payload, 64)
|
||||
if len(prev) != 64 {
|
||||
t.Errorf("worker %d preview len = %d, want 64", idx, len(prev))
|
||||
}
|
||||
hashes[idx] = h
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
wantHash := hashes[0]
|
||||
if wantHash == "" {
|
||||
t.Fatal("expected non-empty blob hash")
|
||||
}
|
||||
for i, h := range hashes {
|
||||
if h != wantHash {
|
||||
t.Fatalf("worker %d hash = %q, want %q", i, h, wantHash)
|
||||
}
|
||||
}
|
||||
blobDir := filepath.Join(tr.dir, "_blobs", "sha256", wantHash[:2])
|
||||
entries, err := os.ReadDir(blobDir)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir(%s): %v", blobDir, err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
if strings.HasSuffix(e.Name(), ".tmp") {
|
||||
t.Fatalf("leftover temp blob file: %s", e.Name())
|
||||
}
|
||||
}
|
||||
got, err := os.ReadFile(filepath.Join(blobDir, wantHash+".bin"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile blob: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, payload) {
|
||||
t.Fatalf("blob content mismatch: got %d bytes, want %d", len(got), len(payload))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,445 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/egress"
|
||||
)
|
||||
|
||||
var (
|
||||
errSOCKS5UDPShortPacket = errors.New("short SOCKS5 UDP packet")
|
||||
errSOCKS5UDPBadReserved = errors.New("non-zero SOCKS5 UDP RSV field")
|
||||
errSOCKS5UDPFragmented = errors.New("SOCKS5 UDP fragmentation (FRAG != 0) is not supported")
|
||||
errSOCKS5UDPUnsupportedAtyp = errors.New("unsupported SOCKS5 UDP address type (IPv4/domain only)")
|
||||
errSOCKS5UDPBadDestination = errors.New("invalid SOCKS5 UDP destination address or port")
|
||||
)
|
||||
|
||||
var fallbackUDPAssocSeq atomic.Int64
|
||||
|
||||
// RawUDPCapture holds one forwarded UDP datagram observed on a SOCKS5 UDP
|
||||
// ASSOCIATE relay. Each datagram is recorded individually as it arrives with
|
||||
// no fragmentation reassembly.
|
||||
type RawUDPCapture struct {
|
||||
AssociationID string
|
||||
Host string
|
||||
Port int
|
||||
Direction string // "c2s" | "s2c"
|
||||
Timestamp time.Time
|
||||
Payload []byte
|
||||
}
|
||||
|
||||
// parseSOCKS5UDPHeader parses an RFC 1928 §7 UDP request header.
|
||||
//
|
||||
// +----+------+------+----------+----------+----------+
|
||||
// |RSV | FRAG | ATYP | DST.ADDR | DST.PORT | DATA |
|
||||
// +----+------+------+----------+----------+----------+
|
||||
// | 2 | 1 | 1 | Variable | 2 | Variable |
|
||||
// +----+------+------+----------+----------+----------+
|
||||
//
|
||||
// Fragmentation reassembly is intentionally unsupported: any datagram with
|
||||
// FRAG != 0 returns errSOCKS5UDPFragmented and must be dropped per RFC 1928 §7.
|
||||
// P4.1c covers IPv4 destinations (ATYP=0x01) and domain names resolved to IPv4
|
||||
// (ATYP=0x03); IPv6 (ATYP=0x04) is rejected with errSOCKS5UDPUnsupportedAtyp.
|
||||
func parseSOCKS5UDPHeader(pkt []byte) (atyp byte, host string, port int, payload []byte, err error) {
|
||||
if len(pkt) < 4 {
|
||||
return 0, "", 0, nil, errSOCKS5UDPShortPacket
|
||||
}
|
||||
if pkt[0] != 0x00 || pkt[1] != 0x00 {
|
||||
return 0, "", 0, nil, errSOCKS5UDPBadReserved
|
||||
}
|
||||
if pkt[2] != 0x00 {
|
||||
return 0, "", 0, nil, errSOCKS5UDPFragmented
|
||||
}
|
||||
atyp = pkt[3]
|
||||
off := 4
|
||||
switch atyp {
|
||||
case socks5AtypIPv4:
|
||||
if len(pkt) < off+4+2 {
|
||||
return atyp, "", 0, nil, errSOCKS5UDPShortPacket
|
||||
}
|
||||
ip := net.IPv4(pkt[off], pkt[off+1], pkt[off+2], pkt[off+3]).To4()
|
||||
if ip == nil || ip.IsUnspecified() {
|
||||
return atyp, "", 0, nil, errSOCKS5UDPBadDestination
|
||||
}
|
||||
host = ip.String()
|
||||
off += 4
|
||||
case socks5AtypDomain:
|
||||
if len(pkt) < off+1 {
|
||||
return atyp, "", 0, nil, errSOCKS5UDPShortPacket
|
||||
}
|
||||
domLen := int(pkt[off])
|
||||
off++
|
||||
if domLen == 0 || len(pkt) < off+domLen+2 {
|
||||
return atyp, "", 0, nil, errSOCKS5UDPShortPacket
|
||||
}
|
||||
host = strings.TrimSpace(string(pkt[off : off+domLen]))
|
||||
if host == "" {
|
||||
return atyp, "", 0, nil, errSOCKS5UDPBadDestination
|
||||
}
|
||||
off += domLen
|
||||
default:
|
||||
return atyp, "", 0, nil, errSOCKS5UDPUnsupportedAtyp
|
||||
}
|
||||
|
||||
port = int(binary.BigEndian.Uint16(pkt[off : off+2]))
|
||||
off += 2
|
||||
if port <= 0 || port > 65535 {
|
||||
return atyp, "", 0, nil, errSOCKS5UDPBadDestination
|
||||
}
|
||||
return atyp, host, port, pkt[off:], nil
|
||||
}
|
||||
|
||||
// buildSOCKS5UDPPacket wraps an IPv4 upstream reply datagram in an RFC 1928 §7
|
||||
// UDP header (RSV=0x0000, FRAG=0x00, ATYP=0x01).
|
||||
func buildSOCKS5UDPPacket(srcIP net.IP, srcPort int, payload []byte) ([]byte, error) {
|
||||
ip4 := srcIP.To4()
|
||||
if ip4 == nil || srcPort <= 0 || srcPort > 65535 {
|
||||
return nil, errSOCKS5UDPBadDestination
|
||||
}
|
||||
out := make([]byte, 10+len(payload))
|
||||
out[0] = 0x00
|
||||
out[1] = 0x00
|
||||
out[2] = 0x00
|
||||
out[3] = socks5AtypIPv4
|
||||
copy(out[4:8], ip4)
|
||||
binary.BigEndian.PutUint16(out[8:10], uint16(srcPort))
|
||||
copy(out[10:], payload)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *socks5Server) newUDPAssociationID(started time.Time) string {
|
||||
var n int64
|
||||
if s.t != nil {
|
||||
n = s.t.seq.Add(1)
|
||||
} else {
|
||||
n = fallbackUDPAssocSeq.Add(1)
|
||||
}
|
||||
return fmt.Sprintf("udpa-%d-%04d", started.UnixMilli(), n%10000)
|
||||
}
|
||||
|
||||
func (s *socks5Server) handleUDPAssociate(client net.Conn, reqAtyp byte, reqHost string, reqPort int) {
|
||||
// 1. Enforce max concurrent sessions before allocating UDP sockets.
|
||||
select {
|
||||
case s.sem <- struct{}{}:
|
||||
defer func() { <-s.sem }()
|
||||
default:
|
||||
_ = writeSOCKS5Reply(client, socks5RepGeneralFailure, nil)
|
||||
return
|
||||
}
|
||||
|
||||
// 2. Bind client-facing IPv4 UDP relay socket on the TCP local interface IP.
|
||||
bindIP := net.IPv4(127, 0, 0, 1)
|
||||
if ltcp, ok := client.LocalAddr().(*net.TCPAddr); ok && ltcp != nil {
|
||||
if ip4 := ltcp.IP.To4(); ip4 != nil && !ip4.IsUnspecified() {
|
||||
bindIP = ip4
|
||||
}
|
||||
}
|
||||
relayConn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: bindIP, Port: 0})
|
||||
if err != nil {
|
||||
_ = writeSOCKS5Reply(client, socks5RepGeneralFailure, nil)
|
||||
return
|
||||
}
|
||||
if !s.trackConn(relayConn) {
|
||||
_ = relayConn.Close()
|
||||
_ = writeSOCKS5Reply(client, socks5RepGeneralFailure, nil)
|
||||
return
|
||||
}
|
||||
defer s.untrackConn(relayConn)
|
||||
defer relayConn.Close()
|
||||
|
||||
// 3. Bind upstream IPv4 UDP socket for forwarding to verified targets.
|
||||
upstreamConn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
|
||||
if err != nil {
|
||||
_ = writeSOCKS5Reply(client, socks5RepGeneralFailure, nil)
|
||||
return
|
||||
}
|
||||
if !s.trackConn(upstreamConn) {
|
||||
_ = upstreamConn.Close()
|
||||
_ = writeSOCKS5Reply(client, socks5RepGeneralFailure, nil)
|
||||
return
|
||||
}
|
||||
defer s.untrackConn(upstreamConn)
|
||||
defer upstreamConn.Close()
|
||||
|
||||
// Clear handshake deadline and send BND.ADDR + BND.PORT on TCP control conn.
|
||||
_ = client.SetDeadline(time.Time{})
|
||||
relayAddr := relayConn.LocalAddr().(*net.UDPAddr)
|
||||
if err := writeSOCKS5Reply(client, socks5RepSuccess, relayAddr); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Determine allowed client UDP source IP/port.
|
||||
var expectedClientIP net.IP
|
||||
expectedClientPort := 0
|
||||
if reqAtyp == socks5AtypIPv4 {
|
||||
if ip := net.ParseIP(reqHost).To4(); ip != nil && !ip.IsUnspecified() {
|
||||
expectedClientIP = ip
|
||||
}
|
||||
if reqPort > 0 && reqPort <= 65535 {
|
||||
expectedClientPort = reqPort
|
||||
}
|
||||
}
|
||||
if expectedClientIP == nil {
|
||||
if rtcp, ok := client.RemoteAddr().(*net.TCPAddr); ok && rtcp != nil {
|
||||
expectedClientIP = rtcp.IP.To4()
|
||||
}
|
||||
}
|
||||
|
||||
assocID := s.newUDPAssociationID(time.Now().UTC())
|
||||
|
||||
baseCtx := context.Background()
|
||||
if s.cfg.BaseContext != nil {
|
||||
if ctx := s.cfg.BaseContext(); ctx != nil {
|
||||
baseCtx = ctx
|
||||
}
|
||||
}
|
||||
assocCtx, assocCancel := context.WithCancel(baseCtx)
|
||||
var closeOnce sync.Once
|
||||
closeAssoc := func() {
|
||||
closeOnce.Do(func() {
|
||||
assocCancel()
|
||||
_ = relayConn.Close()
|
||||
_ = upstreamConn.Close()
|
||||
_ = client.Close()
|
||||
})
|
||||
}
|
||||
defer closeAssoc()
|
||||
|
||||
// Close association when server stops.
|
||||
stopWatchDone := make(chan struct{})
|
||||
go func() {
|
||||
select {
|
||||
case <-s.stop:
|
||||
closeAssoc()
|
||||
case <-assocCtx.Done():
|
||||
}
|
||||
close(stopWatchDone)
|
||||
}()
|
||||
defer func() { <-stopWatchDone }()
|
||||
|
||||
// Idle timer bounds inactive associations.
|
||||
var idleTimer *time.Timer
|
||||
if s.cfg.UDPIdleTimeout > 0 {
|
||||
idleTimer = time.AfterFunc(s.cfg.UDPIdleTimeout, closeAssoc)
|
||||
defer idleTimer.Stop()
|
||||
}
|
||||
touchIdle := func() {
|
||||
if idleTimer != nil {
|
||||
idleTimer.Reset(s.cfg.UDPIdleTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
assocMu sync.Mutex
|
||||
clientUDPAddr *net.UDPAddr
|
||||
allowedPeers = map[string]string{} // resolved "ip:port" -> logical dst host
|
||||
datagramCount int
|
||||
byteCount int64
|
||||
)
|
||||
|
||||
// consumeBudget reserves one datagram + payload length against the
|
||||
// association's bounds. Returns false if either limit would be exceeded.
|
||||
consumeBudget := func(payloadLen int) bool {
|
||||
assocMu.Lock()
|
||||
defer assocMu.Unlock()
|
||||
if datagramCount+1 > s.cfg.UDPMaxDatagrams || byteCount+int64(payloadLen) > s.cfg.UDPMaxBytes {
|
||||
return false
|
||||
}
|
||||
datagramCount++
|
||||
byteCount += int64(payloadLen)
|
||||
return true
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(3)
|
||||
|
||||
// Goroutine 1: tie association lifetime strictly to the TCP control connection.
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
var discard [64]byte
|
||||
for {
|
||||
if _, err := client.Read(discard[:]); err != nil {
|
||||
closeAssoc()
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Goroutine 2: client -> upstream UDP relay (c2s).
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
buf := make([]byte, DefaultSOCKS5UDPMaxDatagramSize)
|
||||
for {
|
||||
n, from, err := relayConn.ReadFromUDP(buf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if !s.acceptClientUDPSource(from, expectedClientIP, expectedClientPort, &assocMu, &clientUDPAddr) {
|
||||
continue
|
||||
}
|
||||
|
||||
_, dstHost, dstPort, payload, err := parseSOCKS5UDPHeader(buf[:n])
|
||||
if err != nil {
|
||||
// Malformed header, FRAG != 0, or unsupported ATYP: drop datagram.
|
||||
continue
|
||||
}
|
||||
|
||||
// Copy payload before forwarding/recording so buf can be reused safely.
|
||||
payloadCopy := append([]byte(nil), payload...)
|
||||
targetAddr := net.JoinHostPort(dstHost, strconv.Itoa(dstPort))
|
||||
|
||||
// Enforce per-datagram destination policy (target gate + egress governor)
|
||||
// BEFORE forwarding. A denied destination drops that datagram cleanly.
|
||||
dstUDPAddr, err := s.resolveAndPaceUDP4(assocCtx, targetAddr)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if !consumeBudget(len(payloadCopy)) {
|
||||
closeAssoc()
|
||||
return
|
||||
}
|
||||
|
||||
assocMu.Lock()
|
||||
allowedPeers[dstUDPAddr.String()] = dstHost
|
||||
assocMu.Unlock()
|
||||
|
||||
if _, err := upstreamConn.WriteToUDP(payloadCopy, dstUDPAddr); err != nil {
|
||||
if assocCtx.Err() != nil {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
touchIdle()
|
||||
s.recordUDPCapture(RawUDPCapture{
|
||||
AssociationID: assocID,
|
||||
Host: dstHost,
|
||||
Port: dstPort,
|
||||
Direction: "c2s",
|
||||
Timestamp: time.Now().UTC(),
|
||||
Payload: payloadCopy,
|
||||
})
|
||||
}
|
||||
}()
|
||||
|
||||
// Goroutine 3: upstream -> client UDP relay (s2c).
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
buf := make([]byte, DefaultSOCKS5UDPMaxDatagramSize)
|
||||
for {
|
||||
n, from, err := upstreamConn.ReadFromUDP(buf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
assocMu.Lock()
|
||||
origHost, allowed := allowedPeers[from.String()]
|
||||
cli := clientUDPAddr
|
||||
assocMu.Unlock()
|
||||
if !allowed || cli == nil {
|
||||
// Drop unsolicited UDP packets from destinations this association
|
||||
// has not sent an allowed datagram to.
|
||||
continue
|
||||
}
|
||||
|
||||
payloadCopy := append([]byte(nil), buf[:n]...)
|
||||
if !consumeBudget(len(payloadCopy)) {
|
||||
closeAssoc()
|
||||
return
|
||||
}
|
||||
|
||||
wrapped, err := buildSOCKS5UDPPacket(from.IP, from.Port, payloadCopy)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if _, err := relayConn.WriteToUDP(wrapped, cli); err != nil {
|
||||
if assocCtx.Err() != nil {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
touchIdle()
|
||||
s.recordUDPCapture(RawUDPCapture{
|
||||
AssociationID: assocID,
|
||||
Host: origHost,
|
||||
Port: from.Port,
|
||||
Direction: "s2c",
|
||||
Timestamp: time.Now().UTC(),
|
||||
Payload: payloadCopy,
|
||||
})
|
||||
}
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (s *socks5Server) acceptClientUDPSource(from *net.UDPAddr, expectedIP net.IP, expectedPort int, mu *sync.Mutex, pinned **net.UDPAddr) bool {
|
||||
if from == nil || from.IP == nil {
|
||||
return false
|
||||
}
|
||||
fromIP4 := from.IP.To4()
|
||||
if fromIP4 == nil {
|
||||
return false
|
||||
}
|
||||
if expectedIP != nil && !fromIP4.Equal(expectedIP) {
|
||||
return false
|
||||
}
|
||||
if expectedPort > 0 && from.Port != expectedPort {
|
||||
return false
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if *pinned == nil {
|
||||
*pinned = &net.UDPAddr{IP: append(net.IP(nil), fromIP4...), Port: from.Port}
|
||||
return true
|
||||
}
|
||||
return (*pinned).IP.Equal(fromIP4) && (*pinned).Port == from.Port
|
||||
}
|
||||
|
||||
func (s *socks5Server) resolveAndPaceUDP4(assocCtx context.Context, targetAddr string) (*net.UDPAddr, error) {
|
||||
// 1. Target gate + fresh DNS resolution (IPv4).
|
||||
resCtx, resCancel := context.WithTimeout(assocCtx, s.cfg.DialTimeout)
|
||||
defer resCancel()
|
||||
dstAddr, err := s.cfg.ResolveUDP4(resCtx, targetAddr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if dstAddr == nil || dstAddr.IP == nil || dstAddr.IP.To4() == nil || dstAddr.Port <= 0 {
|
||||
return nil, errSOCKS5UDPBadDestination
|
||||
}
|
||||
|
||||
// 2. Egress governor (same egress.Do path as HTTP / SOCKS5 TCP / transparent TCP).
|
||||
govCtx, govCancel := context.WithTimeout(assocCtx, s.cfg.PaceTimeout)
|
||||
defer govCancel()
|
||||
targetURL := &url.URL{Scheme: "udp", Host: targetAddr}
|
||||
_, err = egress.Do(egress.Current(), govCtx, targetURL, func() (struct{}, error) {
|
||||
return struct{}{}, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dstAddr, nil
|
||||
}
|
||||
|
||||
func (s *socks5Server) recordUDPCapture(rec RawUDPCapture) {
|
||||
if s.cfg.OnRecordUDP != nil {
|
||||
s.cfg.OnRecordUDP(rec)
|
||||
return
|
||||
}
|
||||
if s.t != nil && s.t.db != nil {
|
||||
if _, err := s.t.RecordRawUDPDatagram(rec, s.cfg.UDPPreviewBytes); err != nil {
|
||||
log.Printf("[traffic/socks5-udp] failed to record datagram assoc=%s dst=%s:%d: %v",
|
||||
rec.AssociationID, rec.Host, rec.Port, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,749 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/egress"
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
)
|
||||
|
||||
func startUDPEchoServer(t *testing.T, addr string) (*net.UDPConn, *atomic.Int32) {
|
||||
t.Helper()
|
||||
uaddr, err := net.ResolveUDPAddr("udp4", addr)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve udp %s: %v", addr, err)
|
||||
}
|
||||
conn, err := net.ListenUDP("udp4", uaddr)
|
||||
if err != nil {
|
||||
t.Fatalf("listen udp %s: %v", addr, err)
|
||||
}
|
||||
var received atomic.Int32
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
buf := make([]byte, 65535)
|
||||
for {
|
||||
n, from, err := conn.ReadFromUDP(buf)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
received.Add(1)
|
||||
reply := append([]byte("UDP-ECHO:"), buf[:n]...)
|
||||
_, _ = conn.WriteToUDP(reply, from)
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
_ = conn.Close()
|
||||
<-done
|
||||
})
|
||||
return conn, &received
|
||||
}
|
||||
|
||||
func socks5UDPAssociate(t *testing.T, proxyAddr string, useAuth bool, atyp byte, dstHost string, dstPort int) (net.Conn, *net.UDPAddr, byte) {
|
||||
t.Helper()
|
||||
c, err := net.DialTimeout("tcp4", proxyAddr, 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("dial socks5 %s: %v", proxyAddr, err)
|
||||
}
|
||||
_ = c.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
if useAuth {
|
||||
if _, err := c.Write([]byte{socks5Version, 1, socks5AuthUserPass}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var sel [2]byte
|
||||
if _, err := io.ReadFull(c, sel[:]); err != nil || sel != [2]byte{socks5Version, socks5AuthUserPass} {
|
||||
t.Fatalf("auth selection = %v err=%v", sel, err)
|
||||
}
|
||||
user := egress.PaceUser
|
||||
authReq := []byte{socks5UserPassVer, byte(len(user))}
|
||||
authReq = append(authReq, []byte(user)...)
|
||||
authReq = append(authReq, 0)
|
||||
if _, err := c.Write(authReq); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ares [2]byte
|
||||
if _, err := io.ReadFull(c, ares[:]); err != nil || ares != [2]byte{socks5UserPassVer, 0x00} {
|
||||
t.Fatalf("auth reply = %v err=%v", ares, err)
|
||||
}
|
||||
} else {
|
||||
if _, err := c.Write([]byte{socks5Version, 1, socks5AuthNone}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var sel [2]byte
|
||||
if _, err := io.ReadFull(c, sel[:]); err != nil || sel != [2]byte{socks5Version, socks5AuthNone} {
|
||||
t.Fatalf("no-auth selection = %v err=%v", sel, err)
|
||||
}
|
||||
}
|
||||
|
||||
req := []byte{socks5Version, socks5CmdUDPAssociate, 0x00, atyp}
|
||||
switch atyp {
|
||||
case socks5AtypIPv4:
|
||||
ip4 := net.ParseIP(dstHost).To4()
|
||||
if ip4 == nil {
|
||||
ip4 = net.IPv4zero.To4()
|
||||
}
|
||||
req = append(req, ip4...)
|
||||
case socks5AtypIPv6:
|
||||
ip16 := net.ParseIP(dstHost).To16()
|
||||
if ip16 == nil {
|
||||
ip16 = net.IPv6zero
|
||||
}
|
||||
req = append(req, ip16...)
|
||||
case socks5AtypDomain:
|
||||
req = append(req, byte(len(dstHost)))
|
||||
req = append(req, []byte(dstHost)...)
|
||||
}
|
||||
var p [2]byte
|
||||
binary.BigEndian.PutUint16(p[:], uint16(dstPort))
|
||||
req = append(req, p[:]...)
|
||||
if _, err := c.Write(req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var head [4]byte
|
||||
if _, err := io.ReadFull(c, head[:]); err != nil {
|
||||
t.Fatalf("read reply head: %v", err)
|
||||
}
|
||||
var bndIP net.IP
|
||||
switch head[3] {
|
||||
case socks5AtypIPv4:
|
||||
var ip [4]byte
|
||||
if _, err := io.ReadFull(c, ip[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bndIP = net.IP(ip[:])
|
||||
case socks5AtypIPv6:
|
||||
var ip [16]byte
|
||||
if _, err := io.ReadFull(c, ip[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bndIP = net.IP(ip[:])
|
||||
default:
|
||||
t.Fatalf("unexpected reply atyp 0x%02x", head[3])
|
||||
}
|
||||
var bp [2]byte
|
||||
if _, err := io.ReadFull(c, bp[:]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = c.SetDeadline(time.Time{})
|
||||
return c, &net.UDPAddr{IP: bndIP, Port: int(binary.BigEndian.Uint16(bp[:]))}, head[1]
|
||||
}
|
||||
|
||||
func makeClientUDPPacketIPv4(ip net.IP, port int, frag byte, payload []byte) []byte {
|
||||
out := make([]byte, 10+len(payload))
|
||||
out[0] = 0x00
|
||||
out[1] = 0x00
|
||||
out[2] = frag
|
||||
out[3] = socks5AtypIPv4
|
||||
copy(out[4:8], ip.To4())
|
||||
binary.BigEndian.PutUint16(out[8:10], uint16(port))
|
||||
copy(out[10:], payload)
|
||||
return out
|
||||
}
|
||||
|
||||
func makeClientUDPPacketDomain(domain string, port int, payload []byte) []byte {
|
||||
out := make([]byte, 0, 4+1+len(domain)+2+len(payload))
|
||||
out = append(out, 0x00, 0x00, 0x00, socks5AtypDomain, byte(len(domain)))
|
||||
out = append(out, []byte(domain)...)
|
||||
var p [2]byte
|
||||
binary.BigEndian.PutUint16(p[:], uint16(port))
|
||||
out = append(out, p[:]...)
|
||||
out = append(out, payload...)
|
||||
return out
|
||||
}
|
||||
|
||||
func waitForUDPDatagrams(t *testing.T, tr *Traffic, want int) []RawUDPDatagram {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
rows, total, err := tr.ListRawUDPDatagrams(RawUDPFilter{Limit: 100})
|
||||
if err != nil {
|
||||
t.Fatalf("ListRawUDPDatagrams: %v", err)
|
||||
}
|
||||
if total >= want {
|
||||
return rows
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
rows, total, _ := tr.ListRawUDPDatagrams(RawUDPFilter{Limit: 100})
|
||||
t.Fatalf("timed out waiting for %d raw UDP datagrams (got %d: %+v)", want, total, rows)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestSOCKS5UDPHeaderParsingAndBuild(t *testing.T) {
|
||||
t.Run("ipv4_roundtrip", func(t *testing.T) {
|
||||
payload := []byte("dns-query-bytes")
|
||||
pkt, err := buildSOCKS5UDPPacket(net.IPv4(198, 51, 100, 53), 53, payload)
|
||||
if err != nil {
|
||||
t.Fatalf("buildSOCKS5UDPPacket: %v", err)
|
||||
}
|
||||
atyp, host, port, got, err := parseSOCKS5UDPHeader(pkt)
|
||||
if err != nil {
|
||||
t.Fatalf("parseSOCKS5UDPHeader: %v", err)
|
||||
}
|
||||
if atyp != socks5AtypIPv4 || host != "198.51.100.53" || port != 53 || !bytes.Equal(got, payload) {
|
||||
t.Fatalf("got (%d, %q, %d, %q)", atyp, host, port, got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("domain_valid", func(t *testing.T) {
|
||||
pkt := makeClientUDPPacketDomain("udp.example.test", 5353, []byte("hi"))
|
||||
atyp, host, port, got, err := parseSOCKS5UDPHeader(pkt)
|
||||
if err != nil {
|
||||
t.Fatalf("parse domain: %v", err)
|
||||
}
|
||||
if atyp != socks5AtypDomain || host != "udp.example.test" || port != 5353 || string(got) != "hi" {
|
||||
t.Fatalf("got (%d, %q, %d, %q)", atyp, host, port, got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("fragmented_rejected", func(t *testing.T) {
|
||||
pkt := makeClientUDPPacketIPv4(net.IPv4(127, 0, 0, 1), 53, 0x01, []byte("frag"))
|
||||
if _, _, _, _, err := parseSOCKS5UDPHeader(pkt); !errors.Is(err, errSOCKS5UDPFragmented) {
|
||||
t.Fatalf("err = %v, want errSOCKS5UDPFragmented", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("bad_rsv_short_ipv6_bad_dst", func(t *testing.T) {
|
||||
if _, _, _, _, err := parseSOCKS5UDPHeader([]byte{0, 0, 0}); !errors.Is(err, errSOCKS5UDPShortPacket) {
|
||||
t.Fatalf("short err = %v", err)
|
||||
}
|
||||
badRsv := makeClientUDPPacketIPv4(net.IPv4(127, 0, 0, 1), 53, 0, nil)
|
||||
badRsv[1] = 0x01
|
||||
if _, _, _, _, err := parseSOCKS5UDPHeader(badRsv); !errors.Is(err, errSOCKS5UDPBadReserved) {
|
||||
t.Fatalf("bad rsv err = %v", err)
|
||||
}
|
||||
ipv6Pkt := make([]byte, 22)
|
||||
ipv6Pkt[3] = socks5AtypIPv6
|
||||
ipv6Pkt[19] = 1
|
||||
ipv6Pkt[21] = 53
|
||||
if _, _, _, _, err := parseSOCKS5UDPHeader(ipv6Pkt); !errors.Is(err, errSOCKS5UDPUnsupportedAtyp) {
|
||||
t.Fatalf("ipv6 err = %v", err)
|
||||
}
|
||||
portZero := makeClientUDPPacketIPv4(net.IPv4(127, 0, 0, 1), 0, 0, nil)
|
||||
if _, _, _, _, err := parseSOCKS5UDPHeader(portZero); !errors.Is(err, errSOCKS5UDPBadDestination) {
|
||||
t.Fatalf("port 0 err = %v", err)
|
||||
}
|
||||
unspec := makeClientUDPPacketIPv4(net.IPv4zero, 53, 0, nil)
|
||||
if _, _, _, _, err := parseSOCKS5UDPHeader(unspec); !errors.Is(err, errSOCKS5UDPBadDestination) {
|
||||
t.Fatalf("0.0.0.0 err = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestSOCKS5UDPFlagOffMeansNoListenerAndRefusesAssociate(t *testing.T) {
|
||||
// 1. All flags off -> no listener.
|
||||
t.Setenv(SOCKS5EnabledEnv, "")
|
||||
t.Setenv(SOCKS5AddrEnv, "")
|
||||
t.Setenv(SOCKS5UDPEnabledEnv, "")
|
||||
if cfg, ok := SOCKS5ConfigFromEnv(); ok || cfg.EnableUDP {
|
||||
t.Fatalf("all off = (%+v, %v), want disabled", cfg, ok)
|
||||
}
|
||||
|
||||
// 2. Explicit ARTEX_SOCKS5_ENABLED=0 overrides ARTEX_SOCKS5_UDP_ENABLED=1 (no listener).
|
||||
t.Setenv(SOCKS5EnabledEnv, "0")
|
||||
t.Setenv(SOCKS5UDPEnabledEnv, "1")
|
||||
if cfg, ok := SOCKS5ConfigFromEnv(); ok {
|
||||
t.Fatalf("explicit socks5=0 with udp=1 = (%+v, %v), want disabled", cfg, ok)
|
||||
}
|
||||
|
||||
// 3. ARTEX_SOCKS5_UDP_ENABLED=1 alone enables SOCKS5 on DefaultSOCKS5Addr with EnableUDP=true.
|
||||
t.Setenv(SOCKS5EnabledEnv, "")
|
||||
t.Setenv(SOCKS5AddrEnv, "")
|
||||
t.Setenv(SOCKS5UDPEnabledEnv, "1")
|
||||
if cfg, ok := SOCKS5ConfigFromEnv(); !ok || !cfg.EnableUDP || cfg.Addr != DefaultSOCKS5Addr {
|
||||
t.Fatalf("udp=1 = (%+v, %v), want (EnableUDP=true, Addr=%s)", cfg, ok, DefaultSOCKS5Addr)
|
||||
}
|
||||
|
||||
// 4. SOCKS5 enabled with EnableUDP=false refuses UDP ASSOCIATE with 0x07.
|
||||
trOff, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer trOff.Close()
|
||||
addrOff, err := trOff.StartSOCKS5(SOCKS5Config{Addr: "127.0.0.1:0", EnableUDP: false})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if trOff.SOCKS5UDPEnabled() {
|
||||
t.Fatal("expected SOCKS5UDPEnabled=false")
|
||||
}
|
||||
c, _, rep := socks5UDPAssociate(t, addrOff, false, socks5AtypIPv4, "0.0.0.0", 0)
|
||||
_ = c.Close()
|
||||
if rep != socks5RepCmdNotSupported {
|
||||
t.Fatalf("UDP ASSOCIATE with flag off rep = 0x%02x, want 0x%02x", rep, socks5RepCmdNotSupported)
|
||||
}
|
||||
|
||||
// 5. With EnableUDP=true, IPv6 UDP ASSOCIATE is rejected with 0x08 (IPv4 first).
|
||||
trOn, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer trOn.Close()
|
||||
addrOn, err := trOn.StartSOCKS5(SOCKS5Config{Addr: "127.0.0.1:0", EnableUDP: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !trOn.SOCKS5UDPEnabled() {
|
||||
t.Fatal("expected SOCKS5UDPEnabled=true")
|
||||
}
|
||||
c6, _, rep6 := socks5UDPAssociate(t, addrOn, false, socks5AtypIPv6, "::", 0)
|
||||
_ = c6.Close()
|
||||
if rep6 != socks5RepAtypNotSupported {
|
||||
t.Fatalf("IPv6 UDP ASSOCIATE rep = 0x%02x, want 0x%02x", rep6, socks5RepAtypNotSupported)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSOCKS5UDPAssociateRelayStoreAndControlConnClose(t *testing.T) {
|
||||
prev := egress.CurrentProfile()
|
||||
t.Cleanup(func() { egress.SetProfile(prev) })
|
||||
egress.SetProfile(egress.Profile{Name: "off"})
|
||||
|
||||
echo, received := startUDPEchoServer(t, "127.0.0.1:0")
|
||||
echoAddr := echo.LocalAddr().(*net.UDPAddr)
|
||||
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
proxyAddr, err := tr.StartSOCKS5(SOCKS5Config{
|
||||
Addr: "127.0.0.1:0",
|
||||
EnableUDP: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctrl, relayAddr, rep := socks5UDPAssociate(t, proxyAddr, true, socks5AtypIPv4, "0.0.0.0", 0)
|
||||
if rep != socks5RepSuccess {
|
||||
t.Fatalf("UDP ASSOCIATE rep = 0x%02x, want 0x00", rep)
|
||||
}
|
||||
|
||||
cliUDP, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cliUDP.Close()
|
||||
|
||||
// 1. Send a FRAG != 0 datagram first: must be dropped silently (no fragmentation reassembly).
|
||||
fragPkt := makeClientUDPPacketIPv4(echoAddr.IP, echoAddr.Port, 0x01, []byte("fragment-must-drop"))
|
||||
if _, err := cliUDP.WriteToUDP(fragPkt, relayAddr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 2. Send a valid IPv4 datagram and verify echoed reply.
|
||||
msg1 := []byte("hello-socks5-udp-ipv4")
|
||||
if _, err := cliUDP.WriteToUDP(makeClientUDPPacketIPv4(echoAddr.IP, echoAddr.Port, 0x00, msg1), relayAddr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = cliUDP.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
buf := make([]byte, 4096)
|
||||
n, _, err := cliUDP.ReadFromUDP(buf)
|
||||
if err != nil {
|
||||
t.Fatalf("read udp reply 1: %v", err)
|
||||
}
|
||||
_, repHost, repPort, repPayload, err := parseSOCKS5UDPHeader(buf[:n])
|
||||
if err != nil {
|
||||
t.Fatalf("parse reply 1 header: %v", err)
|
||||
}
|
||||
want1 := append([]byte("UDP-ECHO:"), msg1...)
|
||||
if repHost != "127.0.0.1" || repPort != echoAddr.Port || !bytes.Equal(repPayload, want1) {
|
||||
t.Fatalf("reply 1 = (%s:%d, %q), want (127.0.0.1:%d, %q)", repHost, repPort, repPayload, echoAddr.Port, want1)
|
||||
}
|
||||
|
||||
// 3. Send a valid domain-addressed datagram ("localhost") and verify echoed reply.
|
||||
msg2 := []byte("hello-socks5-udp-domain")
|
||||
if _, err := cliUDP.WriteToUDP(makeClientUDPPacketDomain("localhost", echoAddr.Port, msg2), relayAddr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = cliUDP.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
n, _, err = cliUDP.ReadFromUDP(buf)
|
||||
if err != nil {
|
||||
t.Fatalf("read udp reply 2: %v", err)
|
||||
}
|
||||
_, _, _, repPayload2, err := parseSOCKS5UDPHeader(buf[:n])
|
||||
if err != nil {
|
||||
t.Fatalf("parse reply 2 header: %v", err)
|
||||
}
|
||||
want2 := append([]byte("UDP-ECHO:"), msg2...)
|
||||
if !bytes.Equal(repPayload2, want2) {
|
||||
t.Fatalf("reply 2 = %q, want %q", repPayload2, want2)
|
||||
}
|
||||
|
||||
// Only the 2 valid datagrams reached upstream (fragmented datagram was dropped).
|
||||
if got := received.Load(); got != 2 {
|
||||
t.Fatalf("upstream received %d datagrams, want 2", got)
|
||||
}
|
||||
|
||||
// Verify 4 SQLite raw_udp_datagrams rows (2 c2s + 2 s2c), same association_id, 0 HTTP exchanges.
|
||||
rows := waitForUDPDatagrams(t, tr, 4)
|
||||
if len(rows) != 4 {
|
||||
t.Fatalf("want 4 raw UDP datagram rows, got %d", len(rows))
|
||||
}
|
||||
assocID := rows[0].AssociationID
|
||||
if !strings.HasPrefix(assocID, "udpa-") {
|
||||
t.Fatalf("unexpected association_id %q", assocID)
|
||||
}
|
||||
var c2sCount, s2cCount int
|
||||
for _, r := range rows {
|
||||
if r.AssociationID != assocID || r.Protocol != "udp" || r.Port != echoAddr.Port || r.Timestamp <= 0 {
|
||||
t.Fatalf("unexpected row metadata: %+v", r)
|
||||
}
|
||||
switch r.Direction {
|
||||
case "c2s":
|
||||
c2sCount++
|
||||
case "s2c":
|
||||
s2cCount++
|
||||
default:
|
||||
t.Fatalf("unexpected direction %q", r.Direction)
|
||||
}
|
||||
}
|
||||
if c2sCount != 2 || s2cCount != 2 {
|
||||
t.Fatalf("direction counts = (c2s=%d, s2c=%d), want (2, 2)", c2sCount, s2cCount)
|
||||
}
|
||||
var httpRows int
|
||||
if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchanges`).Scan(&httpRows); err != nil || httpRows != 0 {
|
||||
t.Fatalf("HTTP exchanges rows = (%d, %v), want 0", httpRows, err)
|
||||
}
|
||||
|
||||
// 4. Closing the TCP control connection must tear down the UDP association.
|
||||
// The close is processed asynchronously by the association goroutine, so a
|
||||
// single immediate probe races it: probe until a datagram is no longer
|
||||
// forwarded, then assert the relay is closed.
|
||||
_ = ctrl.Close()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
closed := false
|
||||
for time.Now().Before(deadline) {
|
||||
before := received.Load()
|
||||
_, _ = cliUDP.WriteToUDP(makeClientUDPPacketIPv4(echoAddr.IP, echoAddr.Port, 0x00, []byte("after-close")), relayAddr)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if received.Load() == before {
|
||||
closed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !closed {
|
||||
t.Fatal("UDP association still forwards datagrams 2s after the TCP control connection closed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSOCKS5UDPPolicyPerDestinationTargetGateAndGovernor(t *testing.T) {
|
||||
t.Run("target_gate_drops_denied_destination_without_passthrough", func(t *testing.T) {
|
||||
prev := egress.CurrentProfile()
|
||||
t.Cleanup(func() { egress.SetProfile(prev) })
|
||||
egress.SetProfile(egress.Profile{Name: "off"})
|
||||
|
||||
deniedEcho, deniedCount := startUDPEchoServer(t, "127.0.0.1:0")
|
||||
allowedEcho, allowedCount := startUDPEchoServer(t, "127.0.0.1:0")
|
||||
deniedAddr := deniedEcho.LocalAddr().(*net.UDPAddr)
|
||||
allowedAddr := allowedEcho.LocalAddr().(*net.UDPAddr)
|
||||
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
deniedKey := net.JoinHostPort(deniedAddr.IP.String(), strconv.Itoa(deniedAddr.Port))
|
||||
proxyAddr, err := tr.StartSOCKS5(SOCKS5Config{
|
||||
Addr: "127.0.0.1:0",
|
||||
EnableUDP: true,
|
||||
ResolveUDP4: func(ctx context.Context, addr string) (*net.UDPAddr, error) {
|
||||
if addr == deniedKey {
|
||||
return nil, errors.New("blocked by target gate")
|
||||
}
|
||||
return net.ResolveUDPAddr("udp4", addr)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctrl, relayAddr, rep := socks5UDPAssociate(t, proxyAddr, false, socks5AtypIPv4, "0.0.0.0", 0)
|
||||
if rep != socks5RepSuccess {
|
||||
t.Fatalf("rep = 0x%02x", rep)
|
||||
}
|
||||
defer ctrl.Close()
|
||||
|
||||
cliUDP, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cliUDP.Close()
|
||||
|
||||
// Datagram to denied destination must be dropped cleanly.
|
||||
_, _ = cliUDP.WriteToUDP(makeClientUDPPacketIPv4(deniedAddr.IP, deniedAddr.Port, 0x00, []byte("denied")), relayAddr)
|
||||
// Subsequent datagram to allowed destination on the SAME association succeeds.
|
||||
_, _ = cliUDP.WriteToUDP(makeClientUDPPacketIPv4(allowedAddr.IP, allowedAddr.Port, 0x00, []byte("allowed")), relayAddr)
|
||||
|
||||
_ = cliUDP.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
buf := make([]byte, 2048)
|
||||
n, _, err := cliUDP.ReadFromUDP(buf)
|
||||
if err != nil {
|
||||
t.Fatalf("allowed destination reply failed: %v", err)
|
||||
}
|
||||
_, _, gotPort, gotPayload, err := parseSOCKS5UDPHeader(buf[:n])
|
||||
if err != nil || gotPort != allowedAddr.Port || string(gotPayload) != "UDP-ECHO:allowed" {
|
||||
t.Fatalf("unexpected reply: port=%d payload=%q err=%v", gotPort, gotPayload, err)
|
||||
}
|
||||
|
||||
if deniedCount.Load() != 0 {
|
||||
t.Fatalf("denied target received %d datagrams, want 0", deniedCount.Load())
|
||||
}
|
||||
if allowedCount.Load() != 1 {
|
||||
t.Fatalf("allowed target received %d datagrams, want 1", allowedCount.Load())
|
||||
}
|
||||
|
||||
rows := waitForUDPDatagrams(t, tr, 2)
|
||||
for _, r := range rows {
|
||||
if r.Port == deniedAddr.Port {
|
||||
t.Fatalf("denied destination recorded in store: %+v", r)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("pinned_udp_resolver_fails_closed_without_interceptor", func(t *testing.T) {
|
||||
prev := egress.CurrentProfile()
|
||||
t.Cleanup(func() { egress.SetProfile(prev) })
|
||||
egress.SetProfile(egress.Profile{Name: "off"})
|
||||
|
||||
echo, count := startUDPEchoServer(t, "127.0.0.1:0")
|
||||
echoAddr := echo.LocalAddr().(*net.UDPAddr)
|
||||
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
proxyAddr, err := tr.StartSOCKS5(SOCKS5Config{
|
||||
Addr: "127.0.0.1:0",
|
||||
EnableUDP: true,
|
||||
BaseContext: func() context.Context {
|
||||
return intercept.WithTargetScope(context.Background(), intercept.TargetScope{
|
||||
TaskID: "77",
|
||||
Enabled: func() bool { return true },
|
||||
})
|
||||
},
|
||||
ResolveUDP4: intercept.PinnedUDP4Resolver("socks5-udp", true),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctrl, relayAddr, rep := socks5UDPAssociate(t, proxyAddr, false, socks5AtypIPv4, "0.0.0.0", 0)
|
||||
if rep != socks5RepSuccess {
|
||||
t.Fatalf("rep = 0x%02x", rep)
|
||||
}
|
||||
defer ctrl.Close()
|
||||
|
||||
cliUDP, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cliUDP.Close()
|
||||
|
||||
_, _ = cliUDP.WriteToUDP(makeClientUDPPacketIPv4(echoAddr.IP, echoAddr.Port, 0x00, []byte("gated-out")), relayAddr)
|
||||
_ = cliUDP.SetReadDeadline(time.Now().Add(200 * time.Millisecond))
|
||||
buf := make([]byte, 512)
|
||||
if _, _, rerr := cliUDP.ReadFromUDP(buf); rerr == nil {
|
||||
t.Fatal("expected gated datagram to be dropped")
|
||||
}
|
||||
if count.Load() != 0 {
|
||||
t.Fatalf("gated target received %d datagrams, want 0", count.Load())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("egress_governor_drops_datagram_before_forwarding", func(t *testing.T) {
|
||||
prev := egress.CurrentProfile()
|
||||
t.Cleanup(func() { egress.SetProfile(prev) })
|
||||
egress.SetProfile(egress.Profile{Name: "tight", RPS: 0.01, Burst: 1, MaxPerHost: 1, MaxGlobal: 1})
|
||||
|
||||
echo, count := startUDPEchoServer(t, "127.0.0.1:0")
|
||||
echoAddr := echo.LocalAddr().(*net.UDPAddr)
|
||||
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
proxyAddr, err := tr.StartSOCKS5(SOCKS5Config{
|
||||
Addr: "127.0.0.1:0",
|
||||
EnableUDP: true,
|
||||
PaceTimeout: 40 * time.Millisecond,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctrl, relayAddr, rep := socks5UDPAssociate(t, proxyAddr, true, socks5AtypIPv4, "0.0.0.0", 0)
|
||||
if rep != socks5RepSuccess {
|
||||
t.Fatalf("rep = 0x%02x", rep)
|
||||
}
|
||||
defer ctrl.Close()
|
||||
|
||||
cliUDP, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cliUDP.Close()
|
||||
|
||||
// 1st datagram consumes the single burst token.
|
||||
_, _ = cliUDP.WriteToUDP(makeClientUDPPacketIPv4(echoAddr.IP, echoAddr.Port, 0x00, []byte("first")), relayAddr)
|
||||
_ = cliUDP.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
buf := make([]byte, 512)
|
||||
if _, _, err := cliUDP.ReadFromUDP(buf); err != nil {
|
||||
t.Fatalf("first datagram failed: %v", err)
|
||||
}
|
||||
|
||||
// 2nd datagram exceeds PaceTimeout and is dropped before forwarding.
|
||||
_, _ = cliUDP.WriteToUDP(makeClientUDPPacketIPv4(echoAddr.IP, echoAddr.Port, 0x00, []byte("second-paced-out")), relayAddr)
|
||||
_ = cliUDP.SetReadDeadline(time.Now().Add(200 * time.Millisecond))
|
||||
if _, _, rerr := cliUDP.ReadFromUDP(buf); rerr == nil {
|
||||
t.Fatal("expected governor-refused 2nd datagram to be dropped")
|
||||
}
|
||||
if got := count.Load(); got != 1 {
|
||||
t.Fatalf("upstream received %d datagrams, want 1", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestSOCKS5UDPAssociationBoundsAndBlobSpill(t *testing.T) {
|
||||
prev := egress.CurrentProfile()
|
||||
t.Cleanup(func() { egress.SetProfile(prev) })
|
||||
egress.SetProfile(egress.Profile{Name: "off"})
|
||||
|
||||
echo, _ := startUDPEchoServer(t, "127.0.0.1:0")
|
||||
echoAddr := echo.LocalAddr().(*net.UDPAddr)
|
||||
|
||||
t.Run("blob_spill_and_max_datagrams", func(t *testing.T) {
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
proxyAddr, err := tr.StartSOCKS5(SOCKS5Config{
|
||||
Addr: "127.0.0.1:0",
|
||||
EnableUDP: true,
|
||||
UDPMaxDatagrams: 2, // 1 c2s + 1 s2c
|
||||
UDPPreviewBytes: 64,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctrl, relayAddr, rep := socks5UDPAssociate(t, proxyAddr, false, socks5AtypIPv4, "0.0.0.0", 0)
|
||||
if rep != socks5RepSuccess {
|
||||
t.Fatalf("rep = 0x%02x", rep)
|
||||
}
|
||||
defer ctrl.Close()
|
||||
|
||||
cliUDP, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cliUDP.Close()
|
||||
|
||||
big := bytes.Repeat([]byte("UDPBLOB!"), 64) // 512 bytes > 64 preview
|
||||
if _, err := cliUDP.WriteToUDP(makeClientUDPPacketIPv4(echoAddr.IP, echoAddr.Port, 0x00, big), relayAddr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = cliUDP.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
buf := make([]byte, 4096)
|
||||
if _, _, err := cliUDP.ReadFromUDP(buf); err != nil {
|
||||
t.Fatalf("read reply: %v", err)
|
||||
}
|
||||
|
||||
rows := waitForUDPDatagrams(t, tr, 2)
|
||||
var c2sRow *RawUDPDatagram
|
||||
for i := range rows {
|
||||
if rows[i].Direction == "c2s" {
|
||||
c2sRow = &rows[i]
|
||||
}
|
||||
}
|
||||
if c2sRow == nil || c2sRow.Size != 512 || len(c2sRow.Preview) != 64 || c2sRow.Blob == "" {
|
||||
t.Fatalf("expected 512-byte datagram with 64-byte preview and blob hash, got %+v", c2sRow)
|
||||
}
|
||||
blobFile := filepath.Join(tr.dir, "_blobs", "sha256", c2sRow.Blob[:2], c2sRow.Blob+".bin")
|
||||
blobData, err := os.ReadFile(blobFile)
|
||||
if err != nil || !bytes.Equal(blobData, big) {
|
||||
t.Fatalf("blob read %s: len=%d err=%v", blobFile, len(blobData), err)
|
||||
}
|
||||
var refs int
|
||||
if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM blob_refs WHERE exchange_id = ?`, c2sRow.ID).Scan(&refs); err != nil || refs != 1 {
|
||||
t.Fatalf("blob_refs for %s = (%d, %v), want 1", c2sRow.ID, refs, err)
|
||||
}
|
||||
|
||||
// 3rd datagram exceeds UDPMaxDatagrams=2 and closes the association (and TCP control conn).
|
||||
_, _ = cliUDP.WriteToUDP(makeClientUDPPacketIPv4(echoAddr.IP, echoAddr.Port, 0x00, []byte("over-limit")), relayAddr)
|
||||
_ = ctrl.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
one := make([]byte, 1)
|
||||
if _, rerr := ctrl.Read(one); rerr == nil {
|
||||
t.Fatal("expected TCP control conn to close when UDPMaxDatagrams exceeded")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("max_bytes_and_idle_timeout", func(t *testing.T) {
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
proxyAddr, err := tr.StartSOCKS5(SOCKS5Config{
|
||||
Addr: "127.0.0.1:0",
|
||||
EnableUDP: true,
|
||||
UDPMaxBytes: 16,
|
||||
UDPIdleTimeout: 150 * time.Millisecond,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Idle timeout closes association without any datagrams.
|
||||
ctrlIdle, _, rep := socks5UDPAssociate(t, proxyAddr, false, socks5AtypIPv4, "0.0.0.0", 0)
|
||||
if rep != socks5RepSuccess {
|
||||
t.Fatalf("idle rep = 0x%02x", rep)
|
||||
}
|
||||
_ = ctrlIdle.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
one := make([]byte, 1)
|
||||
if _, rerr := ctrlIdle.Read(one); rerr == nil {
|
||||
t.Fatal("expected idle UDP association to close after UDPIdleTimeout")
|
||||
}
|
||||
_ = ctrlIdle.Close()
|
||||
|
||||
// MaxBytes=16 rejects a 32-byte datagram and closes the association.
|
||||
ctrlBytes, relayAddr, rep := socks5UDPAssociate(t, proxyAddr, false, socks5AtypIPv4, "0.0.0.0", 0)
|
||||
if rep != socks5RepSuccess {
|
||||
t.Fatalf("bytes rep = 0x%02x", rep)
|
||||
}
|
||||
defer ctrlBytes.Close()
|
||||
cliUDP, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cliUDP.Close()
|
||||
|
||||
_, _ = cliUDP.WriteToUDP(makeClientUDPPacketIPv4(echoAddr.IP, echoAddr.Port, 0x00, bytes.Repeat([]byte("A"), 32)), relayAddr)
|
||||
_ = ctrlBytes.SetReadDeadline(time.Now().Add(2 * time.Second))
|
||||
if _, rerr := ctrlBytes.Read(one); rerr == nil {
|
||||
t.Fatal("expected UDP association to close when UDPMaxBytes exceeded")
|
||||
}
|
||||
})
|
||||
}
|
||||
+131
-1
@@ -101,7 +101,10 @@ CREATE TABLE IF NOT EXISTS exchange_rewrites (
|
||||
exchange_id TEXT NOT NULL,
|
||||
seq INTEGER NOT NULL,
|
||||
phase TEXT NOT NULL, -- 'request' | 'response'
|
||||
rule_id INTEGER, -- NULL on a global line
|
||||
source_kind TEXT NOT NULL DEFAULT 'rule', -- 'rule' | 'hook'
|
||||
rule_id INTEGER, -- NULL on a global or hook line
|
||||
hook_id INTEGER, -- NULL on a rule or global line
|
||||
hook_version INTEGER, -- NULL on a rule or global line
|
||||
rule_snapshot TEXT, -- JSON, no sensitive action value
|
||||
action TEXT,
|
||||
field TEXT, -- 'method' | 'url' | 'header:<Name>' | 'body' | 'status'
|
||||
@@ -113,6 +116,51 @@ CREATE TABLE IF NOT EXISTS exchange_rewrites (
|
||||
after_blob TEXT,
|
||||
PRIMARY KEY (exchange_id, seq)
|
||||
);
|
||||
|
||||
-- Raw TCP streams relayed through the opt-in SOCKS5 CONNECT listener (P4.1a).
|
||||
-- Stored separately from HTTP exchanges: a raw TCP stream never creates a row
|
||||
-- in exchanges. Oversized directional payloads spill to _blobs/sha256 and
|
||||
-- register their hashes in blob_refs under the stream's tcp-* id.
|
||||
CREATE TABLE IF NOT EXISTS raw_tcp_streams (
|
||||
id TEXT PRIMARY KEY,
|
||||
protocol TEXT NOT NULL DEFAULT 'tcp',
|
||||
host TEXT NOT NULL,
|
||||
port INTEGER NOT NULL,
|
||||
started_at INTEGER NOT NULL, -- unix milliseconds
|
||||
ended_at INTEGER NOT NULL, -- unix milliseconds
|
||||
duration_ms INTEGER NOT NULL,
|
||||
c2s_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
s2c_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
truncated INTEGER NOT NULL DEFAULT 0,
|
||||
c2s_preview BLOB,
|
||||
c2s_blob TEXT,
|
||||
s2c_preview BLOB,
|
||||
s2c_blob TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_tcp_host ON raw_tcp_streams(host);
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_tcp_started ON raw_tcp_streams(started_at DESC, id DESC);
|
||||
|
||||
-- Raw UDP datagrams relayed through opt-in SOCKS5 UDP ASSOCIATE (P4.1c).
|
||||
-- Stored separately from HTTP exchanges: a UDP datagram never creates a row
|
||||
-- in exchanges. No fragmentation reassembly is performed: each forwarded
|
||||
-- datagram is recorded individually as it arrives. Oversized payloads spill
|
||||
-- atomically to _blobs/sha256 and register their hashes in blob_refs under
|
||||
-- the datagram's udp-* id.
|
||||
CREATE TABLE IF NOT EXISTS raw_udp_datagrams (
|
||||
id TEXT PRIMARY KEY,
|
||||
association_id TEXT NOT NULL,
|
||||
protocol TEXT NOT NULL DEFAULT 'udp',
|
||||
host TEXT NOT NULL,
|
||||
port INTEGER NOT NULL,
|
||||
direction TEXT NOT NULL, -- 'c2s' | 's2c'
|
||||
ts INTEGER NOT NULL, -- unix milliseconds
|
||||
size INTEGER NOT NULL DEFAULT 0, -- datagram payload bytes
|
||||
preview BLOB,
|
||||
blob TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_udp_assoc ON raw_udp_datagrams(association_id, ts);
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_udp_host ON raw_udp_datagrams(host);
|
||||
CREATE INDEX IF NOT EXISTS idx_raw_udp_ts ON raw_udp_datagrams(ts DESC, id DESC);
|
||||
`
|
||||
|
||||
// ftsSchema is applied separately from indexSchema: a driver build without FTS5
|
||||
@@ -226,12 +274,23 @@ type Traffic struct {
|
||||
// rewriteRules is the compiled snapshot of the rewrite rules, replaced as a
|
||||
// whole by SetRewriteRules and applied by the sink (rewrite_flow.go).
|
||||
rewriteRules atomic.Pointer[rewrite.Set]
|
||||
// tamperHooks is the compiled snapshot of enabled Go tampering hooks (P4.2),
|
||||
// replaced by SetTamperHooks. Nil when ARTEX_TAMPER_HOOKS_ENABLED is off or
|
||||
// no hooks are enabled.
|
||||
tamperHooks atomic.Pointer[HookSet]
|
||||
// snaps keeps the original of each request a rule rewrote until its
|
||||
// exchange is recorded (rewrite_flow.go). Nil on the zero value.
|
||||
snaps *snapTable
|
||||
// sweeping tracks the snapshot sweeper, which runs until Close. It is kept
|
||||
// apart from reaping, which only ever holds work that finishes on its own.
|
||||
sweeping sync.WaitGroup
|
||||
// socks holds the opt-in SOCKS5 CONNECT listener when started via StartSOCKS5.
|
||||
socksMu sync.Mutex
|
||||
socks *socks5Server
|
||||
// transparent holds the opt-in Linux transparent TCP listener when started
|
||||
// via StartTransparentTCP (P4.1b).
|
||||
transparentMu sync.Mutex
|
||||
transparent *transparentServer
|
||||
}
|
||||
|
||||
// Open initializes the traffic tree, blob store and SQLite index under dir.
|
||||
@@ -353,6 +412,9 @@ func (t *Traffic) initIndex() error {
|
||||
if _, err := conn.ExecContext(ctx, indexSchema); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureExchangeRewritesHookColumns(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := conn.ExecContext(ctx, ftsSchema); err != nil {
|
||||
log.Printf("[traffic] full-text index unavailable; body search will be disabled (metadata search is unaffected): %v", err)
|
||||
return nil
|
||||
@@ -361,6 +423,60 @@ func (t *Traffic) initIndex() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetTamperHooks atomically swaps the compiled tampering-hook snapshot used by
|
||||
// the capture proxy. Passing nil disables hook execution (baseline fast path).
|
||||
func (t *Traffic) SetTamperHooks(hs *HookSet) {
|
||||
if t == nil {
|
||||
return
|
||||
}
|
||||
t.tamperHooks.Store(hs)
|
||||
}
|
||||
|
||||
// TamperHooks returns the currently active HookSet, or nil when disabled.
|
||||
func (t *Traffic) TamperHooks() *HookSet {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
return t.tamperHooks.Load()
|
||||
}
|
||||
|
||||
func ensureExchangeRewritesHookColumns(ctx context.Context, conn *sql.Conn) error {
|
||||
rows, err := conn.QueryContext(ctx, `PRAGMA table_info(exchange_rewrites)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
existing := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var cid int
|
||||
var name, ctype string
|
||||
var notnull int
|
||||
var dflt any
|
||||
var pk int
|
||||
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
existing[name] = true
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, col := range []struct{ name, ddl string }{
|
||||
{"source_kind", `ALTER TABLE exchange_rewrites ADD COLUMN source_kind TEXT NOT NULL DEFAULT 'rule'`},
|
||||
{"hook_id", `ALTER TABLE exchange_rewrites ADD COLUMN hook_id INTEGER`},
|
||||
{"hook_version", `ALTER TABLE exchange_rewrites ADD COLUMN hook_version INTEGER`},
|
||||
} {
|
||||
if existing[col.name] {
|
||||
continue
|
||||
}
|
||||
if _, err := conn.ExecContext(ctx, col.ddl); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// hostOnly strips an optional :port, so passthrough keys match whether the host
|
||||
// arrives as "example.com:443" (CONNECT) or "example.com" (request URL).
|
||||
func hostOnly(hostport string) string {
|
||||
@@ -445,6 +561,20 @@ func (t *Traffic) Close() error {
|
||||
if t.closed != nil {
|
||||
t.closeOnce.Do(func() { close(t.closed) })
|
||||
}
|
||||
t.socksMu.Lock()
|
||||
srv := t.socks
|
||||
t.socks = nil
|
||||
t.socksMu.Unlock()
|
||||
if srv != nil {
|
||||
_ = srv.Close()
|
||||
}
|
||||
t.transparentMu.Lock()
|
||||
tsrv := t.transparent
|
||||
t.transparent = nil
|
||||
t.transparentMu.Unlock()
|
||||
if tsrv != nil {
|
||||
_ = tsrv.Close()
|
||||
}
|
||||
t.reaping.Wait()
|
||||
t.sweeping.Wait()
|
||||
return t.db.Close()
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user