Compare commits

..
Author SHA1 Message Date
BaaRai 82d8fa2b4a Merge pull request #432 from BaaRai/feat/proxy-transparent-tcp
feat(traffic): opt-in Linux transparent TCP inbound (P4.1b)
2026-11-21 00:20:02 +01:00
BaaRai 5c298aa523 Merge pull request #431 from BaaRai/feat/proxy-socks5-raw-tcp
feat(traffic): opt-in SOCKS5 CONNECT listener and raw TCP recording (P4.1a)
2026-11-21 00:05:31 +01:00
BaaRai 64765c74d7 Merge pull request #430 from BaaRai/chore/p36-hygiene
chore: P3.6 hygiene (web bundle, Go 1.26.11, dead code, dashboard indexes, test cleanup)
2026-11-20 23:20:51 +01:00
BaaRai 2d5906295c docs(traffic): note that IP/CIDR rules apply on transparent TCP (no hostname) 2026-11-20 23:08:42 +01:00
BaaRai 752d0cfab4 test: clean remaining residual rows across db, agent, and server test suites 2026-11-20 22:54:57 +01:00
BaaRai bd59d7d9d5 docs: document Linux transparent TCP inbound (P4.1b) and firewall rules/cleanup 2026-11-20 22:52:51 +01:00
BaaRai 095d3a57d4 test(traffic): add transparent TCP unit, policy, store, and Linux iptables REDIRECT tests 2026-11-20 22:52:51 +01:00
BaaRai 82691f1cc2 feat(traffic): add opt-in Linux IPv4 transparent TCP listener (SO_ORIGINAL_DST) 2026-11-20 22:52:50 +01:00
BaaRai 792e965f26 fix(traffic): atomic temp+rename raw TCP blob spill and document SOCKS5 proxy caveats 2026-11-20 22:44:20 +01:00
BaaRai afa5d0e1a1 docs: document opt-in SOCKS5 raw TCP capture (P4.1a) and update CHANGELOG 2026-11-20 22:32:18 +01:00
BaaRai 026213f216 Merge pull request #398 from YunzhouLi-hub/fix/show-bash-exit-127
fix(web): explain Bash exit 127 in session transcripts
2026-11-20 22:30:55 +01:00
BaaRai 8b49362db4 test(traffic): add SOCKS5 listener and raw TCP recording tests 2026-11-20 22:30:08 +01:00
BaaRai 551c14e644 feat(traffic): record raw TCP streams in SQLite store with previews and blob refs 2026-11-20 22:23:24 +01:00
BaaRai 19af5ba329 feat(traffic): add opt-in RFC 1928 SOCKS5 CONNECT listener with target gate and egress governor 2026-11-20 22:22:13 +01:00
BaaRai 49721523c5 fix(i18n): add the zh-CN translation for the bash exit 127 notice 2026-11-20 22:21:57 +01:00
BaaRai 79119ae9bd test: clean up residual findings, retest conversations, parent assets, and test rows 2026-11-20 22:18:53 +01:00
BaaRai 0d332d18be Merge remote-tracking branch 'origin/main' into pr398-wire
# Conflicts:
#	web/package.json
2026-11-20 22:12:54 +01:00
BaaRai b7e3079059 test(web): run the bash exit 127 test in the web job 2026-11-20 22:12:23 +01:00
BaaRai 9057b36e55 Merge pull request #318 from renaesop/codex/s3-archive-server
feat: connect S3 task archives with recoverable server lifecycle
2026-11-20 21:56:56 +01:00
BaaRai b3b9334dc8 perf(db): add partial covering indexes for dashboard/usage queries (migration 0010) 2026-11-20 21:48:31 +01:00
BaaRai 65876b7abb fix(server): scrub the container authorization token file path too
The SDK reads AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE while LoadDefaultConfig
builds the archive store and captures the path; the file itself is read lazily.
Adding the name to secretEnv keeps that path out of agent child processes, the
same way AWS_WEB_IDENTITY_TOKEN_FILE is already scrubbed, and does not affect
the client.
2026-11-20 21:45:15 +01:00
BaaRai 33e8d79426 chore: remove provably unused unexported Go helpers and dead web template files 2026-11-20 21:43:43 +01:00
BaaRai 42e8d3554d chore(deps): bump Go to 1.26.11 and x/crypto to v0.59.0 (govulncheck) 2026-11-20 21:28:26 +01:00
BaaRai 3593297664 fix(server): do not fsync the journal directory on Windows
writeRemoteArchiveJournal ended with Sync on two directory handles.
Windows cannot flush a directory handle, so the journal write failed
there and no remote archive or remote deletion could start.

Move the directory sync to syncDir: the unix build keeps the fsync, the
other builds return nil (the journal file itself is already synced).
2026-11-20 21:13:55 +01:00
BaaRai 74116993cc fix(server): fail the archive job instead of crashing on a worker panic
The archive worker goroutine had no recover. An S3-compatible endpoint
that answers CreateMultipartUpload with 200 and no UploadId makes the
SDK upload manager dereference a nil pointer, which ended the server.

Run each claimed job under a recover: log the panic value and stack and
return it as the job error, so the job is failed through the same
FailTaskArchiveJob path as any other error. The job's deferred rollbacks
have run by then, so the source data stays in place.

FailTaskArchiveJob now only fails a job that is still in its active
state. A panic recovered after the archive committed would otherwise
turn a ready archive into archive_failed, which can be neither restored
nor deleted.
2026-11-20 21:13:55 +01:00
BaaRai 745204423c fix(server): scrub ambient AWS credential variables from child processes
The archive store falls back to the AWS SDK default chain when no
ARTEX_ARCHIVE_S3_* override is set. Those AWS_* variables stayed in the
environment after startup, so agent shells, custom tools and MCP stdio
servers inherited them. Add them to secretEnv: they are scrubbed after
the store is built and kept out of /proc/<pid>/environ by the seal.

The SDK captures what it needs at config load. Tests cover static keys
(both spellings), container credentials and web identity fetched after
the scrub, and a real New with WithSecretScrub.
2026-11-20 21:05:22 +01:00
BaaRai caa38f6ca8 chore(web): remove dead imports, unused template components, and 18 unused dependencies 2026-11-20 21:02:09 +01:00
BaaRai 0d85fc9b06 Merge pull request #427 from BaaRai/feat/worker-cache-prefix
feat(agent): opt-in worker cache-prefix split (P3.5 phase C)
2026-11-20 20:57:10 +01:00
BaaRai 40a43580fe Merge pull request #410 from tglemser/fix/record-fact-facts-only-schema
fix(agent): publish record_fact as a facts-only contract
2026-11-20 20:48:32 +01:00
BaaRai 5def7bf57e docs(server): correct the record_fact refresh comment 2026-11-20 20:07:50 +01:00
BaaRai 268693066a Merge pull request #413 from BlackGownQ/feat/i18n-zh-cn
feat(i18n): add Simplified Chinese locale using the existing translation layer
2026-11-20 20:00:03 +01:00
BaaRai 0ffb0a68c4 Merge pull request #420 from ldevbr/contrib/api-recon-npm-registry
skills(api-recon): point the lockfile at the public npm registry
2026-11-20 19:59:45 +01:00
BaaRai 892d1330f6 Merge pull request #416 from ldevbr/contrib/npm-ci-ignore-scripts
build: run npm ci with --ignore-scripts in the build/install/update s…
2026-11-20 19:59:28 +01:00
BaaRai 11adad32ce Merge pull request #415 from ldevbr/contrib/bump-vuln-deps
deps: bump klauspost/compress and brotli to fix known advisories
2026-11-20 19:59:10 +01:00
BaaRai 49c5e8fa8a Merge pull request #406 from maheshBawa/fix/compose-llm-private-endpoints
fix(compose): pass ARTEX_LLM_PRIVATE_ENDPOINTS to the artex container
2026-11-20 19:58:40 +01:00
BaaRai f6f9575b19 Merge pull request #401 from YunzhouLi-hub/feat/neon-theme-preset
feat(web): add an optional Neon theme preset
2026-11-20 19:58:29 +01:00
BaaRai 280e99eace Merge pull request #396 from YunzhouLi-hub/docs/tool-input-troubleshooting
docs: troubleshoot empty tool input in session transcripts
2026-11-20 19:58:19 +01:00
BaaRai e2ca0118e7 Merge pull request #400 from YunzhouLi-hub/docs/first-time-contributors
docs: add a first-contribution guide
2026-11-20 19:58:08 +01:00
BaaRai 1e9079267b Merge pull request #411 from adam-33-pixel/fix/llmrec-explicit-stop-status
llmrec: decide usage status when the consumer stops reading
2026-11-20 19:57:57 +01:00
BaaRai aa8ba54efb fix(server): reword the record_fact default-mismatch log line 2026-11-20 19:13:38 +01:00
BaaRai 5e1c55db4d fix(server): recognise the released record_fact descriptions as known defaults
The releases v0.4.0 to v0.6.0 seed a record_fact description without the source bullet and their own facts property description. Neither was in the list of known defaults, so the one-time refresh kept both texts on a database seeded by a release and logged the description as an operator edit.

Both released texts are now known defaults, taken byte for byte from agent/tools.go at those tags, and the log line says the description does not match a known default.
2026-11-20 18:48:06 +01:00
BaaRai 5679618ea7 ci: drop the stale skip note for the llmrec usage status test 2026-11-20 18:17:05 +01:00
BaaRai 9ab895f2b7 docs: link SECURITY.md in the first-contribution guide 2026-11-20 18:16:44 +01:00
BaaRai f758b4d964 fix(server): keep the record_fact refresh flag unset when the refresh fails
refreshRecordFactContract wrote its one-time flag in a defer, so a start on
which GetTool, the schema decode or the UPDATE failed still spent the flag.
No later start retried and the database kept the earlier description.

The flag is now written once the outcome is known: after the UPDATE
succeeded (also when it changed no row), or when the row is not the
built-in one. A failed read, decode or UPDATE leaves it unset and the next
start retries.

Test: TestFix410RefreshFlagNotSetOnFailure (an UPDATE refused by a trigger,
and a stored schema that does not decode; the next start refreshes the
text and sets the flag).
2026-11-20 18:11:42 +01:00
BaaRai c5cb1afad7 fix(agent): keep the record_fact harness opt-out through tool wrappers
record_fact opts out of the harness schema check (norma Spec.RawInput) and
validates its own input. The harness finds the opt-out with a type
assertion on the tool it is handed, and that tool is a wrapper: catalog
tools are metered (meteredTool), and guardPanic wraps every tool last
(guardedTool). Both embed CoreTool, which hides AcceptsRawInput, so the
opt-out was lost and a malformed call was answered by the SDK ("invalid
tool input: facts: expected array") instead of by the handler.

guardedTool and meteredTool now forward AcceptsRawInput, as overriddenTool
already does.

Test: TestFix410AcceptsRawInputSurvivesWrappers covers each wrapper and
record_fact through wireTools, AugmentTools and harness.Query.
2026-11-20 18:11:16 +01:00
Luis 3feeb37a3a deps: bump klauspost/compress and brotli to fix known advisories
- github.com/klauspost/compress 1.18.6 -> 1.20.2 (GO-2026-5841, GO-2026-6865)
- github.com/andybalholm/brotli 1.2.1 -> 1.2.7 (GO-2026-6888)

`go mod tidy` and `go build ./...` pass; the change is limited to these two
modules in go.mod/go.sum.
2026-11-20 10:26:02 -05:00
Luis bbe95d085b skills(api-recon): point the lockfile at the public npm registry
Every `resolved` URL in skills/api-recon/scripts/package-lock.json pointed at
registry.npmmirror.com (a third-party mirror). Repoint them to
registry.npmjs.org. The integrity hashes are unchanged and still valid, so the
tarballs are byte-identical; `npm ci --registry=https://registry.npmjs.org`
installs cleanly and puppeteer-core loads.
2026-11-20 10:26:01 -05:00
Luis 9be6b76420 build: run npm ci with --ignore-scripts in the build/install/update scripts
The Dockerfile already uses `npm ci --ignore-scripts`; the host-side build
helpers (build.sh, install.sh, update.sh) did not. Align them so a lifecycle
script in a dependency cannot run during a local rebuild. No package in the
current lockfile declares install scripts, so this is a hardening no-op today.
2026-11-20 10:26:01 -05:00
BlackGownQ ada0092168 feat(i18n): add Simplified Chinese translations 2026-11-20 23:21:59 +08:00
BlackGownQ 2d6d5c4e2b feat(i18n): register and document Simplified Chinese 2026-11-20 23:21:13 +08:00
BlackGownQ 12f0ec0432 test(i18n): validate catalogs and locale behavior 2026-11-20 23:20:24 +08:00
BlackGownQ 2ff2fc59ac test(i18n): run locale validation and runtime regression tests 2026-11-20 23:19:32 +08:00
renaesop 64a53ea9c1 fix: initialize archive credentials before startup environment scrub 2026-11-20 14:26:07 +00:00
renaesop 3f16151fc8 Merge remote-tracking branch 'upstream/main' into codex/s3-archive-server 2026-11-20 14:21:05 +00:00
Adam Rufaro Dzitiro 2db355389a llmrec: decide usage status when the consumer stops reading 2026-11-20 12:21:22 +02:00
tglemserandClaude Mythos 5.1 6e906b4920 fix(agent): publish record_fact as a facts-only contract
The schema the model sees for record_fact declared every fact field twice:
at the top level (single-fact shape) and inside facts[] (batch shape), with
the handler deciding which shape a call is. A model that fills every
declared property sends the single fact inside facts[] and a copy, or a
placeholder such as "", [] or "x", of each field beside it; since PR #282
every such call is refused by name, and the refusal text ("summary is not
allowed at the top level") leads the model to send a dummy value next, so
the same refusal repeats until the intent ends. Observed with a model over
the OpenAI Responses API: 209 of 212 record_fact calls of two tasks were
refused, 156 of them identical in one intent.

The published schema is now {facts:[{...}]} and nothing else (required:
facts, additionalProperties false). The handler keeps the strict contract
of PR #282: unknown or mis-cased fields, invalid or foreign asset ids,
missing summaries, empty batches and any top-level field beside facts
(other than the legacy intent_id) are refused before anything is written,
with an error that names the fields and the one accepted shape. It still
accepts the legacy single fact and the legacy batch with a top-level
intent_id. An item without intent_id is linked to the intent the worker is
running (SetOwnerNode), as report_finding already does.

record_fact opts out of the harness schema check (norma Spec.RawInput) so
the stricter published schema does not reject the legacy shapes before the
handler sees them; the catalog decorator forwards the opt-out.
RepairRecordFactSchema republishes stored and cached schemas as facts-only
and keeps the annotations of facts and its items. A one-time startup
refresh replaces the stored description and the texts that referred to the
top level when they equal the previous defaults; operator edits stay and
are logged. seedFactSourceSchema adds the source enum on the items.

Tests: the handler contract (TestRecordFactSummaryContract,
TestRecordFactStrictFields, TestRecordFactBatchTopLevelConflicts,
TestRecordFactRefusesTopLevelCopiesBesideFacts), the published schema and
its repair (TestRecordFactSchemaIsFactsOnly, TestRecordFactStoredContractRepair,
TestRecordFactInputSchemaUpgrade, TestRecordFactSourceSchemaUpgrade,
TestWireTools), the running-intent default, the one-time refresh
(TestRefreshRecordFactContract) and a synthetic replay of the call shapes
(TestRecordFactShapesReplay).

Co-Authored-By: Claude Mythos 5.1 <noreply@anthropic.com>
2026-11-20 10:03:52 +00:00
maheshBawaandClaude Opus 5.5 594d1918fd fix(compose): pass ARTEX_LLM_PRIVATE_ENDPOINTS to the artex container
The model-list allow-list (#390) is read from the
environment, but docker-compose.yml never forwarded it, so a Docker install
could not list models from a local Ollama at host.docker.internal:11434.
Empty by default, so the strict public-only policy is unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-11-20 14:43:27 +05:30
李云周 84c77021c4 feat(web): add an optional Neon theme preset 2026-11-20 15:24:57 +08:00
李云周 5e979f102a docs: add a first-contribution guide 2026-11-20 15:18:38 +08:00
李云周 bb22be6972 fix(web): explain Bash exit 127 in session transcripts 2026-11-20 15:03:45 +08:00
李云周 cc465be782 docs: explain empty tool input in session transcripts 2026-11-20 14:59:02 +08:00
renaesop 044fbdf2cb Merge remote-tracking branch 'upstream/main' into codex/s3-archive-server 2026-11-20 02:18:38 +00:00
Chubboy Ren f69eb060e3 Merge remote-tracking branch 'upstream/main' into codex/s3-archive-server 2026-11-19 03:33:51 +00:00
Chubboy Ren 672a38e2be fix: recover partial archives and defer unknown storage labels 2026-11-19 03:33:51 +00:00
Chubboy Ren bdd14ff7f0 feat: wire remote task archive lifecycle and recovery 2026-11-18 02:05:18 +00:00
141 changed files with 9896 additions and 3304 deletions
+2
View File
@@ -21,6 +21,8 @@ ARTEX_LLM_PROVIDER=
ARTEX_LLM_MODEL=
ARTEX_LLM_BASE_URL=
ARTEX_LLM_PROXY=
# Allow the UI model list to reach a local Ollama, e.g. host.docker.internal:11434
ARTEX_LLM_PRIVATE_ENDPOINTS=
# Untrusted-output boundary. Frames every tool output between nonce-tagged markers so
# the model treats it as data, not instructions. On by default (unset or empty = on);
+1 -2
View File
@@ -39,8 +39,6 @@ jobs:
# - TestGraphOverviewExpandsAssociatedCompanyScope,
# TestCaptureApprovalLifecycle: fail on unmodified upstream code with a
# fresh database.
# - TestSideUsageRecordedOnceOnConsumerCancellation: concurrency/cancellation
# test that passes locally but is timing-sensitive on 2-core CI runners.
# They now skip themselves in code (t.Skip with a pointer to the issue).
- name: go test
# -p 1: package test suites share the single postgres service; running
@@ -150,6 +148,7 @@ jobs:
- run: npm run test:replay
- run: npm run test:upload
- run: npm run test:i18n
- run: npm run test:bash
- run: npm run build:static
python:
+3
View File
@@ -23,6 +23,8 @@ All notable changes to this project are documented in this file. The format is b
- **`ask_operator` target warning**: a request whose question names a host or URL outside the goal targets is tagged as containing target-derived text (#197).
- **Opt-in update banner**: `ARTEX_UPDATE_CHECK` makes the backend check the latest GitHub release at most once per 24 hours and the UI show a banner; off by default (#202).
- **Release provenance**: release archives carry GitHub build provenance attestations and a CycloneDX SBOM (`gh attestation verify <zip> --repo BaaRai/ARTEX`); release notes are the matching `CHANGELOG.md` section (#219, #201).
- **Opt-in Linux transparent TCP inbound & raw TCP recording (P4.1b)**: on Linux when traffic capture is enabled, setting `ARTEX_TRANSPARENT_TCP_ENABLED=1` (loopback `127.0.0.1:1081` by default) or `ARTEX_TRANSPARENT_TCP_ADDR=<ipv4:port>` starts an IPv4 transparent TCP listener fed by operator-managed `iptables`/`nftables` `REDIRECT` rules (off by default; ARTEX never touches the host firewall; root/`CAP_NET_ADMIN` is required only for the operator's redirect rules; IPv6 is not covered). Recovers original IPv4 destinations via `SO_ORIGINAL_DST` with self-redirect loop protection, reuses the P4.1a target gate (`intercept.Interceptor.DialSOCKS5`), egress governor (`egress.Do`), half-close relay bounds, and SQLite `raw_tcp_streams` store (atomic `_blobs` spill, no HTTP `exchanges` row, no PostgreSQL migration). See `docs/traffic-transparent.md`.
- **Opt-in SOCKS5 inbound proxy & raw TCP recording (P4.1a)**: when traffic capture is enabled, setting `ARTEX_SOCKS5_ENABLED=1` (loopback `127.0.0.1:1080` by default) or `ARTEX_SOCKS5_ADDR=<addr>` starts an RFC 1928 SOCKS5 `CONNECT` listener alongside the HTTP recording proxy (off by default; HTTP proxy and agent proxy env vars are unchanged; listener does not start if traffic recording is disabled). Accepts loopback no-auth and RFC 1929 username/password without verifying credentials (carrying `artex-pace` in the username; binding to a non-loopback address therefore exposes an unauthenticated proxy, though the target gate and governor still apply), rejects `BIND`/`UDP ASSOCIATE` with `0x07`, enforces the existing target gate (`intercept.Interceptor.DialSOCKS5` / `intercept.PinnedDialer`) and egress governor (`egress.Do`) before dialing, relays half-close-aware TCP with concurrency/byte/idle/handshake bounds, and records one row per connection in SQLite `raw_tcp_streams` with capped directional previews and atomic (temp+rename) `_blobs` refs without writing HTTP `exchanges` rows. See `docs/traffic-socks5.md`.
- **Opt-in worker cache-prefix split (P3.5 Phase C)**: setting `ARTEX_CACHE_PREFIX=1` (off by default) splits the `worker` system prompt at `DynamicBoundary = 1` into a sibling-shareable static prefix (`body + untrustedContentTail + workerTrafficBlock`) and a per-intent dynamic suffix (`workerArtifactSpec(runDir)` through `<available-deferred-tools>`) without reordering sections or altering the visible joined prompt text. Planner, mainagent, chat, and goals are unchanged; custom worker prompts referencing `{{.Now}}` remain volatile in the static segment. See `docs/context-telemetry.md`.
- **Opt-in LLM context size telemetry (P3.5 Phase A)**: setting `ARTEX_CONTEXT_TELEMETRY=1` (off by default) records per-call request section byte sizes (`system_static`/`system_dynamic` split at `DynamicBoundary`, serialized `tools`, `messages` by role, `tool_result` subset, and `last_message_bytes`), tool/message counts, and token/cache usage in the new `llm_context` table (migration `0009_llm_context.sql`), with zero prompt/behavior change and zero content stored, independent of `llm_record`. `GET /api/llm/context/summary` aggregates input percentiles (`p50`/`p90`/`p99`), pro-rata section shares over real `input_tokens`, `cache_read/input` ratios by `(worker, model)`, and top tasks. See `docs/context-telemetry.md`.
- **HAR 1.2 traffic import and export**: `POST /api/traffic/import/har` imports raw HAR 1.2 JSON captures (up to 32 MiB / 2,000 entries / 32 MiB decoded bodies) with deterministic `har-<sha256>` IDs (and `_artexExchangeId` preservation for ARTEX exports so distinct same-second exchanges round-trip without collapsing), base64 decoding, `postData.params` and `cookies` array fallbacks, URL userinfo stripping, blob spilling above 256 KiB, and full-text indexing, without applying capture rewrite rules (`exchange_rewrites` stays empty); per-entry errors are reported alongside `imported`/`skipped`/`failed` counts. `GET /api/traffic/export/har` exports filtered traffic (reusing the `GET /api/traffic` filters) as a compliant HAR 1.2 document (with explicit unknown placeholders for unrecorded timings and header sizes) in stable `(ts, id)` order, encoding UTF-8 bodies as text and binary bodies as base64, capped at 1,000 entries / 20 MiB raw headers+bodies (31 MiB JSON backstop, guaranteed under the 32 MiB import limit) with explicit truncation reporting via `X-ARTEX-HAR-*` headers and `_truncated`/`_truncatedReason`/`_totalMatched` fields. The Traffic page adds **Import HAR** and **Export HAR** controls. See `docs/traffic-har.md`.
@@ -53,6 +55,7 @@ All notable changes to this project are documented in this file. The format is b
### Fixed
- **LLM usage status on early stream exit**: when a consumer stops reading a stream before `message_stop`, the recorder now stores the `llm_usage` row as `error` at that point, whether or not the context was cancelled, instead of inferring the status from `ctx.Err()` in a deferred call. `TestSideUsageRecordedOnceOnConsumerCancellation` runs again (follow-up to #9).
- **Workspace and skill symlink confinement**: workspace file operations, the skill file APIs and the skill loader resolve symlinks in every path component and refuse a target outside their root, so a link planted in the workspace or a skill directory cannot expose another file.
- **Target gate behind proxies**: a target approval is name-only whenever a proxy is in the path, including the recording proxy, the global egress proxy and a custom tool's own proxy, so the gate never claims an IP binding the proxy does not enforce.
- **Remote MCP response bound**: Streamable HTTP JSON-RPC bodies and SSE data payloads are refused above 8 MiB, so a remote MCP server cannot force an unbounded decode.
+7
View File
@@ -1,5 +1,12 @@
# Contributing
## First contribution
- **Share feedback.** Check [open issues](https://github.com/BaaRai/ARTEX/issues) for the same problem or proposal and add any new details there. For a new idea, explain what you were trying to do and what would make it easier.
- **Report a problem.** Include a short reproduction and the relevant environment. For a UI issue, a sanitized screenshot and the browser version can help. Remove credentials, private URLs, and sensitive task data before posting.
- **Report a security issue.** Do not open a public issue for a vulnerability; follow [SECURITY.md](SECURITY.md) to report it privately.
- **Send a small change.** A documentation correction or a focused UI improvement is a good place to start. Check [open pull requests](https://github.com/BaaRai/ARTEX/pulls) for overlapping work and refer to the relevant issue in your PR so readers can find the discussion.
## Pull requests
- **At most 7 open pull requests per contributor.** The repository enforces this with GitHub's pull request creation limit, drafts included: when you reach it, close or merge one before opening another. Review is manual, one change at a time, so a larger batch makes the queue unmanageable and slows everyone down, including you. If you have more changes queued, open one issue that lists them and wait for the current ones to be reviewed.
+19
View File
@@ -545,6 +545,25 @@ a trusted peer can claim any client address.
## Operations
### Troubleshooting an empty tool call
In the task's session transcript, expand the affected tool call and compare **Command** and
**Output**. The `UNTRUSTED_OUTPUT` markers identify the boundary around tool output. They are
expected and should stay enabled while troubleshooting.
- **Command shows `{}`.** The recorded tool input is an empty JSON object. For a tool that
requires a command, this means its command argument was missing. Check which model and version
produced the call.
- **Command shows `(empty)` or `(failed to load)`.** The transcript has no input detail to display
or could not fetch it. If the session record contains a command, include that discrepancy in a
report.
- **Output shows `[]`.** This is the recorded result body. Check the row's success or failure
indicator and its input separately before drawing conclusions about what ran.
For a recurring problem, share the ARTEX version or commit, the model and version, and a sanitized
excerpt of the affected input and result. Remove credentials, target data, and other sensitive
content before sharing.
### Backups
`scripts/backup.sh` dumps the Postgres database (and optionally the data directory) with
+8
View File
@@ -101,6 +101,14 @@ func guardPanic(t actool.CoreTool) actool.CoreTool { return &guardedTool{CoreToo
type guardedTool struct{ actool.CoreTool }
// AcceptsRawInput forwards the wrapped tool's opt-out of the harness schema
// check, as overriddenTool does: this wrapper is the one the harness sees, and
// the embedded interface alone would hide the opt-out from its type assertion.
func (g *guardedTool) AcceptsRawInput() bool {
r, ok := g.CoreTool.(interface{ AcceptsRawInput() bool })
return ok && r.AcceptsRawInput()
}
func (g *guardedTool) Call(ctx context.Context, in json.RawMessage, tc *actool.ToolContext) (res actool.Result, err error) {
defer func() {
if r := recover(); r != nil {
+1 -1
View File
@@ -33,7 +33,7 @@ func TestTaskScopeLabelIsHonest(t *testing.T) {
// denominator and list_untested_assets.
func TestAutoScopeOutsideDeclaredRootExcluded(t *testing.T) {
d := testDB(t)
defer d.Close()
t.Cleanup(func() { _ = d.Close() })
stamp := time.Now().UnixNano()
declared := fmt.Sprintf("app.f8-%d.invalid", stamp)
+2 -2
View File
@@ -194,7 +194,7 @@ func companiesName(t *testing.T, companies *db.CompanyStore, companyID int64) st
func TestBlackboardToolsReadDirectSources(t *testing.T) {
d := testDB(t)
defer d.Close()
t.Cleanup(func() { _ = d.Close() })
grand, err := d.CreateTask("grand", "grand goal", nil, 0, 0)
if err != nil {
@@ -378,7 +378,7 @@ func TestBlackboardToolsReadDirectSources(t *testing.T) {
// model which ids it deferred, after de-duplicating and dropping invalid ids.
func TestGetWorkerTraceStepIDsDegradeGracefully(t *testing.T) {
d := testDB(t)
defer d.Close()
t.Cleanup(func() { _ = d.Close() })
task, err := d.CreateTask("trace-cap", "goal", nil, 0, 0)
if err != nil {
+1 -1
View File
@@ -16,7 +16,7 @@ import (
// write happens even when the stored flag is off, which is the outage behaviour.
func TestCoverageReadFailureKeepsToolsAndWritesAutoScope(t *testing.T) {
d := testDB(t)
defer d.Close()
t.Cleanup(func() { _ = d.Close() })
off := false
task, err := d.CreateTaskWithOptions("coverage outage", "goal", db.TaskCreateOptions{CoverageEnabled: &off})
if err != nil {
+3 -4
View File
@@ -115,10 +115,9 @@ func TestRecordFactAssetAliases(t *testing.T) {
func TestRecordFactAssetAliasSchema(t *testing.T) {
schema := NewToolSet(nil, "").recordFact().InputSchema()
props := schema["properties"].(map[string]any)
for _, name := range []string{"asset_ids", "tested_asset_ids"} {
if props[name] == nil {
t.Fatalf("single schema missing %s", name)
}
// Facts-only contract: the alias lives on the items.
if len(props) != 1 || props["facts"] == nil {
t.Fatalf("top level must hold facts only: %v", props)
}
item := props["facts"].(map[string]any)["items"].(map[string]any)
itemProps, _ := item["properties"].(map[string]any)
+88 -58
View File
@@ -4,10 +4,12 @@ import (
"bytes"
"encoding/json"
"fmt"
"sort"
"strings"
)
// Match strings.TrimSpace, including Unicode whitespace, in the published schema.
const factWhitespace = "\t\n\v\f\r \u0085\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000"
const factWhitespace = "\t\n\v\f\r \u0085   - 

   "
func factAssetIDSchema() map[string]any {
return map[string]any{
@@ -16,10 +18,13 @@ func factAssetIDSchema() map[string]any {
}
}
// factItemSchema is one fact: the shape of every facts[] item. It is also the
// shape of the legacy single-fact call (fields at the top level, no facts),
// which the handler still accepts but the published schema no longer shows.
func factItemSchema() map[string]any {
schema := obj(map[string]any{
"summary": str("Required on every fact: a non-empty one-sentence summary of this conclusion. Batch items do not inherit summary from the top level."),
"intent_id": idp("The intent that produced this fact; only this field inherits from the top level in a batch."),
"summary": str("Required: a non-empty one-sentence summary of this conclusion."),
"intent_id": idp("The intent that produced this fact. Omit it to link the fact to the intent this worker is running; a batch may name a different intent per item."),
"detail": str("Details of this fact, including all related observations."),
"evidence": str("One line of key evidence: the command and the most proving output lines. Put longer output in detail."),
"confidence": str("observed (seen directly in output) | inferred (inferred from phenomena). Label negative conclusions honestly."),
@@ -34,68 +39,67 @@ func factItemSchema() map[string]any {
return schema
}
// recordFactFactsDescription documents the only top-level property of
// record_fact. The wording tells a model that tends to fill every declared
// property that there is nothing else to fill.
const recordFactFactsDescription = "The facts to record, usually exactly one. Every item is a complete fact with its own summary. This is the only top-level property: never send summary, detail, evidence, confidence, source, intent_id or asset ids beside facts. Returned ids match item order; retry only failed indices."
// recordFactContractNote is appended to the record_fact description: the one
// call shape the model is given.
const recordFactContractNote = "\n★Call shape: exactly {\"facts\":[{...}]} with one item per distinct conclusion (usually one item). Each item carries its own summary, detail, evidence, confidence, source, asset_ids and, when it differs from the intent you are running, intent_id. Nothing else at the top level; a top-level fact field beside facts is refused and nothing is written."
// RecordFactContractNote exposes the text to the server's one-time refresh.
func RecordFactContractNote() string { return recordFactContractNote }
// recordFactSchema is the published input contract: facts[] and nothing else.
// Providers reject root-level anyOf, and the earlier contract (fact fields at
// the top level beside facts[]) made models fill both; the handler alone keeps
// the legacy shapes alive.
func recordFactSchema() map[string]any {
props := factItemSchema()["properties"].(map[string]any)
props["facts"] = map[string]any{
"type": "array", "minItems": 1, "items": factItemSchema(),
"description": "A non-empty batch of complete facts. Every item needs its own summary. Only intent_id inherits from the top level. Returned ids match item order; retry only failed indices.",
}
schema := obj(props)
// Providers reject root-level anyOf. The handler enforces single/batch mode.
schema := obj(map[string]any{
"facts": map[string]any{
"type": "array", "minItems": 1, "items": factItemSchema(),
"description": recordFactFactsDescription,
},
}, "facts")
schema["additionalProperties"] = false
return schema
}
// RepairRecordFactSchema restores the single/batch input contract in stored or
// cached schemas, preserving annotations only on fields the handler accepts.
// RepairRecordFactSchema brings a stored or cached record_fact schema to the
// facts-only contract. The structure always comes from the code; of the stored
// schema only the annotations (description, default) of facts and of its known
// item fields survive. Top-level fact fields of the earlier contract are
// dropped: the handler still accepts them, but the model must not see them.
func RepairRecordFactSchema(stored map[string]any) map[string]any {
raw, _ := json.Marshal(stored)
var out map[string]any
_ = json.Unmarshal(raw, &out)
if out == nil {
out = map[string]any{}
out := recordFactSchema()
facts := out["properties"].(map[string]any)["facts"].(map[string]any)
storedProps, _ := stored["properties"].(map[string]any)
storedFacts, _ := storedProps["facts"].(map[string]any)
if storedFacts == nil {
return out
}
mergeFactContract(out, recordFactSchema())
// A batch requires summaries on its items, never a redundant top-level one.
delete(out, "required")
delete(out, "anyOf")
canonical := recordFactSchema()["properties"].(map[string]any)
props := out["properties"].(map[string]any)
for name := range props {
if _, known := canonical[name]; !known {
delete(props, name)
if d, ok := storedFacts["description"].(string); ok && strings.TrimSpace(d) != "" {
facts["description"] = d
}
storedItems, _ := storedFacts["items"].(map[string]any)
storedItemProps, _ := storedItems["properties"].(map[string]any)
itemProps := facts["items"].(map[string]any)["properties"].(map[string]any)
for name, p := range itemProps {
sp, _ := storedItemProps[name].(map[string]any)
if sp == nil {
continue
}
}
items := props["facts"].(map[string]any)["items"].(map[string]any)
itemProps := items["properties"].(map[string]any)
for name := range itemProps {
if _, known := canonical[name]; !known || name == "facts" {
delete(itemProps, name)
prop := p.(map[string]any)
for _, key := range []string{"description", "default"} {
if v, ok := sp[key]; ok && v != nil {
prop[key] = v
}
}
}
return out
}
func mergeFactContract(dst, src map[string]any) {
for key, value := range src {
if key == "description" || key == "default" {
if _, ok := dst[key]; ok {
continue
}
}
if child, ok := value.(map[string]any); ok {
target, _ := dst[key].(map[string]any)
if target == nil {
target = map[string]any{}
dst[key] = target
}
mergeFactContract(target, child)
} else {
dst[key] = value
}
}
}
type recordFactInput struct {
Facts []factItem `json:"facts"`
factItem
@@ -103,6 +107,9 @@ type recordFactInput struct {
// Validate exact field names before decoding: encoding/json otherwise treats
// Summary, SUMMARY, etc. as aliases even with DisallowUnknownFields enabled.
// Three input shapes reach the handler: the published one ({facts:[...]}), the
// legacy single fact (fields at the top level, no facts) and the legacy batch
// with a top-level intent_id as the default of its items.
func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) {
var out recordFactInput
fields, err := exactFactFields(in, true)
@@ -111,11 +118,6 @@ func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) {
}
_, batch := fields["facts"]
if batch {
for name := range fields {
if name != "intent_id" && name != "facts" {
return out, true, fmt.Errorf("field %q is not allowed at the top level in batch mode; only intent_id and facts are accepted", name)
}
}
var items []json.RawMessage
if err := json.Unmarshal(fields["facts"], &items); err != nil {
return out, true, fmt.Errorf("facts must be an array: %w", err)
@@ -123,10 +125,16 @@ func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) {
if len(items) == 0 {
return out, true, fmt.Errorf("facts must be a non-empty array of complete facts, each with its own summary")
}
itemFields := make([]map[string]json.RawMessage, len(items))
for i, item := range items {
if _, err := exactFactFields(item, false); err != nil {
f, err := exactFactFields(item, false)
if err != nil {
return out, true, fmt.Errorf("facts[%d]: %w", i, err)
}
itemFields[i] = f
}
if err := checkBatchTopLevel(fields, itemFields); err != nil {
return out, true, err
}
}
decoder := json.NewDecoder(bytes.NewReader(in))
@@ -137,6 +145,28 @@ func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) {
return out, batch, nil
}
// checkBatchTopLevel enforces the contract at the top level of a facts[] call,
// where only facts and (legacy) intent_id carry meaning. Any other top-level
// field is refused by name before anything is written, including empty
// placeholders and copies of a fact's fields: a value there is at best
// redundant and at worst a second, conflicting claim. The message names the
// one accepted shape, so a model that filled every declared property of an
// older schema learns where the fields belong.
func checkBatchTopLevel(fields map[string]json.RawMessage, items []map[string]json.RawMessage) error {
var extra []string
for name := range fields {
if name == "intent_id" || name == "facts" {
continue
}
extra = append(extra, fmt.Sprintf("%q", name))
}
if len(extra) == 0 {
return nil
}
sort.Strings(extra)
return fmt.Errorf("top-level field(s) %s are not allowed beside facts (%d item(s)): record_fact takes only {\"facts\":[...]}; put every field of a fact inside its facts item and send nothing else at the top level", strings.Join(extra, ", "), len(items))
}
func exactFactFields(in json.RawMessage, topLevel bool) (map[string]json.RawMessage, error) {
var fields map[string]json.RawMessage
if err := json.Unmarshal(in, &fields); err != nil {
+54
View File
@@ -0,0 +1,54 @@
package agent
// Texts of the earlier record_fact contract (fact fields at the top level
// beside facts[]), kept verbatim so the server's one-time refresh
// (server/fact_input_schema.go) can tell a stored default from an operator
// edit. Nothing here is published to a model.
// recordFactLegacyDescription is the record_fact description seeded up to the
// facts-only contract.
const recordFactLegacyDescription = "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n" +
"⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n" +
"★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n" +
"⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n" +
" · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n" +
" · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n" +
" · source = where the content came from: target_response (content the target itself returned: page bodies, headers, error messages, banners, even when a tool fetched it) | tool_output (a tool's own result about the target: port states, scanner verdicts, status codes, timings) | third_party_intel (OSINT, CVE databases, public reports) | operator_input (given by the human operator) | inference (your own reasoning; the default when omitted). Any other value is refused.\n" +
" · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen."
// recordFactReleasedDescription is the record_fact description seeded by the
// releases v0.4.0 to v0.6.0 (agent/tools.go at those tags): the text above
// without the source bullet.
const recordFactReleasedDescription = "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n" +
"⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n" +
"★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n" +
"⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n" +
" · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n" +
" · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n" +
" · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen."
// recordFactLegacyItemDescriptions are the facts[] item descriptions of the
// earlier contract whose wording referred to the top level; the refresh
// replaces exactly these texts with the current ones.
var recordFactLegacyItemDescriptions = map[string]string{
"summary": "Required on every fact: a non-empty one-sentence summary of this conclusion. Batch items do not inherit summary from the top level.",
"intent_id": "The intent that produced this fact; only this field inherits from the top level in a batch.",
}
// recordFactLegacyFactsDescription is the facts property description of the
// earlier contract.
const recordFactLegacyFactsDescription = "A non-empty batch of complete facts. Every item needs its own summary. Only intent_id inherits from the top level. Returned ids match item order; retry only failed indices."
// recordFactReleasedFactsDescription is the facts property description seeded
// by the releases v0.4.0 to v0.6.0. RepairRecordFactSchema keeps a stored facts
// description, so the refresh has to know this text to replace it.
const recordFactReleasedFactsDescription = "[use when there are several distinct conclusions] array of facts; element fields same as the top-level fields below (summary/detail/evidence/confidence/intent_id/asset_ids); omitting intent_id falls back to the top-level intent_id. The returned ids match this array in length and order."
// RecordFactLegacyDescriptions returns the description texts of the earlier
// contract that the refresh may replace: the tool descriptions, the facts
// property descriptions and the item descriptions.
func RecordFactLegacyDescriptions() (tool, facts []string, items map[string]string) {
return []string{recordFactLegacyDescription, recordFactReleasedDescription},
[]string{recordFactLegacyFactsDescription, recordFactReleasedFactsDescription},
recordFactLegacyItemDescriptions
}
+244 -16
View File
@@ -3,9 +3,11 @@ package agent
import (
"encoding/json"
"regexp"
"strconv"
"strings"
"testing"
"github.com/Autumn-27/artex/db"
actool "github.com/Autumn-27/norma/tool"
)
@@ -47,6 +49,10 @@ func TestRecordFactSummaryContract(t *testing.T) {
}
}
// A stored schema of the earlier contract (fact fields at the top level beside
// facts[], operator annotations, a stray property) is republished as the
// facts-only contract; annotations of facts and of its items survive, the top
// level loses every fact field, and the result is idempotent.
func TestRecordFactStoredContractRepair(t *testing.T) {
old := map[string]any{
"type": "object", "required": []any{"summary"},
@@ -55,32 +61,40 @@ func TestRecordFactStoredContractRepair(t *testing.T) {
"summary": map[string]any{"type": "string", "description": "operator summary"},
"confidence": map[string]any{"type": "string", "default": "inferred"},
"custom_field": map[string]any{"type": "string"},
"facts": map[string]any{"type": "array", "items": map[string]any{"type": "object"}},
"facts": map[string]any{"type": "array", "description": "operator facts", "items": map[string]any{"type": "object", "properties": map[string]any{
"summary": map[string]any{"type": "string", "description": "operator item summary"},
"confidence": map[string]any{"type": "string", "default": "observed"},
"custom": map[string]any{"type": "string"},
}}},
},
}
base := NewToolSet(nil, "").recordFact()
repaired := DecorateTool(base, "operator tool", old)
schema := repaired.InputSchema()
if _, required := schema["required"]; required {
t.Fatal("batch still requires a top-level summary")
if req, _ := schema["required"].([]string); len(req) != 1 || req[0] != "facts" {
t.Fatalf("facts must be the only required property: %v", schema["required"])
}
if _, exists := schema["anyOf"]; exists {
t.Fatal("root anyOf reaches the provider")
}
props := schema["properties"].(map[string]any)
item := props["facts"].(map[string]any)["items"].(map[string]any)
if len(props) != 1 || props["facts"] == nil || schema["additionalProperties"] != false {
t.Fatalf("top level must hold facts only: %v", props)
}
facts := props["facts"].(map[string]any)
item := facts["items"].(map[string]any)
required := item["required"].([]string)
if len(required) != 1 || required[0] != "summary" || props["facts"].(map[string]any)["minItems"] != 1 {
if len(required) != 1 || required[0] != "summary" || facts["minItems"] != 1 {
t.Fatal("batch must require a nonempty list with a summary per item")
}
if props["summary"].(map[string]any)["minLength"] != 1 ||
item["properties"].(map[string]any)["summary"].(map[string]any)["minLength"] != 1 {
itemProps := item["properties"].(map[string]any)
if itemProps["summary"].(map[string]any)["minLength"] != 1 {
t.Fatal("empty summaries permitted by schema")
}
if repaired.Description() != "operator tool" ||
props["summary"].(map[string]any)["description"] != "operator summary" ||
props["confidence"].(map[string]any)["default"] != "inferred" || props["custom_field"] != nil {
t.Fatal("operator annotations lost")
if repaired.Description() != "operator tool" || facts["description"] != "operator facts" ||
itemProps["summary"].(map[string]any)["description"] != "operator item summary" ||
itemProps["confidence"].(map[string]any)["default"] != "observed" || itemProps["custom"] != nil {
t.Fatalf("item annotations lost or stray property kept: %v", itemProps)
}
if _, ok := old["properties"].(map[string]any)["tested_asset_ids"]; ok {
t.Fatal("shared old schema mutated")
@@ -94,9 +108,18 @@ func TestRecordFactStoredContractRepair(t *testing.T) {
if err := json.Unmarshal(first, &roundTrip); err != nil {
t.Fatal(err)
}
for _, input := range []string{`{"summary":"single"}`, `{"facts":[{"summary":"batch"}]}`} {
if err := actool.ValidateInput(roundTrip, json.RawMessage(input)); err != nil {
t.Fatalf("SDK rejects valid %s after catalog round trip: %v", input, err)
if err := actool.ValidateInput(roundTrip, json.RawMessage(`{"facts":[{"summary":"batch"}]}`)); err != nil {
t.Fatalf("SDK rejects the published shape after catalog round trip: %v", err)
}
// The legacy single shape no longer passes the published schema; the
// handler validates its own input instead, also behind the catalog wrapper.
if err := actool.ValidateInput(roundTrip, json.RawMessage(`{"summary":"single"}`)); err == nil {
t.Fatal("published schema must require facts")
}
for _, tool := range []actool.CoreTool{base, repaired} {
r, ok := tool.(interface{ AcceptsRawInput() bool })
if !ok || !r.AcceptsRawInput() {
t.Fatalf("%T must opt out of the harness schema check", tool)
}
}
}
@@ -140,7 +163,7 @@ func TestRecordFactStrictFields(t *testing.T) {
}
func TestRecordFactSchemaMatchesStrings(t *testing.T) {
props := recordFactSchema()["properties"].(map[string]any)
props := factItemSchema()["properties"].(map[string]any)
summary := regexp.MustCompile(props["summary"].(map[string]any)["pattern"].(string))
for _, value := range []string{"", " ", "\t\r\n", "\u00a0\u0085", "\u2003\u3000", "conclusion", " observed "} {
if got, want := summary.MatchString(value), strings.TrimSpace(value) != ""; got != want {
@@ -155,7 +178,7 @@ func TestRecordFactSchemaMatchesStrings(t *testing.T) {
}
}
for _, input := range []string{
`{"summary":"single","asset_ids":["1",2]}`,
`{"facts":[{"summary":"batch","asset_ids":["1",2]}]}`,
`{"facts":[{"summary":"batch","tested_asset_ids":["1",2]}]}`,
} {
if err := actool.ValidateInput(recordFactSchema(), json.RawMessage(input)); err != nil {
@@ -163,3 +186,208 @@ func TestRecordFactSchemaMatchesStrings(t *testing.T) {
}
}
}
// A model that fills every declared property of the earlier schema sends the
// single fact inside facts[] and a copy (or a placeholder) of every fact field
// beside it. Such a call is refused by name before any write, whether the
// copies are identical, empty or dummies, and the message names the one shape
// that is accepted. The published schema no longer declares those fields.
func TestRecordFactRefusesTopLevelCopiesBesideFacts(t *testing.T) {
store := guardTaskStore(t)
intentID := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "fingerprint the service"}, "running")
tools := NewToolSet(store, "worker")
before, err := store.ListByKind(db.KindFact, 100)
if err != nil {
t.Fatal(err)
}
id := strconv.FormatInt(intentID, 10)
fact := `{"summary":"TLS 1.0 is offered on port 8443","detail":"the handshake succeeded with TLSv1.0","evidence":"openssl s_client -tls1 → Protocol: TLSv1","confidence":"observed","source":"tool_output","intent_id":` + id + `,"asset_ids":[]}`
for _, tc := range []struct{ call, want string }{
// an exact copy of every field beside facts[]
{`{"summary":"TLS 1.0 is offered on port 8443","detail":"the handshake succeeded with TLSv1.0","evidence":"openssl s_client -tls1 → Protocol: TLSv1","confidence":"observed","source":"tool_output","intent_id":` + id + `,"asset_ids":[],"facts":[` + fact + `]}`, `"asset_ids", "confidence", "detail", "evidence", "source", "summary"`},
// empty placeholders only
{`{"summary":"","detail":"","evidence":"","confidence":"","source":"","asset_ids":[],"tested_asset_ids":[],"intent_id":` + id + `,"facts":[` + fact + `]}`, `"asset_ids", "confidence", "detail", "evidence", "source", "summary", "tested_asset_ids"`},
// dummy values
{`{"summary":"x","facts":[` + fact + `]}`, `"summary"`},
{`{"summary":null,"facts":[` + fact + `]}`, `"summary"`},
} {
w := tools.Writes().Facts
r := callGuardTool(t, tools.recordFact(), tc.call)
if !r.IsError || !strings.Contains(r.Flatten(), tc.want) || !strings.Contains(r.Flatten(), `record_fact takes only {"facts":[...]}`) || tools.Writes().Facts != w {
t.Errorf("%s: want refusal naming %s, got %s (writes %d→%d)", tc.call, tc.want, r.Flatten(), w, tools.Writes().Facts)
}
}
if n, err := store.ListByKind(db.KindFact, 100); err != nil || len(n) != len(before) {
t.Fatalf("refused calls wrote facts: %d → %d %v", len(before), len(n), err)
}
// The same fact in the published shape is written once and linked.
res := callGuardTool(t, tools.recordFact(), `{"facts":[`+fact+`]}`)
if res.IsError || tools.Writes().Facts != 1 {
t.Fatalf("published shape refused: %s", res.Flatten())
}
var out struct {
IDs []int64 `json:"ids"`
}
if err := json.Unmarshal([]byte(res.Flatten()), &out); err != nil || len(out.IDs) != 1 || out.IDs[0] <= 0 {
t.Fatalf("batch result must carry exactly one id: %s", res.Flatten())
}
if yielded, err := store.FactsYielded(intentID); err != nil || len(yielded) != 1 || yielded[0] != out.IDs[0] {
t.Fatalf("fact not linked to the intent: %v %v", yielded, err)
}
node, err := store.GetNode(out.IDs[0])
if err != nil || node == nil {
t.Fatal(err)
}
var payload map[string]any
_ = json.Unmarshal(node.Payload, &payload)
if payload["summary"] != "TLS 1.0 is offered on port 8443" || payload["source"] != "tool_output" || payload["confidence"] != "observed" {
t.Fatalf("fact payload: %s", node.Payload)
}
}
// Conflicting copies, placeholders, unknown fields, bad asset ids, missing
// summaries and empty batches are refused, and nothing is written by a
// refused call.
func TestRecordFactBatchTopLevelConflicts(t *testing.T) {
store := guardTaskStore(t)
tools := NewToolSet(store, "worker")
before, err := store.ListByKind(db.KindFact, 100) // the task's origin fact
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct{ call, want string }{
// dummy summaries a model may send after a refusal
{`{"summary":"x","facts":[{"summary":"real conclusion"}]}`, `"summary"`},
{`{"summary":"placeholder","detail":"test","evidence":"test","facts":[{"summary":"real","detail":"real detail","evidence":"real evidence"}]}`, `"detail", "evidence", "summary"`},
{`{"summary":"contradictory","facts":[{"summary":"different"}]}`, `"summary"`},
{`{"detail":"invented","facts":[{"summary":"legitimate"}]}`, `"detail"`},
{`{"confidence":"observed","facts":[{"summary":"fact says inferred","confidence":"inferred"}]}`, `"confidence"`},
{`{"source":"inference","facts":[{"summary":"fact says tool_output","source":"tool_output"}]}`, `"source"`},
{`{"asset_ids":[1],"facts":[{"summary":"fact has other assets","asset_ids":[2]}]}`, `"asset_ids"`},
{`{"summary":"first","facts":[{"summary":"first"},{"summary":"second"}]}`, `(2 item(s))`},
{`{"summary":"x","typo":1,"facts":[{"summary":"unknown field"}]}`, `"typo"`},
{`{"facts":[{"summary":"unknown nested","typo":1}]}`, `"typo"`},
{`{"facts":[{"summary":"bad asset id","asset_ids":["abc"]}]}`, `asset`},
{`{"summary":"must not inherit","facts":[{"detail":"missing summary"}]}`, `"summary"`},
{`{"summary":"x","facts":[]}`, `non-empty`},
{`{"facts":[]}`, `non-empty`},
{`{"summary":"","facts":[]}`, `non-empty`},
} {
w := tools.Writes().Facts
r := callGuardTool(t, tools.recordFact(), tc.call)
if !r.IsError || !strings.Contains(r.Flatten(), tc.want) || tools.Writes().Facts != w {
t.Errorf("%s: want refusal mentioning %s, got %s (writes %d→%d)", tc.call, tc.want, r.Flatten(), w, tools.Writes().Facts)
}
}
if n, err := store.ListByKind(db.KindFact, 100); err != nil || len(n) != len(before) {
t.Fatalf("refused calls wrote facts: %d → %d %v", len(before), len(n), err)
}
}
// The published schema holds facts[] and nothing else, and the description
// names that one shape; the earlier "two shapes" wording is gone.
func TestRecordFactSchemaIsFactsOnly(t *testing.T) {
schema := recordFactSchema()
props := schema["properties"].(map[string]any)
if len(props) != 1 || props["facts"] == nil {
t.Fatalf("top-level properties: %v", props)
}
if req, _ := schema["required"].([]string); len(req) != 1 || req[0] != "facts" || schema["additionalProperties"] != false {
t.Fatalf("facts must be required and the only property: %v %v", schema["required"], schema["additionalProperties"])
}
facts := props["facts"].(map[string]any)
if d, _ := facts["description"].(string); !strings.Contains(d, "never send summary") {
t.Errorf("facts description must forbid top-level copies: %q", d)
}
item := facts["items"].(map[string]any)["properties"].(map[string]any)
for _, name := range []string{"summary", "intent_id", "detail", "evidence", "confidence", "source", "asset_ids", "tested_asset_ids"} {
if item[name] == nil {
t.Errorf("facts[] item lacks %s", name)
}
}
for name, d := range map[string]string{
"summary": item["summary"].(map[string]any)["description"].(string),
"intent_id": item["intent_id"].(map[string]any)["description"].(string),
} {
if strings.Contains(d, "top level") || strings.Contains(d, "top-level") {
t.Errorf("facts[].%s still refers to a top level: %q", name, d)
}
}
desc := NewToolSet(nil, "").recordFact().Description()
if !strings.HasSuffix(desc, recordFactContractNote) || strings.Contains(desc, "top-level intent_id") {
t.Fatalf("record_fact description must describe the facts-only contract: %q", desc)
}
// The seed the catalog stores is the same contract.
for _, seed := range BuiltinToolSeeds() {
if seed.Key != "record_fact" {
continue
}
if seed.Desc != desc {
t.Fatal("seed description differs from the tool")
}
if p := seed.Schema["properties"].(map[string]any); len(p) != 1 || p["facts"] == nil {
t.Fatalf("seed schema is not facts-only: %v", p)
}
}
}
// A facts[] item without intent_id is linked to the intent the worker runs
// (SetOwnerNode); a legacy top-level intent_id still wins; a planner ToolSet
// (owner = the task root, not an intent) writes an unlinked fact.
func TestRecordFactDefaultsToRunningIntent(t *testing.T) {
store := guardTaskStore(t)
running := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "running intent"}, "running")
other := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "other intent"}, "open")
tools := NewToolSet(store, "worker")
tools.SetOwnerNode(running)
ids := func(res actool.Result) []int64 {
t.Helper()
var out struct {
IDs []int64 `json:"ids"`
}
if err := json.Unmarshal([]byte(res.Flatten()), &out); err != nil || len(out.IDs) != 1 || out.IDs[0] <= 0 {
t.Fatalf("batch result: %s", res.Flatten())
}
return out.IDs
}
res := callGuardTool(t, tools.recordFact(), `{"facts":[{"summary":"linked to the running intent","confidence":"observed","source":"tool_output"}]}`)
if res.IsError {
t.Fatal(res.Flatten())
}
first := ids(res)[0]
if yielded, err := store.FactsYielded(running); err != nil || len(yielded) != 1 || yielded[0] != first {
t.Fatalf("fact %d not linked to the running intent %d: %v %v", first, running, yielded, err)
}
res = callGuardTool(t, tools.recordFact(), `{"intent_id":`+strconv.FormatInt(other, 10)+`,"facts":[{"summary":"legacy top-level intent"}]}`)
if res.IsError {
t.Fatal(res.Flatten())
}
second := ids(res)[0]
if yielded, err := store.FactsYielded(other); err != nil || len(yielded) != 1 || yielded[0] != second {
t.Fatalf("legacy top-level intent_id must win: %v %v", yielded, err)
}
res = callGuardTool(t, tools.recordFact(), `{"facts":[{"summary":"item intent","intent_id":`+strconv.FormatInt(other, 10)+`}]}`)
if res.IsError {
t.Fatal(res.Flatten())
}
third := ids(res)[0]
if yielded, _ := store.FactsYielded(other); len(yielded) != 2 || yielded[1] != third && yielded[0] != third {
t.Fatalf("item intent_id must win: %v", yielded)
}
if yielded, _ := store.FactsYielded(running); len(yielded) != 1 {
t.Fatalf("running intent gained facts it did not produce: %v", yielded)
}
planner := NewToolSet(store, "planner")
origin, err := store.ListByKind(db.KindFact, 1)
if err != nil || len(origin) == 0 {
t.Fatalf("origin fact: %v", err)
}
planner.SetOwnerNode(origin[0].ID)
res = callGuardTool(t, planner.recordFact(), `{"facts":[{"summary":"planner fact without intent"}]}`)
if res.IsError || planner.Writes().Facts != 1 {
t.Fatalf("planner write refused: %s", res.Flatten())
}
if yielded, _ := store.FactsYielded(running); len(yielded) != 1 {
t.Fatalf("planner fact must not be linked to a worker intent: %v", yielded)
}
}
+4 -1
View File
@@ -44,7 +44,10 @@ func TestReportFindingStoresReasoningFields(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer d.DeleteTask(task.ID)
defer func() {
_, _ = d.DeleteFindingsByTask(task.ID)
_ = d.DeleteTask(task.ID)
}()
ts := NewToolSet(nil, "")
ts.ts, ts.taskID = d.Exploration(task.ExplorationID), task.ID
for _, tc := range []struct{ extra, counter, rationale string }{
+8 -2
View File
@@ -25,7 +25,10 @@ func TestReportFindingOptionalTrafficWithoutCapture(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer d.DeleteTask(task.ID)
defer func() {
_, _ = d.DeleteFindingsByTask(task.ID)
_ = d.DeleteTask(task.ID)
}()
ts := NewToolSet(nil, "")
ts.ts, ts.taskID = d.Exploration(task.ExplorationID), task.ID
ts.SetFindingRecorder(evidence.New(d, nil, t.TempDir()))
@@ -60,7 +63,10 @@ func TestReportFindingAtomicContract(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer d.DeleteTask(task.ID)
defer func() {
_, _ = d.DeleteFindingsByTask(task.ID)
_ = d.DeleteTask(task.ID)
}()
ts := NewToolSet(nil, "")
ts.ts = d.Exploration(task.ExplorationID)
ts.taskID = task.ID
+128
View File
@@ -0,0 +1,128 @@
package agent
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"testing"
"github.com/Autumn-27/artex/db"
)
// Synthetic replay of the record_fact call shapes a model produces, replayed
// against the handler in the package test database. The shapes mirror what
// was observed in production before the facts-only contract (a model filling
// every declared top-level property beside facts[]) without using recorded
// data. Checked per call: the verdict, that an accepted call writes exactly one
// node per facts item linked to the intent and anchored to the assets, and
// that a refused call writes nothing and names the top-level fields.
func TestRecordFactShapesReplay(t *testing.T) {
d := testDB(t)
t.Cleanup(func() { _ = d.Close() })
task, err := d.CreateTaskWithOptions("record_fact shapes", "replay synthetic record_fact call shapes", db.TaskCreateOptions{})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = d.DeleteTask(task.ID) })
store := d.Exploration(task.ExplorationID)
intentID := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "enumerate the service"}, "running")
var assets []int64
for i := 1; i <= 2; i++ {
id, err := d.Assets().UpsertIP(db.UpsertIPReq{IP: fmt.Sprintf("203.0.113.%d", i), TaskID: task.ID})
if err != nil {
t.Fatal(err)
}
assets = append(assets, id)
}
t.Cleanup(func() { _, _ = d.Assets().DeleteByIDs(assets) })
tools := NewToolSet(store, "worker")
tools.SetOwnerNode(intentID)
tool := tools.recordFact()
iid := strconv.FormatInt(intentID, 10)
a1, a2 := strconv.FormatInt(assets[0], 10), strconv.FormatInt(assets[1], 10)
item := func(n int) string {
return fmt.Sprintf(`{"summary":"conclusion %d","detail":"detail %d","evidence":"cmd %d → line","confidence":"observed","source":"tool_output","asset_ids":[%s]}`, n, n, n, a1)
}
type shape struct {
name string
call string
accept bool
items int
fields string // named in the refusal
}
shapes := []shape{
{"published: one item", `{"facts":[` + item(1) + `]}`, true, 1, ""},
{"published: two items", `{"facts":[` + item(1) + `,` + item(2) + `]}`, true, 2, ""},
{"published: item with its own intent_id and alias", `{"facts":[{"summary":"alias","tested_asset_ids":["` + a2 + `"],"intent_id":` + iid + `}]}`, true, 1, ""},
{"legacy batch: top-level intent_id", `{"intent_id":` + iid + `,"facts":[` + item(3) + `]}`, true, 1, ""},
{"legacy single", `{"summary":"single fact","confidence":"inferred","asset_ids":[` + a2 + `]}`, true, 1, ""},
{"copies of every field beside facts", `{"summary":"conclusion 1","detail":"detail 1","evidence":"cmd 1 → line","confidence":"observed","source":"tool_output","asset_ids":[` + a1 + `],"facts":[` + item(1) + `]}`, false, 1, `"asset_ids", "confidence", "detail", "evidence", "source", "summary"`},
{"empty placeholders beside facts", `{"summary":"","detail":"","asset_ids":[],"tested_asset_ids":[],"facts":[` + item(1) + `]}`, false, 1, `"asset_ids", "detail", "summary", "tested_asset_ids"`},
{"dummy summary beside facts", `{"summary":"x","facts":[` + item(1) + `]}`, false, 1, `"summary"`},
{"abbreviated copy beside facts", `{"summary":"short","detail":"short detail","facts":[` + item(1) + `]}`, false, 1, `"detail", "summary"`},
{"conflicting source beside facts", `{"source":"inference","facts":[` + item(1) + `]}`, false, 1, `"source"`},
{"top-level summary beside two items", `{"summary":"conclusion 1","facts":[` + item(1) + `,` + item(2) + `]}`, false, 2, `"summary"`},
}
for _, s := range shapes {
before := tools.Writes().Facts
linked, _ := store.FactsYielded(intentID)
res := callGuardTool(t, tool, s.call)
written := tools.Writes().Facts - before
if !res.IsError != s.accept {
t.Errorf("%s: accepted=%v want %v: %s", s.name, !res.IsError, s.accept, res.Flatten())
continue
}
if !s.accept {
if written != 0 || !strings.Contains(res.Flatten(), "top-level field(s) "+s.fields) || !strings.Contains(res.Flatten(), "no facts were written") {
t.Errorf("%s: refusal must write nothing and name %s: %s (writes %d)", s.name, s.fields, res.Flatten(), written)
}
continue
}
if written != s.items {
t.Errorf("%s: wrote %d facts for %d items", s.name, written, s.items)
}
after, _ := store.FactsYielded(intentID)
if len(after) != len(linked)+s.items {
t.Errorf("%s: intent links %d → %d for %d items", s.name, len(linked), len(after), s.items)
}
ids := after[len(linked):]
if strings.HasPrefix(s.call, `{"facts"`) || strings.HasPrefix(s.call, `{"intent_id"`) {
var out struct {
IDs []int64 `json:"ids"`
}
if err := json.Unmarshal([]byte(res.Flatten()), &out); err != nil || len(out.IDs) != s.items {
t.Errorf("%s: ids: %s", s.name, res.Flatten())
}
}
for _, id := range ids {
node, err := store.GetNode(id)
if err != nil || node == nil || node.Kind != db.KindFact {
t.Fatalf("%s: fact %d: %v", s.name, id, err)
}
var payload map[string]any
_ = json.Unmarshal(node.Payload, &payload)
if sum, _ := payload["summary"].(string); sum == "" || sum == "x" {
t.Errorf("%s: stored summary %q", s.name, sum)
}
if strings.Contains(s.call, "asset_ids") {
anchored := false
for _, asset := range assets {
refs, err := store.AssetRefs(asset)
if err != nil {
t.Fatal(err)
}
for _, r := range refs {
if r.ID == id {
anchored = true
}
}
}
if !anchored {
t.Errorf("%s: fact %d not anchored to an asset", s.name, id)
}
}
}
}
}
+3 -3
View File
@@ -22,7 +22,7 @@ func TestIngestScanOutputPG(t *testing.T) {
if err != nil {
t.Fatal(err)
}
const assetsWhere = `ip IN ('198.51.100.77','2001:db8::77') OR domain LIKE '%scan-ingest.example.com' OR url LIKE '%198.51.100.77%' OR url LIKE '%2001:db8::77%'`
const assetsWhere = `ip IN ('198.51.100.77','2001:db8::77') OR domain IN ('198.51.100.77','2001:db8::77','example.com') OR domain LIKE '%scan-ingest.example.com' OR url LIKE '%198.51.100.77%' OR url LIKE '%2001:db8::77%'`
t.Cleanup(func() {
_ = d.DeleteTask(task.ID)
_, _ = d.Exec(`DELETE FROM assets WHERE ` + assetsWhere)
@@ -127,7 +127,7 @@ func TestIngestScanOutputJSONPG(t *testing.T) {
if err != nil {
t.Fatal(err)
}
const assetsWhere = `domain LIKE '%scan-ingest.example.com' OR url LIKE '%scan-ingest.example.com%'`
const assetsWhere = `ip IN ('198.51.100.77','2001:db8::77') OR domain IN ('198.51.100.77','2001:db8::77','example.com') OR domain LIKE '%scan-ingest.example.com' OR url LIKE '%scan-ingest.example.com%'`
t.Cleanup(func() {
_ = d.DeleteTask(task.ID)
_, _ = d.Exec(`DELETE FROM assets WHERE ` + assetsWhere)
@@ -210,7 +210,7 @@ func TestIngestScanOutputUserinfoPG(t *testing.T) {
if err != nil {
t.Fatal(err)
}
const assetsWhere = `domain LIKE '%scan-ingest.example.com' OR url LIKE '%scan-ingest.example.com%'`
const assetsWhere = `ip IN ('198.51.100.77','2001:db8::77') OR domain IN ('198.51.100.77','2001:db8::77','example.com') OR domain LIKE '%scan-ingest.example.com' OR url LIKE '%scan-ingest.example.com%'`
t.Cleanup(func() {
_ = d.DeleteTask(task.ID)
_, _ = d.Exec(`DELETE FROM assets WHERE ` + assetsWhere)
+8
View File
@@ -180,6 +180,14 @@ type overriddenTool struct {
func (o *overriddenTool) Description() string { return o.desc }
func (o *overriddenTool) InputSchema() map[string]any { return o.schema }
// AcceptsRawInput forwards the wrapped tool's opt-out of the harness schema
// check (record_fact validates its own input); the embedded interface alone
// would hide it from the harness's type assertion.
func (o *overriddenTool) AcceptsRawInput() bool {
r, ok := o.CoreTool.(interface{ AcceptsRawInput() bool })
return ok && r.AcceptsRawInput()
}
func (o *overriddenTool) Call(ctx context.Context, in json.RawMessage, tc *actool.ToolContext) (actool.Result, error) {
return o.CoreTool.Call(ctx, injectDefaults(in, o.schema), tc)
}
+44 -5
View File
@@ -394,8 +394,21 @@ func readTool(name, desc string, schema map[string]any, run func(context.Context
}
func writeTool(name, desc string, schema map[string]any, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool {
return buildWriteTool(name, desc, schema, false, run)
}
// writeRawTool is writeTool for a handler that validates its own input: the
// harness skips its schema check (Spec.RawInput), so the published schema can
// be stricter (required properties) than the legacy shapes the handler still
// accepts, and refusals carry the handler's own message instead of the
// SDK's generic one.
func writeRawTool(name, desc string, schema map[string]any, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool {
return buildWriteTool(name, desc, schema, true, run)
}
func buildWriteTool(name, desc string, schema map[string]any, raw bool, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool {
return actool.Build(actool.Spec{
Name: name, Description: desc, Schema: schema,
Name: name, Description: desc, Schema: schema, RawInput: raw,
Permissions: func(context.Context, json.RawMessage, acperm.Context) acperm.Decision { return acperm.Allowed() },
Run: func(ctx context.Context, in json.RawMessage, _ *actool.ToolContext) (actool.Result, error) {
return run(ctx, in)
@@ -419,6 +432,12 @@ func (t *ToolSet) writeExpTool(name, desc string, schema map[string]any, run fun
return writeTool(name, desc, schema, t.needExploration(name, run))
}
// writeExpToolRaw is writeExpTool with the harness schema check skipped (see
// writeRawTool); the handler validates the input itself.
func (t *ToolSet) writeExpToolRaw(name, desc string, schema map[string]any, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool {
return writeRawTool(name, desc, schema, t.needExploration(name, run))
}
// needExploration wraps a handler so it only runs with an exploration store.
// Tools that degrade more usefully than "unavailable" (report_finding points at
// add_task_hint, set_goals/set_constraints at the task itself) keep their own
@@ -1588,15 +1607,30 @@ func (t *ToolSet) recordOneFact(it factItem, defaultIntent int64) (int64, error)
return id, nil
}
// runningIntent is the intent this ToolSet works for (the worker's claimed
// intent, set by SetOwnerNode), or 0 when the owner node is not an intent
// (planner: the task root) or unknown.
func (t *ToolSet) runningIntent() int64 {
if t.ts == nil || t.ownerNode <= 0 {
return 0
}
node, err := t.ts.GetNode(t.ownerNode)
if err != nil || node == nil || node.Kind != db.KindIntent {
return 0
}
return node.ID
}
func (t *ToolSet) recordFact() actool.CoreTool {
return t.writeExpTool("record_fact", "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n"+
return t.writeExpToolRaw("record_fact", "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n"+
"⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n"+
"★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n"+
"★facts is the only parameter: facts:[{...}] with one item per [mutually distinct] conclusion, usually exactly one item. Returns an ids array, same length and order as facts. An item without intent_id is linked to the intent you are running.\n"+
"⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n"+
" · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n"+
" · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n"+
" · source = where the content came from: target_response (content the target itself returned: page bodies, headers, error messages, banners, even when a tool fetched it) | tool_output (a tool's own result about the target: port states, scanner verdicts, status codes, timings) | third_party_intel (OSINT, CVE databases, public reports) | operator_input (given by the human operator) | inference (your own reasoning; the default when omitted). Any other value is refused.\n"+
" · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen.",
" · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen."+
recordFactContractNote,
recordFactSchema(),
func(_ context.Context, in json.RawMessage) (actool.Result, error) {
a, batch, err := decodeRecordFactInput(in)
@@ -1607,7 +1641,12 @@ func (t *ToolSet) recordFact() actool.CoreTool {
if !batch {
items = []factItem{a.factItem}
}
defaultIntent := pid(a.factItem.IntentID) // top-level intent_id = batch default
// Default intent of the items: a legacy top-level intent_id, else the
// intent this worker is running (the published schema has no top level).
defaultIntent := pid(a.factItem.IntentID)
if defaultIntent <= 0 {
defaultIntent = t.runningIntent()
}
ids := make([]int64, len(items))
errs := map[string]string{}
+117
View File
@@ -9,6 +9,7 @@ import (
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
)
@@ -130,3 +131,119 @@ func TestArchiveIncompleteCredentialOverride(t *testing.T) {
}
}
}
// The server removes the AWS credential variables from its environment once the
// store is built, so that child processes do not inherit them. The SDK must have
// captured by then everything it needs to sign: the static keys, and for the
// container and web-identity providers the endpoint, token and file path they
// fetch credentials from later (their first fetch happens after the scrub here).
func TestReview318CredentialsSurviveEnvScrub(t *testing.T) {
scrubbed := []string{
"AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN",
"AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
"AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN",
}
for _, mode := range []string{"environment", "environment-alternate-names", "container", "web-identity"} {
t.Run(mode, func(t *testing.T) {
isolateAWSCredentials(t)
for _, key := range append([]string{
"ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_SESSION_TOKEN",
"ARTEX_ARCHIVE_S3_SSE", "ARTEX_ARCHIVE_S3_KMS_KEY_ID", "ARTEX_ARCHIVE_S3_PREFIX", "AWS_ENDPOINT_URL_STS",
}, scrubbed...) {
t.Setenv(key, "")
}
var envScrubbed atomic.Bool
var fetchedAfterScrub atomic.Int32
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if envScrubbed.Load() {
fetchedAfterScrub.Add(1)
}
expiration := time.Now().Add(time.Hour).UTC().Format(time.RFC3339)
if mode == "container" {
if r.Header.Get("Authorization") != "fixture-container-token" {
t.Error("missing container authorization token")
}
_ = json.NewEncoder(w).Encode(map[string]any{
"AccessKeyId": "fixture-key", "SecretAccessKey": "fixture-secret",
"Token": "fixture-session", "Expiration": expiration,
})
return
}
if err := r.ParseForm(); err != nil || r.PostForm.Get("Action") != "AssumeRoleWithWebIdentity" ||
r.PostForm.Get("WebIdentityToken") != "fixture-web-identity-token" ||
r.PostForm.Get("RoleArn") != "arn:aws:iam::123456789012:role/archive" {
t.Errorf("unexpected STS request: %v %v", err, r.PostForm)
}
w.Header().Set("Content-Type", "text/xml")
fmt.Fprintf(w, `<AssumeRoleWithWebIdentityResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/"><AssumeRoleWithWebIdentityResult><Credentials><AccessKeyId>fixture-key</AccessKeyId><SecretAccessKey>fixture-secret</SecretAccessKey><SessionToken>fixture-session</SessionToken><Expiration>%s</Expiration></Credentials></AssumeRoleWithWebIdentityResult></AssumeRoleWithWebIdentityResponse>`, expiration)
}))
defer provider.Close()
switch mode {
case "environment":
t.Setenv("AWS_ACCESS_KEY_ID", "fixture-key")
t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture-secret")
t.Setenv("AWS_SESSION_TOKEN", "fixture-session")
case "environment-alternate-names":
t.Setenv("AWS_ACCESS_KEY", "fixture-key")
t.Setenv("AWS_SECRET_KEY", "fixture-secret")
t.Setenv("AWS_SESSION_TOKEN", "fixture-session")
case "container":
t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", provider.URL+"/credentials")
t.Setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN", "fixture-container-token")
case "web-identity":
tokenFile := filepath.Join(t.TempDir(), "token")
if err := os.WriteFile(tokenFile, []byte("fixture-web-identity-token"), 0600); err != nil {
t.Fatal(err)
}
t.Setenv("AWS_WEB_IDENTITY_TOKEN_FILE", tokenFile)
t.Setenv("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/archive")
t.Setenv("AWS_ENDPOINT_URL_STS", provider.URL)
}
var authorization, sessionToken atomic.Value
objects := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodHead {
t.Errorf("unexpected S3 operation: %s", r.Method)
}
authorization.Store(r.Header.Get("Authorization"))
sessionToken.Store(r.Header.Get("X-Amz-Security-Token"))
w.WriteHeader(http.StatusNotFound)
}))
defer objects.Close()
t.Setenv("ARTEX_ARCHIVE_STORAGE", "s3")
t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", objects.URL)
t.Setenv("ARTEX_ARCHIVE_S3_REGION", "us-east-1")
t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "fixture")
t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true")
store, err := FromEnv()
if err != nil || store == nil {
t.Fatalf("store from environment: %v", err)
}
for _, key := range scrubbed {
if err := os.Unsetenv(key); err != nil {
t.Fatal(err)
}
}
envScrubbed.Store(true)
// HEAD is the first request of a download. The fixture answers 404, so
// the call fails; only the signature of the request matters here.
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
checksum := strings.Repeat("a", 64)
err = store.Download(ctx, store.URI(1, 2, checksum), filepath.Join(t.TempDir(), "probe"), checksum, 1)
if err == nil {
t.Fatal("the fixture has no object: the download must fail")
}
got, _ := authorization.Load().(string)
if !strings.Contains(got, "Credential=fixture-key/") || !strings.Contains(got, "Signature=") {
t.Fatalf("request not signed with the captured key after the scrub: Authorization=%q (%v)", got, err)
}
if token, _ := sessionToken.Load().(string); token != "fixture-session" {
t.Fatalf("session token lost after the scrub: %q", token)
}
if lazy := mode == "container" || mode == "web-identity"; lazy && fetchedAfterScrub.Load() == 0 {
t.Fatal("the provider was expected to fetch its credentials after the scrub")
}
})
}
}
+1 -1
View File
@@ -118,7 +118,7 @@ else
command -v rsync >/dev/null 2>&1 || die "rsync not detected"
info "Building frontend static assets"
if [ "${ARTEX_SKIP_NPM_CI:-0}" != "1" ]; then
(cd web && npm ci)
(cd web && npm ci --ignore-scripts)
fi
(cd web && npm run build:static)
info "Syncing frontend assets to server/webui/dist"
+2 -2
View File
@@ -22,7 +22,7 @@ func TestFlushActivityOutboxReplaysOnce(t *testing.T) {
if err != nil {
SkipPGUnavailable(t, err)
}
defer d.Close()
t.Cleanup(func() { _ = d.Close() })
expID, err := d.CreateExploration("outbox", "replay")
if err != nil {
t.Fatal(err)
@@ -65,7 +65,7 @@ func TestFlushActivityOutboxDropsStuckRow(t *testing.T) {
if err != nil {
SkipPGUnavailable(t, err)
}
defer d.Close()
t.Cleanup(func() { _ = d.Close() })
const missing = int64(9_000_000_000_001)
t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM activity_outbox WHERE exploration_id=$1`, missing) })
es := d.Exploration(missing)
+1
View File
@@ -1143,6 +1143,7 @@ func TestAssetIPRejectsHostname(t *testing.T) {
t.Fatalf("empty ip rejected: %v", err)
}
deleteAsset(d, id)
_, _ = d.Exec(`DELETE FROM assets WHERE domain IN ('ip-guard.example.com','example.com')`)
id, err = av2.UpsertIP(UpsertIPReq{IP: "203.0.113.7"})
if err != nil {
t.Fatalf("valid ip rejected: %v", err)
+96
View File
@@ -0,0 +1,96 @@
package db
import (
"fmt"
"strings"
"testing"
"time"
)
func TestDashboardUsageIndexesMigration0010(t *testing.T) {
d, err := Open(testDSN(t))
if err != nil {
SkipPGUnavailable(t, err)
}
t.Cleanup(func() { _ = d.Close() })
wantIndexes := []struct {
name string
table string
pred string
}{
{"idx_act_result_created", "activity", "kind = 'result'"},
{"idx_conv_act_result", "conversation_activities", "kind = 'result'"},
{"idx_llm_usage_judge_ts", "llm_usage", "worker = 'judge'"},
}
for _, want := range wantIndexes {
var valid, ready bool
var def string
err := d.QueryRow(`
SELECT i.indisvalid, i.indisready, pg_get_indexdef(i.indexrelid)
FROM pg_index i
JOIN pg_class c ON c.oid = i.indexrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'public' AND c.relname = $1`, want.name).Scan(&valid, &ready, &def)
if err != nil {
t.Fatalf("index public.%s missing after Open: %v", want.name, err)
}
if !valid || !ready {
t.Fatalf("index %s indisvalid=%v indisready=%v, want both true (%s)", want.name, valid, ready, def)
}
if !strings.Contains(def, want.table) || !strings.Contains(def, want.pred) {
t.Fatalf("index %s definition %q missing table %q or predicate %q", want.name, def, want.table, want.pred)
}
}
// Verify sargable TokenByModel matches both specific task_id and empty task_id.
taskID := fmt.Sprintf("idx-test-%d", time.Now().UnixNano())
t.Cleanup(func() {
_, _ = d.Exec(`DELETE FROM llm_usage WHERE task_id = $1 OR profile_name = $2`, taskID, taskID)
})
if err := d.InsertLLMUsage(&LLMUsage{
TaskID: taskID,
Worker: "worker",
Model: "claude-sonnet-4",
ProfileName: taskID,
InputTokens: 1000,
OutputTokens: 200,
Status: "ok",
}); err != nil {
t.Fatal(err)
}
if err := d.InsertLLMUsage(&LLMUsage{
TaskID: "",
Worker: "chat",
Model: "gpt-5",
ProfileName: taskID,
InputTokens: 500,
OutputTokens: 100,
Status: "ok",
}); err != nil {
t.Fatal(err)
}
got, err := d.TokenByModel(taskID)
if err != nil {
t.Fatalf("TokenByModel(%q): %v", taskID, err)
}
if len(got) != 1 || got[0].Model != "claude-sonnet-4" || got[0].InputTokens != 1000 || got[0].OutputTokens != 200 {
t.Fatalf("TokenByModel(%q) = %+v, want 1 claude-sonnet-4 row", taskID, got)
}
emptyGot, err := d.TokenByModel("")
if err != nil {
t.Fatalf("TokenByModel(\"\"): %v", err)
}
foundChat := false
for _, m := range emptyGot {
if m.Model == "gpt-5" && m.InputTokens >= 500 {
foundChat = true
}
}
if !foundChat {
t.Fatalf("TokenByModel(\"\") did not include NULL/empty task_id row: %+v", emptyGot)
}
}
+1
View File
@@ -28,6 +28,7 @@ func TestFindingQualityByNodeID(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer d.DeleteFinding(rec.FindingID)
if err := d.SetFindingVerification(rec.FindingID, VerificationVerified, 3, "reproduced twice on staging"); err != nil {
t.Fatal(err)
}
+2
View File
@@ -24,10 +24,12 @@ func TestFindingReasoningFieldsRoundTrip(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer d.DeleteFinding(with.FindingID)
without, err := es.RecordFinding(t.Context(), RecordFindingInput{TaskID: tk.ID, VulnClass: "IDOR", Severity: "low", Summary: "without reasoning", Worker: "test"})
if err != nil {
t.Fatal(err)
}
defer d.DeleteFinding(without.FindingID)
want := map[int64][2]string{
with.FindingID: {"UUID guessing returned 404", "two accounts, same response body"},
without.FindingID: {"", ""},
+5 -1
View File
@@ -32,7 +32,11 @@ func TestFindingTrafficLegacyArchiveDefaults(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer func() { d.Exec(`DELETE FROM task_archives WHERE task_id=$1`, task.ID); d.DeleteTask(task.ID) }()
defer func() {
_, _ = d.DeleteFindingsByTask(task.ID)
_, _ = d.Exec(`DELETE FROM task_archives WHERE task_id=$1`, task.ID)
_ = d.DeleteTask(task.ID)
}()
f, err := d.Exploration(task.ExplorationID).RecordFinding(t.Context(), RecordFindingInput{TaskID: task.ID, ExplorationID: task.ExplorationID, Summary: "legacy", Severity: "low"})
if err != nil {
t.Fatal(err)
+9 -3
View File
@@ -100,6 +100,7 @@ func TestSetFindingStatusMirrorsNodePayload(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer d.DeleteFinding(fid)
n, err := d.SetFindingStatus(fid, FindingFalsePositive)
if err != nil || n != 1 {
@@ -146,13 +147,13 @@ func TestFindingReportState(t *testing.T) {
if err != nil {
SkipPGUnavailable(t, err)
}
defer d.Close()
t.Cleanup(func() { d.Close() })
tk, err := d.CreateTask("finding report state", "goal", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
defer d.DeleteTask(tk.ID)
t.Cleanup(func() { _ = d.DeleteTask(tk.ID) })
es := d.Exploration(tk.ExplorationID)
nodeID, err := es.AddNode(KindFinding, map[string]any{"summary": "x", "severity": "high"}, 5, "confirmed", "worker", nil)
@@ -167,7 +168,11 @@ func TestFindingReportState(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { d.Exec(`DELETE FROM notification_events WHERE finding_id IN ($1,$2)`, fid, second) })
t.Cleanup(func() {
_, _ = d.Exec(`DELETE FROM notification_events WHERE finding_id IN ($1,$2)`, fid, second)
_, _ = d.DeleteFinding(fid)
_, _ = d.DeleteFinding(second)
})
one, err := d.GetFinding(fid)
if err != nil || one == nil || one.ReportState != FindingReportNone {
@@ -633,6 +638,7 @@ func TestAddFindingFollowUpIntent(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer d.DeleteFinding(findingID)
auditInput := Activity{Worker: "system", Kind: "text", Summary: "Manually submitted vulnerability deep-exploitation intent", Detail: "Verify exploitability and build an evidence chain"}
intentID, audit, err := store.AddFindingFollowUpIntent(findingID, findingNodeID, "Verify exploitability and build an evidence chain", auditInput)
+12 -2
View File
@@ -191,11 +191,14 @@ func TestExtractLessonsScopesSingleCompanyFromTaskScope(t *testing.T) {
t.Fatal(err)
}
t.Cleanup(func() { cleanupErr(t, "extract task", d.DeleteTask(task.ID)) })
if _, err := d.Exec(`
var extractedFID int64
if err := d.QueryRow(`
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
VALUES ($1,'xss','Reflected XSS','reflected in search','high','confirmed')`, task.ID); err != nil {
VALUES ($1,'xss','Reflected XSS','reflected in search','high','confirmed')
RETURNING id`, task.ID).Scan(&extractedFID); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, extractedFID) })
n, err := d.ExtractLessonsForTask(t.Context(), task.ID)
if err != nil {
t.Fatal(err)
@@ -477,6 +480,7 @@ func TestExtractLessonsRetractsReversedDisposition(t *testing.T) {
RETURNING id`, task.ID, cue, label).Scan(&fid); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
n, err := d.ExtractLessonsForTask(t.Context(), task.ID)
if err != nil {
@@ -589,12 +593,14 @@ func TestExtractLessonsRetireOnlyWhenLastEvidence(t *testing.T) {
RETURNING id`, task.ID, cue, label).Scan(&aID); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, aID) })
if err := d.QueryRow(`
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
VALUES ($1,$2,'Reflected XSS',$3,'high','confirmed')
RETURNING id`, task.ID, cue, label).Scan(&bID); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, bID) })
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
t.Fatal(err)
@@ -681,6 +687,7 @@ func TestExtractLessonsRetractsAllRetests(t *testing.T) {
t.Fatal(err)
}
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
r1 := insertLessonRetest(t, d, fid, "reproduced")
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
@@ -752,6 +759,7 @@ func TestExtractLessonsDoesNotUnretireWithEvidence(t *testing.T) {
t.Fatal(err)
}
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
t.Fatal(err)
@@ -798,6 +806,7 @@ func TestExtractLessonsLeavesAdopted(t *testing.T) {
t.Fatal(err)
}
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
t.Fatal(err)
@@ -841,6 +850,7 @@ func TestExtractLessonsReopenRetractsNothing(t *testing.T) {
t.Fatal(err)
}
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
t.Fatal(err)
+21 -2
View File
@@ -1,6 +1,7 @@
package db
import (
"database/sql"
"sort"
"strconv"
"time"
@@ -71,14 +72,32 @@ VALUES (NULLIF($1,''),$2,NULLIF($3,''),NULLIF($4,''),NULLIF($5,''),$6,$7,$8,$9,$
// Accurate even with per-agent model bindings, pool rotation/failover, and
// interrupted runs, since every call (success or error) is metered.
func (d *DB) TokenByModel(taskID string) ([]ModelTokenStat, error) {
rows, err := d.Query(`
// Keep the predicate sargable for non-empty task IDs so PostgreSQL can use
// idx_llm_usage_model(task_id, model) instead of sequentially scanning the
// table via COALESCE(task_id, '').
var (
rows *sql.Rows
err error
)
if taskID == "" {
rows, err = d.Query(`
SELECT COALESCE(NULLIF(model,''),'(unknown)') AS model, COUNT(*) AS calls,
COALESCE(SUM(input_tokens),0), COALESCE(SUM(output_tokens),0),
COALESCE(SUM(cache_read),0), COALESCE(SUM(cache_write),0)
FROM llm_usage
WHERE COALESCE(task_id,'') = $1
WHERE task_id IS NULL OR task_id = ''
GROUP BY model
ORDER BY SUM(input_tokens) + SUM(output_tokens) DESC, model`)
} else {
rows, err = d.Query(`
SELECT COALESCE(NULLIF(model,''),'(unknown)') AS model, COUNT(*) AS calls,
COALESCE(SUM(input_tokens),0), COALESCE(SUM(output_tokens),0),
COALESCE(SUM(cache_read),0), COALESCE(SUM(cache_write),0)
FROM llm_usage
WHERE task_id = $1
GROUP BY model
ORDER BY SUM(input_tokens) + SUM(output_tokens) DESC, model`, taskID)
}
if err != nil {
return nil, err
}
+2 -2
View File
@@ -44,7 +44,7 @@ func TestCostSeparatesFreshReadAndWriteBuckets(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer d.Close()
t.Cleanup(func() { d.Close() })
priced, unpriced := costProfiles(t, d)
// Every case adds 100k output tokens: 0.1 × 4.00 = 0.40.
for _, tc := range []struct {
@@ -82,7 +82,7 @@ func TestUsageByProfileCostAggregatesRows(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer d.Close()
t.Cleanup(func() { d.Close() })
priced, unpriced := costProfiles(t, d)
t.Cleanup(func() { d.Exec(`DELETE FROM llm_usage WHERE profile_name IN ($1,$2)`, priced, unpriced) })
rows := []LLMUsage{
+2 -2
View File
@@ -138,9 +138,9 @@ func assertEmbeddedApplied(t *testing.T, d *DB) []appliedMigration {
name string
}{{1, llmRecordsMigration}, {2, llmUsageMigration}, {3, "0003_task_scope_icp_index.sql"}, {4, "0004_llm_usage_ts_index.sql"},
{5, "0005_traffic_rewrite_rules.sql"}, {6, "0006_activity_outbox.sql"}, {7, "0007_finding_report_state.sql"},
{8, "0008_traffic_evidence_rewrite_flag.sql"}, {9, "0009_llm_context.sql"}}
{8, "0008_traffic_evidence_rewrite_flag.sql"}, {9, "0009_llm_context.sql"}, {10, "0010_dashboard_usage_indexes.sql"}}
if len(applied) != len(want) {
t.Fatalf("schema_migrations = %+v, want versions 1 to 9", applied)
t.Fatalf("schema_migrations = %+v, want versions 1 to 10", applied)
}
for i, w := range want {
content, err := fs.ReadFile(migrationsFS(), w.name)
@@ -0,0 +1,36 @@
-- Partial covering indexes proved via EXPLAIN (ANALYZE, BUFFERS) on seeded
-- production-shaped tables (P3.6 dashboard & usage query hygiene):
--
-- 1. idx_act_result_created: TokenDailyAll (db/exploration.go) aggregates
-- activity WHERE kind='result' AND created_at >= now() - interval. The
-- existing idx_act_result_usage is keyed on exploration_id, so date-bounded
-- global daily rollups previously fell back to a bitmap heap scan across all
-- result rows (~100x fewer shared buffers with an index-only scan).
-- 2. idx_conv_act_result: ConversationTokenSummaries (db/conversation.go,
-- polled every 5s by the dashboard) left-joins conversation_activities on
-- (conversation_id) WHERE kind='result'. Without a partial index on
-- kind='result', every poll sequentially scans the entire
-- conversation_activities table (~65x fewer shared buffers).
-- 3. idx_llm_usage_judge_ts: JudgeUsageStats (db/llm_usage.go) filters
-- llm_usage WHERE worker='judge' (all-time and recent window). Judge rows
-- are a small fraction of the append-only ledger; a partial covering index
-- turns both queries into index-only scans (~60-80x fewer shared buffers).
--
-- Plain CREATE INDEX (migrations run in a transaction at startup before the
-- HTTP server listens). Very large installs can pre-build them online with
-- CREATE INDEX CONCURRENTLY IF NOT EXISTS before upgrading.
CREATE INDEX IF NOT EXISTS idx_act_result_created
ON activity(created_at)
INCLUDE (input_tokens, output_tokens, cache_read_tokens)
WHERE kind = 'result';
CREATE INDEX IF NOT EXISTS idx_conv_act_result
ON conversation_activities(conversation_id)
INCLUDE (input_tokens, output_tokens, cache_read_tokens, cache_write_tokens)
WHERE kind = 'result';
CREATE INDEX IF NOT EXISTS idx_llm_usage_judge_ts
ON llm_usage(ts)
INCLUDE (input_tokens, output_tokens, cache_read, cache_write)
WHERE worker = 'judge';
+17 -6
View File
@@ -534,7 +534,7 @@ func TestSetFindingStatusWithNotifyOnlyEmitsOnRealChange(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer d.DeleteTask(tk.ID)
t.Cleanup(func() { _ = d.DeleteTask(tk.ID) })
es := d.Exploration(tk.ExplorationID)
f, err := es.RecordFinding(ctx, RecordFindingInput{
TaskID: tk.ID, Worker: "test", VulnClass: "SQL injection", Name: "status change case",
@@ -543,7 +543,10 @@ func TestSetFindingStatusWithNotifyOnlyEmitsOnRealChange(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { d.Exec(`DELETE FROM notification_events WHERE finding_id=$1`, f.FindingID) })
t.Cleanup(func() {
_, _ = d.Exec(`DELETE FROM notification_events WHERE finding_id=$1`, f.FindingID)
_, _ = d.DeleteFinding(f.FindingID)
})
// Writing the finding already registered one finding_created event; count it as the baseline.
var base int
@@ -617,7 +620,7 @@ func TestSetFindingStatusesWithNotify(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer d.DeleteTask(tk.ID)
t.Cleanup(func() { _ = d.DeleteTask(tk.ID) })
es := d.Exploration(tk.ExplorationID)
record := func(name string) int64 {
fi, err := es.RecordFinding(ctx, RecordFindingInput{
@@ -627,7 +630,10 @@ func TestSetFindingStatusesWithNotify(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { d.Exec(`DELETE FROM notification_events WHERE finding_id=$1`, fi.FindingID) })
t.Cleanup(func() {
_, _ = d.Exec(`DELETE FROM notification_events WHERE finding_id=$1`, fi.FindingID)
_, _ = d.DeleteFinding(fi.FindingID)
})
return fi.FindingID
}
first, second := record("bulk one"), record("bulk two")
@@ -891,7 +897,7 @@ func TestFinishFindingRetestEmitsStatusChange(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer d.DeleteTask(tk.ID)
t.Cleanup(func() { _ = d.DeleteTask(tk.ID) })
es := d.Exploration(tk.ExplorationID)
f, err := es.RecordFinding(ctx, RecordFindingInput{
TaskID: tk.ID, Worker: "test", VulnClass: "SQL injection", Name: "retest target",
@@ -900,7 +906,10 @@ func TestFinishFindingRetestEmitsStatusChange(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { d.Exec(`DELETE FROM notification_events WHERE finding_id=$1`, f.FindingID) })
t.Cleanup(func() {
_, _ = d.Exec(`DELETE FROM notification_events WHERE finding_id=$1`, f.FindingID)
_, _ = d.DeleteFinding(f.FindingID)
})
// Create a retest record and drive it straight to completion.
rt, _, _, err := d.CreateFindingRetest(ctx, f.FindingID, "review")
@@ -910,6 +919,8 @@ func TestFinishFindingRetestEmitsStatusChange(t *testing.T) {
if rt.ConversationID == nil {
t.Fatal("a retest should be associated with a session")
}
retestConvID := *rt.ConversationID
t.Cleanup(func() { _ = d.DeleteConversation(retestConvID) })
// A retest must enter running before a verdict can land (consistent with the real flow).
if ok, err := d.StartFindingRetest(ctx, rt.ID); err != nil || !ok {
t.Fatalf("failed to start retest: ok=%v err=%v", ok, err)
+11 -1
View File
@@ -60,6 +60,7 @@ type TaskArchive struct {
Warnings json.RawMessage `json:"warnings"`
FormatVersion int `json:"format_version"`
ArchivePath string `json:"-"`
Storage string `json:"storage,omitempty"`
SHA256 string `json:"sha256,omitempty"`
OriginalSize int64 `json:"original_size"`
CompressedSize int64 `json:"compressed_size"`
@@ -161,6 +162,12 @@ func scanTaskArchive(sc interface{ Scan(...any) error }) (*TaskArchive, error) {
if err != nil {
return nil, err
}
if item.ArchivePath != "" {
item.Storage = "local"
if strings.HasPrefix(item.ArchivePath, "s3://") {
item.Storage = "s3"
}
}
if err := json.Unmarshal([]byte(sources), &item.SourceTaskIDs); err != nil {
return nil, err
}
@@ -449,7 +456,10 @@ func (d *DB) FailTaskArchiveJob(id int64, activeState string, cause error) error
if cause != nil {
message = cause.Error()
}
_, err := d.Exec(`UPDATE task_archives SET state=$2,phase='failed',error=$3 WHERE id=$1`, id, failed, message)
// Only a job still in its active state fails. The worker also reports a panic
// it recovered after the job committed: a ready archive marked archive_failed
// could be neither restored nor deleted.
_, err := d.Exec(`UPDATE task_archives SET state=$2,phase='failed',error=$3 WHERE id=$1 AND state=$4`, id, failed, message, activeState)
return err
}
+61 -3
View File
@@ -154,19 +154,28 @@ func TestTaskArchiveDatabaseRoundTrip(t *testing.T) {
VALUES($1,$2,0,'quota_exhausted','balance exhausted',$3,$4,$3)`, task.ID, llmProfileID, exhaustedAt, chainCreatedAt); err != nil {
t.Fatal(err)
}
var assetID, findingID int64
defer func() {
_, _ = d.Exec(`DELETE FROM llm_usage WHERE task_id=$1 OR exploration_id=$2`, fmt.Sprint(task.ID), task.ExplorationID)
_, _ = d.Exec(`DELETE FROM llm_records WHERE task_id=$1`, fmt.Sprint(task.ID))
_, _ = d.Exec(`DELETE FROM skill_usage WHERE task_id=$1 OR exploration_id=$2`, task.ID, task.ExplorationID)
_, _ = d.Exec(`DELETE FROM tool_usage WHERE task_id=$1 OR exploration_id=$2`, task.ID, task.ExplorationID)
_, _ = d.DeleteFindingsByTask(task.ID)
if findingID != 0 {
_, _ = d.DeleteFinding(findingID)
}
_, _ = d.Exec(`DELETE FROM task_archives WHERE task_id=$1`, task.ID)
_ = d.DeleteTask(task.ID)
if assetID != 0 {
_, _ = d.Exec(`DELETE FROM assets WHERE id=$1`, assetID)
}
_, _ = d.Exec(`DELETE FROM llm_profiles WHERE id=$1`, llmProfileID)
_, _ = d.Exec(`DELETE FROM companies WHERE id=$1`, companyID)
}()
if err := d.SetPaused(task.ID, true); err != nil {
t.Fatal(err)
}
assetID, err := d.Assets().UpsertRootDomain(UpsertRootDomainReq{Domain: fmt.Sprintf("archive-%d.example", task.ID), TaskID: task.ID})
assetID, err = d.Assets().UpsertRootDomain(UpsertRootDomainReq{Domain: fmt.Sprintf("archive-%d.example", task.ID), TaskID: task.ID})
if err != nil {
t.Fatal(err)
}
@@ -197,13 +206,20 @@ VALUES($1,$2,0,'quota_exhausted','balance exhausted',$3,$4,$3)`, task.ID, llmPro
}); err != nil {
t.Fatal(err)
}
if _, err := d.AddFinding(task.ID, nodeID, vulnclass, "archive finding", SeverityCritical, "summary", "evidence", "worker", []int64{assetID}); err != nil {
if findingID, err = d.AddFinding(task.ID, nodeID, vulnclass, "archive finding", SeverityCritical, "summary", "evidence", "worker", []int64{assetID}); err != nil {
t.Fatal(err)
}
archive, err := d.QueueTaskArchive(task.ID)
if err != nil {
t.Fatal(err)
}
if archive.Storage != "" {
t.Fatalf("queued archive without a location has storage %q", archive.Storage)
}
encoded, err := json.Marshal(archive)
if err != nil || bytes.Contains(encoded, []byte(`"storage"`)) {
t.Fatalf("unknown storage must be omitted from JSON: %s, %v", encoded, err)
}
claimed, err := d.ClaimTaskArchiveJob(t.Context())
if err != nil || claimed == nil || claimed.ID != archive.ID || claimed.State != Archiving {
t.Fatalf("claim = %+v, %v", claimed, err)
@@ -242,7 +258,7 @@ VALUES($1,$2,0,'quota_exhausted','balance exhausted',$3,$4,$3)`, task.ID, llmPro
t.Fatalf("hot compaction left nodes=%d assets=%d usage=%d", nodes, assets, usage)
}
ready, err := d.GetTaskArchive(archive.ID)
if err != nil || ready == nil || ready.State != ArchiveReady {
if err != nil || ready == nil || ready.State != ArchiveReady || ready.Storage != "local" {
t.Fatalf("ready archive = %+v, %v", ready, err)
}
var stats map[string]any
@@ -517,3 +533,45 @@ func TestFillArchiveDefaults(t *testing.T) {
t.Fatalf("activity changed: %s err=%v", out, err)
}
}
// FailTaskArchiveJob fails a job that is still running. A failure reported after
// the archive committed (the worker recovers a panic in the cleanup that follows
// the commit) must not turn a ready archive into archive_failed: it could then
// be neither restored nor deleted.
func TestFailTaskArchiveJobKeepsCommittedState(t *testing.T) {
d, err := Open(testDSN(t))
if err != nil {
SkipPGUnavailable(t, err)
}
defer d.Close()
task, err := d.CreateTaskWithOptions("late archive failure", "must stay ready", TaskCreateOptions{})
if err != nil {
t.Fatal(err)
}
defer func() {
_, _ = d.Exec(`DELETE FROM task_archives WHERE task_id=$1`, task.ID)
_ = d.DeleteTask(task.ID)
}()
if err := d.SetPaused(task.ID, true); err != nil {
t.Fatal(err)
}
queued, err := d.QueueTaskArchive(task.ID)
if err != nil {
t.Fatal(err)
}
for _, c := range []struct{ state, want string }{{Archiving, ArchiveFailed}, {ArchiveReady, ArchiveReady}} {
if _, err := d.Exec(`UPDATE task_archives SET state=$2,error='' WHERE id=$1`, queued.ID, c.state); err != nil {
t.Fatal(err)
}
if err := d.FailTaskArchiveJob(queued.ID, Archiving, errors.New("late failure")); err != nil {
t.Fatal(err)
}
item, err := d.GetTaskArchive(queued.ID)
if err != nil || item == nil {
t.Fatalf("archive %d: %v", queued.ID, err)
}
if item.State != c.want || (c.want == ArchiveFailed) != (item.Error == "late failure") {
t.Fatalf("failing a job in state %s: state=%s error=%q, want state %s", c.state, item.State, item.Error, c.want)
}
}
}
+5 -1
View File
@@ -168,8 +168,9 @@ func TestIntentAssetsIncludesDirectSourceProvenance(t *testing.T) {
cleanupErr(t, "source task", d.DeleteTask(source.ID))
})
intentHost := fmt.Sprintf("intent-%d.example.test", time.Now().UnixNano())
assetID, err := d.Assets().UpsertHTTPService(UpsertHTTPServiceReq{
URL: fmt.Sprintf("https://intent-%d.example.test", time.Now().UnixNano()), TaskID: source.ID,
URL: "https://" + intentHost, TaskID: source.ID,
})
if err != nil {
t.Fatal(err)
@@ -177,6 +178,9 @@ func TestIntentAssetsIncludesDirectSourceProvenance(t *testing.T) {
t.Cleanup(func() {
_, err := d.Assets().DeleteByIDs([]int64{assetID})
cleanupErr(t, "asset", err)
// UpsertHTTPService also creates the parent subdomain and root_domain rows.
_, err = d.Exec(`DELETE FROM assets WHERE domain IN ($1, 'example.test')`, intentHost)
cleanupErr(t, "parent assets", err)
})
intentID, err := d.Exploration(source.ExplorationID).AddIntent(map[string]any{"summary": "source worker"}, 5, []int64{assetID}, "planner")
if err != nil {
+1 -1
View File
@@ -13,7 +13,7 @@ func TestTaskLLMSpendCountsJudgeNotChat(t *testing.T) {
if err != nil {
SkipPGUnavailable(t, err)
}
defer d.Close()
t.Cleanup(func() { _ = d.Close() })
taskID := fmt.Sprintf("spend-%d", time.Now().UnixNano())
t.Cleanup(func() {
+2
View File
@@ -38,6 +38,8 @@ services:
ARTEX_LLM_MODEL: ${ARTEX_LLM_MODEL:-}
ARTEX_LLM_BASE_URL: ${ARTEX_LLM_BASE_URL:-}
ARTEX_LLM_PROXY: ${ARTEX_LLM_PROXY:-}
# Exact host:port list the model-list fetch may reach on private addresses (e.g. host.docker.internal:11434 for Ollama).
ARTEX_LLM_PRIVATE_ENDPOINTS: ${ARTEX_LLM_PRIVATE_ENDPOINTS:-}
ARTEX_JWT_KEY_DIR: /app/keys
# Optional: key for secrets at rest; empty = keys/secret.key (generated at first start).
ARTEX_SECRET_KEY: ${ARTEX_SECRET_KEY:-}
+56
View File
@@ -0,0 +1,56 @@
# Opt-in SOCKS5 Inbound Proxy & Raw TCP Recording (P4.1a)
ARTEX ships an opt-in **RFC 1928 SOCKS5 `CONNECT`** inbound listener in `traffic/` that records raw TCP connections separately from HTTP traffic.
## Opt-in Configuration (Off by Default)
| Environment Variable | Default | Behavior |
|---|---|---|
| `ARTEX_SOCKS5_ENABLED` | unset (off) | Set to `1` / `true` / `on` / `yes` to start the SOCKS5 listener on `127.0.0.1:1080` (loopback). Explicit `0` / `false` / `off` / `no` forces it off even if `ARTEX_SOCKS5_ADDR` is set. |
| `ARTEX_SOCKS5_ADDR` | `127.0.0.1:1080` | Explicit TCP bind address; setting a non-empty value also enables the listener unless `ARTEX_SOCKS5_ENABLED` is explicitly false. |
- **Flag off**: no SOCKS5 listener is started and the HTTP MITM recording proxy is byte-identical.
- **Requires traffic recorder**: the server only starts the SOCKS5 listener when the traffic recorder is enabled (inside the traffic proxy block in `server/manager.go`). If traffic capture is disabled, setting `ARTEX_SOCKS5_ENABLED` / `ARTEX_SOCKS5_ADDR` does not open a listener.
- **Agents unchanged**: workers and tools keep `HTTP_PROXY`, `HTTPS_PROXY`, and `ALL_PROXY` pointing to the existing HTTP proxy; ARTEX does not silently switch agents to SOCKS5.
- **Non-loopback bind warning**: RFC 1929 username/password (`0x02`) accepts **any** username and password without verifying credentials (so the pace marker `artex-pace` can ride in the username). Binding `ARTEX_SOCKS5_ADDR` to a non-loopback address therefore exposes an **unauthenticated proxy** to anyone who can reach that port. The target gate and egress governor still apply to every connection, but keep the listener on loopback (`127.0.0.1`) or restrict access at the host firewall.
## Protocol Scope (P4.1a)
- **Commands**: `CONNECT` (`0x01`) only. `BIND` (`0x02`), `UDP ASSOCIATE` (`0x03`, planned for P4.1c), and unknown commands return RFC 1928 reply `0x07` (*Command not supported*).
- **Addressing**: IPv4 (`0x01`), domain name (`0x03`), and IPv6 (`0x04`). Unknown address types return `0x08` (*Address type not supported*).
- **Authentication**:
- Loopback peers (`net.IP.IsLoopback()`) may connect with **No Auth** (`0x00`). Non-loopback peers offering only `0x00` are rejected with `0x05 0xFF`.
- **Username/Password** (`0x02`, RFC 1929) is accepted without requiring credentials so clients can pass the pace marker (`artex-pace`, `egress.PaceUser`) in the username.
- **Relay**: Full-duplex, half-close aware (`CloseWrite` on EOF in each direction).
## Policy Enforcement Before Dialing
Before opening any outbound TCP socket, the listener invokes the same controls used by the HTTP path:
1. **Egress governor (`egress.Do(egress.Current(), ...)`)**: enforces the active noise profile (`off` is a zero-delay passthrough). If waiting for a slot/token exceeds `PaceTimeout`, the connection is refused before dialing with SOCKS5 reply `0x02` (*Connection not allowed by ruleset*).
2. **Target gate (`intercept.Interceptor.DialSOCKS5` / `intercept.PinnedDialer`)**: enforces global asset block rules (before and after fresh DNS resolution) and any enabled `intercept.TargetScope` pins/exemptions, dialing only verified addresses. Blocked targets return `0x02` before dialing; DNS lookup failures return `0x04` (*Host unreachable*).
## Raw TCP Storage (SQLite)
Each completed relayed connection writes one row to the `raw_tcp_streams` table in the SQLite traffic index (`traffic/traffic.go`, `indexSchema` — no PostgreSQL migration):
- Columns: `id` (`tcp-<ms>-<seq>`), `protocol` (`'tcp'`), `host`, `port`, `started_at`, `ended_at`, `duration_ms`, `c2s_bytes`, `s2c_bytes`, `truncated`, `c2s_preview`, `c2s_blob`, `s2c_preview`, `s2c_blob`.
- **No HTTP exchange row**: raw TCP streams never insert into `exchanges`.
- **Preview & blob spilling**:
- Directional payloads up to `PreviewBytes` (default `8 KiB`) stay inline in `c2s_preview` / `s2c_preview`.
- Payloads above `PreviewBytes` keep the leading `PreviewBytes` inline and spill the captured bytes (up to `MaxBytesPerDir`) atomically (temp file + rename) into `_blobs/sha256/` with references registered in `blob_refs` under the stream's `tcp-*` ID.
- Exceeding `MaxBytesPerDir` sets `truncated = 1` while continuing to count exact total relayed `c2s_bytes` / `s2c_bytes`.
## Operational Bounds
| Parameter | Default |
|---|---|
| `MaxConcurrent` | `64` active streams (excess connections receive `0x01`) |
| `MaxBytesPerDir` | `4 MiB` captured per direction |
| `PreviewBytes` | `8 KiB` inline preview per direction |
| `IdleTimeout` | `2m` per-direction read/write deadline |
| `HandshakeTimeout` | `10s` for greeting, auth, and `CONNECT` request |
| `DialTimeout` | `30s` outbound TCP connect timeout |
| `PaceTimeout` | `30s` egress governor wait bound |
Closing `Traffic` stops accepting new SOCKS5 connections, closes active sockets, and waits for all handler goroutines to exit before closing SQLite.
+81
View File
@@ -0,0 +1,81 @@
# Opt-in Linux Transparent TCP Inbound & Raw TCP Recording (P4.1b)
ARTEX ships an opt-in **Linux-only IPv4 transparent TCP inbound listener** in `traffic/` that accepts connections redirected by `iptables` or `nftables` `REDIRECT` rules, recovers each socket's original IPv4 destination via `getsockopt(SOL_IP, SO_ORIGINAL_DST)`, enforces the existing target gate and egress governor before dialing, and records the connection in SQLite `raw_tcp_streams`.
## Opt-in Configuration (Off by Default)
| Environment Variable | Default | Behavior |
|---|---|---|
| `ARTEX_TRANSPARENT_TCP_ENABLED` | unset (off) | Set to `1` / `true` / `on` / `yes` to start the transparent TCP listener on `127.0.0.1:1081` (IPv4 loopback). Explicit `0` / `false` / `off` / `no` forces it off even if `ARTEX_TRANSPARENT_TCP_ADDR` is set. |
| `ARTEX_TRANSPARENT_TCP_ADDR` | `127.0.0.1:1081` | Explicit IPv4 TCP bind address; setting a non-empty value also enables the listener unless `ARTEX_TRANSPARENT_TCP_ENABLED` is explicitly false. Use `0.0.0.0:1081` only when redirecting traffic from other interfaces (`PREROUTING`) and protect the port with a host firewall. |
- **Flag off**: no transparent TCP listener is started, no firewall rules are touched, and the HTTP MITM proxy is unchanged.
- **Requires traffic recorder**: the server only starts the transparent TCP listener when the traffic recorder is enabled (inside the traffic proxy block in `server/manager.go`). If traffic capture is disabled, setting these environment variables does not open a listener.
- **Linux only**: on non-Linux operating systems (`windows`, `darwin`), leaving the flag off is a no-op; explicitly enabling it logs `ErrTransparentTCPNotLinux` and leaves the listener disabled.
- **IPv4 only (IPv6 not covered)**: P4.1b recovers `SO_ORIGINAL_DST` (`SOL_IP`, `AF_INET`) for IPv4 TCP sockets only. IPv6 (`IP6T_SO_ORIGINAL_DST`) is **not covered**; IPv6 bind addresses and IPv6 sockets are refused.
- **Firewall never touched by ARTEX**: ARTEX never installs, modifies, or deletes `iptables`/`nftables` rules. Root or `CAP_NET_ADMIN` is required **only** by the operator when installing and cleaning up the host redirect rules—never to run ARTEX itself.
## Firewall Rules & Cleanup (Operator-Managed)
Choose **either** `iptables` **or** `nftables`. Always exclude the UID running ARTEX (example: `artex`) from local `OUTPUT` redirect rules so ARTEX's own outbound dials do not loop back into the transparent listener.
### Option 1: `iptables` (IPv4 local `OUTPUT` REDIRECT)
Install into a dedicated chain so cleanup is atomic:
```bash
# Requires root or CAP_NET_ADMIN
sudo iptables -t nat -N ARTEX_TRANSPARENT
# Never redirect ARTEX's own outbound dials or loopback traffic
sudo iptables -t nat -A ARTEX_TRANSPARENT -m owner --uid-owner artex -j RETURN
sudo iptables -t nat -A ARTEX_TRANSPARENT -d 127.0.0.0/8 -j RETURN
# Example: redirect outbound TCP to lab target 198.51.100.0/24 into ARTEX on 127.0.0.1:1081
sudo iptables -t nat -A ARTEX_TRANSPARENT -p tcp -d 198.51.100.0/24 -j REDIRECT --to-ports 1081
sudo iptables -t nat -A OUTPUT -p tcp -j ARTEX_TRANSPARENT
```
Cleanup (exact removal):
```bash
sudo iptables -t nat -D OUTPUT -p tcp -j ARTEX_TRANSPARENT
sudo iptables -t nat -F ARTEX_TRANSPARENT
sudo iptables -t nat -X ARTEX_TRANSPARENT
```
*(For traffic arriving on another interface—for example a container or routed VM—bind `ARTEX_TRANSPARENT_TCP_ADDR=0.0.0.0:1081`, attach `ARTEX_TRANSPARENT` to `PREROUTING` instead of `OUTPUT`, and drop direct non-redirected packets to port 1081 at the host firewall.)*
### Option 2: `nftables` (IPv4 local `OUTPUT` REDIRECT)
Install into a dedicated table so cleanup is a single command:
```bash
# Requires root or CAP_NET_ADMIN
sudo nft -f - <<'NFT'
table ip artex_transparent {
chain output {
type nat hook output priority -100; policy accept;
meta skuid "artex" return
ip daddr 127.0.0.0/8 return
ip daddr 198.51.100.0/24 tcp redirect to :1081
}
}
NFT
```
Cleanup (exact removal):
```bash
sudo nft delete table ip artex_transparent
```
## Policy Enforcement, Loop Guard & Storage (Shared with P4.1a)
For every accepted TCP connection:
1. **Original destination recovery**: reads `getsockopt(SOL_IP, SO_ORIGINAL_DST)` under `HandshakeTimeout` (default `10s`), parses `struct sockaddr_in` (rejecting short buffers, non-`AF_INET` families including IPv6, port `0`, and `0.0.0.0`), and rejects **self-redirects** (direct non-redirected connections where `SO_ORIGINAL_DST` equals the listener's own address/port) before dialing.
2. **Egress governor (`egress.Do(egress.Current(), ...)`)**: enforces the active noise profile (`PaceTimeout`, default `30s`) before dialing.
3. **Target gate (`intercept.Interceptor.DialSOCKS5` / `intercept.PinnedDialer`)**: enforces global asset block rules and any enabled `intercept.TargetScope` pins/exemptions before opening any outbound socket. Because the transparent listener only knows the recovered destination IP (never a DNS hostname), hostname-based block or allow rules cannot match—IP and CIDR rules are the ones that apply. Refused connections are closed immediately without dialing and without writing a store row.
4. **Relay & bounds**: reuses the P4.1a full-duplex half-close-aware relay (`CloseWrite`) and defaults (`MaxConcurrent=64`, `MaxBytesPerDir=4 MiB`, `PreviewBytes=8 KiB`, `IdleTimeout=2m`, `HandshakeTimeout=10s`, `DialTimeout=30s`, `PaceTimeout=30s`).
5. **SQLite storage**: records one row per completed connection in `raw_tcp_streams` (`protocol='tcp'`, atomic temp+rename `_blobs/sha256/` spill + `blob_refs`), with **zero HTTP `exchanges` rows** and **no PostgreSQL migration**.
Closing `Traffic` stops accepting new transparent connections, closes active sockets, and waits for all handler goroutines to exit before closing SQLite.
-14
View File
@@ -281,20 +281,6 @@ func hostOf(rawURL string) string {
return u.Hostname()
}
func isBlank(v any) bool {
switch x := v.(type) {
case nil:
return true
case string:
return x == ""
case float64:
return x == 0
case int:
return x == 0
}
return false
}
func uniq(in []string) []string {
seen := map[string]struct{}{}
out := in[:0]
+9 -9
View File
@@ -1,6 +1,6 @@
module github.com/Autumn-27/artex
go 1.26.10
go 1.26.11
require (
github.com/Autumn-27/norma v0.4.11
@@ -13,19 +13,19 @@ require (
github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.5.3
github.com/jackc/pgx/v5 v5.10.0
github.com/klauspost/compress v1.18.6
github.com/klauspost/compress v1.20.2
github.com/lqqyt2423/go-mitmproxy v1.9.2
github.com/miekg/dns v1.1.72
github.com/projectdiscovery/dnsx v1.2.3
golang.org/x/crypto v0.58.0
golang.org/x/crypto v0.59.0
golang.org/x/net v0.62.0
golang.org/x/text v0.43.0
golang.org/x/text v0.44.0
modernc.org/sqlite v1.52.0
mvdan.cc/sh/v3 v3.14.1
)
require (
github.com/andybalholm/brotli v1.2.1 // indirect
github.com/andybalholm/brotli v1.2.7 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect
@@ -70,10 +70,10 @@ require (
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect
golang.org/x/mod v0.41.0 // indirect
golang.org/x/sync v0.24.0 // indirect
golang.org/x/sys v0.49.0 // indirect
golang.org/x/tools v0.51.0 // indirect
golang.org/x/mod v0.42.0 // indirect
golang.org/x/sync v0.25.0 // indirect
golang.org/x/sys v0.50.0 // indirect
golang.org/x/tools v0.52.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
modernc.org/libc v1.72.3 // indirect
modernc.org/mathutil v1.7.1 // indirect
+16 -16
View File
@@ -1,7 +1,7 @@
github.com/BaaRai/Norma v0.4.11 h1:Z5lSRFKBDI0EzAw1YDA5iQ5GXsa9e7J19YmIe0zaVcQ=
github.com/BaaRai/Norma v0.4.11/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw=
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/andybalholm/brotli v1.2.7 h1:UgafRPGMam5ySHfRkhthMqOFA6IFoeF3D7mWIDLjG5A=
github.com/andybalholm/brotli v1.2.7/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4=
@@ -81,8 +81,8 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo=
github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -143,23 +143,23 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
golang.org/x/crypto v0.59.0 h1:xS2jhqYbzNrpvkWP2YTDf4TJwBy3ykR03i8vFDxEc7c=
golang.org/x/crypto v0.59.0/go.mod h1:veDyUwATGs3c+vN3VfTbzViMGXeTBk3WyhBOkn+F/4U=
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA=
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU=
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY=
golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs=
golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI=
golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA=
golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs=
golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo=
golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY=
golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+1 -1
View File
@@ -106,7 +106,7 @@ JSON
# embedding the frontend requires node to produce the static assets
if command -v npm >/dev/null 2>&1; then
info "building frontend static assets..."
( cd web && npm ci && npm run build:static )
( cd web && npm ci --ignore-scripts && npm run build:static )
rm -rf server/webui/dist && cp -r web/out server/webui/dist
info "compiling embedded single binary..."
CGO_ENABLED=0 go build -tags embedui -trimpath -o artex ./cmd/artex
+59
View File
@@ -3,12 +3,15 @@ package intercept
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"net/netip"
"strconv"
"strings"
"time"
"github.com/Autumn-27/artex/db"
)
// The HTTP clients the platform owns (WebFetch and the custom http tool)
@@ -178,3 +181,59 @@ func (e *dialError) Error() string {
}
return msg + " Do not fetch the target with another tool or by its address."
}
// IsTargetBlocked reports whether err came from PinnedDialer / DialSOCKS5
// refusing a target before dialing (rule block, unpinned target, or unavailable
// target gate), returning its platform control code.
func IsTargetBlocked(err error) (string, bool) {
var de *dialError
if errors.As(err, &de) {
return de.code, true
}
return "", false
}
// DialSOCKS5 enforces the platform's target gate before dialing a SOCKS5
// CONNECT destination, reusing the same policy functions as PinnedDialer:
// - When ctx carries an enabled TargetScope, delegates directly to dialTarget
// (asset block rules, exemptions, fresh DNS resolution, and pin checks).
// - Otherwise enforces global asset block rules (domain, IP, and CIDR, before
// and after fresh DNS resolution) and dials only the resolved addresses that
// passed those rules.
func (i *Interceptor) DialSOCKS5(ctx context.Context, network, addr string) (net.Conn, error) {
if i == nil {
return baseDial(ctx, network, addr)
}
if scope, ok := TargetScopeFromContext(ctx); ok && scope.Enabled != nil && scope.Enabled() {
return i.dialTarget(ctx, scope, "socks5", network, addr)
}
t, err := addrTarget(network, addr)
if err != nil {
return nil, &dialError{code: TargetGateUnavailable, target: addr, reason: "the address of the connection could not be read (" + err.Error() + ")"}
}
if i.targets == nil {
return i.baseDialer()(ctx, network, addr)
}
rules, err := i.targets.BlockRules(db.TargetScopeRef{})
if err != nil {
return nil, &dialError{code: TargetGateUnavailable, target: t.String(), reason: "the asset rules could not be read: " + err.Error()}
}
ct := &checkedTarget{Target: t}
if t.Kind != TargetDomain {
ct.ips, ct.resolver, ct.resolvedAt = []string{t.Host}, literalResolver, time.Now().UTC()
}
if rule, hit := blockRule(rules, ct); hit {
return nil, &dialError{code: TargetRuleBlocked, target: t.String(), reason: fmt.Sprintf("%s is blocked by an asset rule. %s", t, rule.Reason())}
}
if t.Kind == TargetDomain {
c := &targetCheck{i: i, ctx: ctx, tool: "socks5", fresh: true}
c.resolve(ct, true)
if ct.err != nil {
return nil, ct.err
}
if rule, hit := blockRule(rules, ct); hit {
return nil, &dialError{code: TargetRuleBlocked, target: t.String(), reason: fmt.Sprintf("%s is blocked by an asset rule. %s", t, rule.Reason())}
}
}
return i.dialAddrs(ctx, network, ct.ips, t.Port)
}
+20
View File
@@ -6,6 +6,7 @@ package llmrec
import (
"context"
"encoding/json"
"errors"
"iter"
"log"
"strings"
@@ -16,6 +17,11 @@ import (
"github.com/Autumn-27/norma/transcript"
)
// errStreamAbandoned marks a stream whose consumer stopped reading before
// message_stop while its context was still live. The call did not complete,
// so it is metered as an error rather than inferred from ctx.Err().
var errStreamAbandoned = errors.New("llmrec: consumer stopped reading before message_stop")
type taskIDContextKey struct{}
type workerContextKey struct{}
@@ -190,6 +196,7 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S
usage llm.Usage
stopReason string
streamErr error
complete bool // message_stop was seen
)
finished := false
@@ -246,8 +253,21 @@ func (r *Recorder) Stream(ctx context.Context, req llm.CompletionRequest) iter.S
stopReason = ev.StopReason
}
usage.MergeEcho(ev.Usage)
case llm.SEMessageStop:
complete = true
}
if !yield(ev, err) {
// The consumer stopped early. Decide the status here from what
// is known, not later in the deferred finish: the call only
// counts as "ok" if message_stop arrived and ctx is live.
switch {
case ctx.Err() != nil:
finish(ctx.Err())
case !complete:
finish(errStreamAbandoned)
default:
finish(nil)
}
return
}
}
+12 -14
View File
@@ -151,15 +151,9 @@ func (meteredStreamProvider) Stream(context.Context, llm.CompletionRequest) iter
}
func TestSideUsageRecordedOnceOnConsumerCancellation(t *testing.T) {
// Timing-sensitive on 2-core CI runners (BaaRai/ARTEX#9). Repro:
// ARTEX_PG_DSN=postgres://artex:testpass@localhost:5432/artextest_race \
// go test ./llmrec -run TestSideUsageRecordedOnceOnConsumerCancellation -count=20 -race -v
// The `early=` t.Fatalf assertion below compares the single llm_usage row's
// count/input/output/worker/status; in the `early` branch the status
// occasionally comes back "ok" instead of "error" because the consumer's
// cancellation and the recorder's deferred finish race on a loaded runner.
// Keep the skip until the metering status is made cancellation-explicit.
t.Skip("timing-sensitive on CI runners; repro and failing assertion documented above; BaaRai/ARTEX#9")
// Previously skipped as timing-sensitive (BaaRai/ARTEX#9). The recorder now
// decides the status at the point the consumer stops, so the result no
// longer depends on when the deferred finish runs.
dsn := os.Getenv("ARTEX_PG_DSN")
if dsn == "" {
t.Skip("requires isolated ARTEX_PG_DSN")
@@ -169,7 +163,9 @@ func TestSideUsageRecordedOnceOnConsumerCancellation(t *testing.T) {
t.Fatal(err)
}
defer pg.Close()
for _, early := range []bool{false, true} {
// full: read to the end. cancel: cancel ctx, then stop reading.
// abandon: stop reading before message_stop with ctx still live.
for _, mode := range []string{"full", "cancel", "abandon"} {
profile := "btw-metering-" + uuid.NewString()
ctx, cancel := context.WithCancel(transcript.WithSessionID(t.Context(), "exp0-btw-test"))
recorder := Wrap(meteredStreamProvider{}, pg, "fixture", profile, "", "", func() bool { return false })
@@ -177,8 +173,10 @@ func TestSideUsageRecordedOnceOnConsumerCancellation(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if early && event.Type == llm.SEMessageDelta {
cancel()
if mode != "full" && event.Type == llm.SEMessageDelta {
if mode == "cancel" {
cancel()
}
break // Consumer exits before the provider can return a cancellation event.
}
}
@@ -192,11 +190,11 @@ FROM llm_usage WHERE profile_name=$1`, profile).Scan(&count, &input, &output, &w
t.Fatal(err)
}
wantStatus := "ok"
if early {
if mode != "full" {
wantStatus = "error"
}
if count != 1 || input != 23 || output != 5 || worker != "btw" || status != wantStatus {
t.Fatalf("early=%v: count=%d usage=%d/%d worker=%q status=%q", early, count, input, output, worker, status)
t.Fatalf("mode=%s: count=%d usage=%d/%d worker=%q status=%q", mode, count, input, output, worker, status)
}
}
}
+2 -2
View File
@@ -16,7 +16,7 @@ func TestChatUnavailableReasonDistinguishesStates(t *testing.T) {
if err != nil {
db.SkipPGUnavailable(t, err)
}
defer m.Close()
t.Cleanup(func() { _ = m.Close() })
td := t.TempDir()
ctx, cancel := context.WithCancel(context.Background())
s := New(ctx, m, td, td, td)
@@ -68,7 +68,7 @@ func TestResolveChatAgentHonoursConversationProfile(t *testing.T) {
if err != nil {
db.SkipPGUnavailable(t, err)
}
defer m.Close()
t.Cleanup(func() { _ = m.Close() })
td := t.TempDir()
ctx, cancel := context.WithCancel(context.Background())
s := New(ctx, m, td, td, td)
+3
View File
@@ -73,6 +73,9 @@ func TestCoreTaskLifecyclePG(t *testing.T) {
if id == "" || expID == 0 {
t.Fatalf("bad task payload: %v", out)
}
defer func() {
_, _ = m.pg.Exec(`DELETE FROM assets WHERE type = 'ip' AND ip = '127.0.0.1'`)
}()
// the exploration owns goal node(s) — goal seeding is async, poll briefly
var goals []*db.Node
+1 -1
View File
@@ -84,7 +84,7 @@ func TestResetSettledRunReseedsBudgetFromUsage(t *testing.T) {
if err != nil {
pgdb.SkipPGUnavailable(t, err)
}
defer m.Close()
t.Cleanup(func() { _ = m.Close() })
taskID := strconv.FormatInt(time.Now().UnixNano(), 10)
t.Cleanup(func() {
+13 -3
View File
@@ -5,13 +5,23 @@ import "os"
// secretEnv lists the credentials read from the environment only at startup:
// ARTEX_PG_DSN and ARTEX_SECRET_KEY by NewManager, the API keys by agent.FromEnv inside New.
// agent.FromEnv also reads ARTEX_LLM_PROXY and ARTEX_LLM_BASE_URL: both are URLs that
// can carry credentials in their userinfo (http://user:pass@proxy:3128). The ARTEX_ARCHIVE_S3_*
// keys are not read by the server, but the container may carry them and the agent's
// shells would inherit them.
// can carry credentials in their userinfo (http://user:pass@proxy:3128). The archive
// store also captures ARTEX_ARCHIVE_S3_* credentials during New, before the scrub,
// so the client can use them without passing them to agent child processes.
//
// Without that override the AWS SDK's default chain reads the AWS_* names below
// while the store is built (LoadDefaultConfig) and keeps what it needs: the static
// keys, or the container endpoint and token and the web-identity role and token
// file path it fetches credentials from later. Credentials that come from a shared
// credentials or config file, or from the instance metadata service, are not in
// the environment: the scrub does not hide them from child processes.
var secretEnv = []string{
"ARTEX_PG_DSN", "ARTEX_SECRET_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY",
"ARTEX_LLM_PROXY", "ARTEX_LLM_BASE_URL",
"ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_SESSION_TOKEN",
"AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN",
"AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
"AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN",
}
// ScrubSecretEnv removes secretEnv from the process environment once startup has
+12
View File
@@ -60,6 +60,18 @@ func TestSealedEnvironmentHandoff(t *testing.T) {
"ARTEX_ARCHIVE_S3_ACCESS_KEY_ID": "AKIATESTONLY",
"ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY": "s3-secret-test",
"ARTEX_ARCHIVE_S3_SESSION_TOKEN": "",
// Ambient AWS credentials the archive store reads when it has no override.
"AWS_ACCESS_KEY_ID": "AKIAAMBIENTTEST",
"AWS_ACCESS_KEY": "AKIAALTERNATETEST",
"AWS_SECRET_ACCESS_KEY": "ambient-secret-test",
"AWS_SECRET_KEY": "alternate-secret-test",
"AWS_SESSION_TOKEN": "",
"AWS_CONTAINER_CREDENTIALS_FULL_URI": "http://127.0.0.1:8080/credentials?id=1",
"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI": "/v2/credentials/test",
"AWS_CONTAINER_AUTHORIZATION_TOKEN": "container-token-test",
"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE": "/var/run/secrets/token-file",
"AWS_WEB_IDENTITY_TOKEN_FILE": "/var/run/secrets/token",
"AWS_ROLE_ARN": "arn:aws:iam::123456789012:role/test",
}
env := os.Environ()
for k, v := range want {
+117
View File
@@ -0,0 +1,117 @@
package server
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"testing"
)
// The record_fact texts seeded by the releases v0.4.0 to v0.6.0, copied from
// agent/tools.go at those tags. They are kept here, apart from the agent
// package's list of known defaults, so the test fails when that list drifts
// from the released bytes.
const fix410ReleasedDescription = "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n" +
"⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n" +
"★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n" +
"⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n" +
" · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n" +
" · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n" +
" · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen."
const fix410ReleasedFactsDescription = "[use when there are several distinct conclusions] array of facts; element fields same as the top-level fields below (summary/detail/evidence/confidence/intent_id/asset_ids); omitting intent_id falls back to the top-level intent_id. The returned ids match this array in length and order."
// A database seeded by a release stores the released record_fact description,
// the released facts property description and facts items without properties.
// The one-time refresh replaces both texts with the current ones and sets its
// flag; texts an operator edited match no known default and stay as stored.
func TestFix410ReleasedDefaultsRefreshed(t *testing.T) {
for _, c := range []struct {
name, text, sum string
size int
}{
{"description", fix410ReleasedDescription, "0da5cec5cde7f5e6f41343418fcc2a7d132d6520f336efb39fcbf1ab6ac51957", 2038},
{"facts description", fix410ReleasedFactsDescription, "eb054843a6b33dd39b94ba0648f292babb9cf9bdd106ee4eff6bf593d3f3edf4", 286},
} {
sum := sha256.Sum256([]byte(c.text))
if len(c.text) != c.size || hex.EncodeToString(sum[:]) != c.sum {
t.Fatalf("the released %s in this test is not the released text: %d bytes, sha256 %x", c.name, len(c.text), sum)
}
}
skipWithoutDSN(t)
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
defer m.Close()
s := &Server{m: m}
const flag = "tool_schema_refresh_record_fact_facts_only"
seed := recordFactSeed(t)
seedSchema, _ := json.Marshal(seed.Schema)
seedAgents, _ := json.Marshal(seed.Agents)
defer func() {
_ = m.pg.UpsertToolForce("record_fact", seed.Desc, seedSchema, seedAgents)
_, _ = m.pg.Exec(`DELETE FROM settings WHERE key=$1`, flag)
}()
currentFacts := seed.Schema["properties"].(map[string]any)["facts"].(map[string]any)["description"]
// start stores the row as a release seeded it, with the given two texts,
// runs the startup steps and returns the two texts as stored afterwards.
start := func(desc, factsDesc string) (string, any) {
t.Helper()
str := func(d string) map[string]any { return map[string]any{"type": "string", "description": d} }
schema, _ := json.Marshal(map[string]any{"type": "object", "properties": map[string]any{
"facts": map[string]any{"type": "array", "description": factsDesc, "items": map[string]any{"type": "object"}},
"summary": str("a [one-sentence summary] of this exploration's conclusion (a condensation of detail)"),
"intent_id": map[string]any{"type": "integer", "description": "the id of the intent that produced this fact (the intent you claimed; the default for each item in batch mode)"},
"detail": str("details related to this fact: write all of this exploration's observations here"),
"evidence": str("[one line] of key evidence: the command + the one or two most proving output lines. Keep it brief; don't paste large output blocks (details go in detail)."),
"confidence": str("observed (seen directly in output) | inferred (inferred from phenomena). Always label negative conclusions honestly."),
"asset_ids": map[string]any{"type": "array", "items": map[string]any{"type": "integer"}, "description": "related asset ids (optional, 0/1/many): which assets this fact involves"},
}})
if err := m.pg.UpsertToolForce("record_fact", desc, schema, seedAgents); err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exec(`DELETE FROM settings WHERE key=$1`, flag); err != nil {
t.Fatal(err)
}
s.seedFactInputSchema()
s.seedFactSourceSchema()
s.refreshRecordFactContract()
if v, _, _ := m.pg.GetSetting(flag); v != "true" {
t.Errorf("flag = %q after the refresh, want true", v)
}
row, err := m.pg.GetTool("record_fact")
if err != nil || row == nil {
t.Fatalf("row: %v", err)
}
var stored struct {
Properties map[string]map[string]any `json:"properties"`
}
if err := json.Unmarshal(row.Schema, &stored); err != nil {
t.Fatal(err)
}
if len(stored.Properties) != 1 || stored.Properties["facts"] == nil {
t.Fatalf("top level still holds fact fields: %s", row.Schema)
}
return row.Description, stored.Properties["facts"]["description"]
}
desc, factsDesc := start(fix410ReleasedDescription, fix410ReleasedFactsDescription)
if desc != seed.Desc {
t.Errorf("released description not replaced by the current one: %.60q", desc)
}
if factsDesc != currentFacts {
t.Errorf("released facts description not replaced by the current one: %v", factsDesc)
}
editedDesc := fix410ReleasedDescription + "\nOperator note: record every negative result."
editedFacts := fix410ReleasedFactsDescription + " Operator note: one fact per host."
desc, factsDesc = start(editedDesc, editedFacts)
if desc != editedDesc {
t.Errorf("operator description changed: %.60q", desc)
}
if factsDesc != editedFacts {
t.Errorf("operator facts description changed: %v", factsDesc)
}
}
+93
View File
@@ -9,6 +9,99 @@ import (
// SeedTool only inserts new rows. Repair existing built-ins too, while keeping
// descriptions, bindings, enabled/deferred flags and parameter annotations.
// refreshRecordFactContract brings an already-seeded record_fact row to the
// facts-only contract (one-time, settings flag). SeedTool is first-insert-only
// and ToolResolve serves the stored text, so without this an old database keeps
// describing the earlier contract (fact fields at the top level beside facts[])
// to the model. Only text equal to the previous code default is replaced:
// - the tool description;
// - the facts property and item descriptions that referred to the top level.
//
// Operator edits stay; a description that matches no known default is logged
// as such and left in place. The schema structure itself is repaired by
// seedFactInputSchema on every start (RepairRecordFactSchema).
//
// The flag is written only once the outcome is known: the UPDATE succeeded
// (also when it changed no row) or the row is not the built-in one. A failed
// read, decode or UPDATE leaves it unset, so the next start retries.
func (s *Server) refreshRecordFactContract() {
const flag = "tool_schema_refresh_record_fact_facts_only"
if v, _, _ := s.m.pg.GetSetting(flag); v == "true" {
return
}
done := func() { _ = s.m.pg.SetSetting(flag, "true") }
var seed *agent.ToolSeed
for _, sd := range agent.BuiltinToolSeeds() {
if sd.Key == "record_fact" {
seed = &sd
break
}
}
row, err := s.m.pg.GetTool("record_fact")
if err != nil {
log.Printf("[facts] read record_fact for the contract refresh: %v", err)
return
}
if row == nil || seed == nil {
return
}
if !row.System || row.Kind != "builtin" {
done()
return
}
legacyTool, legacyFacts, legacyItems := agent.RecordFactLegacyDescriptions()
desc := row.Description
switch {
case desc == seed.Desc:
case sliceHas(legacyTool, desc):
desc = seed.Desc
default:
log.Printf("[facts] record_fact description does not match a known default (operator edit or an older release); the facts-only schema is published anyway, review the text in the tool catalog")
}
var schema map[string]any
if err := json.Unmarshal(row.Schema, &schema); err != nil {
log.Printf("[facts] decode record_fact schema: %v", err)
return
}
repaired := agent.RepairRecordFactSchema(schema)
facts := repaired["properties"].(map[string]any)["facts"].(map[string]any)
canonFacts := seed.Schema["properties"].(map[string]any)["facts"].(map[string]any)
if d, ok := facts["description"].(string); ok && sliceHas(legacyFacts, d) {
facts["description"] = canonFacts["description"]
}
items := facts["items"].(map[string]any)["properties"].(map[string]any)
canonItems := canonFacts["items"].(map[string]any)["properties"].(map[string]any)
for name, old := range legacyItems {
prop, _ := items[name].(map[string]any)
want, _ := canonItems[name].(map[string]any)
if prop != nil && want != nil && prop["description"] == old {
prop["description"] = want["description"]
}
}
raw, err := json.Marshal(repaired)
if err != nil {
return
}
if _, err := s.m.pg.Exec(`UPDATE tools SET description=$1, schema=$2::jsonb, updated_at=now()
WHERE key='record_fact' AND system AND kind='builtin'
AND (description IS DISTINCT FROM $1 OR schema IS DISTINCT FROM $2::jsonb)`, desc, string(raw)); err != nil {
log.Printf("[facts] refresh record_fact contract: %v", err)
return
}
done()
log.Printf("[facts] record_fact now publishes the facts-only contract (one-time)")
}
func sliceHas(list []string, s string) bool {
for _, v := range list {
if v == s {
return true
}
}
return false
}
func (s *Server) seedFactInputSchema() {
row, err := s.m.pg.GetTool("record_fact")
if err != nil {
+228 -8
View File
@@ -3,6 +3,7 @@ package server
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
@@ -42,10 +43,16 @@ func TestRecordFactInputSchemaUpgrade(t *testing.T) {
if err := pg.UpdateTool("record_fact", "operator tool", raw, bindings, old.Enabled); err != nil {
t.Fatal(err)
}
// Facts-only contract: the top level holds facts and
// nothing else; the earlier top-level fact fields and their operator
// annotations are dropped (the handler still accepts the legacy shapes),
// item annotations survive.
check := func(schema map[string]any) {
t.Helper()
if _, ok := schema["required"]; ok {
t.Fatal("top-level summary still required for batches")
if req, _ := schema["required"].([]any); len(req) != 1 || req[0] != "facts" {
if reqS, _ := schema["required"].([]string); len(reqS) != 1 || reqS[0] != "facts" {
t.Fatalf("facts must be the only required property: %v", schema["required"])
}
}
if _, ok := schema["anyOf"]; ok {
t.Fatal("root anyOf still present")
@@ -54,14 +61,13 @@ func TestRecordFactInputSchemaUpgrade(t *testing.T) {
t.Fatal("unknown fields allowed")
}
props := schema["properties"].(map[string]any)
if len(props) != 1 || props["facts"] == nil {
t.Fatalf("top level must hold facts only: %v", props)
}
items := props["facts"].(map[string]any)["items"].(map[string]any)
itemProps := items["properties"].(map[string]any)
if props["tested_asset_ids"] == nil || itemProps["tested_asset_ids"] == nil || itemProps["summary"] == nil {
t.Fatal("repaired schema missing alias or batch summary")
}
if props["summary"].(map[string]any)["description"] != "operator summary" ||
props["confidence"].(map[string]any)["default"] != "inferred" || props["custom"] != nil {
t.Fatal("operator annotations lost")
if itemProps["tested_asset_ids"] == nil || itemProps["summary"] == nil || itemProps["source"] == nil {
t.Fatal("repaired schema missing alias, batch summary or source")
}
if err := actool.ValidateInput(schema, json.RawMessage(`{"facts":[{"summary":"complete batch item"}]}`)); err != nil {
t.Fatal("valid batch rejected", err)
@@ -104,3 +110,217 @@ func TestRecordFactInputSchemaUpgrade(t *testing.T) {
t.Fatal("second migration changed the row")
}
}
// recordFactLegacySchema is the stored record_fact schema of the earlier
// contract: fact fields at the top level beside facts[], with the texts that
// referred to the top level, plus an operator annotation on an item field.
func recordFactLegacySchema(t *testing.T) (string, map[string]string) {
t.Helper()
_, legacyFacts, legacyItems := agent.RecordFactLegacyDescriptions()
item := map[string]any{"type": "object", "required": []string{"summary"}, "additionalProperties": false, "properties": map[string]any{
"summary": map[string]any{"type": "string", "minLength": 1, "description": legacyItems["summary"]},
"intent_id": map[string]any{"type": "integer", "description": legacyItems["intent_id"]},
"detail": map[string]any{"type": "string", "description": "Details of this fact, including all related observations."},
"confidence": map[string]any{"type": "string", "description": "operator confidence note", "default": "inferred"},
"source": map[string]any{"type": "string", "enum": []string{"target_response", "tool_output", "third_party_intel", "operator_input", "inference"}, "description": "where the content came from"},
}}
schema := map[string]any{"type": "object", "additionalProperties": false, "properties": map[string]any{
"summary": map[string]any{"type": "string", "minLength": 1, "description": legacyItems["summary"]},
"intent_id": map[string]any{"type": "integer", "description": legacyItems["intent_id"]},
"detail": map[string]any{"type": "string", "description": "Details of this fact, including all related observations."},
"source": map[string]any{"type": "string", "enum": []string{"inference"}},
"facts": map[string]any{"type": "array", "minItems": 1, "description": legacyFacts[0], "items": item},
}}
raw, _ := json.Marshal(schema)
return string(raw), legacyItems
}
func recordFactSeed(t *testing.T) agent.ToolSeed {
t.Helper()
for _, sd := range agent.BuiltinToolSeeds() {
if sd.Key == "record_fact" {
return sd
}
}
t.Fatal("record_fact seed missing")
return agent.ToolSeed{}
}
// A database that still stores the earlier record_fact contract (text and
// schema) is brought to the facts-only contract once: description replaced
// when it is the previous default, facts/item descriptions replaced when they
// are the earlier texts, operator annotations kept, top-level fact fields
// gone, source enum present on the items. A second run changes nothing, and
// an operator-edited description is left alone.
func TestRefreshRecordFactContract(t *testing.T) {
skipWithoutDSN(t)
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable (%v)", err)
}
defer m.Close()
s := &Server{m: m}
seed := recordFactSeed(t)
seedSchema, _ := json.Marshal(seed.Schema)
seedAgents, _ := json.Marshal(seed.Agents)
restore := func() {
_ = m.pg.UpsertToolForce("record_fact", seed.Desc, seedSchema, seedAgents)
_, _ = m.pg.Exec(`DELETE FROM settings WHERE key='tool_schema_refresh_record_fact_facts_only'`)
}
defer restore()
legacyTool, _, legacyItems := agent.RecordFactLegacyDescriptions()
legacySchema, _ := recordFactLegacySchema(t)
run := func(desc string) {
t.Helper()
if err := m.pg.UpsertToolForce("record_fact", desc, json.RawMessage(legacySchema), seedAgents); err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exec(`DELETE FROM settings WHERE key='tool_schema_refresh_record_fact_facts_only'`); err != nil {
t.Fatal(err)
}
s.seedFactInputSchema()
s.seedFactSourceSchema()
s.refreshRecordFactContract()
}
for _, desc := range legacyTool {
run(desc)
row, err := m.pg.GetTool("record_fact")
if err != nil || row == nil {
t.Fatalf("row: %v", err)
}
if row.Description != seed.Desc {
t.Fatalf("description not refreshed from a known default: %q", row.Description[:60])
}
var schema map[string]any
if err := json.Unmarshal(row.Schema, &schema); err != nil {
t.Fatal(err)
}
props := schema["properties"].(map[string]any)
if len(props) != 1 || props["facts"] == nil {
t.Fatalf("top level still holds fact fields: %v", props)
}
facts := props["facts"].(map[string]any)
items := facts["items"].(map[string]any)["properties"].(map[string]any)
canonItems := seed.Schema["properties"].(map[string]any)["facts"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any)
if facts["description"] != seed.Schema["properties"].(map[string]any)["facts"].(map[string]any)["description"] {
t.Fatalf("facts description not refreshed: %v", facts["description"])
}
for name := range legacyItems {
if items[name].(map[string]any)["description"] != canonItems[name].(map[string]any)["description"] {
t.Fatalf("item %s description not refreshed: %v", name, items[name])
}
}
if items["confidence"].(map[string]any)["description"] != "operator confidence note" || items["confidence"].(map[string]any)["default"] != "inferred" {
t.Fatalf("operator annotation lost: %v", items["confidence"])
}
if enum, _ := items["source"].(map[string]any)["enum"].([]any); len(enum) != 5 {
t.Fatalf("source enum missing on the items: %v", items["source"])
}
if v, _, _ := m.pg.GetSetting("tool_schema_refresh_record_fact_facts_only"); v != "true" {
t.Fatal("flag not set")
}
// Second start: nothing changes.
var updated1, updated2 string
_ = m.pg.QueryRow(`SELECT updated_at::text FROM tools WHERE key='record_fact'`).Scan(&updated1)
s.seedFactInputSchema()
s.seedFactSourceSchema()
s.refreshRecordFactContract()
_ = m.pg.QueryRow(`SELECT updated_at::text FROM tools WHERE key='record_fact'`).Scan(&updated2)
if updated1 != updated2 {
t.Fatal("second start rewrote the row")
}
}
// An operator-edited description is kept; the schema is still repaired.
run("my own record_fact text")
row, _ := m.pg.GetTool("record_fact")
if row == nil || row.Description != "my own record_fact text" {
t.Fatalf("operator description replaced: %v", row)
}
if !strings.Contains(string(row.Schema), `"facts"`) || strings.Contains(string(row.Schema), legacyItems["intent_id"]) {
t.Fatalf("schema not repaired beside an operator description: %s", row.Schema)
}
}
// A start on which the refresh fails must not spend the one-time flag: the
// next start retries and the stored default text is refreshed then.
func TestFix410RefreshFlagNotSetOnFailure(t *testing.T) {
skipWithoutDSN(t)
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Skipf("postgres unavailable (%v)", err)
}
defer m.Close()
s := &Server{m: m}
const flag = "tool_schema_refresh_record_fact_facts_only"
seed := recordFactSeed(t)
seedSchema, _ := json.Marshal(seed.Schema)
seedAgents, _ := json.Marshal(seed.Agents)
legacyTool, _, _ := agent.RecordFactLegacyDescriptions()
legacySchema, _ := recordFactLegacySchema(t)
exec := func(query string, args ...any) {
t.Helper()
if _, err := m.pg.Exec(query, args...); err != nil {
t.Fatal(err)
}
}
const dropTrigger = `DROP TRIGGER IF EXISTS fix410_fail_update ON tools`
defer func() {
_, _ = m.pg.Exec(dropTrigger)
_, _ = m.pg.Exec(`DROP FUNCTION IF EXISTS fix410_fail_update()`)
_ = m.pg.UpsertToolForce("record_fact", seed.Desc, seedSchema, seedAgents)
_, _ = m.pg.Exec(`DELETE FROM settings WHERE key=$1`, flag)
}()
exec(`CREATE OR REPLACE FUNCTION fix410_fail_update() RETURNS trigger LANGUAGE plpgsql AS
'BEGIN RAISE EXCEPTION ''fix410: simulated transient failure''; END'`)
start := func() {
s.seedFactInputSchema()
s.seedFactSourceSchema()
s.refreshRecordFactContract()
}
for _, tc := range []struct {
name string
fail, clear func()
}{
{"update fails", func() {
exec(`CREATE TRIGGER fix410_fail_update BEFORE UPDATE ON tools
FOR EACH ROW WHEN (OLD.key = 'record_fact') EXECUTE FUNCTION fix410_fail_update()`)
}, func() { exec(dropTrigger) }},
{"schema does not decode", func() {
exec(`UPDATE tools SET schema='[]'::jsonb WHERE key='record_fact'`)
}, func() {
exec(`UPDATE tools SET schema=$1::jsonb WHERE key='record_fact'`, legacySchema)
}},
} {
t.Run(tc.name, func(t *testing.T) {
// A database that stores the earlier default text and has not run the refresh.
if err := m.pg.UpsertToolForce("record_fact", legacyTool[0], json.RawMessage(legacySchema), seedAgents); err != nil {
t.Fatal(err)
}
exec(`DELETE FROM settings WHERE key=$1`, flag)
tc.fail()
start()
if v, ok, _ := m.pg.GetSetting(flag); ok {
t.Errorf("flag is %q after a failed refresh: later starts skip it", v)
}
if row, _ := m.pg.GetTool("record_fact"); row == nil || row.Description != legacyTool[0] {
t.Fatal("the simulated failure did not leave the earlier description in place")
}
tc.clear()
start()
row, err := m.pg.GetTool("record_fact")
if err != nil || row == nil {
t.Fatalf("row: %v", err)
}
if row.Description != seed.Desc {
t.Errorf("the next start did not refresh the description: %.60q", row.Description)
}
if v, _, _ := m.pg.GetSetting(flag); v != "true" {
t.Errorf("flag = %q after a successful refresh, want true", v)
}
})
}
}
+19 -8
View File
@@ -35,12 +35,13 @@ func TestRecordFactSourceSchemaUpgrade(t *testing.T) {
t.Cleanup(func() { pg.UpdateTool(old.Key, old.Description, old.Schema, bindings, old.Enabled) })
// Simulate a pre-provenance row customised by the operator: no source
// property, an edited description and an edited parameter description.
// property on the facts items, an edited description and an edited
// parameter description (facts-only contract: the items carry the fields).
var schema map[string]any
if err := json.Unmarshal(old.Schema, &schema); err != nil {
t.Fatal(err)
}
props := schema["properties"].(map[string]any)
props := schema["properties"].(map[string]any)["facts"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any)
delete(props, "source")
props["confidence"].(map[string]any)["description"] = "operator confidence text"
custom, _ := json.Marshal(schema)
@@ -72,16 +73,23 @@ func TestRecordFactSourceSchemaUpgrade(t *testing.T) {
t.Fatal(err)
}
var got struct {
Properties map[string]map[string]any `json:"properties"`
Properties struct {
Facts struct {
Items struct {
Properties map[string]map[string]any `json:"properties"`
} `json:"items"`
} `json:"facts"`
} `json:"properties"`
}
if err := json.Unmarshal(tool.Schema, &got); err != nil {
t.Fatal(err)
}
itemProps := got.Properties.Facts.Items.Properties
wantEnum := "[target_response tool_output third_party_intel operator_input inference]"
if src := got.Properties["source"]; src == nil || fmt.Sprint(src["enum"]) != wantEnum {
if src := itemProps["source"]; src == nil || fmt.Sprint(src["enum"]) != wantEnum {
t.Fatalf("upgrade did not add the source enum: %s", tool.Schema)
}
if tool.Description != "operator description" || got.Properties["confidence"]["description"] != "operator confidence text" || len(got.Properties) != len(props)+1 {
if tool.Description != "operator description" || itemProps["confidence"]["description"] != "operator confidence text" || len(itemProps) != len(props)+1 {
t.Fatalf("operator configuration overwritten: %q %s", tool.Description, tool.Schema)
}
@@ -92,15 +100,18 @@ func TestRecordFactSourceSchemaUpgrade(t *testing.T) {
}
}
// resolvedRecordFactProps returns the properties of record_fact as ToolResolve
// serves it to the worker.
// resolvedRecordFactProps returns the facts[] item properties of record_fact as
// ToolResolve serves it to the worker.
func resolvedRecordFactProps(t *testing.T) map[string]any {
t.Helper()
tools := agent.ToolResolve()(context.Background(), "worker", agent.NewToolSet(nil, "").WorkerTools())
for _, tl := range tools {
if tl.Name() == "record_fact" {
props, _ := tl.InputSchema()["properties"].(map[string]any)
return props
facts, _ := props["facts"].(map[string]any)
items, _ := facts["items"].(map[string]any)
itemProps, _ := items["properties"].(map[string]any)
return itemProps
}
}
t.Fatal("worker lost record_fact")
+6 -1
View File
@@ -38,7 +38,11 @@ func trafficEvidenceServer(t *testing.T) (*Server, *db.RecordedFinding, func(str
t.Fatal(err)
}
tid, _ := strconv.ParseInt(task.ID, 10, 64)
t.Cleanup(func() { m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, tid); m.pg.DeleteTask(tid) })
t.Cleanup(func() {
_, _ = m.pg.DeleteFindingsByTask(tid)
_, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, tid)
_ = m.pg.DeleteTask(tid)
})
ctx, cancel := context.WithCancel(context.Background())
cancel()
s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
@@ -278,6 +282,7 @@ func TestFindingTrafficArchiveV3RoundTripAndRetry(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer s.m.pg.DeleteFinding(shared.FindingID)
if _, err = s.m.traffic.DeleteHost("evidence.local"); err != nil {
t.Fatal(err)
}
+5 -1
View File
@@ -286,15 +286,19 @@ func TestDeepenAdmissionFailureDiscardsFollowUpIntent(t *testing.T) {
t.Fatal(err)
}
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
var findingID int64
defer func() {
_, _ = m.pg.Exec(`UPDATE tasks SET deleted_at=NULL WHERE id=$1`, taskID)
if findingID > 0 {
_, _ = m.pg.DeleteFinding(findingID)
}
_, _ = m.DeleteTask(task.ID, DeleteTaskOptions{})
}()
findingNodeID, err := task.Store.AddNode(db.KindFinding, map[string]any{"summary": "source"}, 5, "confirmed", "worker", nil)
if err != nil {
t.Fatal(err)
}
findingID, err := m.pg.AddFinding(taskID, findingNodeID, "test", "source", db.SeverityHigh, "source", "", "worker", nil)
findingID, err = m.pg.AddFinding(taskID, findingNodeID, "test", "source", db.SeverityHigh, "source", "", "worker", nil)
if err != nil {
t.Fatal(err)
}
+8 -2
View File
@@ -36,7 +36,10 @@ func TestTaskLessonsExtractionAndIdempotency(t *testing.T) {
t.Fatal(err)
}
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() { _ = m.pg.DeleteTask(taskID) }()
defer func() {
_, _ = m.pg.DeleteFindingsByTask(taskID)
_ = m.pg.DeleteTask(taskID)
}()
defer func() { _, _ = m.pg.Exec(`DELETE FROM rsi_lessons`) }()
if _, err := m.pg.Exec(`
@@ -138,7 +141,10 @@ func TestTerminalStatusExtractsLessons(t *testing.T) {
t.Fatal(err)
}
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() { _ = m.pg.DeleteTask(taskID) }()
defer func() {
_, _ = m.pg.DeleteFindingsByTask(taskID)
_ = m.pg.DeleteTask(taskID)
}()
if _, err := m.pg.Exec(`
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
VALUES ($1,'xss','Reflected XSS','reflected in search','high','confirmed')`, taskID); err != nil {
+1
View File
@@ -74,6 +74,7 @@ func TestRecorderPersistsRawWireBodies(t *testing.T) {
if _, err := pg.Exec(`DELETE FROM llm_records WHERE session_id = $1`, session); err != nil {
t.Logf("cleanup: %v", err)
}
_, _ = pg.Exec(`DELETE FROM llm_usage WHERE task_id = '999999' AND profile_name = 'raw-test-profile'`)
}()
ctx := transcript.WithSessionID(context.Background(), session)
+16
View File
@@ -463,6 +463,22 @@ func NewManager(dir, proxyAddr string) (*Manager, error) {
if tr != nil {
m.traffic = tr
m.reloadRewriteRules() // in place before the proxy accepts a connection
if scfg, ok := traffic.SOCKS5ConfigFromEnv(); ok {
scfg.DialContext = m.interceptor.DialSOCKS5
if bound, serr := tr.StartSOCKS5(scfg); serr != nil {
log.Printf("[traffic/socks5] disabled: %v", serr)
} else {
log.Printf("[traffic/socks5] opt-in SOCKS5 CONNECT listener on %s", bound)
}
}
if tcfg, ok := traffic.TransparentTCPConfigFromEnv(); ok {
tcfg.DialContext = m.interceptor.DialSOCKS5
if bound, terr := tr.StartTransparentTCP(tcfg); terr != nil {
log.Printf("[traffic/transparent] disabled: %v", terr)
} else {
log.Printf("[traffic/transparent] opt-in transparent TCP listener on %s (Linux IPv4 SO_ORIGINAL_DST)", bound)
}
}
go func() {
log.Printf("[traffic] recording proxy on %s (set HTTP_PROXY=%s + trust _ca CA)", proxyAddr, tr.ProxyAddr())
if err := tr.Start(); err != nil {
+1 -1
View File
@@ -18,7 +18,7 @@ func TestSeedAssociatesTargetAssetWithTask(t *testing.T) {
if err != nil {
pgdb.SkipPGUnavailable(t, err)
}
defer m.Close()
t.Cleanup(func() { _ = m.Close() })
task, err := m.CreateTask("seed ownership", "seed ownership", nil, 0, 0)
if err != nil {
t.Fatal(err)
+4 -1
View File
@@ -72,7 +72,10 @@ func newNotifyFixture(t *testing.T) *notifyFixture {
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { pg.Exec(`DELETE FROM tasks WHERE id=$1`, taskID) })
t.Cleanup(func() {
_, _ = pg.DeleteFindingsByTask(taskID)
_ = pg.DeleteTask(taskID)
})
var mark int64
if err := pg.QueryRow(`SELECT COALESCE(max(id),0) FROM notification_events`).Scan(&mark); err != nil {
+16 -3
View File
@@ -573,6 +573,7 @@ func (s *Server) seedOrchestrationTools() {
s.seedFindingEvidenceSchema() // optional counterevidence / confidence_rationale on report_finding, added once per DB
s.seedFactInputSchema() // restore record_fact single/batch contracts
s.seedFactSourceSchema() // source enum on record_fact, added once per DB
s.refreshRecordFactContract() // record_fact publishes the facts-only contract (one-time)
s.seedTaskScopeDescription() // add_task_scope: "coverage scope, not enforced" replaces the authorization-boundary text
// Note: pentest's default tool bindings need no migration — BuiltinToolSeeds already seeds
// list_assets/insert_assets/report_finding/list_findings/list_companies together with
@@ -602,11 +603,23 @@ func (s *Server) seedFactSourceSchema() {
if err := json.Unmarshal(obj["properties"], &properties); err != nil {
return
}
if len(properties["source"]) == 0 {
// The source enum lives on the facts[] items (facts-only contract); the
// top level has no fact fields any more.
var facts map[string]json.RawMessage
var items map[string]json.RawMessage
var itemProps map[string]json.RawMessage
if err := json.Unmarshal(properties["facts"], &facts); err != nil {
return
}
_, err = s.m.pg.Exec(`UPDATE tools SET schema=jsonb_set(schema,ARRAY['properties','source'],$1::jsonb,true),updated_at=now()
WHERE key='record_fact' AND system AND NOT(COALESCE(schema->'properties','{}'::jsonb) ? 'source')`, string(properties["source"]))
if err := json.Unmarshal(facts["items"], &items); err != nil {
return
}
if err := json.Unmarshal(items["properties"], &itemProps); err != nil || len(itemProps["source"]) == 0 {
return
}
_, err = s.m.pg.Exec(`UPDATE tools SET schema=jsonb_set(schema,ARRAY['properties','facts','items','properties','source'],$1::jsonb,true),updated_at=now()
WHERE key='record_fact' AND system AND (schema->'properties'->'facts'->'items') ? 'properties'
AND NOT(COALESCE(schema->'properties'->'facts'->'items'->'properties','{}'::jsonb) ? 'source')`, string(itemProps["source"]))
if err != nil {
log.Printf("[facts] upgrade record_fact.source: %v", err)
}
+2
View File
@@ -24,6 +24,7 @@ func TestPlannerQuietSkipEdge(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _, _ = m.DeleteTask(created.ID, DeleteTaskOptions{}) })
tt, ok := m.Task(created.ID)
if !ok {
t.Fatal("task not found after creation")
@@ -93,6 +94,7 @@ func TestPlannerQuietModeToggleMidRun(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _, _ = m.DeleteTask(created.ID, DeleteTaskOptions{}) })
tt, ok := m.Task(created.ID)
if !ok {
t.Fatal("task not found")
+119
View File
@@ -0,0 +1,119 @@
package server
import (
"context"
"strings"
"testing"
"github.com/Autumn-27/norma/harness"
"github.com/Autumn-27/norma/llm"
actool "github.com/Autumn-27/norma/tool"
"github.com/Autumn-27/artex/agent"
"github.com/Autumn-27/artex/db"
)
// acceptsRawInput is the assertion norma's harness makes before its schema
// check: only a tool that reports AcceptsRawInput skips it.
func acceptsRawInput(t actool.CoreTool) bool {
r, ok := t.(interface{ AcceptsRawInput() bool })
return ok && r.AcceptsRawInput()
}
// record_fact opts out of the harness schema check and validates its own
// input. Every wrapper between the handler and the harness has to forward
// that opt-out: an embedded CoreTool alone hides it from the harness's type
// assertion, and the stricter published schema then rejects the calls the
// handler exists to answer.
func TestFix410AcceptsRawInputSurvivesWrappers(t *testing.T) {
oldAugment, oldResolve := agent.ToolAugment(), agent.ToolResolve()
t.Cleanup(func() {
agent.SetToolAugment(oldAugment)
agent.SetToolResolve(oldResolve)
})
ctx := context.Background()
probe := func(raw bool) actool.CoreTool {
return actool.Build(actool.Spec{Name: "fix410_probe", Schema: map[string]any{"type": "object"}, RawInput: raw})
}
// AugmentTools without hooks applies guardPanic and nothing else.
t.Run("guardedTool", func(t *testing.T) {
agent.SetToolAugment(nil)
agent.SetToolResolve(nil)
for _, raw := range []bool{true, false} {
out, _, cleanup := agent.AugmentTools(ctx, "worker", []actool.CoreTool{probe(raw)})
cleanup()
if len(out) != 1 {
t.Fatalf("AugmentTools returned %d tools, want 1", len(out))
}
if got := acceptsRawInput(out[0]); got != raw {
t.Errorf("%T: AcceptsRawInput = %v, wrapped tool reports %v", out[0], got, raw)
}
}
})
t.Run("meteredTool", func(t *testing.T) {
for _, raw := range []bool{true, false} {
wrapped := meterTool(probe(raw), &recordingToolUsage{}, "fix410_probe", "worker", agent.RunInfo{})
if got := acceptsRawInput(wrapped); got != raw {
t.Errorf("%T: AcceptsRawInput = %v, wrapped tool reports %v", wrapped, got, raw)
}
}
})
// The chain a worker's record_fact goes through on the server: the catalog
// decorator and the usage meter (ToolResolve, wireTools), then guardPanic
// (AugmentTools), then the harness itself.
t.Run("server chain", func(t *testing.T) {
dsn, _, err := db.DSN()
if err != nil {
db.SkipPGUnavailable(t, err)
}
pg, err := db.Open(dsn)
if err != nil {
db.SkipPGUnavailable(t, err)
}
t.Cleanup(func() { _ = pg.Close() })
agent.SetToolAugment(nil)
wireTools(pg, nil)
task, err := pg.CreateTaskWithOptions("fix410", "record_fact through the wrapper chain", db.TaskCreateOptions{})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = pg.DeleteTask(task.ID) })
ts := agent.NewToolSet(pg.Exploration(task.ExplorationID), "worker")
tools, _, cleanup := agent.AugmentTools(ctx, "worker", ts.WorkerTools())
defer cleanup()
recordFact := names(tools)["record_fact"]
if recordFact == nil {
t.Fatal("worker has no record_fact")
}
if !acceptsRawInput(recordFact) {
t.Errorf("record_fact lost the harness opt-out behind %T", recordFact)
}
// The first call is malformed (facts is not an array): the harness
// schema check answers it with its generic error unless the tool opted
// out. The second is the legacy single fact, which the published
// schema does not describe and the handler still accepts.
p := &toolScript{results: make(chan string, 4), calls: []scriptedCall{
{"record_fact", `{"facts":{"summary":"not an array"}}`},
{"record_fact", `{"summary":"legacy single fact"}`},
}}
for _, err := range harness.Query(ctx, harness.QueryInput{
Provider: p,
Tools: actool.NewRegistry(recordFact),
Messages: []llm.Message{llm.UserText("record the facts")},
}, harness.QueryDeps{}) {
if err != nil {
t.Fatal(err)
}
}
if refusal := p.next(t); !strings.Contains(refusal, "invalid record_fact input: facts must be an array") {
t.Errorf("malformed call: want the handler's own refusal, not the SDK schema error, got: %s", refusal)
}
if legacy := p.next(t); strings.HasPrefix(legacy, "[error]") || ts.Writes().Facts != 1 {
t.Errorf("legacy single fact not written (facts=%d): %s", ts.Writes().Facts, legacy)
}
})
}
+10 -5
View File
@@ -24,6 +24,7 @@ import (
"time"
"github.com/Autumn-27/artex/agent"
"github.com/Autumn-27/artex/archivestore"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/egress"
"github.com/Autumn-27/artex/intercept"
@@ -158,9 +159,11 @@ type Server struct {
// Cold task archives run through one persistent FIFO worker. The buffered wake
// channel coalesces enqueue bursts; the database remains the source of truth.
archiveWake chan struct{}
archiveWG sync.WaitGroup
side *sideQuestionState
archiveWake chan struct{}
archiveWG sync.WaitGroup
archiveStore *archivestore.Store
archiveStoreErr error
side *sideQuestionState
}
// provEntry is a cached provider + its config for one LLM profile id.
@@ -302,8 +305,10 @@ func New(ctx context.Context, m *Manager, skillDir string, dataDir string, keyDi
} else {
log.Printf("[engine] no LLM provider configured — engine idle until set via /api/llm or env")
}
// Both env readers have now run (NewManager for the DSN, FromEnv above for the
// keys). Drop the secrets before anything below can spawn a child process:
s.initTaskArchiveStore()
// Startup env readers have now run (NewManager for the DSN, the LLM provider
// and archive store for their credentials). Drop the secrets before anything
// below can spawn a child process:
// restoring runtimes resumes workers that run shells, and MCP discovery starts
// stdio servers.
if s.scrubSecrets {
+108
View File
@@ -0,0 +1,108 @@
package server
import (
"encoding/binary"
"fmt"
"io"
"net"
"strconv"
"sync/atomic"
"testing"
"time"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/traffic"
)
func TestManagerSOCKS5OptInFlagAndAssetBlockRuleRefusal(t *testing.T) {
// 1. Flag off (default): Manager starts NO SOCKS5 listener.
t.Setenv(traffic.SOCKS5EnabledEnv, "")
t.Setenv(traffic.SOCKS5AddrEnv, "")
mOff, err := NewManager(t.TempDir(), "127.0.0.1:0")
if err != nil {
db.SkipPGUnavailable(t, err)
}
t.Cleanup(func() { _ = mOff.Close() })
if mOff.traffic == nil {
t.Fatal("expected HTTP recording proxy to start")
}
if got := mOff.traffic.SOCKS5Addr(); got != "" {
t.Fatalf("SOCKS5Addr with flag off = %q, want empty", got)
}
// 2. Flag on (ARTEX_SOCKS5_ADDR=127.0.0.1:0): listener starts and enforces
// global asset block rules via m.interceptor.DialSOCKS5 BEFORE dialing.
t.Setenv(traffic.SOCKS5AddrEnv, "127.0.0.1:0")
mOn, err := NewManager(t.TempDir(), "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = mOn.Close() })
socksAddr := mOn.traffic.SOCKS5Addr()
if socksAddr == "" {
t.Fatal("expected SOCKS5 listener when ARTEX_SOCKS5_ADDR is set")
}
// Add a global asset block rule for 127.0.0.1 and ensure cleanup.
rule, err := mOn.pg.CreateAssetInterceptRule(
"exact_ip",
"127.0.0.1",
fmt.Sprintf("p41a-block-test-%d", time.Now().UnixNano()),
true,
)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = mOn.pg.DeleteAssetInterceptRule(rule.ID) })
// Target TCP server on 127.0.0.1: must NEVER receive a connection.
targetLn, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer targetLn.Close()
var accepted atomic.Int32
go func() {
for {
c, err := targetLn.Accept()
if err != nil {
return
}
accepted.Add(1)
_ = c.Close()
}
}()
_, portStr, _ := net.SplitHostPort(targetLn.Addr().String())
port, _ := strconv.Atoi(portStr)
c, err := net.DialTimeout("tcp", socksAddr, 3*time.Second)
if err != nil {
t.Fatal(err)
}
defer c.Close()
_ = c.SetDeadline(time.Now().Add(5 * time.Second))
// Greeting: VER=5, NMETHODS=1, NO AUTH
if _, err := c.Write([]byte{0x05, 0x01, 0x00}); err != nil {
t.Fatal(err)
}
var sel [2]byte
if _, err := io.ReadFull(c, sel[:]); err != nil || sel != [2]byte{0x05, 0x00} {
t.Fatalf("method selection = %v err=%v", sel, err)
}
// CONNECT 127.0.0.1:<port>
req := []byte{0x05, 0x01, 0x00, 0x01, 127, 0, 0, 1, 0, 0}
binary.BigEndian.PutUint16(req[8:10], uint16(port))
if _, err := c.Write(req); err != nil {
t.Fatal(err)
}
var rep [10]byte
if _, err := io.ReadFull(c, rep[:]); err != nil {
t.Fatal(err)
}
if rep[1] != 0x02 {
t.Fatalf("blocked target SOCKS5 reply = 0x%02x, want 0x02 (not allowed by ruleset)", rep[1])
}
if accepted.Load() != 0 {
t.Fatalf("blocked target accepted %d connections, want 0", accepted.Load())
}
}
+8
View File
@@ -0,0 +1,8 @@
//go:build !unix
package server
// syncDir does nothing here: Windows does not support fsync on a directory
// handle (FlushFileBuffers fails on it), and the contents of the file written in
// the directory are already synced by the caller.
func syncDir(path string) error { return nil }
+18
View File
@@ -0,0 +1,18 @@
//go:build unix
package server
import (
"errors"
"os"
)
// syncDir flushes a directory, so that an entry just created or renamed in it
// survives a crash.
func syncDir(path string) error {
d, err := os.Open(path)
if err != nil {
return err
}
return errors.Join(d.Sync(), d.Close())
}
+18
View File
@@ -0,0 +1,18 @@
//go:build unix
package server
import (
"path/filepath"
"testing"
)
func TestSyncDir(t *testing.T) {
dir := t.TempDir()
if err := syncDir(dir); err != nil {
t.Fatalf("sync of an existing directory: %v", err)
}
if err := syncDir(filepath.Join(dir, "missing")); err == nil {
t.Fatal("sync of a missing directory must fail")
}
}
+1
View File
@@ -83,6 +83,7 @@ func (g *gateServer) cleanupTask(t *testing.T, id string) {
_ = g.m.pg.DeleteTask(n)
_, _ = g.m.pg.Exec(`DELETE FROM target_pins WHERE task_id=$1`, n)
_, _ = g.m.pg.Exec(`DELETE FROM intercept_pending WHERE task_id=$1`, id)
_, _ = g.m.pg.Exec(`DELETE FROM assets WHERE type = 'ip' AND ip = '127.0.0.1'`)
})
}
+1
View File
@@ -174,6 +174,7 @@ func (f *targetScopeFixture) newTask(t *testing.T) *Task {
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _, _ = f.m.DeleteTask(created.ID, DeleteTaskOptions{}) })
task, ok := f.m.Task(created.ID)
if !ok {
t.Fatal("task not found after creation")
+9
View File
@@ -650,6 +650,15 @@ func TestTaskLLMResolutionReportsDatabaseFailure(t *testing.T) {
if err != nil {
t.Fatal(err)
}
taskID := mustTaskID(t, task.ID)
t.Cleanup(func() {
if dsn, _, err := db.DSN(); err == nil {
if pg, err := db.Open(dsn); err == nil {
_ = pg.DeleteTask(taskID)
_ = pg.Close()
}
}
})
if err := m.pg.Close(); err != nil {
t.Fatal(err)
}
+403
View File
@@ -0,0 +1,403 @@
package server
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"sync"
"testing"
"github.com/Autumn-27/artex/archivestore"
pgdb "github.com/Autumn-27/artex/db"
)
// Recreate persisted states at operation boundaries, then run the same recovery
// functions used at startup. PostgreSQL, files, and HTTP object storage are real
// isolated fixtures; no production failpoint hooks are needed.
func TestRemoteArchiveInterruptionRecovery(t *testing.T) {
if os.Getenv("ARTEX_PG_DSN") == "" {
t.Skip("requires isolated PostgreSQL")
}
phases := []string{
"staged_before_compression", "compression_partial", "compressed_before_journal", "journal_before_upload",
"uploaded_before_verification", "verified_before_database",
"database_before_local_cleanup", "package_cleaned_before_stage_cleanup",
"restore_download_before_install", "restore_files_before_database", "restore_database_before_object_delete", "restore_object_deleted_before_stub",
"delete_journal_before_database", "delete_database_before_object",
}
for _, phase := range phases {
t.Run(phase, func(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
defer m.Close()
store, objects := newRecoveryObjectStore(t)
s := &Server{ctx: context.Background(), m: m, archiveStore: store}
task, err := m.CreateTask("interruption "+phase, "preserve source", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
id, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }()
if err := m.pg.SetPaused(id, true); err != nil {
t.Fatal(err)
}
job, err := m.pg.QueueTaskArchive(id)
if err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Archiving); err != nil {
t.Fatal(err)
}
source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt")
if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(source, []byte("original task data"), 0600); err != nil {
t.Fatal(err)
}
snapshot, err := m.pg.SnapshotTaskArchive(id)
if err != nil {
t.Fatal(err)
}
stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID)
if err != nil {
t.Fatal(err)
}
packagePath := taskArchivePath(m.dir, job.ID, task.ID)
if phase == "compression_partial" {
if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(packagePath+".partial", []byte("incomplete compressed package"), 0600); err != nil {
t.Fatal(err)
}
}
var original, compressed int64
var checksum, uri, journal string
if phase != "staged_before_compression" && phase != "compression_partial" {
original, compressed, checksum, err = writeTaskArchivePackage(packagePath, stage.payload, snapshot)
if err != nil {
t.Fatal(err)
}
uri = store.URI(id, job.ID, checksum)
}
if phase != "staged_before_compression" && phase != "compression_partial" && phase != "compressed_before_journal" {
journal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, LocalPath: packagePath, Operation: "upload"})
if err != nil {
t.Fatal(err)
}
}
committed := phase != "staged_before_compression" && phase != "compression_partial" && phase != "compressed_before_journal" &&
phase != "journal_before_upload" && phase != "uploaded_before_verification" && phase != "verified_before_database"
if phase == "uploaded_before_verification" {
data, err := os.ReadFile(packagePath)
if err != nil {
t.Fatal(err)
}
objects.put(uri, data)
} else if phase == "verified_before_database" || committed {
if err := store.Upload(s.ctx, uri, packagePath, checksum); err != nil {
t.Fatal(err)
}
}
if committed {
if err := m.pg.CompleteTaskArchive(job.ID, snapshot, uri, checksum, original, compressed); err != nil {
t.Fatal(err)
}
}
afterArchive := phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" ||
phase == "restore_object_deleted_before_stub" || phase == "delete_journal_before_database" || phase == "delete_database_before_object"
if phase == "package_cleaned_before_stage_cleanup" || afterArchive {
if err := os.Remove(packagePath); err != nil {
t.Fatal(err)
}
if err := os.Remove(journal); err != nil {
t.Fatal(err)
}
}
if afterArchive {
if err := stage.commit(); err != nil {
t.Fatal(err)
}
}
restored := false
remoteDeleted := false
if phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" {
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Restoring); err != nil {
t.Fatal(err)
}
parent := filepath.Join(taskArchiveRoot(m.dir), ".restore")
if err := os.MkdirAll(parent, 0700); err != nil {
t.Fatal(err)
}
extracted, err := os.MkdirTemp(parent, "interrupted-")
if err != nil {
t.Fatal(err)
}
downloaded := filepath.Join(extracted, "download.tar.zst")
if err := store.Download(s.ctx, uri, downloaded, checksum, compressed); err != nil {
t.Fatal(err)
}
if err := extractTaskArchivePackage(downloaded, checksum, extracted); err != nil {
t.Fatal(err)
}
if phase != "restore_download_before_install" {
if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID, s.execPriv); err != nil {
t.Fatal(err)
}
}
if phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" {
if _, err := m.pg.RestoreTaskArchive(job.ID, snapshot, 0); err != nil {
t.Fatal(err)
}
restored = true
}
if phase == "restore_object_deleted_before_stub" {
if err := store.Delete(s.ctx, uri); err != nil {
t.Fatal(err)
}
remoteDeleted = true
}
}
deleting := phase == "delete_journal_before_database" || phase == "delete_database_before_object"
if deleting {
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Deleting); err != nil {
t.Fatal(err)
}
if _, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, Operation: "delete"}); err != nil {
t.Fatal(err)
}
if phase == "delete_database_before_object" {
if err := m.pg.DeleteTaskArchiveStub(job.ID); err != nil {
t.Fatal(err)
}
remoteDeleted = true
}
}
if err := m.pg.RecoverTaskArchiveJobs(); err != nil {
t.Fatal(err)
}
if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if err := s.recoverRemoteArchiveJournals(); err != nil {
t.Fatal(err)
}
wantSource := !committed || restored
data, fileErr := os.ReadFile(source)
if wantSource && (fileErr != nil || string(data) != "original task data") {
t.Fatalf("source data lost: %v", fileErr)
}
if !wantSource && !os.IsNotExist(fileErr) {
t.Fatalf("archived source resurrected: %v", fileErr)
}
if _, err := os.Stat(packagePath); !os.IsNotExist(err) {
t.Fatalf("temporary package remains: %v", err)
}
if _, err := os.Stat(packagePath + ".partial"); !os.IsNotExist(err) {
t.Fatalf("partial package remains: %v", err)
}
if got, want := objects.has(uri), committed && !remoteDeleted; got != want {
t.Fatalf("remote object exists=%v, want=%v", got, want)
}
if committed && !deleting {
actual, err := m.pg.IsTaskArchiveRestored(job.ID)
if err != nil || actual != restored {
t.Fatalf("restored=%v want=%v err=%v", actual, restored, err)
}
}
// A second startup must be harmless at every boundary.
if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if err := s.recoverRemoteArchiveJournals(); err != nil {
t.Fatal(err)
}
})
}
}
type recoveryObjects struct {
mu sync.Mutex
data map[string][]byte
}
func (o *recoveryObjects) put(uri string, data []byte) {
o.mu.Lock()
defer o.mu.Unlock()
// Test URIs always use the fixture bucket with path-style addressing.
o.data["/test/"+uri[len("s3://test/"):]] = append([]byte(nil), data...)
}
func (o *recoveryObjects) has(uri string) bool {
o.mu.Lock()
defer o.mu.Unlock()
if uri == "" {
return false
}
_, ok := o.data["/test/"+uri[len("s3://test/"):]]
return ok
}
func newRecoveryObjectStore(t *testing.T) (*archivestore.Store, *recoveryObjects) {
t.Helper()
objects := &recoveryObjects{data: map[string][]byte{}}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
objects.mu.Lock()
defer objects.mu.Unlock()
data, exists := objects.data[r.URL.Path]
w.Header().Set("ETag", `"recovery-etag"`)
switch r.Method {
case "PUT":
if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" {
t.Error("missing SSE")
}
objects.data[r.URL.Path], _ = io.ReadAll(r.Body)
case "HEAD":
if !exists {
w.WriteHeader(404)
return
}
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
w.Header().Set("X-Amz-Server-Side-Encryption", "AES256")
case "GET":
if !exists {
w.WriteHeader(404)
return
}
var start, end int
if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end >= len(data) || r.Header.Get("If-Match") != `"recovery-etag"` {
w.WriteHeader(416)
return
}
w.Header().Set("Content-Length", strconv.Itoa(end-start+1))
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
w.WriteHeader(206)
_, _ = w.Write(data[start : end+1])
case "DELETE":
delete(objects.data, r.URL.Path)
w.WriteHeader(204)
default:
t.Errorf("unexpected object request %s", r.Method)
w.WriteHeader(400)
}
}))
t.Cleanup(srv.Close)
store, err := archivestore.New(archivestore.Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
if err != nil {
t.Fatal(err)
}
return store, objects
}
// Neither a rollback conflict nor a temporary-file cleanup failure may discard
// the staging journal: recovery must be safe to retry after the obstruction ends.
func TestArchivePartialRecoveryRetainsJournalOnFailure(t *testing.T) {
if os.Getenv("ARTEX_PG_DSN") == "" {
t.Skip("requires isolated PostgreSQL")
}
for _, failure := range []string{"rollback-conflict", "cleanup-failure"} {
t.Run(failure, func(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
defer m.Close()
task, err := m.CreateTask("partial recovery "+failure, "retain source", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
id, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }()
if err := m.pg.SetPaused(id, true); err != nil {
t.Fatal(err)
}
job, err := m.pg.QueueTaskArchive(id)
if err != nil {
t.Fatal(err)
}
source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt")
if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(source, []byte("original"), 0600); err != nil {
t.Fatal(err)
}
snapshot, err := m.pg.SnapshotTaskArchive(id)
if err != nil {
t.Fatal(err)
}
stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID)
if err != nil {
t.Fatal(err)
}
partial := taskArchivePath(m.dir, job.ID, task.ID) + ".partial"
if err := os.MkdirAll(filepath.Dir(partial), 0700); err != nil {
t.Fatal(err)
}
var obstruction string
if failure == "rollback-conflict" {
if len(stage.journal.Moves) == 0 {
t.Fatal("no staged source")
}
obstruction = stage.journal.Moves[0].Source
if err := os.MkdirAll(obstruction, 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(partial, []byte("partial"), 0600); err != nil {
t.Fatal(err)
}
} else {
if err := os.Mkdir(partial, 0700); err != nil {
t.Fatal(err)
}
obstruction = filepath.Join(partial, "block-remove")
if err := os.WriteFile(obstruction, []byte("fixture"), 0600); err != nil {
t.Fatal(err)
}
}
if err := recoverTaskArchiveStages(m.dir, m.pg); err == nil {
t.Fatal("expected recovery obstruction")
}
if _, err := os.Stat(filepath.Join(stage.root, "journal.json")); err != nil {
t.Fatal("lost retry journal", err)
}
if _, err := os.Stat(partial); err != nil {
t.Fatal("removed partial before successful recovery", err)
}
if failure == "cleanup-failure" {
if got, err := os.ReadFile(source); err != nil || string(got) != "original" {
t.Fatal("source was not rolled back", err)
}
}
if err := os.RemoveAll(obstruction); err != nil {
t.Fatal(err)
}
if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if got, err := os.ReadFile(source); err != nil || string(got) != "original" {
t.Fatal("source lost on retry", err)
}
for _, path := range []string{stage.root, partial} {
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("recovery left %s: %v", path, err)
}
}
if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
})
}
}
+27
View File
@@ -0,0 +1,27 @@
package server
import (
"encoding/json"
"os"
"testing"
)
// Writing the journal must succeed on every platform the server builds for. It
// needs no database: on Windows the directory sync it ended with failed, so no
// remote archive or remote deletion could start.
func TestRemoteArchiveJournalWrite(t *testing.T) {
s := &Server{m: &Manager{dir: t.TempDir()}}
want := remoteArchiveJournal{ArchiveID: 7, URI: "s3://bucket/tasks/package.tar.zst", LocalPath: "package.tar.zst", Operation: "upload"}
name, err := s.writeRemoteArchiveJournal(want)
if err != nil {
t.Fatal(err)
}
raw, err := os.ReadFile(name)
if err != nil {
t.Fatal(err)
}
var got remoteArchiveJournal
if err := json.Unmarshal(raw, &got); err != nil || got != want {
t.Fatalf("journal content %q: %v", raw, err)
}
}
+29 -10
View File
@@ -193,6 +193,16 @@ func (s *taskArchiveFileStage) rollback() error {
if s == nil || s.done {
return nil
}
err := s.rollbackFiles()
if err == nil {
err = os.RemoveAll(s.root)
}
s.done = true
return err
}
// Keep the journal until the caller has also cleaned temporary packages.
func (s *taskArchiveFileStage) rollbackFiles() error {
var errs []error
for i := len(s.journal.Moves) - 1; i >= 0; i-- {
move := s.journal.Moves[i]
@@ -218,12 +228,6 @@ func (s *taskArchiveFileStage) rollback() error {
errs = append(errs, err)
}
}
if len(errs) == 0 {
if err := os.RemoveAll(s.root); err != nil {
errs = append(errs, err)
}
}
s.done = true
return errors.Join(errs...)
}
@@ -464,18 +468,33 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error {
errs = append(errs, getErr)
continue
}
if archive != nil && archive.State == pgdb.ArchiveReady {
if archive != nil && archive.ArchivePath != "" && archive.SHA256 != "" {
if err := os.RemoveAll(root); err != nil {
errs = append(errs, err)
}
continue
}
stage := &taskArchiveFileStage{root: root, payload: filepath.Join(root, "payload"), journal: journal}
if err := stage.rollback(); err != nil {
if err := stage.rollbackFiles(); err != nil {
errs = append(errs, err)
continue
}
if archive != nil && archive.ArchivePath != "" {
_ = os.Remove(archive.ArchivePath)
// The process can exit after compression but before an upload journal or
// database location exists. Remove that temporary package only after the
// source-file rollback succeeds, for both local and remote backends.
packagePath := taskArchivePath(dataDir, journal.ArchiveID, journal.TaskID)
if err := validateArchivePath(dataDir, packagePath); err != nil {
errs = append(errs, err)
continue
}
// A crash inside compression leaves .partial, before any upload journal.
// Retain this staging journal on cleanup failure so startup can retry.
if err := errors.Join(removeLocalArchivePackage(packagePath), removeLocalArchivePackage(packagePath+".partial")); err != nil {
errs = append(errs, err)
continue
}
if err := os.RemoveAll(root); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
+131
View File
@@ -0,0 +1,131 @@
package server
import (
"context"
"crypto/rand"
"crypto/sha256"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/Autumn-27/artex/archivestore"
pgdb "github.com/Autumn-27/artex/db"
)
// An S3-compatible endpoint that answers CreateMultipartUpload with 200 and no
// UploadId makes the SDK's upload manager dereference a nil pointer. The panic
// happens in the archive worker goroutine: it must fail that one job like any
// other upload error, keep the source data, and leave the process running.
func TestReview318ArchiveWorkerSurvivesUploadPanic(t *testing.T) {
skipWithoutDSN(t)
t.Setenv("ARTEX_ARCHIVE_STORAGE", "local")
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
defer m.Close()
ctx, cancel := context.WithCancel(context.Background())
cancel()
s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
s.archiveWG.Wait()
var creates, unexpected atomic.Int32
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
switch {
case r.Method == http.MethodPost && r.URL.Query().Has("uploads"):
creates.Add(1)
w.Header().Set("Content-Type", "application/xml")
_, _ = io.WriteString(w, `<?xml version="1.0" encoding="UTF-8"?><InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Bucket>test</Bucket></InitiateMultipartUploadResult>`)
case r.Method == http.MethodDelete:
// The worker's deferred cleanup of the upload that never committed.
w.WriteHeader(http.StatusNoContent)
default:
unexpected.Add(1)
w.WriteHeader(http.StatusBadRequest)
}
}))
defer remote.Close()
s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
if err != nil {
t.Fatal(err)
}
task, err := m.CreateTask("multipart panic", "keep the source", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
id, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }()
if err := m.pg.SetPaused(id, true); err != nil {
t.Fatal(err)
}
// The package must exceed the 8 MiB part size of archivestore.Upload for the
// upload manager to choose multipart: 9 MiB of random bytes do not compress.
source := make([]byte, 9<<20)
if _, err := rand.Read(source); err != nil {
t.Fatal(err)
}
file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "large.bin")
if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(file, source, 0600); err != nil {
t.Fatal(err)
}
job, err := m.pg.QueueTaskArchive(id)
if err != nil {
t.Fatal(err)
}
// Run the server's own worker goroutine, as New starts it.
workerCtx, stopWorker := context.WithCancel(context.Background())
s.ctx = workerCtx
s.startTaskArchiveWorker()
defer func() { stopWorker(); s.archiveWG.Wait() }()
deadline := time.Now().Add(2 * time.Minute)
var item *pgdb.TaskArchive
for {
item, err = m.pg.GetTaskArchive(job.ID)
if err != nil {
t.Fatal(err)
}
if item == nil {
t.Fatal("archive job disappeared")
}
if item.State != pgdb.ArchiveQueued && item.State != pgdb.Archiving {
break
}
if time.Now().After(deadline) {
t.Fatalf("archive job still %s (%s)", item.State, item.Phase)
}
time.Sleep(50 * time.Millisecond)
}
if creates.Load() != 1 {
t.Fatalf("expected one CreateMultipartUpload, got %d: the package did not take the multipart path", creates.Load())
}
if item.State != pgdb.ArchiveFailed || !strings.Contains(item.Error, "panic") {
t.Fatalf("job must be archive_failed with the panic as its error: state=%s error=%q", item.State, item.Error)
}
if item.ArchivePath != "" || item.SHA256 != "" || item.ArchivedAt != nil {
t.Fatalf("a location was committed for a failed archive: path=%q sha256=%q archived_at=%v", item.ArchivePath, item.SHA256, item.ArchivedAt)
}
got, err := os.ReadFile(file)
if err != nil || sha256.Sum256(got) != sha256.Sum256(source) {
t.Fatalf("source file lost or changed after the panic: %v", err)
}
if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil {
t.Fatalf("task lost after the panic: %v", err)
}
if unexpected.Load() != 0 {
t.Fatalf("%d unexpected S3 requests", unexpected.Load())
}
}
+163
View File
@@ -0,0 +1,163 @@
package server
import (
"encoding/json"
"errors"
"fmt"
"log"
"os"
"path/filepath"
"strings"
pgdb "github.com/Autumn-27/artex/db"
)
// An outbox bridges the local PostgreSQL transaction and object storage. Never
// delete the only package before metadata permits it. Unfinished cleanup is
// retried after a restart (and before subsequent jobs).
type remoteArchiveJournal struct {
ArchiveID int64 `json:"archive_id"`
URI string `json:"uri"`
LocalPath string `json:"local_path,omitempty"`
Operation string `json:"operation"`
}
func (s *Server) validateStoredArchive(uri string) error {
if strings.HasPrefix(uri, "s3://") {
return s.archiveStore.Validate(uri)
}
return validateArchivePath(s.m.dir, uri)
}
func removeLocalArchivePackage(filename string) error {
if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
func (s *Server) removeStoredArchive(uri string) error {
if err := s.validateStoredArchive(uri); err != nil {
return err
}
if strings.HasPrefix(uri, "s3://") {
return s.archiveStore.Delete(s.ctx, uri)
}
return removeLocalArchivePackage(uri)
}
func (s *Server) writeRemoteArchiveJournal(j remoteArchiveJournal) (string, error) {
dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote")
if err := os.MkdirAll(dir, archiveDirMode); err != nil {
return "", err
}
f, err := os.CreateTemp(dir, ".journal-")
if err != nil {
return "", err
}
tmp := f.Name()
defer os.Remove(tmp)
encodeErr := json.NewEncoder(f).Encode(j)
syncErr := f.Sync()
if err := errors.Join(encodeErr, syncErr, f.Close()); err != nil {
return "", err
}
name := tmp + ".json"
if err := os.Rename(tmp, name); err != nil {
return "", err
}
// Persist the directory entry before an upload or database deletion starts.
if err := syncDir(dir); err != nil {
return "", err
}
// The .remote directory itself may have just been created.
return name, syncDir(filepath.Dir(dir))
}
func (s *Server) recoverRemoteArchiveJournals() error {
dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote")
entries, err := os.ReadDir(dir)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
var errs []error
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
continue
}
name := filepath.Join(dir, entry.Name())
if err := s.recoverRemoteArchiveJournal(name); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
func (s *Server) recoverRemoteArchiveJournal(name string) error {
raw, err := os.ReadFile(name)
if err != nil {
return err
}
var j remoteArchiveJournal
if err := json.Unmarshal(raw, &j); err != nil {
return err
}
if j.ArchiveID <= 0 {
return errors.New("invalid remote archive journal ID")
}
if err := s.archiveStore.Validate(j.URI); err != nil {
return err
}
item, err := s.m.pg.GetTaskArchive(j.ArchiveID)
if err != nil {
return err
}
switch j.Operation {
case "upload":
if item == nil || item.ArchivePath != j.URI {
if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil {
return err
}
}
if err := validateArchivePath(s.m.dir, j.LocalPath); err != nil {
return err
}
// An old failed upload may outlive a retry after switching back to
// local storage. The retry reuses this pathname: keep its committed
// package instead of treating it as the old upload's temporary copy.
if item == nil || item.ArchivePath != j.LocalPath {
if err := removeLocalArchivePackage(j.LocalPath); err != nil {
return err
}
}
case "delete":
if item == nil {
if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil {
return err
}
}
default:
return fmt.Errorf("unknown remote archive operation %q", j.Operation)
}
return os.Remove(name)
}
func (s *Server) deleteRemoteTaskArchive(job *pgdb.TaskArchive) error {
journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, Operation: "delete"})
if err != nil {
return err
}
if err := s.m.pg.DeleteTaskArchiveStub(job.ID); err != nil {
// A connection failure can leave the transaction's commit ambiguous.
// Recovery consults PostgreSQL before deciding whether to delete.
return err
}
if err := s.recoverRemoteArchiveJournal(journal); err != nil {
// The task is gone; the durable outbox retries the remaining object cleanup.
log.Printf("[task-archive] archive %d deleted; remote cleanup pending: %v", job.ID, err)
}
return nil
}
+419
View File
@@ -0,0 +1,419 @@
package server
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"github.com/Autumn-27/artex/archivestore"
pgdb "github.com/Autumn-27/artex/db"
)
func TestRemoteArchiveLifecycle(t *testing.T) {
if os.Getenv("ARTEX_PG_DSN") == "" {
t.Skip("requires isolated PostgreSQL")
}
t.Setenv("ARTEX_ARCHIVE_STORAGE", "local")
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
defer m.Close()
ctx, cancel := context.WithCancel(context.Background())
cancel()
s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
s.archiveWG.Wait()
s.ctx = context.Background()
var mu sync.Mutex
objects := map[string][]byte{}
corrupt, failDelete := false, false
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
defer mu.Unlock()
switch r.Method {
case "PUT":
objects[r.URL.Path], _ = io.ReadAll(r.Body)
w.Header().Set("ETag", `"test"`)
case "HEAD":
w.Header().Set("X-Amz-Server-Side-Encryption", "AES256")
w.Header().Set("ETag", `"test"`)
w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path])))
case "GET":
data := objects[r.URL.Path]
if corrupt && len(data) > 0 {
data = bytes.Clone(data)
data[0] ^= 0xff
}
var start, end int64
if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) || r.Header.Get("If-Match") != `"test"` {
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return
}
w.Header().Set("ETag", `"test"`)
w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10))
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
w.WriteHeader(http.StatusPartialContent)
_, _ = w.Write(data[start : end+1])
case "DELETE":
if failDelete {
w.WriteHeader(403)
return
}
delete(objects, r.URL.Path)
w.WriteHeader(204)
default:
w.WriteHeader(400)
}
}))
defer remote.Close()
s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
if err != nil {
t.Fatal(err)
}
task, err := m.CreateTask("S3 round trip", "preserve data", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
id, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }()
if err := m.pg.SetPaused(id, true); err != nil {
t.Fatal(err)
}
file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "important.txt")
if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(file, []byte("keep this content"), 0600); err != nil {
t.Fatal(err)
}
pgTask, err := m.pg.GetTask(id)
if err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exploration(pgTask.ExplorationID).AppendActivity(pgdb.Activity{Worker: "system", Kind: "system", Summary: "archive proof"}); err != nil {
t.Fatal(err)
}
job, err := m.pg.QueueTaskArchive(id)
if err != nil {
t.Fatal(err)
}
// A configured-but-invalid backend must fail before moving any source data.
s.archiveStoreErr = errors.New("invalid fixture configuration")
if err := s.runOneTaskArchiveJob(); !errors.Is(err, s.archiveStoreErr) {
t.Fatalf("configuration error was not returned: %v", err)
}
if got, err := os.ReadFile(file); err != nil || string(got) != "keep this content" {
t.Fatalf("bad configuration lost source: %v", err)
}
if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil {
t.Fatal("bad configuration lost task", err)
}
s.archiveStoreErr = nil
if _, err := m.pg.QueueTaskArchive(id); err != nil {
t.Fatal(err)
}
// A corrupt readback must roll back filesystem staging and leave PG intact.
mu.Lock()
corrupt = true
mu.Unlock()
if err := s.runOneTaskArchiveJob(); err == nil {
t.Fatal("expected integrity failure")
}
if got, _ := os.ReadFile(file); string(got) != "keep this content" {
t.Fatal("upload failure lost source")
}
if task, err := m.pg.GetTask(id); err != nil || task == nil {
t.Fatal("upload failure lost task")
}
mu.Lock()
corrupt = false
mu.Unlock()
if err := s.recoverRemoteArchiveJournals(); err != nil {
t.Fatal(err)
}
if _, err := m.pg.QueueTaskArchive(id); err != nil {
t.Fatal(err)
}
if err := s.runOneTaskArchiveJob(); err != nil {
t.Fatal(err)
}
job, err = m.pg.GetTaskArchive(job.ID)
if err != nil {
t.Fatal(err)
}
if job.State != pgdb.ArchiveReady || job.Storage != "s3" || !strings.HasPrefix(job.ArchivePath, "s3://") {
t.Fatalf("unexpected archive %+v", job)
}
if _, err := os.Stat(file); !os.IsNotExist(err) {
t.Fatal("workspace not released")
}
if _, err := os.Stat(taskArchivePath(m.dir, job.ID, task.ID)); !os.IsNotExist(err) {
t.Fatal("local package retained")
}
// Simulate the crash window after commit: cleanup keeps the remote object.
journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, LocalPath: taskArchivePath(m.dir, job.ID, task.ID), Operation: "upload"})
if err != nil {
t.Fatal(err)
}
if err := s.recoverRemoteArchiveJournal(journal); err != nil {
t.Fatal(err)
}
// A corrupted download must leave the archive and its remote package intact.
mu.Lock()
corrupt = true
mu.Unlock()
if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
t.Fatal(err)
}
if err := s.runOneTaskArchiveJob(); err == nil {
t.Fatal("expected corrupted restore to fail")
}
if _, err := os.Stat(file); !os.IsNotExist(err) {
t.Fatal("corrupted archive installed workspace files")
}
if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || restored {
t.Fatalf("corrupted archive restored database: %v %v", restored, err)
}
mu.Lock()
if len(objects) == 0 {
t.Error("corrupted restore deleted the only remote package")
}
corrupt = false
mu.Unlock()
// Restoration commits files and PG first. Failed remote deletion is retryable.
mu.Lock()
failDelete = true
mu.Unlock()
if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
t.Fatal(err)
}
if err := s.runOneTaskArchiveJob(); err == nil {
t.Fatal("expected remote delete failure")
}
if got, _ := os.ReadFile(file); string(got) != "keep this content" {
t.Fatal("restored content differs")
}
if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || !restored {
t.Fatalf("PG restore %v %v", restored, err)
}
mu.Lock()
failDelete = false
mu.Unlock()
if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
t.Fatal(err)
}
if err := s.runOneTaskArchiveJob(); err != nil {
t.Fatal(err)
}
if item, err := m.pg.GetTaskArchive(job.ID); err != nil || item != nil {
t.Fatal("restore did not consume stub")
}
var count int
if err := m.pg.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='archive proof'`, pgTask.ExplorationID).Scan(&count); err != nil || count != 1 {
t.Fatalf("activity restore count=%d err=%v", count, err)
}
// Re-archive, then crash/retry the deletion outbox after PG removed its stub.
job, err = m.pg.QueueTaskArchive(id)
if err != nil {
t.Fatal(err)
}
if err := s.runOneTaskArchiveJob(); err != nil {
t.Fatal(err)
}
mu.Lock()
failDelete = true
mu.Unlock()
if _, err := m.pg.QueueTaskArchiveDelete(job.ID); err != nil {
t.Fatal(err)
}
if err := s.runOneTaskArchiveJob(); err != nil {
t.Fatal(err)
}
mu.Lock()
failDelete = false
mu.Unlock()
if err := s.recoverRemoteArchiveJournals(); err != nil {
t.Fatal(err)
}
mu.Lock()
remaining := len(objects)
mu.Unlock()
if remaining != 0 {
t.Fatalf("left %d objects", remaining)
}
}
// Once PostgreSQL has committed an archive location and checksum, a subsequent
// queued operation must not make startup recovery resurrect the old workspace.
func TestTaskArchiveStageRecoveryAfterCommit(t *testing.T) {
if os.Getenv("ARTEX_PG_DSN") == "" {
t.Skip("requires isolated PostgreSQL")
}
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
defer m.Close()
task, err := m.CreateTask("remote stage recovery", "test", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() {
_, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID)
_ = m.pg.DeleteTask(taskID)
}()
if err := m.pg.SetPaused(taskID, true); err != nil {
t.Fatal(err)
}
job, err := m.pg.QueueTaskArchive(taskID)
if err != nil {
t.Fatal(err)
}
pgTask, err := m.pg.GetTask(taskID)
if err != nil {
t.Fatal(err)
}
for _, backend := range []string{"local", "s3"} {
for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} {
t.Run(backend+"/"+state, func(t *testing.T) {
committed := state != pgdb.ArchiveQueued
checksum := ""
if committed {
checksum = strings.Repeat("a", 64)
}
packagePath := taskArchivePath(m.dir, job.ID, task.ID)
if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(packagePath, []byte("verified package"), 0600); err != nil {
t.Fatal(err)
}
location := packagePath
if backend == "s3" {
location = "s3://test/tasks/package.tar.zst"
}
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, location, checksum); err != nil {
t.Fatal(err)
}
file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt")
if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil {
t.Fatal(err)
}
stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID)
if err != nil {
t.Fatal(err)
}
if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stage.root); !os.IsNotExist(err) {
t.Fatalf("staging cleanup incomplete: %v", err)
}
if backend == "local" && committed {
if data, err := os.ReadFile(packagePath); err != nil || string(data) != "verified package" {
t.Fatal("committed local package lost", err)
}
} else if !committed {
if _, err := os.Stat(packagePath); !os.IsNotExist(err) {
t.Fatal("uncommitted temporary package retained", err)
}
}
if committed {
if _, err := os.Stat(file); !os.IsNotExist(err) {
t.Fatalf("resurrected committed workspace: %v", err)
}
} else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" {
t.Fatalf("uncommitted archive lost source: %v", err)
}
})
}
}
}
// Failed remote cleanup can survive switching to local storage and retrying the
// same archive ID. Its journal must never remove the newly committed local copy.
func TestRemoteArchiveJournalPreservesLocalRetry(t *testing.T) {
if os.Getenv("ARTEX_PG_DSN") == "" {
t.Skip("requires isolated PostgreSQL")
}
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
defer m.Close()
task, err := m.CreateTask("local retry", "test", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() {
_, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID)
_ = m.pg.DeleteTask(taskID)
}()
if err := m.pg.SetPaused(taskID, true); err != nil {
t.Fatal(err)
}
job, err := m.pg.QueueTaskArchive(taskID)
if err != nil {
t.Fatal(err)
}
filename := taskArchivePath(m.dir, job.ID, task.ID)
if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte("committed local archive"), 0600); err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, pgdb.ArchiveReady, filename, strings.Repeat("a", 64)); err != nil {
t.Fatal(err)
}
var deletes atomic.Int32
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
deletes.Add(1)
w.WriteHeader(http.StatusNoContent)
}))
defer remote.Close()
store, err := archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
if err != nil {
t.Fatal(err)
}
s := &Server{ctx: context.Background(), m: m, archiveStore: store}
journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{
ArchiveID: job.ID, URI: store.URI(taskID, job.ID, strings.Repeat("b", 64)),
LocalPath: filename, Operation: "upload",
})
if err != nil {
t.Fatal(err)
}
if err := s.recoverRemoteArchiveJournal(journal); err != nil {
t.Fatal(err)
}
if data, err := os.ReadFile(filename); err != nil || string(data) != "committed local archive" {
t.Fatalf("old upload cleanup removed local retry: %v", err)
}
if deletes.Load() != 1 {
t.Fatal("uncommitted remote object was not removed")
}
if _, err := os.Stat(journal); !os.IsNotExist(err) {
t.Fatalf("finished journal remains: %v", err)
}
}
+175
View File
@@ -0,0 +1,175 @@
package server
import (
"context"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
)
func TestArchiveStoreLoadsCredentialsBeforeSecretScrub(t *testing.T) {
skipWithoutDSN(t)
// New scrubs all startup secrets; restore the test database/key environment
// afterward so the remaining package tests keep their isolated configuration.
for _, key := range secretEnv {
t.Setenv(key, os.Getenv(key))
}
var signed atomic.Bool
endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete {
t.Errorf("unexpected S3 operation: %s", r.Method)
}
signed.Store(strings.Contains(r.Header.Get("Authorization"), "Credential=archive-fixture-key/") &&
r.Header.Get("X-Amz-Security-Token") == "archive-fixture-session")
w.WriteHeader(http.StatusNoContent)
}))
defer endpoint.Close()
t.Setenv("ARTEX_ARCHIVE_STORAGE", "s3")
t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", endpoint.URL)
t.Setenv("ARTEX_ARCHIVE_S3_REGION", "us-east-1")
t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "fixture")
t.Setenv("ARTEX_ARCHIVE_S3_PREFIX", "archives")
t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true")
t.Setenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "archive-fixture-key")
t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "archive-fixture-secret")
t.Setenv("ARTEX_ARCHIVE_S3_SESSION_TOKEN", "archive-fixture-session")
t.Setenv("ARTEX_ARCHIVE_S3_SSE", "AES256")
t.Setenv("ARTEX_ARCHIVE_S3_KMS_KEY_ID", "")
// If initialization happens after the scrub, the SDK would silently select
// this different ambient identity instead of the dedicated override.
t.Setenv("AWS_ACCESS_KEY_ID", "ambient-fixture-key")
t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-fixture-secret")
t.Setenv("AWS_SESSION_TOKEN", "")
t.Setenv("AWS_PROFILE", "")
t.Setenv("AWS_DEFAULT_PROFILE", "")
t.Setenv("AWS_CA_BUNDLE", "")
t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "no-config"))
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "no-credentials"))
t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
defer m.Close()
ctx, cancel := context.WithCancel(context.Background())
cancel() // Exercise real startup without leaving archive/scheduler workers alive.
s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir(), WithSecretScrub())
s.archiveWG.Wait()
if s.archiveStoreErr != nil || s.archiveStore == nil {
t.Fatalf("S3 initialization failed: %v", s.archiveStoreErr)
}
for _, key := range []string{"ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_SESSION_TOKEN"} {
if _, present := os.LookupEnv(key); present {
t.Fatalf("%s remains in the child-process environment", key)
}
}
requestCtx, stop := context.WithTimeout(context.Background(), 3*time.Second)
defer stop()
if err := s.archiveStore.Delete(requestCtx, s.archiveStore.URI(1, 2, "startup-probe")); err != nil {
t.Fatal(err)
}
if !signed.Load() {
t.Fatal("archive client lost its dedicated credentials during startup scrub")
}
}
// The archive store may take its identity from the ambient AWS credential
// variables instead of the ARTEX_ARCHIVE_S3_* override. Those variables are as
// sensitive as the override: after startup no child process may inherit them,
// and the S3 client built before the scrub must keep signing with them.
func TestReview318AmbientAWSCredentialsScrubbedAfterArchiveInit(t *testing.T) {
skipWithoutDSN(t)
sh, err := exec.LookPath("sh")
if err != nil {
t.Skip("no sh in PATH")
}
awsNames := []string{
"AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN",
"AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
"AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN",
}
// New scrubs all startup secrets; t.Setenv restores them for the tests that follow.
for _, key := range secretEnv {
t.Setenv(key, os.Getenv(key))
}
for _, key := range awsNames {
t.Setenv(key, "probe-"+key)
}
var authorization, sessionToken atomic.Value
endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodHead {
t.Errorf("unexpected S3 operation: %s", r.Method)
}
authorization.Store(r.Header.Get("Authorization"))
sessionToken.Store(r.Header.Get("X-Amz-Security-Token"))
w.WriteHeader(http.StatusNotFound)
}))
defer endpoint.Close()
t.Setenv("ARTEX_ARCHIVE_STORAGE", "s3")
t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", endpoint.URL)
t.Setenv("ARTEX_ARCHIVE_S3_REGION", "us-east-1")
t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "fixture")
t.Setenv("ARTEX_ARCHIVE_S3_PREFIX", "archives")
t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true")
// No dedicated override: the SDK's default chain reads the AWS_* variables.
t.Setenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "")
t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "")
t.Setenv("ARTEX_ARCHIVE_S3_SESSION_TOKEN", "")
t.Setenv("ARTEX_ARCHIVE_S3_SSE", "AES256")
t.Setenv("ARTEX_ARCHIVE_S3_KMS_KEY_ID", "")
t.Setenv("AWS_ACCESS_KEY_ID", "ambient-fixture-key")
t.Setenv("AWS_SECRET_ACCESS_KEY", "ambient-fixture-secret")
t.Setenv("AWS_SESSION_TOKEN", "ambient-fixture-session")
t.Setenv("AWS_PROFILE", "")
t.Setenv("AWS_DEFAULT_PROFILE", "")
t.Setenv("AWS_CA_BUNDLE", "")
t.Setenv("AWS_CONFIG_FILE", filepath.Join(t.TempDir(), "no-config"))
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "no-credentials"))
t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
defer m.Close()
ctx, cancel := context.WithCancel(context.Background())
cancel() // Exercise real startup without leaving archive/scheduler workers alive.
s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir(), WithSecretScrub())
s.archiveWG.Wait()
if s.archiveStoreErr != nil || s.archiveStore == nil {
t.Fatalf("S3 initialization failed: %v", s.archiveStoreErr)
}
out, err := exec.Command(sh, "-c", "env").Output()
if err != nil {
t.Fatal(err)
}
env := "\n" + string(out)
for _, key := range awsNames {
if strings.Contains(env, "\n"+key+"=") {
t.Errorf("%s still reaches child processes after startup", key)
}
}
// HEAD is the first request of a download. The fixture answers 404, so the
// call fails; only the signature of the request matters here.
requestCtx, stop := context.WithTimeout(context.Background(), 5*time.Second)
defer stop()
checksum := strings.Repeat("a", 64)
err = s.archiveStore.Download(requestCtx, s.archiveStore.URI(1, 2, checksum), filepath.Join(t.TempDir(), "probe"), checksum, 1)
if err == nil {
t.Fatal("the fixture has no object: the download must fail")
}
got, _ := authorization.Load().(string)
if !strings.Contains(got, "Credential=ambient-fixture-key/") || !strings.Contains(got, "Signature=") {
t.Fatalf("archive client did not sign with the ambient key after the scrub: Authorization=%q (%v)", got, err)
}
if token, _ := sessionToken.Load().(string); token != "ambient-fixture-session" {
t.Fatalf("archive client lost the ambient session token after the scrub: %q", token)
}
}
+92 -19
View File
@@ -9,11 +9,13 @@ import (
"net/http"
"os"
"path/filepath"
"runtime/debug"
"strconv"
"strings"
"time"
"github.com/Autumn-27/artex/agent"
"github.com/Autumn-27/artex/archivestore"
pgdb "github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/traffic"
)
@@ -33,6 +35,19 @@ type archiveBatchItem struct {
Archive int64 `json:"archive_id,omitempty"`
}
// initTaskArchiveStore captures credentials before New scrubs the process environment.
func (s *Server) initTaskArchiveStore() {
if s.m == nil || s.m.pg == nil {
return
}
s.archiveStore, s.archiveStoreErr = archivestore.FromEnv()
if s.archiveStoreErr != nil {
log.Printf("[task-archive] storage configuration: %v", s.archiveStoreErr)
} else if s.archiveStore != nil {
log.Printf("[task-archive] new archives use S3 object storage with full SHA-256 readback verification")
}
}
func (s *Server) startTaskArchiveWorker() {
if s.m == nil || s.m.pg == nil {
return
@@ -88,22 +103,19 @@ func (s *Server) notifyTaskArchiveWorker() {
}
func (s *Server) runOneTaskArchiveJob() error {
if s.ctx.Err() != nil {
return s.ctx.Err()
}
// Retry deferred remote cleanup before claiming more work. Failed cleanup
// keeps its durable journal and does not block unrelated local archives.
if err := s.recoverRemoteArchiveJournals(); err != nil {
log.Printf("[task-archive] deferred object cleanup: %v", err)
}
job, err := s.m.pg.ClaimTaskArchiveJob(s.ctx)
if err != nil || job == nil {
return err
}
var runErr error
switch job.State {
case pgdb.Archiving:
runErr = s.archiveTask(job)
case pgdb.Restoring:
runErr = s.restoreTaskArchive(job)
case pgdb.Deleting:
runErr = s.deleteTaskArchive(job)
default:
runErr = fmt.Errorf("unknown claimed archive state %q", job.State)
}
if runErr != nil {
if runErr := s.runClaimedTaskArchiveJob(job); runErr != nil {
if failErr := s.m.pg.FailTaskArchiveJob(job.ID, job.State, runErr); failErr != nil {
return errors.Join(runErr, failErr)
}
@@ -114,7 +126,35 @@ func (s *Server) runOneTaskArchiveJob() error {
return nil
}
// runClaimedTaskArchiveJob runs one claimed job and turns a panic below it into
// an ordinary job failure. The worker goroutine has no other recover: a panic
// there (the S3 upload manager dereferences a nil UploadId when an endpoint
// answers CreateMultipartUpload without one) would end the whole server with the
// job half done. The job's deferred rollbacks have already run when the panic is
// recovered here, so the caller fails the job with its source data in place.
func (s *Server) runClaimedTaskArchiveJob(job *pgdb.TaskArchive) (runErr error) {
defer func() {
if r := recover(); r != nil {
log.Printf("[task-archive] archive job %d (%s) panicked: %v\n%s", job.ID, job.State, r, debug.Stack())
runErr = fmt.Errorf("archive worker panic: %v", r)
}
}()
switch job.State {
case pgdb.Archiving:
return s.archiveTask(job)
case pgdb.Restoring:
return s.restoreTaskArchive(job)
case pgdb.Deleting:
return s.deleteTaskArchive(job)
default:
return fmt.Errorf("unknown claimed archive state %q", job.State)
}
}
func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
if s.archiveStoreErr != nil {
return s.archiveStoreErr
}
taskID := strconv.FormatInt(job.TaskID, 10)
if !s.beginTaskDelete(taskID) {
return errors.New("task is already undergoing another archive or delete operation")
@@ -199,6 +239,21 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
_ = os.Remove(archivePath)
}
}()
storedPath := archivePath
var remoteJournal string
if s.archiveStore != nil {
storedPath = s.archiveStore.URI(job.TaskID, job.ID, checksum)
remoteJournal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{
ArchiveID: job.ID, URI: storedPath, LocalPath: archivePath, Operation: "upload",
})
if err != nil {
return err
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "upload_and_verify", 65)
if err := s.archiveStore.Upload(s.ctx, storedPath, archivePath, checksum); err != nil {
return err
}
}
var trafficStage *traffic.HostDeleteStage
if s.m.traffic != nil && len(snapshot.ExclusiveHosts) > 0 {
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_traffic", 68)
@@ -213,11 +268,18 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
}()
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_database", 78)
if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, archivePath, checksum, originalSize, compressedSize); err != nil {
if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, storedPath, checksum, originalSize, compressedSize); err != nil {
return err
}
committed = true
removePackage = false
removePackage = s.archiveStore != nil
if remoteJournal != "" {
// The journal also covers a crash after database commit but before local
// package removal. Keep it if cleanup fails, so the worker can retry.
if err := removeLocalArchivePackage(archivePath); err == nil {
_ = os.Remove(remoteJournal)
}
}
if trafficStage != nil {
if err := trafficStage.Commit(); err != nil {
warning := "archive completed, but cleaning the exclusive traffic hot store failed (the archive package can still restore it): " + err.Error()
@@ -239,7 +301,7 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
}
func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil {
if err := s.validateStoredArchive(job.ArchivePath); err != nil {
return err
}
// A prior attempt may have committed PostgreSQL and files, then failed only
@@ -247,7 +309,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
if restored, err := s.m.pg.IsTaskArchiveRestored(job.ID); err != nil {
return err
} else if restored {
if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) {
if err := s.removeStoredArchive(job.ArchivePath); err != nil {
return err
}
if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil {
@@ -266,7 +328,15 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
return err
}
defer func() { _ = os.RemoveAll(extracted) }()
if err := extractTaskArchivePackage(job.ArchivePath, job.SHA256, extracted); err != nil {
packagePath := job.ArchivePath
if strings.HasPrefix(packagePath, "s3://") {
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "download_and_verify", 12)
packagePath = filepath.Join(extracted, "download.tar.zst")
if err := s.archiveStore.Download(s.ctx, job.ArchivePath, packagePath, job.SHA256, job.CompressedSize); err != nil {
return err
}
}
if err := extractTaskArchivePackage(packagePath, job.SHA256, extracted); err != nil {
return err
}
manifestRaw, err := os.ReadFile(filepath.Join(extracted, "manifest.json"))
@@ -356,7 +426,7 @@ func (s *Server) restoreTaskArchivePayload(job *pgdb.TaskArchive, snapshot *pgdb
return err
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "consume_package", 95)
if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) {
if err := s.removeStoredArchive(job.ArchivePath); err != nil {
return err
}
if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil {
@@ -393,9 +463,12 @@ func (s *Server) loadRestoredTask(taskID int64) {
}
func (s *Server) deleteTaskArchive(job *pgdb.TaskArchive) error {
if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil {
if err := s.validateStoredArchive(job.ArchivePath); err != nil {
return err
}
if strings.HasPrefix(job.ArchivePath, "s3://") {
return s.deleteRemoteTaskArchive(job)
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "stage_package_delete", 35)
staged, moved, err := stageTaskArchivePackageDelete(job.ArchivePath, job.ID)
if err != nil {
+8 -1
View File
@@ -49,7 +49,14 @@ func TestTaskCredentialsFlow(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { s.m.PG().Exec(`DELETE FROM tasks WHERE id=$1`, id) })
t.Cleanup(func() {
_ = s.m.pg.DeleteTask(id)
// The first POST with an invalid credential entry returns 400 after
// creating a task/exploration row; remove any remaining fixture rows.
_, _ = s.m.PG().Exec(`DELETE FROM tasks WHERE description = 'credential flow test'`)
_, _ = s.m.PG().Exec(`DELETE FROM explorations WHERE description = 'credential flow test'`)
_, _ = s.m.PG().Exec(`DELETE FROM assets WHERE type = 'ip' AND ip = '127.0.0.1'`)
})
// List: both rows, user source, order stable.
resp = request("GET", fmt.Sprintf("/api/tasks/%s/credentials", created.ID), "")
+1 -1
View File
@@ -21,7 +21,7 @@ func TestReviveClearsBudgetEndReason(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { pg.Exec(`DELETE FROM tasks WHERE id=$1`, id) })
t.Cleanup(func() { _ = pg.DeleteTask(id) })
endReason := func() string {
t.Helper()
var r string
+9 -11
View File
@@ -193,20 +193,18 @@ func healthIDFor(prov llm.Provider, profileID int64) int64 {
return profileID
}
// current resolves the provider this role runs on, by precedence:
// Agent binding → task LLM profile chain → global/environment configuration.
// currentFor resolves the provider this role runs on, by precedence:
// Agent binding → task LLM profile chain → global/environment configuration,
// honoring a per-run profile pin set by the engine for per-intent rotation (#85).
// Binding wins over the task chain: a role explicitly assigned a model keeps running
// on that model; only when the binding is missing or fails to build does it fall back
// to the task chain, and to the global configuration when the chain is empty.
// The returned profile id is non-zero only on the task-chain path — streamTaskLLM uses it to decide whether a quota error should advance the task's failover state (binding/global paths leave the chain state untouched, preserving existing semantics).
func (r *taskLLMRuntime) current() (taskLLMSelection, error) {
return r.currentFor(context.Background())
}
// currentFor resolves like current, honoring a per-run profile pin set by the
// engine for per-intent rotation (#85). A pin is only a dispatch preference:
// the chain must still be in a ready state (an exhausted chain with no active
// head stays exhausted) and the pinned id must belong to the task's chain.
// A pin is only a dispatch preference: the chain must still be in a ready state
// (an exhausted chain with no active head stays exhausted) and the pinned id must
// belong to the task's chain. The returned profile id is non-zero only on the
// task-chain path — streamTaskLLM uses it to decide whether a quota error should
// advance the task's failover state (binding/global paths leave the chain state
// untouched, preserving existing semantics).
func (r *taskLLMRuntime) currentFor(ctx context.Context) (taskLLMSelection, error) {
taskNum, err := parseTaskID(r.taskID)
if err != nil {
+8 -2
View File
@@ -75,7 +75,10 @@ func TestRotationConcurrentProvidersAndPrecedence(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer m.DeleteTask(task.ID, DeleteTaskOptions{})
defer func() {
_, _ = m.pg.Exec(`DELETE FROM llm_usage WHERE task_id = $1`, task.ID)
_, _ = m.DeleteTask(task.ID, DeleteTaskOptions{})
}()
s := &Server{m: m, provByProfile: map[int64]*provEntry{}}
runtime := &taskLLMRuntime{s: s, taskID: task.ID, agentKey: "worker"}
done := make(chan error, 3)
@@ -343,7 +346,10 @@ func TestRunIntentRotatesPinnedProfileWithCachedExecCtx(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer m.DeleteTask(task.ID, DeleteTaskOptions{})
defer func() {
_, _ = m.pg.Exec(`DELETE FROM llm_usage WHERE task_id = $1`, task.ID)
_, _ = m.DeleteTask(task.ID, DeleteTaskOptions{})
}()
e := NewEngine(m)
// Warm the per-task cached execCtx before any worker intent runs.
+1 -1
View File
@@ -656,7 +656,7 @@ func TestProfileDeleteRestoresQuotaBlockedIntentWhenFallbackAvailable(t *testing
if err != nil {
db.SkipPGUnavailable(t, err)
}
defer m.Close()
t.Cleanup(func() { _ = m.Close() })
profileID, err := m.pg.SaveProfile(&db.LLMProfile{
Name: fmt.Sprintf("delete-quota-fallback-%d", time.Now().UnixNano()),
+4 -1
View File
@@ -22,7 +22,10 @@ func TestTaskOutcomeCard(t *testing.T) {
t.Fatal(err)
}
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() { _ = m.pg.DeleteTask(taskID) }()
defer func() {
_, _ = m.pg.DeleteFindingsByTask(taskID)
_ = m.pg.DeleteTask(taskID)
}()
// skill_usage has no foreign keys, so DeleteTask leaves these rows behind,
// and their misses would outrank TestSkillGapsReport's fixture in later runs.
defer func() { _, _ = m.pg.Exec(`DELETE FROM skill_usage WHERE task_id=$1`, taskID) }()
+1 -1
View File
@@ -24,7 +24,7 @@ func TestPauseShiftsTaskDeadlineOnResume(t *testing.T) {
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { pg.Exec(`DELETE FROM tasks WHERE id=$1`, id) })
t.Cleanup(func() { _ = pg.DeleteTask(id) })
// A running task with a stamped deadline (first_run_at + 1h budget).
if _, err := m.StampTaskFirstRun(task.ID); err != nil {
+1 -1
View File
@@ -42,7 +42,7 @@ func (s *Server) resolutionFromProfile(p *db.LLMProfile, source string) taskLLMR
return result
}
// resolveTaskRoleLLM mirrors taskLLMRuntime.current without creating a provider:
// resolveTaskRoleLLM mirrors taskLLMRuntime.currentFor without creating a provider:
// role Agent binding -> explicit task chain -> active/global environment config.
// Database failures are returned rather than represented as an unavailable model.
func (s *Server) resolveTaskRoleLLM(t *Task, agentKey string) (taskLLMResolution, error) {
+1 -1
View File
@@ -91,7 +91,7 @@ func TestConversationPatchReturnsPinState(t *testing.T) {
if err != nil {
db.SkipPGUnavailable(t, err)
}
defer m.Close()
t.Cleanup(func() { _ = m.Close() })
ctx, cancel := context.WithCancel(context.Background())
s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
defer stopTestServer(t, s, cancel)
+8
View File
@@ -33,6 +33,14 @@ func meterTool(t actool.CoreTool, recorder toolUsageRecorder, toolKey, agentKey
return &meteredTool{CoreTool: t, recorder: recorder, toolKey: toolKey, agentKey: agentKey, ri: ri}
}
// AcceptsRawInput forwards the resolved tool's opt-out of the harness schema
// check; the embedded interface alone would hide it from the harness's type
// assertion.
func (m *meteredTool) AcceptsRawInput() bool {
r, ok := m.CoreTool.(interface{ AcceptsRawInput() bool })
return ok && r.AcceptsRawInput()
}
func (m *meteredTool) Call(ctx context.Context, input json.RawMessage, tc *actool.ToolContext) (actool.Result, error) {
if err := m.recorder.InsertToolUsage(&db.ToolUsage{
ToolKey: m.toolKey,
+13 -7
View File
@@ -73,12 +73,19 @@ func TestWireTools(t *testing.T) {
// Edit description + add a default to a scalar param, then confirm the resolved
// worker tool reflects both. Restore from code default on cleanup.
t.Cleanup(func() { _ = pg.UpsertToolForce(rf.Key, rf.Description, rf.Schema, mustJSON(rf.Agents)) })
// Facts-only contract: the fact fields, and so the
// editable scalar annotations, live on the facts[] items.
var schema map[string]any
_ = json.Unmarshal(rf.Schema, &schema)
if props, ok := schema["properties"].(map[string]any); ok {
if conf, ok := props["confidence"].(map[string]any); ok {
conf["default"] = "inferred"
}
itemProps := func(sc map[string]any) map[string]any {
props, _ := sc["properties"].(map[string]any)
facts, _ := props["facts"].(map[string]any)
items, _ := facts["items"].(map[string]any)
ip, _ := items["properties"].(map[string]any)
return ip
}
if conf, ok := itemProps(schema)["confidence"].(map[string]any); ok {
conf["default"] = "inferred"
}
edited := mustJSON(schema)
if err := pg.UpdateTool(rf.Key, "EDITED DESC", edited, mustJSON([]string{"worker"}), true); err != nil {
@@ -96,9 +103,8 @@ func TestWireTools(t *testing.T) {
// Default injection: omit confidence → handler input should carry the default.
// We can't run the real handler (nil store), but InputSchema must show the default.
sc := got.InputSchema()
props := sc["properties"].(map[string]any)
conf := props["confidence"].(map[string]any)
if conf["default"] != "inferred" {
conf, _ := itemProps(sc)["confidence"].(map[string]any)
if conf == nil || conf["default"] != "inferred" {
t.Errorf("confidence.default = %v, want inferred", conf["default"])
}
}
+1
View File
@@ -269,6 +269,7 @@ VALUES(?,?, 'HTTP/1.1 200 OK', ?, ?)`, id, "GET "+path+" HTTP/1.1\nHost: "+host,
}
tidV1, _ := strconv.ParseInt(taskV1.ID, 10, 64)
t.Cleanup(func() {
_, _ = s.m.pg.DeleteFindingsByTask(tidV1)
_, _ = s.m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, tidV1)
_ = s.m.pg.DeleteTask(tidV1)
})
+48
View File
@@ -0,0 +1,48 @@
package server
import (
"runtime"
"testing"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/traffic"
)
func TestManagerTransparentTCPOptInFlag(t *testing.T) {
// 1. Flag off (default): Manager starts NO transparent TCP listener.
t.Setenv(traffic.TransparentTCPEnabledEnv, "")
t.Setenv(traffic.TransparentTCPAddrEnv, "")
mOff, err := NewManager(t.TempDir(), "127.0.0.1:0")
if err != nil {
db.SkipPGUnavailable(t, err)
}
defer mOff.Close()
if mOff.traffic == nil {
t.Fatal("expected traffic recorder to be initialized")
}
if got := mOff.traffic.TransparentTCPAddr(); got != "" {
t.Fatalf("TransparentTCPAddr with flag off = %q, want empty", got)
}
// 2. Flag on (ARTEX_TRANSPARENT_TCP_ADDR=127.0.0.1:0): starts on Linux,
// stays cleanly disabled on non-Linux without crashing NewManager.
t.Setenv(traffic.TransparentTCPAddrEnv, "127.0.0.1:0")
mOn, err := NewManager(t.TempDir(), "127.0.0.1:0")
if err != nil {
db.SkipPGUnavailable(t, err)
}
defer mOn.Close()
gotAddr := mOn.traffic.TransparentTCPAddr()
if runtime.GOOS == "linux" {
if gotAddr == "" {
t.Fatal("expected transparent TCP listener on Linux when ARTEX_TRANSPARENT_TCP_ADDR is set")
}
} else {
if gotAddr != "" {
t.Fatalf("expected transparent TCP listener disabled on %s, got %q", runtime.GOOS, gotAddr)
}
}
}
-10
View File
@@ -69,16 +69,6 @@ func workspacePathDenied(realRoot, realPath string) bool {
return false
}
// wsResolve maps a user-supplied relative path to an absolute path INSIDE the work
// dir. It returns ok=false if the path would escape the root (including via symlinks).
func (s *Server) wsResolve(rel string) (string, bool) {
cp, ok := s.wsResolveConfined(rel)
if !ok {
return "", false
}
return cp.realPath, true
}
// wsRel renders an absolute path back as a workspace-relative path (forward slashes).
func (s *Server) wsRel(abs string) string {
base, err := resolveRealRoot(s.m.dir)
+85 -85
View File
@@ -13,7 +13,7 @@
},
"node_modules/@puppeteer/browsers": {
"version": "2.6.1",
"resolved": "https://registry.npmmirror.com/@puppeteer/browsers/-/browsers-2.6.1.tgz",
"resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.6.1.tgz",
"integrity": "sha512-aBSREisdsGH890S2rQqK82qmQYU3uFpSH8wcZWHgHzl3LfzsxAKbLNiAG9mO8v1Y0UICBeClICxPJvyr0rcuxg==",
"license": "Apache-2.0",
"dependencies": {
@@ -35,13 +35,13 @@
},
"node_modules/@tootallnate/quickjs-emscripten": {
"version": "0.23.0",
"resolved": "https://registry.npmmirror.com/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz",
"resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz",
"integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==",
"license": "MIT"
},
"node_modules/@types/node": {
"version": "26.0.1",
"resolved": "https://registry.npmmirror.com/@types/node/-/node-26.0.1.tgz",
"resolved": "https://registry.npmjs.org/@types/node/-/node-26.0.1.tgz",
"integrity": "sha512-fc3KiUoBt6kie0N9bIW3E47vZsuaMf0PM2AaUpLCLT0s/LvX1nxAim6Fc049cNxODPpGm6qRAuUOB86SkRuPQw==",
"license": "MIT",
"optional": true,
@@ -51,7 +51,7 @@
},
"node_modules/@types/yauzl": {
"version": "2.10.3",
"resolved": "https://registry.npmmirror.com/@types/yauzl/-/yauzl-2.10.3.tgz",
"resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz",
"integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==",
"license": "MIT",
"optional": true,
@@ -61,7 +61,7 @@
},
"node_modules/agent-base": {
"version": "7.1.4",
"resolved": "https://registry.npmmirror.com/agent-base/-/agent-base-7.1.4.tgz",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz",
"integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==",
"license": "MIT",
"engines": {
@@ -70,7 +70,7 @@
},
"node_modules/ansi-regex": {
"version": "5.0.1",
"resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-5.0.1.tgz",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
"integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
"license": "MIT",
"engines": {
@@ -79,7 +79,7 @@
},
"node_modules/ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-4.3.0.tgz",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"license": "MIT",
"dependencies": {
@@ -94,7 +94,7 @@
},
"node_modules/ast-types": {
"version": "0.13.4",
"resolved": "https://registry.npmmirror.com/ast-types/-/ast-types-0.13.4.tgz",
"resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz",
"integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==",
"license": "MIT",
"dependencies": {
@@ -106,7 +106,7 @@
},
"node_modules/b4a": {
"version": "1.8.1",
"resolved": "https://registry.npmmirror.com/b4a/-/b4a-1.8.1.tgz",
"resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.1.tgz",
"integrity": "sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==",
"license": "Apache-2.0",
"peerDependencies": {
@@ -120,7 +120,7 @@
},
"node_modules/bare-events": {
"version": "2.9.1",
"resolved": "https://registry.npmmirror.com/bare-events/-/bare-events-2.9.1.tgz",
"resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.9.1.tgz",
"integrity": "sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==",
"license": "Apache-2.0",
"peerDependencies": {
@@ -134,7 +134,7 @@
},
"node_modules/bare-fs": {
"version": "4.7.2",
"resolved": "https://registry.npmmirror.com/bare-fs/-/bare-fs-4.7.2.tgz",
"resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.7.2.tgz",
"integrity": "sha512-aTvMFUWkBmjzKtEQMDGGDNF8bkfpD5N1b/FCwt7A3wrU4t1o/e/85Wzkluh6JlODCjqVESYCkQCdTXqZ9G7VFg==",
"license": "Apache-2.0",
"dependencies": {
@@ -158,7 +158,7 @@
},
"node_modules/bare-os": {
"version": "3.9.3",
"resolved": "https://registry.npmmirror.com/bare-os/-/bare-os-3.9.3.tgz",
"resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.9.3.tgz",
"integrity": "sha512-fF4Q7QsyKVF5Rj0qvI8BgUNjqzC2JvQlpTaPLjVJVxYVUX5Zr9un+y3w1HmA4nNKdFmRBT8z/WmrjvXzXVerKQ==",
"license": "Apache-2.0",
"engines": {
@@ -167,7 +167,7 @@
},
"node_modules/bare-path": {
"version": "3.0.1",
"resolved": "https://registry.npmmirror.com/bare-path/-/bare-path-3.0.1.tgz",
"resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.1.tgz",
"integrity": "sha512-ghj2DSK/2e99a1anTVPCV4m4YIYtrbXhfM7V3D7XZLOTsybnYyaJloymGqssQc8l/or0UoDyRtNQkmkEF/ysgQ==",
"license": "Apache-2.0",
"dependencies": {
@@ -176,7 +176,7 @@
},
"node_modules/bare-stream": {
"version": "2.13.3",
"resolved": "https://registry.npmmirror.com/bare-stream/-/bare-stream-2.13.3.tgz",
"resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.13.3.tgz",
"integrity": "sha512-Kc+brLqvEqGkjyfiwJmImAOqLZL7OsoLKuavx+hJjgVV3nLTOjloJyPMFxjUPerGGHrNH0fLU06jjykMLWrERQ==",
"license": "Apache-2.0",
"dependencies": {
@@ -203,7 +203,7 @@
},
"node_modules/bare-url": {
"version": "2.4.5",
"resolved": "https://registry.npmmirror.com/bare-url/-/bare-url-2.4.5.tgz",
"resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.4.5.tgz",
"integrity": "sha512-K+y9xF1tN+CdPu4qWwr0QiK1Al07eFPGYK5M2pDXcmHdMdgC/tT/bpmMe1hrmRHaidKLkXrC+cRNYf3XVDUhSQ==",
"license": "Apache-2.0",
"dependencies": {
@@ -212,7 +212,7 @@
},
"node_modules/base64-js": {
"version": "1.5.1",
"resolved": "https://registry.npmmirror.com/base64-js/-/base64-js-1.5.1.tgz",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
"funding": [
{
@@ -232,7 +232,7 @@
},
"node_modules/basic-ftp": {
"version": "5.3.1",
"resolved": "https://registry.npmmirror.com/basic-ftp/-/basic-ftp-5.3.1.tgz",
"resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.3.1.tgz",
"integrity": "sha512-bopVNp6ugyA150DDuZfPFdt1KZ5a94ZDiwX4hMgZDzF+GttD80lEy8kj98kbyhLXnPvhtIo93mdnLIjpCAeeOw==",
"license": "MIT",
"engines": {
@@ -241,7 +241,7 @@
},
"node_modules/buffer": {
"version": "5.7.1",
"resolved": "https://registry.npmmirror.com/buffer/-/buffer-5.7.1.tgz",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
"funding": [
{
@@ -265,7 +265,7 @@
},
"node_modules/buffer-crc32": {
"version": "0.2.13",
"resolved": "https://registry.npmmirror.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
"integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==",
"license": "MIT",
"engines": {
@@ -274,7 +274,7 @@
},
"node_modules/chromium-bidi": {
"version": "0.11.0",
"resolved": "https://registry.npmmirror.com/chromium-bidi/-/chromium-bidi-0.11.0.tgz",
"resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.11.0.tgz",
"integrity": "sha512-6CJWHkNRoyZyjV9Rwv2lYONZf1Xm0IuDyNq97nwSsxxP3wf5Bwy15K5rOvVKMtJ127jJBmxFUanSAOjgFRxgrA==",
"license": "Apache-2.0",
"dependencies": {
@@ -287,7 +287,7 @@
},
"node_modules/cliui": {
"version": "8.0.1",
"resolved": "https://registry.npmmirror.com/cliui/-/cliui-8.0.1.tgz",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
"integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==",
"license": "ISC",
"dependencies": {
@@ -301,7 +301,7 @@
},
"node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-2.0.1.tgz",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"license": "MIT",
"dependencies": {
@@ -313,13 +313,13 @@
},
"node_modules/color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.4.tgz",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
"license": "MIT"
},
"node_modules/data-uri-to-buffer": {
"version": "6.0.2",
"resolved": "https://registry.npmmirror.com/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz",
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz",
"integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==",
"license": "MIT",
"engines": {
@@ -328,7 +328,7 @@
},
"node_modules/debug": {
"version": "4.4.3",
"resolved": "https://registry.npmmirror.com/debug/-/debug-4.4.3.tgz",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
"license": "MIT",
"dependencies": {
@@ -345,7 +345,7 @@
},
"node_modules/degenerator": {
"version": "5.0.1",
"resolved": "https://registry.npmmirror.com/degenerator/-/degenerator-5.0.1.tgz",
"resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz",
"integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==",
"license": "MIT",
"dependencies": {
@@ -359,19 +359,19 @@
},
"node_modules/devtools-protocol": {
"version": "0.0.1367902",
"resolved": "https://registry.npmmirror.com/devtools-protocol/-/devtools-protocol-0.0.1367902.tgz",
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1367902.tgz",
"integrity": "sha512-XxtPuC3PGakY6PD7dG66/o8KwJ/LkH2/EKe19Dcw58w53dv4/vSQEkn/SzuyhHE2q4zPgCkxQBxus3VV4ql+Pg==",
"license": "BSD-3-Clause"
},
"node_modules/emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-8.0.0.tgz",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
"license": "MIT"
},
"node_modules/end-of-stream": {
"version": "1.4.5",
"resolved": "https://registry.npmmirror.com/end-of-stream/-/end-of-stream-1.4.5.tgz",
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz",
"integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==",
"license": "MIT",
"dependencies": {
@@ -380,7 +380,7 @@
},
"node_modules/escalade": {
"version": "3.2.0",
"resolved": "https://registry.npmmirror.com/escalade/-/escalade-3.2.0.tgz",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
"integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==",
"license": "MIT",
"engines": {
@@ -389,7 +389,7 @@
},
"node_modules/escodegen": {
"version": "2.1.0",
"resolved": "https://registry.npmmirror.com/escodegen/-/escodegen-2.1.0.tgz",
"resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz",
"integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==",
"license": "BSD-2-Clause",
"dependencies": {
@@ -410,7 +410,7 @@
},
"node_modules/esprima": {
"version": "4.0.1",
"resolved": "https://registry.npmmirror.com/esprima/-/esprima-4.0.1.tgz",
"resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
"integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
"license": "BSD-2-Clause",
"bin": {
@@ -423,7 +423,7 @@
},
"node_modules/estraverse": {
"version": "5.3.0",
"resolved": "https://registry.npmmirror.com/estraverse/-/estraverse-5.3.0.tgz",
"resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
"integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
"license": "BSD-2-Clause",
"engines": {
@@ -432,7 +432,7 @@
},
"node_modules/esutils": {
"version": "2.0.3",
"resolved": "https://registry.npmmirror.com/esutils/-/esutils-2.0.3.tgz",
"resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
"integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
"license": "BSD-2-Clause",
"engines": {
@@ -441,7 +441,7 @@
},
"node_modules/events-universal": {
"version": "1.0.1",
"resolved": "https://registry.npmmirror.com/events-universal/-/events-universal-1.0.1.tgz",
"resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz",
"integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==",
"license": "Apache-2.0",
"dependencies": {
@@ -450,7 +450,7 @@
},
"node_modules/extract-zip": {
"version": "2.0.1",
"resolved": "https://registry.npmmirror.com/extract-zip/-/extract-zip-2.0.1.tgz",
"resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz",
"integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==",
"license": "BSD-2-Clause",
"dependencies": {
@@ -470,13 +470,13 @@
},
"node_modules/fast-fifo": {
"version": "1.3.2",
"resolved": "https://registry.npmmirror.com/fast-fifo/-/fast-fifo-1.3.2.tgz",
"resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz",
"integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==",
"license": "MIT"
},
"node_modules/fd-slicer": {
"version": "1.1.0",
"resolved": "https://registry.npmmirror.com/fd-slicer/-/fd-slicer-1.1.0.tgz",
"resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz",
"integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==",
"license": "MIT",
"dependencies": {
@@ -485,7 +485,7 @@
},
"node_modules/get-caller-file": {
"version": "2.0.5",
"resolved": "https://registry.npmmirror.com/get-caller-file/-/get-caller-file-2.0.5.tgz",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
"license": "ISC",
"engines": {
@@ -494,7 +494,7 @@
},
"node_modules/get-stream": {
"version": "5.2.0",
"resolved": "https://registry.npmmirror.com/get-stream/-/get-stream-5.2.0.tgz",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
"integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
"license": "MIT",
"dependencies": {
@@ -509,7 +509,7 @@
},
"node_modules/get-uri": {
"version": "6.0.5",
"resolved": "https://registry.npmmirror.com/get-uri/-/get-uri-6.0.5.tgz",
"resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz",
"integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==",
"license": "MIT",
"dependencies": {
@@ -523,7 +523,7 @@
},
"node_modules/http-proxy-agent": {
"version": "7.0.2",
"resolved": "https://registry.npmmirror.com/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz",
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz",
"integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==",
"license": "MIT",
"dependencies": {
@@ -536,7 +536,7 @@
},
"node_modules/https-proxy-agent": {
"version": "7.0.6",
"resolved": "https://registry.npmmirror.com/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz",
"integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==",
"license": "MIT",
"dependencies": {
@@ -549,7 +549,7 @@
},
"node_modules/ieee754": {
"version": "1.2.1",
"resolved": "https://registry.npmmirror.com/ieee754/-/ieee754-1.2.1.tgz",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
"integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
"funding": [
{
@@ -569,7 +569,7 @@
},
"node_modules/ip-address": {
"version": "10.2.0",
"resolved": "https://registry.npmmirror.com/ip-address/-/ip-address-10.2.0.tgz",
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz",
"integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==",
"license": "MIT",
"engines": {
@@ -578,7 +578,7 @@
},
"node_modules/is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmmirror.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
"license": "MIT",
"engines": {
@@ -587,7 +587,7 @@
},
"node_modules/lru-cache": {
"version": "7.18.3",
"resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-7.18.3.tgz",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
"integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
"license": "ISC",
"engines": {
@@ -596,19 +596,19 @@
},
"node_modules/mitt": {
"version": "3.0.1",
"resolved": "https://registry.npmmirror.com/mitt/-/mitt-3.0.1.tgz",
"resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz",
"integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==",
"license": "MIT"
},
"node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"license": "MIT"
},
"node_modules/netmask": {
"version": "2.1.1",
"resolved": "https://registry.npmmirror.com/netmask/-/netmask-2.1.1.tgz",
"resolved": "https://registry.npmjs.org/netmask/-/netmask-2.1.1.tgz",
"integrity": "sha512-eonl3sLUha+S1GzTPxychyhnUzKyeQkZ7jLjKrBagJgPla13F+uQ71HgpFefyHgqrjEbCPkDArxYsjY8/+gLKA==",
"license": "MIT",
"engines": {
@@ -617,7 +617,7 @@
},
"node_modules/once": {
"version": "1.4.0",
"resolved": "https://registry.npmmirror.com/once/-/once-1.4.0.tgz",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
"license": "ISC",
"dependencies": {
@@ -626,7 +626,7 @@
},
"node_modules/pac-proxy-agent": {
"version": "7.2.0",
"resolved": "https://registry.npmmirror.com/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz",
"resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz",
"integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==",
"license": "MIT",
"dependencies": {
@@ -645,7 +645,7 @@
},
"node_modules/pac-resolver": {
"version": "7.0.1",
"resolved": "https://registry.npmmirror.com/pac-resolver/-/pac-resolver-7.0.1.tgz",
"resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz",
"integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==",
"license": "MIT",
"dependencies": {
@@ -658,13 +658,13 @@
},
"node_modules/pend": {
"version": "1.2.0",
"resolved": "https://registry.npmmirror.com/pend/-/pend-1.2.0.tgz",
"resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
"integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==",
"license": "MIT"
},
"node_modules/progress": {
"version": "2.0.3",
"resolved": "https://registry.npmmirror.com/progress/-/progress-2.0.3.tgz",
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
"license": "MIT",
"engines": {
@@ -673,7 +673,7 @@
},
"node_modules/proxy-agent": {
"version": "6.5.0",
"resolved": "https://registry.npmmirror.com/proxy-agent/-/proxy-agent-6.5.0.tgz",
"resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz",
"integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==",
"license": "MIT",
"dependencies": {
@@ -692,13 +692,13 @@
},
"node_modules/proxy-from-env": {
"version": "1.1.0",
"resolved": "https://registry.npmmirror.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==",
"license": "MIT"
},
"node_modules/pump": {
"version": "3.0.4",
"resolved": "https://registry.npmmirror.com/pump/-/pump-3.0.4.tgz",
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz",
"integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==",
"license": "MIT",
"dependencies": {
@@ -708,7 +708,7 @@
},
"node_modules/puppeteer-core": {
"version": "23.11.1",
"resolved": "https://registry.npmmirror.com/puppeteer-core/-/puppeteer-core-23.11.1.tgz",
"resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-23.11.1.tgz",
"integrity": "sha512-3HZ2/7hdDKZvZQ7dhhITOUg4/wOrDRjyK2ZBllRB0ZCOi9u0cwq1ACHDjBB+nX+7+kltHjQvBRdeY7+W0T+7Gg==",
"license": "Apache-2.0",
"dependencies": {
@@ -725,7 +725,7 @@
},
"node_modules/require-directory": {
"version": "2.1.1",
"resolved": "https://registry.npmmirror.com/require-directory/-/require-directory-2.1.1.tgz",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
"integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==",
"license": "MIT",
"engines": {
@@ -734,7 +734,7 @@
},
"node_modules/semver": {
"version": "7.8.5",
"resolved": "https://registry.npmmirror.com/semver/-/semver-7.8.5.tgz",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz",
"integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==",
"license": "ISC",
"bin": {
@@ -746,7 +746,7 @@
},
"node_modules/smart-buffer": {
"version": "4.2.0",
"resolved": "https://registry.npmmirror.com/smart-buffer/-/smart-buffer-4.2.0.tgz",
"resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
"integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
"license": "MIT",
"engines": {
@@ -756,7 +756,7 @@
},
"node_modules/socks": {
"version": "2.8.9",
"resolved": "https://registry.npmmirror.com/socks/-/socks-2.8.9.tgz",
"resolved": "https://registry.npmjs.org/socks/-/socks-2.8.9.tgz",
"integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==",
"license": "MIT",
"dependencies": {
@@ -770,7 +770,7 @@
},
"node_modules/socks-proxy-agent": {
"version": "8.0.5",
"resolved": "https://registry.npmmirror.com/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz",
"resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz",
"integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==",
"license": "MIT",
"dependencies": {
@@ -784,7 +784,7 @@
},
"node_modules/source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
"license": "BSD-3-Clause",
"optional": true,
@@ -794,7 +794,7 @@
},
"node_modules/streamx": {
"version": "2.28.0",
"resolved": "https://registry.npmmirror.com/streamx/-/streamx-2.28.0.tgz",
"resolved": "https://registry.npmjs.org/streamx/-/streamx-2.28.0.tgz",
"integrity": "sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==",
"license": "MIT",
"dependencies": {
@@ -805,7 +805,7 @@
},
"node_modules/string-width": {
"version": "4.2.3",
"resolved": "https://registry.npmmirror.com/string-width/-/string-width-4.2.3.tgz",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
"license": "MIT",
"dependencies": {
@@ -819,7 +819,7 @@
},
"node_modules/strip-ansi": {
"version": "6.0.1",
"resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-6.0.1.tgz",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
"license": "MIT",
"dependencies": {
@@ -831,7 +831,7 @@
},
"node_modules/tar-fs": {
"version": "3.1.3",
"resolved": "https://registry.npmmirror.com/tar-fs/-/tar-fs-3.1.3.tgz",
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.3.tgz",
"integrity": "sha512-/hU4AXnIdZu+Gvl1pk0oI5f5HxWsCJRtY2aFaJdk9VvyL48DWU6iU5WAIPG+wIi1YvWA6eTJvIviP/tMAZZNwQ==",
"license": "MIT",
"dependencies": {
@@ -845,7 +845,7 @@
},
"node_modules/tar-stream": {
"version": "3.2.0",
"resolved": "https://registry.npmmirror.com/tar-stream/-/tar-stream-3.2.0.tgz",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.2.0.tgz",
"integrity": "sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==",
"license": "MIT",
"dependencies": {
@@ -857,7 +857,7 @@
},
"node_modules/teex": {
"version": "1.0.1",
"resolved": "https://registry.npmmirror.com/teex/-/teex-1.0.1.tgz",
"resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz",
"integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==",
"license": "MIT",
"dependencies": {
@@ -866,7 +866,7 @@
},
"node_modules/text-decoder": {
"version": "1.2.7",
"resolved": "https://registry.npmmirror.com/text-decoder/-/text-decoder-1.2.7.tgz",
"resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz",
"integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==",
"license": "Apache-2.0",
"dependencies": {
@@ -875,25 +875,25 @@
},
"node_modules/through": {
"version": "2.3.8",
"resolved": "https://registry.npmmirror.com/through/-/through-2.3.8.tgz",
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
"integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==",
"license": "MIT"
},
"node_modules/tslib": {
"version": "2.8.1",
"resolved": "https://registry.npmmirror.com/tslib/-/tslib-2.8.1.tgz",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
"license": "0BSD"
},
"node_modules/typed-query-selector": {
"version": "2.12.2",
"resolved": "https://registry.npmmirror.com/typed-query-selector/-/typed-query-selector-2.12.2.tgz",
"resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.2.tgz",
"integrity": "sha512-EOPFbyIub4ngnEdqi2yOcNeDLaX/0jcE1JoAXQDDMIthap7FoN795lc/SHfIq2d416VufXpM8z/lD+WRm2gfOQ==",
"license": "MIT"
},
"node_modules/unbzip2-stream": {
"version": "1.4.3",
"resolved": "https://registry.npmmirror.com/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz",
"resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz",
"integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==",
"license": "MIT",
"dependencies": {
@@ -903,14 +903,14 @@
},
"node_modules/undici-types": {
"version": "8.3.0",
"resolved": "https://registry.npmmirror.com/undici-types/-/undici-types-8.3.0.tgz",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz",
"integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==",
"license": "MIT",
"optional": true
},
"node_modules/wrap-ansi": {
"version": "7.0.0",
"resolved": "https://registry.npmmirror.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
"license": "MIT",
"dependencies": {
@@ -927,13 +927,13 @@
},
"node_modules/wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmmirror.com/wrappy/-/wrappy-1.0.2.tgz",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
"license": "ISC"
},
"node_modules/ws": {
"version": "8.21.0",
"resolved": "https://registry.npmmirror.com/ws/-/ws-8.21.0.tgz",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz",
"integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==",
"license": "MIT",
"engines": {
@@ -954,7 +954,7 @@
},
"node_modules/y18n": {
"version": "5.0.8",
"resolved": "https://registry.npmmirror.com/y18n/-/y18n-5.0.8.tgz",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
"integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
"license": "ISC",
"engines": {
@@ -963,7 +963,7 @@
},
"node_modules/yargs": {
"version": "17.7.3",
"resolved": "https://registry.npmmirror.com/yargs/-/yargs-17.7.3.tgz",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz",
"integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==",
"license": "MIT",
"dependencies": {
@@ -981,7 +981,7 @@
},
"node_modules/yargs-parser": {
"version": "21.1.1",
"resolved": "https://registry.npmmirror.com/yargs-parser/-/yargs-parser-21.1.1.tgz",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
"integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
"license": "ISC",
"engines": {
@@ -990,7 +990,7 @@
},
"node_modules/yauzl": {
"version": "2.10.0",
"resolved": "https://registry.npmmirror.com/yauzl/-/yauzl-2.10.0.tgz",
"resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz",
"integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==",
"license": "MIT",
"dependencies": {
@@ -1000,7 +1000,7 @@
},
"node_modules/zod": {
"version": "3.23.8",
"resolved": "https://registry.npmmirror.com/zod/-/zod-3.23.8.tgz",
"resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz",
"integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==",
"license": "MIT",
"funding": {
+296
View File
@@ -0,0 +1,296 @@
package traffic
import (
"crypto/sha256"
"database/sql"
"encoding/hex"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"time"
)
// RawTCPStream is one recorded raw TCP connection relayed by the opt-in SOCKS5
// listener. It never creates an HTTP exchanges row.
type RawTCPStream struct {
ID string `json:"id"`
Protocol string `json:"protocol"`
Host string `json:"host"`
Port int `json:"port"`
StartedAt int64 `json:"started_at"` // unix milliseconds
EndedAt int64 `json:"ended_at"` // unix milliseconds
DurationMs int64 `json:"duration_ms"` // non-negative duration in milliseconds
C2SBytes int64 `json:"c2s_bytes"`
S2CBytes int64 `json:"s2c_bytes"`
Truncated bool `json:"truncated"`
C2SPreview []byte `json:"c2s_preview,omitempty"`
C2SBlob string `json:"c2s_blob,omitempty"`
S2CPreview []byte `json:"s2c_preview,omitempty"`
S2CBlob string `json:"s2c_blob,omitempty"`
}
// RawTCPFilter narrows ListRawTCPStreams queries.
type RawTCPFilter struct {
Host string
Port int
Limit int
Offset int
}
func (t *Traffic) newRawTCPStreamID(started time.Time) string {
n := t.seq.Add(1)
return fmt.Sprintf("tcp-%d-%04d", started.UnixMilli(), n%10000)
}
// spillRawDir stores one direction's captured bytes: up to previewCap stays
// inline in preview with no blob; beyond previewCap, the full captured bytes
// are written to the content-addressed _blobs store and preview holds the
// leading previewCap bytes.
func (t *Traffic) spillRawDir(data []byte, previewCap int) (preview []byte, blobHash string) {
if len(data) == 0 {
return nil, ""
}
if previewCap <= 0 {
previewCap = DefaultSOCKS5PreviewBytes
}
if len(data) <= previewCap {
cp := make([]byte, len(data))
copy(cp, data)
return cp, ""
}
sum := sha256.Sum256(data)
h := hex.EncodeToString(sum[:])
blobDir := filepath.Join(t.dir, "_blobs", "sha256", h[:2])
if err := os.MkdirAll(blobDir, 0o755); err != nil {
log.Printf("[traffic/socks5] failed to create blob directory: %v", err)
return append([]byte(nil), data[:previewCap]...), ""
}
blobPath := filepath.Join(blobDir, h+".bin")
if _, err := os.Stat(blobPath); os.IsNotExist(err) {
if err := writeBlobAtomic(blobDir, blobPath, h, data); err != nil {
log.Printf("[traffic/socks5] failed to write blob %s: %v", h, err)
return append([]byte(nil), data[:previewCap]...), ""
}
}
return append([]byte(nil), data[:previewCap]...), h
}
// writeBlobAtomic writes data to a temporary file inside blobDir and renames it
// into place at blobPath so a crash mid-write cannot leave a partial file under
// a content-addressed SHA-256 hash.
func writeBlobAtomic(blobDir, blobPath, hash string, data []byte) error {
tmp, err := os.CreateTemp(blobDir, hash+".*.tmp")
if err != nil {
return err
}
tmpPath := tmp.Name()
cleanup := true
defer func() {
if cleanup {
_ = os.Remove(tmpPath)
}
}()
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
return err
}
if err := tmp.Chmod(0o644); err != nil {
_ = tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
if err := os.Rename(tmpPath, blobPath); err != nil {
// Another goroutine may have won the race writing the same
// content-addressed hash (notably on Windows where Rename can fail
// if the destination already exists).
if _, statErr := os.Stat(blobPath); statErr == nil {
return nil
}
return err
}
cleanup = false
return nil
}
// RecordRawTCPStream persists one raw TCP connection into raw_tcp_streams and
// registers any spilled directional blobs in blob_refs. It never writes to the
// HTTP exchanges table.
func (t *Traffic) RecordRawTCPStream(rec RawTCPCapture, previewBytes int) (string, error) {
if t == nil || t.db == nil {
return "", fmt.Errorf("traffic store unavailable")
}
started := rec.StartedAt.UTC()
if started.IsZero() {
started = time.Now().UTC()
}
ended := rec.EndedAt.UTC()
if ended.IsZero() || ended.Before(started) {
ended = started
}
durMs := ended.Sub(started).Milliseconds()
if durMs < 0 {
durMs = 0
}
c2sPrev, c2sBlob := t.spillRawDir(rec.C2SData, previewBytes)
s2cPrev, s2cBlob := t.spillRawDir(rec.S2CData, previewBytes)
t.wmu.Lock()
defer t.wmu.Unlock()
id := t.newRawTCPStreamID(started)
trunc := 0
if rec.Truncated {
trunc = 1
}
tx, err := t.db.Begin()
if err != nil {
return "", fmt.Errorf("begin raw tcp tx: %w", err)
}
defer tx.Rollback() //nolint:errcheck
if _, err := tx.Exec(`
INSERT INTO raw_tcp_streams(
id, protocol, host, port, started_at, ended_at, duration_ms,
c2s_bytes, s2c_bytes, truncated, c2s_preview, c2s_blob, s2c_preview, s2c_blob
) VALUES(?, 'tcp', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
id, strings.ToLower(strings.TrimSpace(rec.Host)), rec.Port,
started.UnixMilli(), ended.UnixMilli(), durMs,
rec.C2SBytes, rec.S2CBytes, trunc,
c2sPrev, nullIfEmpty(c2sBlob), s2cPrev, nullIfEmpty(s2cBlob),
); err != nil {
return "", fmt.Errorf("insert raw_tcp_streams: %w", err)
}
for _, h := range []string{c2sBlob, s2cBlob} {
if h == "" {
continue
}
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash, exchange_id) VALUES(?, ?)`, h, id); err != nil {
return "", fmt.Errorf("register raw tcp blob ref: %w", err)
}
}
if err := tx.Commit(); err != nil {
return "", fmt.Errorf("commit raw tcp tx: %w", err)
}
return id, nil
}
// GetRawTCPStream fetches a single recorded raw TCP stream by ID. Returns nil, nil
// when not found.
func (t *Traffic) GetRawTCPStream(id string) (*RawTCPStream, error) {
if t == nil || t.db == nil {
return nil, fmt.Errorf("traffic store unavailable")
}
var s RawTCPStream
var trunc int
var c2sBlob, s2cBlob sql.NullString
err := t.db.QueryRow(`
SELECT id, protocol, host, port, started_at, ended_at, duration_ms,
c2s_bytes, s2c_bytes, truncated, c2s_preview, c2s_blob, s2c_preview, s2c_blob
FROM raw_tcp_streams WHERE id = ?`, id).Scan(
&s.ID, &s.Protocol, &s.Host, &s.Port, &s.StartedAt, &s.EndedAt, &s.DurationMs,
&s.C2SBytes, &s.S2CBytes, &trunc, &s.C2SPreview, &c2sBlob, &s.S2CPreview, &s2cBlob,
)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, err
}
s.Truncated = trunc != 0
s.C2SBlob = c2sBlob.String
s.S2CBlob = s2cBlob.String
return &s, nil
}
// ListRawTCPStreams returns raw TCP streams newest-first along with the total
// count matching the filter.
func (t *Traffic) ListRawTCPStreams(f RawTCPFilter) ([]RawTCPStream, int, error) {
if t == nil || t.db == nil {
return nil, 0, fmt.Errorf("traffic store unavailable")
}
var conds []string
var args []any
if h := strings.ToLower(strings.TrimSpace(f.Host)); h != "" {
conds = append(conds, "host = ?")
args = append(args, h)
}
if f.Port > 0 {
conds = append(conds, "port = ?")
args = append(args, f.Port)
}
where := ""
if len(conds) > 0 {
where = " WHERE " + strings.Join(conds, " AND ")
}
var total int
if err := t.db.QueryRow(`SELECT COUNT(*) FROM raw_tcp_streams`+where, args...).Scan(&total); err != nil {
return nil, 0, err
}
limit := f.Limit
if limit <= 0 {
limit = 50
}
if limit > 500 {
limit = 500
}
offset := f.Offset
if offset < 0 {
offset = 0
}
qArgs := append(append([]any{}, args...), limit, offset)
rows, err := t.db.Query(`
SELECT id, protocol, host, port, started_at, ended_at, duration_ms,
c2s_bytes, s2c_bytes, truncated, c2s_preview, c2s_blob, s2c_preview, s2c_blob
FROM raw_tcp_streams`+where+`
ORDER BY started_at DESC, id DESC
LIMIT ? OFFSET ?`, qArgs...)
if err != nil {
return nil, 0, err
}
defer rows.Close()
out := []RawTCPStream{}
for rows.Next() {
var s RawTCPStream
var trunc int
var c2sBlob, s2cBlob sql.NullString
if err := rows.Scan(
&s.ID, &s.Protocol, &s.Host, &s.Port, &s.StartedAt, &s.EndedAt, &s.DurationMs,
&s.C2SBytes, &s.S2CBytes, &trunc, &s.C2SPreview, &c2sBlob, &s.S2CPreview, &s2cBlob,
); err != nil {
return nil, 0, err
}
s.Truncated = trunc != 0
s.C2SBlob = c2sBlob.String
s.S2CBlob = s2cBlob.String
out = append(out, s)
}
return out, total, rows.Err()
}
// CountRawTCPStreams returns the number of recorded raw TCP streams (all hosts
// when host is empty, or filtered to one exact host).
func (t *Traffic) CountRawTCPStreams(host string) (int, error) {
if t == nil || t.db == nil {
return 0, fmt.Errorf("traffic store unavailable")
}
var n int
h := strings.ToLower(strings.TrimSpace(host))
if h == "" {
err := t.db.QueryRow(`SELECT COUNT(*) FROM raw_tcp_streams`).Scan(&n)
return n, err
}
err := t.db.QueryRow(`SELECT COUNT(*) FROM raw_tcp_streams WHERE host = ?`, h).Scan(&n)
return n, err
}
+693
View File
@@ -0,0 +1,693 @@
package traffic
import (
"context"
"encoding/binary"
"errors"
"fmt"
"io"
"log"
"net"
"net/url"
"os"
"strconv"
"strings"
"sync"
"syscall"
"time"
"github.com/Autumn-27/artex/egress"
"github.com/Autumn-27/artex/intercept"
)
// Opt-in environment flags for the SOCKS5 inbound listener (P4.1a).
// Off by default: when neither is set, no SOCKS5 listener is started and the
// HTTP recording proxy is unchanged.
const (
// SOCKS5EnabledEnv turns on the SOCKS5 CONNECT listener on DefaultSOCKS5Addr
// (loopback) when set to 1/true/on/yes.
SOCKS5EnabledEnv = "ARTEX_SOCKS5_ENABLED"
// SOCKS5AddrEnv optionally overrides the bind address (default 127.0.0.1:1080).
// Setting a non-empty address also enables the listener unless
// ARTEX_SOCKS5_ENABLED is explicitly false/0/off/no.
SOCKS5AddrEnv = "ARTEX_SOCKS5_ADDR"
DefaultSOCKS5Addr = "127.0.0.1:1080"
DefaultSOCKS5MaxConcurrent = 64
DefaultSOCKS5MaxBytesPerDir = 4 * 1024 * 1024
DefaultSOCKS5PreviewBytes = blobPreview
DefaultSOCKS5IdleTimeout = 2 * time.Minute
DefaultSOCKS5HandshakeTimeout = 10 * time.Second
DefaultSOCKS5DialTimeout = 30 * time.Second
DefaultSOCKS5PaceTimeout = paceWaitMax
)
// RFC 1928 / RFC 1929 protocol constants.
const (
socks5Version = 0x05
socks5AuthNone = 0x00
socks5AuthUserPass = 0x02
socks5AuthNoAccept = 0xFF
socks5UserPassVer = 0x01
socks5CmdConnect = 0x01
socks5CmdBind = 0x02
socks5CmdUDPAssociate = 0x03
socks5AtypIPv4 = 0x01
socks5AtypDomain = 0x03
socks5AtypIPv6 = 0x04
socks5RepSuccess = 0x00
socks5RepGeneralFailure = 0x01
socks5RepNotAllowed = 0x02
socks5RepNetworkUnreachable = 0x03
socks5RepHostUnreachable = 0x04
socks5RepConnectionRefused = 0x05
socks5RepTTLExpired = 0x06
socks5RepCmdNotSupported = 0x07
socks5RepAtypNotSupported = 0x08
)
// SOCKS5Config configures the opt-in RFC 1928 SOCKS5 CONNECT listener.
type SOCKS5Config struct {
// Addr is the TCP bind address (default DefaultSOCKS5Addr, loopback-bound).
Addr string
// MaxConcurrent bounds simultaneous active SOCKS5 streams (default 64).
MaxConcurrent int
// MaxBytesPerDir caps bytes captured for storage per direction (default 4 MiB).
// Relay continues counting total bytes; exceeding the cap sets Truncated=true.
MaxBytesPerDir int64
// PreviewBytes caps the inline preview per direction (default 8 KiB).
PreviewBytes int
// IdleTimeout closes a relayed stream after no bytes move in either direction.
IdleTimeout time.Duration
// HandshakeTimeout bounds the RFC 1928 greeting, auth, and CONNECT request.
HandshakeTimeout time.Duration
// DialTimeout bounds the outbound TCP dial.
DialTimeout time.Duration
// PaceTimeout bounds waiting on the egress governor before refusing the dial.
PaceTimeout time.Duration
// DialContext opens the outbound TCP connection after policy checks.
// Defaults to intercept.PinnedDialer("socks5", false).
DialContext func(ctx context.Context, network, addr string) (net.Conn, error)
// BaseContext, when non-nil, derives the context passed to the egress
// governor and DialContext (e.g. carrying an intercept.TargetScope).
BaseContext func() context.Context
// OnRecord, when non-nil, receives each completed raw TCP capture.
// When nil and attached to a Traffic store, the store's raw-TCP recorder runs.
OnRecord func(RawTCPCapture)
}
// RawTCPCapture is the completed metadata and bounded directional payloads of
// one relayed SOCKS5 TCP connection.
type RawTCPCapture struct {
Host string
Port int
StartedAt time.Time
EndedAt time.Time
C2SBytes int64
S2CBytes int64
C2SData []byte
S2CData []byte
Truncated bool
Paced bool
}
// SOCKS5ConfigFromEnv parses ARTEX_SOCKS5_ENABLED and ARTEX_SOCKS5_ADDR.
// Returns (cfg, false) when disabled (the default).
func SOCKS5ConfigFromEnv() (SOCKS5Config, bool) {
rawEnabled := strings.ToLower(strings.TrimSpace(os.Getenv(SOCKS5EnabledEnv)))
rawAddr := strings.TrimSpace(os.Getenv(SOCKS5AddrEnv))
switch rawEnabled {
case "0", "false", "off", "no":
return SOCKS5Config{}, false
case "1", "true", "on", "yes":
addr := rawAddr
if addr == "" {
addr = DefaultSOCKS5Addr
}
return SOCKS5Config{Addr: addr}, true
case "":
if rawAddr == "" {
return SOCKS5Config{}, false
}
return SOCKS5Config{Addr: rawAddr}, true
default:
log.Printf("[traffic] %s=%q is not a boolean; leaving SOCKS5 disabled", SOCKS5EnabledEnv, rawEnabled)
return SOCKS5Config{}, false
}
}
func (c SOCKS5Config) withDefaults() SOCKS5Config {
if strings.TrimSpace(c.Addr) == "" {
c.Addr = DefaultSOCKS5Addr
}
if c.MaxConcurrent <= 0 {
c.MaxConcurrent = DefaultSOCKS5MaxConcurrent
}
if c.MaxBytesPerDir <= 0 {
c.MaxBytesPerDir = DefaultSOCKS5MaxBytesPerDir
}
if c.PreviewBytes <= 0 {
c.PreviewBytes = DefaultSOCKS5PreviewBytes
}
if c.IdleTimeout <= 0 {
c.IdleTimeout = DefaultSOCKS5IdleTimeout
}
if c.HandshakeTimeout <= 0 {
c.HandshakeTimeout = DefaultSOCKS5HandshakeTimeout
}
if c.DialTimeout <= 0 {
c.DialTimeout = DefaultSOCKS5DialTimeout
}
if c.PaceTimeout <= 0 {
c.PaceTimeout = DefaultSOCKS5PaceTimeout
}
if c.DialContext == nil {
c.DialContext = intercept.PinnedDialer("socks5", false)
}
return c
}
type socks5Server struct {
t *Traffic
cfg SOCKS5Config
ln net.Listener
sem chan struct{}
stop chan struct{}
once sync.Once
wg sync.WaitGroup
mu sync.Mutex
active map[net.Conn]struct{}
}
// StartSOCKS5 binds and starts the opt-in RFC 1928 SOCKS5 CONNECT listener on t.
// Returns the actual bound address (useful when cfg.Addr uses port 0).
func (t *Traffic) StartSOCKS5(cfg SOCKS5Config) (string, error) {
cfg = cfg.withDefaults()
ln, err := net.Listen("tcp", cfg.Addr)
if err != nil {
return "", fmt.Errorf("socks5 listen %s: %w", cfg.Addr, err)
}
srv := &socks5Server{
t: t,
cfg: cfg,
ln: ln,
sem: make(chan struct{}, cfg.MaxConcurrent),
stop: make(chan struct{}),
active: map[net.Conn]struct{}{},
}
t.socksMu.Lock()
if t.socks != nil {
t.socksMu.Unlock()
_ = ln.Close()
return "", errors.New("socks5 listener already running")
}
t.socks = srv
t.socksMu.Unlock()
srv.wg.Add(1)
go srv.serve()
return ln.Addr().String(), nil
}
// SOCKS5Addr returns the bound SOCKS5 listener address, or "" when disabled.
func (t *Traffic) SOCKS5Addr() string {
t.socksMu.Lock()
defer t.socksMu.Unlock()
if t.socks == nil || t.socks.ln == nil {
return ""
}
return t.socks.ln.Addr().String()
}
func (s *socks5Server) Close() error {
var closeErr error
s.once.Do(func() {
close(s.stop)
closeErr = s.ln.Close()
s.mu.Lock()
for c := range s.active {
_ = c.Close()
}
s.mu.Unlock()
})
s.wg.Wait()
return closeErr
}
func (s *socks5Server) stopping() bool {
select {
case <-s.stop:
return true
default:
return false
}
}
func (s *socks5Server) trackConn(c net.Conn) bool {
s.mu.Lock()
defer s.mu.Unlock()
if s.stopping() {
return false
}
s.active[c] = struct{}{}
return true
}
func (s *socks5Server) untrackConn(c net.Conn) {
s.mu.Lock()
delete(s.active, c)
s.mu.Unlock()
}
func (s *socks5Server) serve() {
defer s.wg.Done()
for {
conn, err := s.ln.Accept()
if err != nil {
if s.stopping() || errors.Is(err, net.ErrClosed) {
return
}
var ne net.Error
if errors.As(err, &ne) && ne.Timeout() {
continue
}
log.Printf("[traffic/socks5] accept error: %v", err)
return
}
if !s.trackConn(conn) {
_ = conn.Close()
return
}
s.wg.Add(1)
go func(c net.Conn) {
defer s.wg.Done()
defer s.untrackConn(c)
defer c.Close()
s.handleConn(c)
}(conn)
}
}
func (s *socks5Server) handleConn(client net.Conn) {
// 1. Handshake (greeting + optional RFC 1929 auth + CONNECT request) under HandshakeTimeout.
_ = client.SetDeadline(time.Now().Add(s.cfg.HandshakeTimeout))
paced, ok := s.negotiateAuth(client)
if !ok {
return
}
host, port, rep := readSOCKS5Request(client)
if rep != socks5RepSuccess {
_ = writeSOCKS5Reply(client, rep, nil)
return
}
// 2. Enforce max concurrent streams before touching policy or dialing.
select {
case s.sem <- struct{}{}:
defer func() { <-s.sem }()
default:
_ = writeSOCKS5Reply(client, socks5RepGeneralFailure, nil)
return
}
// 3. Apply the egress governor and target gate before dialing.
targetAddr := net.JoinHostPort(host, strconv.Itoa(port))
upstream, rep := s.dialTarget(host, port, targetAddr)
if rep != socks5RepSuccess {
_ = writeSOCKS5Reply(client, rep, nil)
return
}
if !s.trackConn(upstream) {
_ = upstream.Close()
_ = writeSOCKS5Reply(client, socks5RepGeneralFailure, nil)
return
}
defer s.untrackConn(upstream)
defer upstream.Close()
// Clear handshake deadline before replying and starting the relay.
_ = client.SetDeadline(time.Time{})
if err := writeSOCKS5Reply(client, socks5RepSuccess, upstream.LocalAddr()); err != nil {
return
}
// 4. Bidirectional half-close-aware relay with idle timeout and bounded capture.
startedAt := time.Now().UTC()
c2sCap := &limitedCapture{max: s.cfg.MaxBytesPerDir}
s2cCap := &limitedCapture{max: s.cfg.MaxBytesPerDir}
var relayWG sync.WaitGroup
relayWG.Add(2)
go func() {
defer relayWG.Done()
relayHalf(upstream, client, c2sCap, s.cfg.IdleTimeout)
closeWrite(upstream)
}()
go func() {
defer relayWG.Done()
relayHalf(client, upstream, s2cCap, s.cfg.IdleTimeout)
closeWrite(client)
}()
relayWG.Wait()
endedAt := time.Now().UTC()
rec := RawTCPCapture{
Host: host,
Port: port,
StartedAt: startedAt,
EndedAt: endedAt,
C2SBytes: c2sCap.total,
S2CBytes: s2cCap.total,
C2SData: c2sCap.buf,
S2CData: s2cCap.buf,
Truncated: c2sCap.truncated || s2cCap.truncated,
Paced: paced,
}
s.recordCapture(rec)
}
func (s *socks5Server) recordCapture(rec RawTCPCapture) {
if s.t != nil && s.t.db != nil {
if _, err := s.t.RecordRawTCPStream(rec, s.cfg.PreviewBytes); err != nil {
log.Printf("[traffic/socks5] failed to record raw tcp stream %s:%d: %v", rec.Host, rec.Port, err)
}
}
if s.cfg.OnRecord != nil {
s.cfg.OnRecord(rec)
}
}
func (s *socks5Server) negotiateAuth(conn net.Conn) (paced bool, ok bool) {
var hdr [2]byte
if _, err := io.ReadFull(conn, hdr[:]); err != nil {
return false, false
}
if hdr[0] != socks5Version || hdr[1] == 0 {
return false, false
}
methods := make([]byte, int(hdr[1]))
if _, err := io.ReadFull(conn, methods); err != nil {
return false, false
}
var hasNone, hasUserPass bool
for _, m := range methods {
switch m {
case socks5AuthNone:
hasNone = true
case socks5AuthUserPass:
hasUserPass = true
}
}
// Prefer username/password when offered so clients carrying the pace marker
// (egress.PaceUser) can pass it through without requiring real credentials.
// Accept no-auth for loopback clients.
if hasUserPass {
if _, err := conn.Write([]byte{socks5Version, socks5AuthUserPass}); err != nil {
return false, false
}
return readSOCKS5UserPass(conn)
}
if hasNone && isLoopbackRemote(conn.RemoteAddr()) {
if _, err := conn.Write([]byte{socks5Version, socks5AuthNone}); err != nil {
return false, false
}
return false, true
}
_, _ = conn.Write([]byte{socks5Version, socks5AuthNoAccept})
return false, false
}
// readSOCKS5UserPass parses an RFC 1929 username/password subnegotiation.
// Credentials are never required (any username/password succeeds); a username
// equal to egress.PaceUser marks the connection as paced.
func readSOCKS5UserPass(conn net.Conn) (paced bool, ok bool) {
var head [2]byte
if _, err := io.ReadFull(conn, head[:]); err != nil {
return false, false
}
if head[0] != socks5UserPassVer {
_, _ = conn.Write([]byte{socks5UserPassVer, 0x01})
return false, false
}
uname := make([]byte, int(head[1]))
if _, err := io.ReadFull(conn, uname); err != nil {
return false, false
}
var plen [1]byte
if _, err := io.ReadFull(conn, plen[:]); err != nil {
return false, false
}
if plen[0] > 0 {
pass := make([]byte, int(plen[0]))
if _, err := io.ReadFull(conn, pass); err != nil {
return false, false
}
}
if _, err := conn.Write([]byte{socks5UserPassVer, 0x00}); err != nil {
return false, false
}
return string(uname) == egress.PaceUser, true
}
func readSOCKS5Request(conn net.Conn) (host string, port int, rep byte) {
var hdr [4]byte
if _, err := io.ReadFull(conn, hdr[:]); err != nil {
return "", 0, socks5RepGeneralFailure
}
if hdr[0] != socks5Version || hdr[2] != 0x00 {
return "", 0, socks5RepGeneralFailure
}
cmd := hdr[1]
atyp := hdr[3]
// Read the address + port before rejecting unsupported commands so the stream
// stays cleanly framed if the peer reads our reply.
switch atyp {
case socks5AtypIPv4:
var ip [4]byte
if _, err := io.ReadFull(conn, ip[:]); err != nil {
return "", 0, socks5RepGeneralFailure
}
host = net.IP(ip[:]).String()
case socks5AtypDomain:
var l [1]byte
if _, err := io.ReadFull(conn, l[:]); err != nil {
return "", 0, socks5RepGeneralFailure
}
if l[0] == 0 {
return "", 0, socks5RepGeneralFailure
}
dom := make([]byte, int(l[0]))
if _, err := io.ReadFull(conn, dom); err != nil {
return "", 0, socks5RepGeneralFailure
}
host = strings.TrimSpace(string(dom))
if host == "" {
return "", 0, socks5RepGeneralFailure
}
case socks5AtypIPv6:
var ip [16]byte
if _, err := io.ReadFull(conn, ip[:]); err != nil {
return "", 0, socks5RepGeneralFailure
}
host = net.IP(ip[:]).String()
default:
return "", 0, socks5RepAtypNotSupported
}
var pbuf [2]byte
if _, err := io.ReadFull(conn, pbuf[:]); err != nil {
return "", 0, socks5RepGeneralFailure
}
port = int(binary.BigEndian.Uint16(pbuf[:]))
if cmd != socks5CmdConnect {
return "", 0, socks5RepCmdNotSupported
}
if port <= 0 || port > 65535 {
return "", 0, socks5RepGeneralFailure
}
return host, port, socks5RepSuccess
}
func writeSOCKS5Reply(conn net.Conn, rep byte, bindAddr net.Addr) error {
var bndIP net.IP
var bndPort int
if tcpAddr, ok := bindAddr.(*net.TCPAddr); ok && tcpAddr != nil {
bndIP = tcpAddr.IP
bndPort = tcpAddr.Port
}
var out []byte
if ip4 := bndIP.To4(); ip4 != nil {
out = make([]byte, 0, 10)
out = append(out, socks5Version, rep, 0x00, socks5AtypIPv4)
out = append(out, ip4...)
} else if ip16 := bndIP.To16(); ip16 != nil {
out = make([]byte, 0, 22)
out = append(out, socks5Version, rep, 0x00, socks5AtypIPv6)
out = append(out, ip16...)
} else {
out = make([]byte, 0, 10)
out = append(out, socks5Version, rep, 0x00, socks5AtypIPv4, 0, 0, 0, 0)
}
var p [2]byte
binary.BigEndian.PutUint16(p[:], uint16(bndPort))
out = append(out, p[:]...)
_, err := conn.Write(out)
return err
}
func (s *socks5Server) dialTarget(host string, port int, targetAddr string) (net.Conn, byte) {
baseCtx := context.Background()
if s.cfg.BaseContext != nil {
if ctx := s.cfg.BaseContext(); ctx != nil {
baseCtx = ctx
}
}
stopCtx, stopCancel := context.WithCancel(baseCtx)
defer stopCancel()
go func() {
select {
case <-s.stop:
stopCancel()
case <-stopCtx.Done():
}
}()
// Run through the same egress governor (egress.Do) and target gate dialer
// (intercept.PinnedDialer / Interceptor.DialSOCKS5) the HTTP path uses.
govCtx, govCancel := context.WithTimeout(stopCtx, s.cfg.PaceTimeout)
defer govCancel()
targetURL := &url.URL{Scheme: "tcp", Host: targetAddr}
dialCalled := false
conn, err := egress.Do(egress.Current(), govCtx, targetURL, func() (net.Conn, error) {
dialCalled = true
dialCtx, dialCancel := context.WithTimeout(stopCtx, s.cfg.DialTimeout)
defer dialCancel()
return s.cfg.DialContext(dialCtx, "tcp", targetAddr)
})
if err != nil {
if !dialCalled {
// Refused by the egress governor before dialing.
return nil, socks5RepNotAllowed
}
return nil, mapDialErrorToSOCKS5Reply(err)
}
return conn, socks5RepSuccess
}
func mapDialErrorToSOCKS5Reply(err error) byte {
if _, blocked := intercept.IsTargetBlocked(err); blocked {
return socks5RepNotAllowed
}
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) {
return socks5RepHostUnreachable
}
if errors.Is(err, syscall.ECONNREFUSED) {
return socks5RepConnectionRefused
}
if errors.Is(err, syscall.ENETUNREACH) {
return socks5RepNetworkUnreachable
}
if errors.Is(err, syscall.EHOSTUNREACH) {
return socks5RepHostUnreachable
}
if errors.Is(err, context.DeadlineExceeded) {
return socks5RepTTLExpired
}
low := strings.ToLower(err.Error())
switch {
case strings.Contains(low, "connection refused"):
return socks5RepConnectionRefused
case strings.Contains(low, "network is unreachable"):
return socks5RepNetworkUnreachable
case strings.Contains(low, "no route to host"), strings.Contains(low, "no such host"), strings.Contains(low, "host is unreachable"):
return socks5RepHostUnreachable
}
return socks5RepGeneralFailure
}
func isLoopbackRemote(addr net.Addr) bool {
if addr == nil {
return false
}
switch a := addr.(type) {
case *net.TCPAddr:
return a.IP != nil && a.IP.IsLoopback()
default:
h, _, err := net.SplitHostPort(addr.String())
if err != nil {
h = addr.String()
}
ip := net.ParseIP(strings.Trim(h, "[]"))
return ip != nil && ip.IsLoopback()
}
}
type limitedCapture struct {
max int64
total int64
buf []byte
truncated bool
}
func (c *limitedCapture) record(p []byte) {
if len(p) == 0 {
return
}
c.total += int64(len(p))
rem := c.max - int64(len(c.buf))
if rem <= 0 {
c.truncated = true
return
}
if int64(len(p)) > rem {
c.buf = append(c.buf, p[:rem]...)
c.truncated = true
return
}
c.buf = append(c.buf, p...)
}
func relayHalf(dst, src net.Conn, cap *limitedCapture, idle time.Duration) {
buf := make([]byte, 32*1024)
for {
if idle > 0 {
_ = src.SetReadDeadline(time.Now().Add(idle))
}
n, rerr := src.Read(buf)
if n > 0 {
chunk := buf[:n]
cap.record(chunk)
if idle > 0 {
_ = dst.SetWriteDeadline(time.Now().Add(idle))
}
if _, werr := dst.Write(chunk); werr != nil {
return
}
}
if rerr != nil {
return
}
}
}
func closeWrite(c net.Conn) {
type halfCloser interface {
CloseWrite() error
}
if hc, ok := c.(halfCloser); ok {
_ = hc.CloseWrite()
}
}

Some files were not shown because too many files have changed in this diff Show More