mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-23 00:20:55 +08:00
Compare commits
82
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4f6aaef02a | ||
|
|
2677ff32ef | ||
|
|
8bacfc0245 | ||
|
|
90cd290d42 | ||
|
|
449065aaaa | ||
|
|
5df46e22c1 | ||
|
|
83b35f844b | ||
|
|
9d7983f1a2 | ||
|
|
3ca8f217a3 | ||
|
|
132af98029 | ||
|
|
6ef01d4f24 | ||
|
|
f72311994e | ||
|
|
4171a041e6 | ||
|
|
db867f63a4 | ||
|
|
d59227c5d4 | ||
|
|
f37d09e365 | ||
|
|
5ce7ae6006 | ||
|
|
7b8be967cc | ||
|
|
09eb73c547 | ||
|
|
43ace17ad7 | ||
|
|
4f4bf201f7 | ||
|
|
444bca59d5 | ||
|
|
d86b22719a | ||
|
|
1e7696c572 | ||
|
|
f4c7184dcc | ||
|
|
0c55774613 | ||
|
|
a133b79a0e | ||
|
|
19110a4727 | ||
|
|
c8b771881e | ||
|
|
0eb66f4c44 | ||
|
|
9a8f366574 | ||
|
|
5bed53b6ae | ||
|
|
25217afdce | ||
|
|
a4a2ee4a51 | ||
|
|
cc573132c4 | ||
|
|
ca9aef6692 | ||
|
|
09a28a3bf8 | ||
|
|
55e2a4d037 | ||
|
|
8b4f890e61 | ||
|
|
ff287467aa | ||
|
|
c65034c50f | ||
|
|
df9536b7e5 | ||
|
|
e435b6b0bd | ||
|
|
13073ce18e | ||
|
|
1adfc21346 | ||
|
|
c3a055135d | ||
|
|
c90e2c980c | ||
|
|
b56ad38b43 | ||
|
|
bfaec521ac | ||
|
|
a3c082fdac | ||
|
|
d15166c476 | ||
|
|
59d514e37c | ||
|
|
2411943de6 | ||
|
|
2a2deb30fd | ||
|
|
e2c98b6b00 | ||
|
|
cf2f3c92a7 | ||
|
|
b5bf99f5fb | ||
|
|
da23d7df47 | ||
|
|
9f2c290e9f | ||
|
|
61478bb90b | ||
|
|
42ce6ea5c8 | ||
|
|
c92c784987 | ||
|
|
22ae88afab | ||
|
|
d941a356a5 | ||
|
|
0c5cec0064 | ||
|
|
149d52e762 | ||
|
|
e12c00e2c9 | ||
|
|
9a4116be71 | ||
|
|
aa4a1bfab9 | ||
|
|
2be58efd50 | ||
|
|
d638b047dc | ||
|
|
8f3b0541ab | ||
|
|
30fce2621f | ||
|
|
9ffdecbf28 | ||
|
|
de802016e3 | ||
|
|
801b1e1550 | ||
|
|
d8f8f6c1e7 | ||
|
|
811c2e48b8 | ||
|
|
af68f44695 | ||
|
|
b422eb905c | ||
|
|
091209798f | ||
|
|
1ecb676e34 |
+7
-1
@@ -1,5 +1,6 @@
|
||||
# Docker deployment configuration (copy to .env)
|
||||
# The image is built locally by `docker compose build`; there is no registry tag.
|
||||
# docker-compose.yml builds the image locally. docker-compose.ghcr.yml pulls the
|
||||
# released GHCR image; pin it with ARTEX_IMAGE_TAG (latest or a vX.Y.Z tag).
|
||||
# Address the web UI is published on. 127.0.0.1 keeps it host-local; set 0.0.0.0
|
||||
# to reach it from another machine or from Windows when ARTEX runs inside WSL.
|
||||
ARTEX_BIND_ADDR=127.0.0.1
|
||||
@@ -11,8 +12,13 @@ ARTEX_TRUSTED_PROXIES=
|
||||
# Set it when ./keys is not persistent. Losing the key loses the stored secrets.
|
||||
# ARTEX_SECRET_KEY=
|
||||
POSTGRES_USER=artex
|
||||
# The server percent-encodes this into the connection URL, so / ? % @ : and
|
||||
# spaces are safe. Quote a value that contains #, and write $$ for a literal $,
|
||||
# because Compose reads this file that way.
|
||||
POSTGRES_PASSWORD=change-me-please
|
||||
POSTGRES_DB=artex
|
||||
# Prebuilt GHCR image tag for docker-compose.ghcr.yml. Unset uses latest.
|
||||
# ARTEX_IMAGE_TAG=latest
|
||||
|
||||
# LLM (fill one or both; you can also leave empty and configure in the UI)
|
||||
ANTHROPIC_API_KEY=
|
||||
|
||||
@@ -73,12 +73,12 @@ jobs:
|
||||
with:
|
||||
name: pkg-${{ matrix.goos }}-${{ matrix.goarch }}
|
||||
path: "dist/*.zip"
|
||||
# Upload raw Linux binaries separately for reuse by the docker job (avoids rebuilding the frontend + Go inside the image)
|
||||
- if: matrix.goos == 'linux'
|
||||
# Upload the raw linux/amd64 binary separately for the release job's Syft SBOM scan
|
||||
- if: matrix.goos == 'linux' && matrix.goarch == 'amd64'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: bin-linux-${{ matrix.goarch }}
|
||||
path: dist/artex-linux-${{ matrix.goarch }}/artex
|
||||
name: bin-linux-amd64
|
||||
path: dist/artex-linux-amd64/artex
|
||||
|
||||
# ③ Collect all zips → create the GitHub Release
|
||||
release:
|
||||
@@ -152,45 +152,52 @@ jobs:
|
||||
generate_release_notes: false
|
||||
body_path: ${{ runner.temp }}/release-body.md
|
||||
|
||||
# ④ Multi-arch image → push to Docker Hub (reuses the Linux binaries built by the
|
||||
# binaries job; the image only installs tools + ships the binary; arm64 only needs
|
||||
# to emulate the apt layer, so builds are much faster)
|
||||
# Publishing the image needs Docker Hub credentials and a namespace we own, so the
|
||||
# job stays skipped unless the repository defines DOCKERHUB_ENABLED (set it to true
|
||||
# and provide DOCKERHUB_USERNAME / DOCKERHUB_TOKEN secrets plus DOCKERHUB_IMAGE).
|
||||
# ④ Multi-arch image → push to GitHub Container Registry (and optionally Docker Hub
|
||||
# when DOCKERHUB_ENABLED is set to 'true' alongside DOCKERHUB_IMAGE and
|
||||
# DOCKERHUB_USERNAME / DOCKERHUB_TOKEN secrets). Dry runs (workflow_dispatch)
|
||||
# build both platforms without logging in or pushing.
|
||||
docker:
|
||||
needs: binaries
|
||||
if: ${{ vars.DOCKERHUB_ENABLED == 'true' && github.event_name == 'push' }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4 # fetch the Dockerfile and skills/
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: bin-linux-amd64
|
||||
path: dist/amd64
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: bin-linux-arm64
|
||||
path: dist/arm64
|
||||
- uses: actions/checkout@v4
|
||||
- uses: docker/setup-qemu-action@v3
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/login-action@v3
|
||||
- name: Log in to GitHub Container Registry
|
||||
if: github.event_name == 'push'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Log in to Docker Hub
|
||||
if: github.event_name == 'push' && vars.DOCKERHUB_ENABLED == 'true'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- uses: docker/metadata-action@v5
|
||||
id: meta
|
||||
with:
|
||||
images: ${{ vars.DOCKERHUB_IMAGE }}
|
||||
images: |
|
||||
ghcr.io/${{ github.repository }}
|
||||
${{ vars.DOCKERHUB_ENABLED == 'true' && vars.DOCKERHUB_IMAGE || '' }}
|
||||
tags: |
|
||||
type=ref,event=tag
|
||||
type=semver,pattern={{version}}
|
||||
type=raw,value=latest
|
||||
- uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
push: ${{ github.event_name == 'push' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: ARTEX_VERSION=${{ github.ref_name }}
|
||||
provenance: true
|
||||
sbom: true
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
+41
-1
@@ -4,8 +4,30 @@ All notable changes to this project are documented in this file. The format is b
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Live log subscribers can no longer crash the process**: a client disconnect closes its log channel. The sink now delivers a line while it still holds the subscriber lock, so that close cannot land in the middle of the send.
|
||||
- **Notification delivery ignores HTTP_PROXY**: the dial guard checks the address it connects to. An environment proxy made that address the proxy, so a link-local target could still be reached. Notification HTTP now connects directly.
|
||||
- **Captured traffic blobs are owner-only**: HTTP and raw-stream bodies spilled out of the database are written `0600`, and their bucket directory is `0700`. An existing blob is tightened when it is seen again.
|
||||
- **Asset URLs drop userinfo when parsing fails**: a URL `url.Parse` rejects is no longer stored unchanged. The `scheme://userinfo@` prefix is removed first, so a password in that prefix does not land in the asset row.
|
||||
- **Hook `range` counts against the memory budget**: ranging over a body, slice, string, or map charges the iteration before the loop. A body larger than the hook budget now stops with the memory limit instead of allocating one element at a time with no check.
|
||||
- **Chat upload name search stops**: choosing `name`, `name-1`, `name-2`, … now returns an error when the filesystem rejects the path or after 10000 collisions, instead of looping while `Stat` keeps failing.
|
||||
- **Uploads do not follow a symlink**: chat and workspace uploads `Lstat` the destination and refuse to write when it is already a symlink. A dangling link is no longer treated as a free name.
|
||||
- **Hardening follow-up for the upload and capture fixes**: existing spill blob directories and files are tightened on startup, hook `range` charges 32 bytes per pair, and uploads open with `O_NOFOLLOW` on Unix. The Windows upload path keeps the `Lstat` check.
|
||||
- **Docker Compose database passwords with URL-special characters**: `docker-compose.yml` no longer pastes `POSTGRES_PASSWORD` into `ARTEX_PG_DSN`. The server builds the URL from `ARTEX_PG_HOST`, `ARTEX_PG_PORT`, `ARTEX_PG_USER`, `ARTEX_PG_PASSWORD`, `ARTEX_PG_DBNAME` and `ARTEX_PG_SSLMODE`, percent-encoding the user and password, so `/`, `?`, `%`, `@`, `:` and spaces connect. An explicit `ARTEX_PG_DSN` still wins and is not rewritten. `ARTEX_PG_PASSWORD` is removed from the environment with the other startup secrets (#450).
|
||||
|
||||
## [0.7.0] - 2026-11-21
|
||||
|
||||
Agent-driven traffic capture (SOCKS5, transparent TCP and raw UDP), operator tampering hooks with their admin panel, the agentic injection and proof hardening from the #124 audit, LLM context telemetry and the worker cache-prefix split, capture-proxy rewrite rules, HAR import and export, and a large security batch across the interception gate, sessions, logs, the workspace file API and the model endpoint probe.
|
||||
|
||||
### Added
|
||||
|
||||
- **Traffic > Hooks admin panel (P4.2 PR2)**: the Traffic page gains an operator-only Hooks tab for the constrained Go tampering hooks: list, create (always created disabled), edit, delete, toggle, version history with rollback, and a dry-run against a captured exchange or a synthetic input. A dry-run shows the mutated request or response, the fields the hook changed, and the per-line trace, with masked fields shown as their keyed HMAC. Binary and compressed bodies are flagged and skipped, bodies over 1 MiB are flagged, and the UI never renders hook or exchange text as HTML (#435).
|
||||
- **Concurrent multi-LLM worker rotation** (#270): a task can rotate several bound LLM profiles across its workers, with per-lane exhaustion tracked and shown on the task controls.
|
||||
- **Encrypted S3 archive storage** (#285, #318): an optional S3 backend stores task archives encrypted, connects with the existing archive lifecycle, and lets an interrupted archive job be recovered on the next start.
|
||||
- **Turkish and Simplified Chinese UI, optional Neon theme** (#346, #413, #401): the web console gains a translation layer with a Turkish and a Simplified Chinese locale, and an opt-in Neon theme preset in Settings.
|
||||
- **Readiness probe** (#363): `GET /api/ready` checks PostgreSQL and reports readiness separately from the health route.
|
||||
- **Operator console browser protections** (#349): the embedded console restricts framing to the same origin and keeps referrer information on the same origin.
|
||||
- **Target noise governor**: one shared egress governor paces ARTEX's own HTTP requests (WebFetch, the http custom tools and the enrich probe) against a noise profile (`off` by default, `quiet`, `balanced`, `aggressive`): a per-host token bucket with jitter, per-host and global in-flight caps, 429/503 backoff, GET deduplication with a short cache, and a configurable target User-Agent (`ARTEX_TARGET_USER_AGENT`). Off is a strict passthrough (#205).
|
||||
- **Low-and-slow prompt block**: when a noise profile is set, the planner, worker, main agent and chat prompts carry the per-host limits of the profile and the exact rate or delay flags for nmap, nuclei, httpx, katana, ffuf, dalfox and sqlmap; with the profile off the prompts are unchanged (#212).
|
||||
- **Noise profile at runtime**: System settings has a **Target noise** card. It sets the profile (`off`, `quiet`, `balanced`, `aggressive`) through the `noise_profile` setting, which takes precedence over `ARTEX_NOISE_PROFILE`. The change applies to the next WebFetch, http custom tool and enrich request, and to the next agent prompt, without a restart. The card also shows the per-host counters, which `GET /api/settings/noise` returns (#233).
|
||||
@@ -43,6 +65,9 @@ All notable changes to this project are documented in this file. The format is b
|
||||
|
||||
### Changed
|
||||
|
||||
- **`record_fact` is facts-only** (#410, #282): the tool publishes and accepts facts only; asset writes go through `insert_assets`.
|
||||
- **Web dependency lockfile** (#438): `source-map-js` 1.2.4 and `baseline-browser-mapping` 2.11.39, past their denial-of-service advisories; `npm audit --omit=dev` is clean.
|
||||
- **Build and supply-chain hardening** (#408, #416, #419, #420): tool and Node digests are pinned instead of fetching their checksums, `npm ci` runs with `--ignore-scripts` in the build, install and update stages, the web build disables Next.js telemetry, and the api-recon skill lockfile points at the public npm registry.
|
||||
- **Password changes revoke sessions**: a password change stores a monotonic `auth.password_version` in the same transaction as the hash. Tokens carry that version, so a token from an older version is rejected, including when two changes land in the same second. `reset-password.sh` writes the same counter and requires an ARTEX restart, because the process reads the version at startup. A failed write leaves the previous session valid. Log, activity, and side-question streams re-check the token and close after the version moves. If the version cannot be read at startup, sessions are rejected until the next login reads it.
|
||||
- **Task budgets survive a restart**: planner, worker, judge, and compactor rows in `llm_usage` are counted back into `ARTEX_TASK_BUDGET_*`. A restored task already inside the wrap-up reserve only has that reserve left, so the run wraps up instead of starting another worker. The per-worker estimate is not restored and starts from the defaults (50,000 tokens and 10 turns) until a worker finishes in this process. Chat, goal decomposition, and side questions stay outside the cap.
|
||||
- **Untrusted-output boundary is now on by default**: Norma frames tool results for the worker, planner, chat and main agent unless `ARTEX_UNTRUSTED_BOUNDARY` is set to `0`, `false`, `off` or `no`. It was opt-in via `ARTEX_UNTRUSTED_BOUNDARY=1` in 0.6.0 (#166); the default flips here (#203).
|
||||
@@ -57,7 +82,10 @@ All notable changes to this project are documented in this file. The format is b
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Compose private model endpoints** (#406): `ARTEX_LLM_PRIVATE_ENDPOINTS` is passed to the artex container, so the private endpoint allow-list works in the Compose deployment.
|
||||
- **Bash exit 127 in transcripts** (#398): the session transcript explains a 127 exit as a missing command and shows the matching hint.
|
||||
- **LLM usage status on early stream exit**: when a consumer stops reading a stream before `message_stop`, the recorder now stores the `llm_usage` row as `error` at that point, whether or not the context was cancelled, instead of inferring the status from `ctx.Err()` in a deferred call. `TestSideUsageRecordedOnceOnConsumerCancellation` runs again (follow-up to #9).
|
||||
- **RSI lesson trust**: the planner overview carries each finding's triage status, and `prove_goal` refuses a `false_positive`, `duplicate`, or `ignored` finding. A lesson with no company is visible only to tasks that also have no company. Retracted evidence stays on the row with `withdrawn_at` set, extraction runs when a disposition changes or a retest completes, and it still withdraws evidence after a task gains a second company. The rendered lesson block is capped in characters, not bytes.
|
||||
- **Workspace and skill symlink confinement**: workspace file operations, the skill file APIs and the skill loader resolve symlinks in every path component and refuse a target outside their root, so a link planted in the workspace or a skill directory cannot expose another file.
|
||||
- **Target gate behind proxies**: a target approval is name-only whenever a proxy is in the path, including the recording proxy, the global egress proxy and a custom tool's own proxy, so the gate never claims an IP binding the proxy does not enforce.
|
||||
- **Remote MCP response bound**: Streamable HTTP JSON-RPC bodies and SSE data payloads are refused above 8 MiB, so a remote MCP server cannot force an unbounded decode.
|
||||
@@ -102,6 +130,17 @@ All notable changes to this project are documented in this file. The format is b
|
||||
|
||||
- **Secrets at rest**: the credential-bearing columns and settings (LLM API keys and proxy, task credentials, MCP server env, notification channel config, search and proxy settings), the exec spec of custom tools and the evidence snapshot heads are encrypted in PostgreSQL with AES-256-GCM (`enc:v1:`) using `ARTEX_SECRET_KEY`, else `keys/secret.key`; existing plaintext rows are backfilled once at startup, and a missing or wrong key while encrypted data exists refuses to start. This protects database dumps and backups, not a running instance (#213, #216, #217, #218).
|
||||
- **URL credentials are dropped before storage**: scan and bbot ingestion, the asset store and archive restore strip URL userinfo, so `https://user:pass@host/app` is stored and matched as `https://host/app` (#222, #223, #224).
|
||||
- **Agentic injection and proof hardening** (#437): the four surfaces that still carried target or agent text outside the per-message data envelope now go through it: the worker intent assets move out of the system prompt into the framed user message, and the planner stagnation warning, agent-written task credentials and planner todos are framed. Scanner output is opened with `os.Root` and every symlink and hard-linked file is refused. `prove_goal` closes a goal with a finding only when a bound `traffic_ref` still exists in the live capture and its host matches the finding's linked asset, and refuses fail-closed when that check is unavailable, so a planted or unbound finding stays recorded but cannot close a goal. An agent scope addition outside the declared scope, or one whose classification read fails, is recorded as `expanded` with its source and reason. A deterministic adversarial bench runs in CI against a hostile local target.
|
||||
- **Interception, session, log and endpoint hardening** (#436): when the interception rules cannot be read, tool calls are blocked instead of allowed, and a rule whose pattern does not compile blocks the calls it would target. A cookie-authenticated POST, PUT, PATCH or DELETE must carry a same-origin `Origin` or `Sec-Fetch-Site`, otherwise it answers 403; bearer tokens, GET and HEAD are unaffected. The one-time setup token is no longer written to the log the API exposes, and proxy credentials are masked at write time, on stderr and on read-back. The workspace refuses the capture CA, index and blob store for read and write, and refuses a delete that would remove them. `POST /api/llm/models` refuses carrier-grade NAT, benchmark, IETF protocol and reserved ranges for literal hosts, resolved names, redirects and at dial time, judges NAT64 and 6to4 addresses by the IPv4 they embed, and no longer echoes an upstream error body.
|
||||
|
||||
### Thanks
|
||||
|
||||
Thanks to the contributors whose work is in this release:
|
||||
|
||||
- Security and core reliability: @Coff0xc, @Protonmatter, @liuxiaobo0920, @cyberd4ch, @Dev-next-gen, @Michael-JRead, @anqorithm, @renaesop, @tglemser, @adam-33-pixel
|
||||
- Features, UI and locales: @Toys0125, @cocodedk, @BlackGownQ, @Amineharrabi, @chetanvhsk-wq, @YunzhouLi-hub, @mervedalgin
|
||||
- Build, packaging and tests: @ldevbr, @Lusitaniae, @mp-web3, @Alexandre200-lab, @maheshBawa
|
||||
- Documentation: @err0rgod, @yeting1107, @henefesto, @mathkartel
|
||||
|
||||
## [0.6.0] - 2026-11-12
|
||||
|
||||
@@ -1176,7 +1215,8 @@ First release of this fork. The engine underneath is still upstream's, but this
|
||||
- [@Autumn-27](https://github.com/Autumn-27)
|
||||
- [@neouks](https://github.com/neouks)
|
||||
|
||||
[Unreleased]: https://github.com/BaaRai/ARTEX/compare/v0.6.0...HEAD
|
||||
[Unreleased]: https://github.com/BaaRai/ARTEX/compare/v0.7.0...HEAD
|
||||
[0.7.0]: https://github.com/BaaRai/ARTEX/compare/v0.6.0...v0.7.0
|
||||
[0.6.0]: https://github.com/BaaRai/ARTEX/compare/v0.5.1...v0.6.0
|
||||
[0.5.1]: https://github.com/BaaRai/ARTEX/compare/v0.5.0...v0.5.1
|
||||
[0.5.0]: https://github.com/BaaRai/ARTEX/compare/v0.4.0...v0.5.0
|
||||
|
||||
+5
-4
@@ -22,10 +22,11 @@ COPY web/ ./
|
||||
RUN npm run build:static
|
||||
|
||||
# ------------------------------------------------------------- Go binary ---
|
||||
# Pinned to the Go 1.26 line (matches go.mod and the release binaries job). The
|
||||
# binary is CGO-free, so this stage runs on the build platform and cross-compiles
|
||||
# to $TARGETARCH without QEMU.
|
||||
FROM --platform=$BUILDPLATFORM golang:1.26-bookworm AS build
|
||||
# Pinned to the exact Go release go.mod requires. The official image sets
|
||||
# GOTOOLCHAIN=local, so a floating 1.26 tag that lags go.mod fails the build;
|
||||
# bump this together with the go line in go.mod. The binary is CGO-free, so this
|
||||
# stage runs on the build platform and cross-compiles to $TARGETARCH without QEMU.
|
||||
FROM --platform=$BUILDPLATFORM golang:1.26.11-bookworm AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
@@ -57,8 +57,9 @@ printed at startup:
|
||||
docker compose logs artex | grep "Setup token"
|
||||
```
|
||||
|
||||
> There is no prebuilt Docker image to pull: the image is always built from your checkout.
|
||||
> For all installation paths (script, binaries, source, release archives), see [Installation](#installation).
|
||||
> Multi-arch release images (`linux/amd64`, `linux/arm64`) are published to GitHub Container
|
||||
> Registry at `ghcr.io/baarai/artex`. For pulling the prebuilt image and all other installation
|
||||
> paths (script, binaries, source, release archives), see [Installation](#installation).
|
||||
|
||||
---
|
||||
|
||||
@@ -331,7 +332,9 @@ The UI listens on `127.0.0.1` by default. To reach it from another machine (or f
|
||||
ARTEX runs inside WSL), set `ARTEX_BIND_ADDR=0.0.0.0` in `.env` (the setup token still protects the
|
||||
first-run claim), or keep loopback and use a reverse proxy or SSH tunnel.
|
||||
|
||||
### Option 2: Docker Compose (manual)
|
||||
### Option 2: Docker (Compose checkout build or prebuilt GHCR image)
|
||||
|
||||
Build from a checkout with Docker Compose:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/BaaRai/ARTEX.git
|
||||
@@ -341,8 +344,73 @@ docker compose up -d --build
|
||||
# → http://localhost:8787
|
||||
```
|
||||
|
||||
Each `v*` release publishes a multi-arch image (`linux/amd64`, `linux/arm64`) to GitHub
|
||||
Container Registry with tags `:latest`, `:v<version>`, and `:<version>`. GHCR packages start
|
||||
private. After the first release, maintainers should check that the `artex` package visibility
|
||||
is set to Public and verify the pull below on a machine without GHCR credentials before
|
||||
announcing anonymous installation.
|
||||
|
||||
The standalone container requires an existing PostgreSQL database reachable from the container.
|
||||
Set the `ARTEX_PG_*` variables below. The example uses Docker Desktop's
|
||||
`host.docker.internal` to reach PostgreSQL on the host. On native Linux Docker Engine, add
|
||||
`--add-host=host.docker.internal:host-gateway` to `docker run` for a host database, or use
|
||||
the reachable hostname of your database server.
|
||||
|
||||
```bash
|
||||
docker pull ghcr.io/baarai/artex:latest
|
||||
|
||||
mkdir -p data keys
|
||||
docker run -d --name artex \
|
||||
--security-opt no-new-privileges:true \
|
||||
-p 127.0.0.1:8787:8787 \
|
||||
-p 127.0.0.1:8788:8788 \
|
||||
-e ARTEX_PG_HOST=host.docker.internal \
|
||||
-e ARTEX_PG_PORT=5432 \
|
||||
-e ARTEX_PG_USER=artex \
|
||||
-e ARTEX_PG_PASSWORD='yourpass' \
|
||||
-e ARTEX_PG_DBNAME=artex \
|
||||
-e ARTEX_PG_SSLMODE=disable \
|
||||
-e ARTEX_JWT_KEY_DIR=/app/keys \
|
||||
-v "$PWD/data:/app/data" \
|
||||
-v "$PWD/keys:/app/keys" \
|
||||
ghcr.io/baarai/artex:latest
|
||||
# → http://localhost:8787 (docker logs artex | grep "Setup token")
|
||||
```
|
||||
|
||||
In `docker-compose.yml` and `docker-compose.ghcr.yml`, `POSTGRES_PASSWORD` is passed to ARTEX as
|
||||
its own variable. The server percent-encodes it into the connection URL, so a password may contain
|
||||
`/`, `?`, `%`, `@`, `:` or spaces. In `.env`, quote a value that contains `#`, and write `$$` for a
|
||||
literal `$`. The `ARTEX_PG_*` variables take the password raw; quote it in a shell.
|
||||
|
||||
The image includes a scanner toolchain (`nmap`, `nuclei`, `httpx`, `katana`, `ffuf`, `dalfox`,
|
||||
`sqlmap`, `ripgrep`, `curl`, `npm`, …); `./skills` and `./data` are persisted via bind mounts.
|
||||
`sqlmap`, `ripgrep`, `curl`, `npm`, …) and built-in `/app/skills`. The GHCR Compose file below
|
||||
persists `./keys` and `./data`; it does not mount a host `skills/` directory, so the image's
|
||||
built-in skills are used.
|
||||
|
||||
#### Compose stack from the prebuilt image
|
||||
|
||||
`docker-compose.ghcr.yml` runs the same ARTEX + PostgreSQL stack from the released image, with no
|
||||
checkout build. Download it and `.env.example` once:
|
||||
|
||||
```bash
|
||||
curl -fsSLO https://raw.githubusercontent.com/BaaRai/ARTEX/main/docker-compose.ghcr.yml
|
||||
curl -fsSLO https://raw.githubusercontent.com/BaaRai/ARTEX/main/.env.example
|
||||
cp .env.example .env # set POSTGRES_PASSWORD, optionally an LLM key
|
||||
docker compose -f docker-compose.ghcr.yml pull
|
||||
docker compose -f docker-compose.ghcr.yml up -d
|
||||
# → http://localhost:8787 (docker compose -f docker-compose.ghcr.yml logs artex | grep "Setup token")
|
||||
```
|
||||
|
||||
Update the stack without a git checkout:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.ghcr.yml pull
|
||||
docker compose -f docker-compose.ghcr.yml up -d
|
||||
```
|
||||
|
||||
Pin a version with `ARTEX_IMAGE_TAG=v0.7.1` in `.env` (or `0.7.1`); unset uses `latest`. To use
|
||||
custom skills, copy them into a host directory and add `- ./skills:/app/skills` to the `artex`
|
||||
volumes, which overrides the built-in `/app/skills`.
|
||||
|
||||
For remote MCP you can choose `http` (Streamable HTTP) or `sse` (legacy SSE) in system settings.
|
||||
Legacy SSE servers usually establish the event stream with `GET /sse` and receive JSON-RPC over
|
||||
@@ -405,7 +473,7 @@ runtime supports it.
|
||||
|
||||
## Configuration
|
||||
|
||||
**Database** (`config.json`, or the `ARTEX_PG_DSN` environment variable):
|
||||
**Database** (`config.json`, `ARTEX_PG_DSN`, or `ARTEX_PG_*` environment variables — see [External PostgreSQL](#external-postgresql)):
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -436,6 +504,12 @@ catalog (`GET /v1/models`) and pick a model; models the provider adds later appe
|
||||
next refresh. For example, [NotHoly](https://notholy.com) (offensive-security models) uses
|
||||
base URL `https://api.notholy.com/v1` with an API key from its console.
|
||||
|
||||
**Task archive storage**: local disk by default. Set `ARTEX_ARCHIVE_STORAGE=s3` to
|
||||
store cold task packages in an S3-compatible bucket and release their local
|
||||
package space after verification. Existing local archives remain usable. See
|
||||
[Task archive storage](docs/task-archive-storage.md) for configuration, permissions,
|
||||
integrity checks, temporary disk requirements and recovery.
|
||||
|
||||
**Concurrency**: work agents per task, in "System Settings" (default 3).
|
||||
|
||||
**Untrusted-output boundary (on by default)**: tool results for the worker, planner, chat and main
|
||||
@@ -484,6 +558,70 @@ from the defaults until a worker finishes in this process.
|
||||
**Common flags**: `./start.sh -addr :8787 -proxy :8788` (`-addr` frontend/API, `-proxy` traffic
|
||||
recording proxy).
|
||||
|
||||
### External PostgreSQL
|
||||
|
||||
ARTEX already supports an external PostgreSQL 16+ instance — the `postgres` service in
|
||||
`docker-compose.yml` is only the bundled default. Because the database is startup configuration and
|
||||
the system's single source of truth, there is no runtime database switcher in the UI: change the
|
||||
environment variables and restart the server to point at another instance.
|
||||
|
||||
Configure the connection with either a full connection string or individual parts:
|
||||
|
||||
- **`ARTEX_PG_DSN`**: full PostgreSQL connection URL (for example
|
||||
`postgres://artex:pass@db.example.internal:5432/artex?sslmode=require`). When set, it takes
|
||||
precedence over the `ARTEX_PG_*` variables and you are responsible for percent-encoding any
|
||||
URL-special characters in the username or password.
|
||||
- **`ARTEX_PG_*` parts** (recommended when passwords contain `/`, `?`, `%`, `@`, `:`, or spaces —
|
||||
ARTEX percent-encodes the user and password and removes `ARTEX_PG_PASSWORD` from the environment
|
||||
at startup):
|
||||
- `ARTEX_PG_HOST` — PostgreSQL host
|
||||
- `ARTEX_PG_PORT` — PostgreSQL port (default `5432`)
|
||||
- `ARTEX_PG_USER` — database user
|
||||
- `ARTEX_PG_PASSWORD` — database password (raw, not URL-encoded)
|
||||
- `ARTEX_PG_DBNAME` — database name
|
||||
- `ARTEX_PG_SSLMODE` — TLS mode (`disable`, `require`, `verify-ca`, `verify-full`)
|
||||
|
||||
#### Compose without the bundled `postgres` service
|
||||
|
||||
To run ARTEX against an external PostgreSQL server with Docker Compose, remove the `postgres`
|
||||
service, the `depends_on` block, and the `pgdata` volume from `docker-compose.yml`, and point
|
||||
`artex` at your database:
|
||||
|
||||
```yaml
|
||||
services:
|
||||
artex:
|
||||
image: ghcr.io/baarai/artex:latest
|
||||
security_opt:
|
||||
- "no-new-privileges:true"
|
||||
environment:
|
||||
# Option A: individual parts (recommended; percent-encodes special characters)
|
||||
ARTEX_PG_HOST: ${ARTEX_PG_HOST:?Set ARTEX_PG_HOST in .env}
|
||||
ARTEX_PG_PORT: ${ARTEX_PG_PORT:-5432}
|
||||
ARTEX_PG_USER: ${ARTEX_PG_USER:-artex}
|
||||
ARTEX_PG_PASSWORD: ${ARTEX_PG_PASSWORD:?Set ARTEX_PG_PASSWORD in .env}
|
||||
ARTEX_PG_DBNAME: ${ARTEX_PG_DBNAME:-artex}
|
||||
ARTEX_PG_SSLMODE: ${ARTEX_PG_SSLMODE:-require}
|
||||
# Option B: full DSN (overrides ARTEX_PG_* when set)
|
||||
# ARTEX_PG_DSN: ${ARTEX_PG_DSN}
|
||||
ARTEX_JWT_KEY_DIR: /app/keys
|
||||
ports:
|
||||
- "${ARTEX_BIND_ADDR:-127.0.0.1}:8787:8787"
|
||||
- "127.0.0.1:8788:8788"
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
- ./keys:/app/keys
|
||||
restart: unless-stopped
|
||||
```
|
||||
|
||||
#### Local disk usage (`data/`)
|
||||
|
||||
Moving PostgreSQL to an external host does **not** move the runtime `data/` directory off local
|
||||
disk. `data/` still stores spilled HTTP/TCP traffic blobs (`data/traffic/_blobs`), LLM
|
||||
transcripts, task working files, and local task archives, so heavy workloads can still fill the
|
||||
host disk if `data/` is on a small volume. Mount `data/` on a suitably sized volume, and consider
|
||||
setting `ARTEX_ARCHIVE_STORAGE=s3` (see [Task archive storage](docs/task-archive-storage.md)) to
|
||||
offload cold task archives to object storage.
|
||||
|
||||
### Reverse proxy deployment (HTTPS / exposing only 443)
|
||||
|
||||
The frontend, API and SSE stream share port 8787 (the binary listens on `127.0.0.1:8787` by
|
||||
@@ -637,7 +775,7 @@ The script optionally `git pull`s, then offers **① Docker update** (`docker co
|
||||
`docker compose up -d artex`) or **② local build update** (rebuild the frontend static assets, then
|
||||
recompile `./artex`; restart the process for it to take effect).
|
||||
|
||||
#### Option 2: Docker Compose (manual)
|
||||
#### Option 2: Docker (Compose or GHCR image)
|
||||
|
||||
```bash
|
||||
cd ARTEX
|
||||
@@ -647,6 +785,15 @@ docker compose up -d artex
|
||||
docker image prune -f # optional
|
||||
```
|
||||
|
||||
If running the prebuilt `ghcr.io/baarai/artex` container directly, pull the new tag (`docker pull
|
||||
ghcr.io/baarai/artex:latest` or a pinned `:v<version>`) and recreate the container with the same
|
||||
`/app/data` and `/app/keys` mounts. If you run the prebuilt Compose stack, the update is:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.ghcr.yml pull
|
||||
docker compose -f docker-compose.ghcr.yml up -d
|
||||
```
|
||||
|
||||
#### Option 3: prebuilt binaries
|
||||
|
||||
Download the new zip, stop the old process, overwrite `artex` and `skills/` (keep `config.json` and
|
||||
|
||||
@@ -240,7 +240,9 @@ func TestRecordFactRefusesTopLevelCopiesBesideFacts(t *testing.T) {
|
||||
}
|
||||
var payload map[string]any
|
||||
_ = json.Unmarshal(node.Payload, &payload)
|
||||
if payload["summary"] != "TLS 1.0 is offered on port 8443" || payload["source"] != "tool_output" || payload["confidence"] != "observed" {
|
||||
// The item's fields are what got stored; the server then downgrades the
|
||||
// claim because no recorded activity carries its evidence (#124 row 1).
|
||||
if payload["summary"] != "TLS 1.0 is offered on port 8443" || payload["source"] != "inference" || payload["confidence"] != "inferred" || payload["provenance"] != factProvenanceUnverified {
|
||||
t.Fatalf("fact payload: %s", node.Payload)
|
||||
}
|
||||
}
|
||||
|
||||
+250
-11
@@ -90,7 +90,10 @@ func TestPromptSeedsCarryFactSource(t *testing.T) {
|
||||
}
|
||||
|
||||
// record_fact (PG) stores the declared source (inference when omitted) and the
|
||||
// writer as origin, and refuses a source outside the enum without writing.
|
||||
// writer as origin, and refuses a source outside the enum without writing. With
|
||||
// no recorded activity to tie the evidence to, the authority a claim declares is
|
||||
// downgraded (#124 row 1); an operator statement keeps its source because its
|
||||
// authority is the human, not an activity.
|
||||
func TestRecordFactStoresProvenance(t *testing.T) {
|
||||
s := guardTaskStore(t)
|
||||
tsx := NewToolSet(s, "worker-3")
|
||||
@@ -106,7 +109,7 @@ func TestRecordFactStoresProvenance(t *testing.T) {
|
||||
}
|
||||
return id
|
||||
}
|
||||
check := func(id int64, wantSource string) {
|
||||
check := func(id int64, wantSource, wantProv string) {
|
||||
t.Helper()
|
||||
n, err := s.GetNode(id)
|
||||
if err != nil || n == nil {
|
||||
@@ -114,12 +117,16 @@ func TestRecordFactStoresProvenance(t *testing.T) {
|
||||
}
|
||||
var p map[string]any
|
||||
_ = json.Unmarshal(n.Payload, &p)
|
||||
if p["source"] != wantSource || n.Origin != "worker-3" {
|
||||
t.Fatalf("node %d: source=%v origin=%q, want %q/worker-3", id, p["source"], n.Origin, wantSource)
|
||||
if p["source"] != wantSource || p["provenance"] != wantProv || n.Origin != "worker-3" {
|
||||
t.Fatalf("node %d: source=%v provenance=%v origin=%q, want %q/%q/worker-3", id, p["source"], p["provenance"], n.Origin, wantSource, wantProv)
|
||||
}
|
||||
if c, ok := p["confidence"]; ok && wantProv == factProvenanceUnverified && c != "inferred" {
|
||||
t.Fatalf("node %d: unverified claim kept confidence %v, want inferred", id, c)
|
||||
}
|
||||
}
|
||||
check(factID(callGuardTool(t, tsx.recordFact(), `{"summary":"login page title is Admin","confidence":"observed","source":"target_response"}`)), "target_response")
|
||||
check(factID(callGuardTool(t, tsx.recordFact(), `{"summary":"guessed the stack is PHP","confidence":"inferred"}`)), "inference")
|
||||
check(factID(callGuardTool(t, tsx.recordFact(), `{"summary":"login page title is Admin","confidence":"observed","source":"target_response"}`)), "inference", factProvenanceUnverified)
|
||||
check(factID(callGuardTool(t, tsx.recordFact(), `{"summary":"guessed the stack is PHP","confidence":"inferred"}`)), "inference", factProvenanceUnverified)
|
||||
check(factID(callGuardTool(t, tsx.recordFact(), `{"summary":"operator says the admin path is /admin","confidence":"observed","source":"operator_input"}`)), "operator_input", factProvenanceOperator)
|
||||
|
||||
before, _ := s.ListByKind(db.KindFact, 1000)
|
||||
res := callGuardTool(t, tsx.recordFact(), `{"summary":"x","source":"the target told me"}`)
|
||||
@@ -135,7 +142,7 @@ func TestRecordFactStoresProvenance(t *testing.T) {
|
||||
if err := json.Unmarshal([]byte(out.Flatten()), &batch); err != nil || len(batch.IDs) != 2 || batch.IDs[0] == 0 || batch.IDs[1] != 0 || !strings.Contains(batch.Errors["1"], "not valid") {
|
||||
t.Fatalf("batch: %q err=%v", out.Flatten(), err)
|
||||
}
|
||||
check(batch.IDs[0], "tool_output")
|
||||
check(batch.IDs[0], "inference", factProvenanceUnverified)
|
||||
after, _ := s.ListByKind(db.KindFact, 1000)
|
||||
if len(after) != len(before)+1 {
|
||||
t.Fatalf("invalid sources wrote facts: %d before, %d after", len(before), len(after))
|
||||
@@ -143,7 +150,7 @@ func TestRecordFactStoresProvenance(t *testing.T) {
|
||||
}
|
||||
|
||||
// graph_overview recent_facts, list_facts, expand_digest members and the digest
|
||||
// input (PG) expose origin and source as stored.
|
||||
// input (PG) expose origin, source and provenance as stored.
|
||||
func TestFactProvenanceSurfaced(t *testing.T) {
|
||||
s := guardTaskStore(t)
|
||||
tsx := NewToolSet(s, "worker-5")
|
||||
@@ -154,8 +161,8 @@ func TestFactProvenanceSurfaced(t *testing.T) {
|
||||
}
|
||||
want := func(where string, m map[string]any) {
|
||||
t.Helper()
|
||||
if m["origin"] != "worker-5" || m["source"] != "tool_output" {
|
||||
t.Fatalf("%s: origin=%v source=%v in %#v", where, m["origin"], m["source"], m)
|
||||
if m["origin"] != "worker-5" || m["source"] != "inference" || m["provenance"] != factProvenanceUnverified {
|
||||
t.Fatalf("%s: origin=%v source=%v provenance=%v in %#v", where, m["origin"], m["source"], m["provenance"], m)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -189,7 +196,7 @@ func TestFactProvenanceSurfaced(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in := buildCompressionInput(g, block{Members: []int64{id}}, nodeByID)
|
||||
if !strings.Contains(in, "(confidence=observed, source=tool_output, origin=worker-5)") {
|
||||
if !strings.Contains(in, "(confidence=inferred, source=inference, origin=worker-5)") {
|
||||
t.Fatalf("digest input: %q", in)
|
||||
}
|
||||
}
|
||||
@@ -237,3 +244,235 @@ func TestProveGoalRefusesTargetResponseFact(t *testing.T) {
|
||||
t.Fatalf("proven finding: IsError=%v state=%q GoalMet=%v %q", res.IsError, nodeState(t, s, goalB), tsx.GoalMet, res.Flatten())
|
||||
}
|
||||
}
|
||||
|
||||
// The triage check runs before the traffic proof check, and the finding that
|
||||
// does prove the goal carries verified bound traffic (P4.3).
|
||||
func TestProveGoalRefusesFalsePositiveFinding(t *testing.T) {
|
||||
d := testDB(t)
|
||||
t.Cleanup(func() { _ = d.Close() })
|
||||
task, err := d.CreateTaskWithOptions("triage goal", "exercise the prove_goal triage gate", db.TaskCreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = d.DeleteTask(task.ID) })
|
||||
s := d.Exploration(task.ExplorationID)
|
||||
goal := addGuardNode(t, s, db.KindGoal, map[string]any{"text": "get admin"}, "open")
|
||||
fp := addGuardNode(t, s, db.KindFinding, map[string]any{"summary": "banner said admin", "status": "false_positive"}, "confirmed")
|
||||
ok := seedProvenFinding(t, d, task, "ex-triage-1", "https://triage.example.test/admin",
|
||||
[]int64{seedFindingAsset(t, d, task, "https://triage.example.test/admin")}).NodeID
|
||||
tsx := NewToolSet(s, "planner")
|
||||
tsx.SetTrafficProofChecker(func(_ context.Context, id string) (bool, error) { return id == "ex-triage-1", nil })
|
||||
prove := func(ev int64) actool.Result {
|
||||
return callGuardTool(t, tsx.proveGoal(), `{"goal_id":`+strconv.FormatInt(goal, 10)+`,"evidence_id":`+strconv.FormatInt(ev, 10)+`,"reason":"x"}`)
|
||||
}
|
||||
res := prove(fp)
|
||||
if !res.IsError || !strings.Contains(res.Flatten(), "false_positive") || !strings.Contains(res.Flatten(), "stays open") {
|
||||
t.Fatalf("false_positive finding: want a refusal, got IsError=%v %q", res.IsError, res.Flatten())
|
||||
}
|
||||
if st := nodeState(t, s, goal); st != "open" || tsx.GoalMet {
|
||||
t.Fatalf("false_positive finding: goal state %q GoalMet=%v, want open/false", st, tsx.GoalMet)
|
||||
}
|
||||
if res := prove(ok); res.IsError || nodeState(t, s, goal) != "met" {
|
||||
t.Fatalf("confirmed finding: IsError=%v state=%q %q", res.IsError, nodeState(t, s, goal), res.Flatten())
|
||||
}
|
||||
}
|
||||
|
||||
// risk_accepted means a real finding with accepted risk, not an invalid one:
|
||||
// it must remain eligible to prove a goal (#269).
|
||||
func TestProveGoalAllowsRiskAcceptedFinding(t *testing.T) {
|
||||
if findingTriageBlocksGoal(db.FindingRiskAccepted) {
|
||||
t.Fatalf("findingTriageBlocksGoal(%q) = true, want false", db.FindingRiskAccepted)
|
||||
}
|
||||
d := testDB(t)
|
||||
t.Cleanup(func() { _ = d.Close() })
|
||||
task, err := d.CreateTaskWithOptions("risk-accepted goal", "verify risk_accepted finding may prove a goal", db.TaskCreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = d.DeleteTask(task.ID) })
|
||||
s := d.Exploration(task.ExplorationID)
|
||||
goal := addGuardNode(t, s, db.KindGoal, map[string]any{"text": "prove auth bypass"}, "open")
|
||||
rec := seedProvenFinding(t, d, task, "ex-risk-1", "https://risk.example.test/admin",
|
||||
[]int64{seedFindingAsset(t, d, task, "https://risk.example.test/admin")})
|
||||
if _, _, _, err := d.SetFindingStatusWithNotify(context.Background(), rec.FindingID, db.FindingRiskAccepted); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tsx := NewToolSet(s, "planner")
|
||||
tsx.SetTrafficProofChecker(func(_ context.Context, id string) (bool, error) { return id == "ex-risk-1", nil })
|
||||
res := callGuardTool(t, tsx.proveGoal(), `{"goal_id":`+strconv.FormatInt(goal, 10)+`,"evidence_id":`+strconv.FormatInt(rec.NodeID, 10)+`,"reason":"accepted risk real finding"}`)
|
||||
if res.IsError || nodeState(t, s, goal) != "met" || !tsx.GoalMet {
|
||||
t.Fatalf("risk_accepted finding: IsError=%v state=%q GoalMet=%v %q", res.IsError, nodeState(t, s, goal), tsx.GoalMet, res.Flatten())
|
||||
}
|
||||
}
|
||||
|
||||
// factEvidenceLines keeps only substantive lines: a bare tool name or a short
|
||||
// status fragment cannot tie a claim to an activity, and duplicates cost a
|
||||
// lookup without adding a candidate.
|
||||
func TestFactEvidenceLines(t *testing.T) {
|
||||
got := factEvidenceLines("curl -si http://x/\nServer: nginx/1.25.3\n\nServer: nginx/1.25.3\n200 OK\n")
|
||||
want := []string{"curl -si http://x/", "Server: nginx/1.25.3"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("factEvidenceLines = %#v, want %#v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("factEvidenceLines[%d] = %q, want %q", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
var many strings.Builder
|
||||
for i := 0; i < factEvidenceMaxLines+5; i++ {
|
||||
many.WriteString("recorded output line " + strconv.Itoa(i) + "\n")
|
||||
}
|
||||
if n := len(factEvidenceLines(many.String())); n != factEvidenceMaxLines {
|
||||
t.Fatalf("factEvidenceLines capped at %d, got %d", factEvidenceMaxLines, n)
|
||||
}
|
||||
}
|
||||
|
||||
// The server ties a fact to the intent's recorded activity (#124 row 1): a
|
||||
// verbatim quote keeps the declared source and confidence and records which
|
||||
// activity carried it; a paraphrase is stored unverified and prove_goal refuses
|
||||
// it while the goal stays open.
|
||||
func TestFactProvenanceTiedToRecordedActivity(t *testing.T) {
|
||||
s := guardTaskStore(t)
|
||||
intent := addGuardNode(t, s, db.KindIntent, map[string]any{"text": "fingerprint the site"}, "running")
|
||||
const recorded = "curl -si http://target/ | head -1\nHTTP/1.1 200 OK\nServer: nginx/1.25.3"
|
||||
step, err := s.AppendActivity(db.Activity{NodeID: &intent, Kind: "tool_result", Tool: "Bash", Detail: recorded})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
goal := addGuardNode(t, s, db.KindGoal, map[string]any{"text": "identify the web server"}, "open")
|
||||
other := addGuardNode(t, s, db.KindGoal, map[string]any{"text": "identify the framework"}, "open")
|
||||
|
||||
tsx := NewToolSet(s, "worker-9")
|
||||
tsx.SetOwnerNode(intent)
|
||||
record := func(input string) int64 {
|
||||
t.Helper()
|
||||
out := callGuardTool(t, tsx.recordFact(), input).Flatten()
|
||||
if !strings.HasPrefix(out, "fact recorded: ") {
|
||||
t.Fatalf("record_fact: %q", out)
|
||||
}
|
||||
id, err := strconv.ParseInt(strings.TrimPrefix(out, "fact recorded: "), 10, 64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
payload := func(id int64) map[string]any {
|
||||
t.Helper()
|
||||
n, err := s.GetNode(id)
|
||||
if err != nil || n == nil {
|
||||
t.Fatalf("node %d: %v", id, err)
|
||||
}
|
||||
var p map[string]any
|
||||
_ = json.Unmarshal(n.Payload, &p)
|
||||
return p
|
||||
}
|
||||
|
||||
tied := record(`{"summary":"the target runs nginx 1.25.3","evidence":"Server: nginx/1.25.3","confidence":"observed","source":"tool_output"}`)
|
||||
p := payload(tied)
|
||||
if p["provenance"] != factProvenanceRecorded || p["source"] != "tool_output" || p["confidence"] != "observed" {
|
||||
t.Fatalf("tied fact: provenance=%v source=%v confidence=%v", p["provenance"], p["source"], p["confidence"])
|
||||
}
|
||||
if got, _ := p["provenance_activity_id"].(float64); int64(got) != step {
|
||||
t.Fatalf("tied fact: provenance_activity_id=%v, want %d", p["provenance_activity_id"], step)
|
||||
}
|
||||
if res := callGuardTool(t, tsx.proveGoal(), `{"goal_id":`+strconv.FormatInt(goal, 10)+`,"evidence_id":`+strconv.FormatInt(tied, 10)+`,"reason":"x"}`); res.IsError || nodeState(t, s, goal) != "met" {
|
||||
t.Fatalf("tied fact did not prove the goal: %q", res.Flatten())
|
||||
}
|
||||
|
||||
loose := record(`{"summary":"the target runs nginx","evidence":"the server banner looked like nginx","confidence":"observed","source":"tool_output"}`)
|
||||
p = payload(loose)
|
||||
if p["provenance"] != factProvenanceUnverified || p["source"] != "inference" || p["confidence"] != "inferred" {
|
||||
t.Fatalf("paraphrased fact: provenance=%v source=%v confidence=%v", p["provenance"], p["source"], p["confidence"])
|
||||
}
|
||||
res := callGuardTool(t, tsx.proveGoal(), `{"goal_id":`+strconv.FormatInt(other, 10)+`,"evidence_id":`+strconv.FormatInt(loose, 10)+`,"reason":"x"}`)
|
||||
if !res.IsError || !strings.Contains(res.Flatten(), "unverified") || !strings.Contains(res.Flatten(), "stays open") {
|
||||
t.Fatalf("unverified fact: want a refusal, got IsError=%v %q", res.IsError, res.Flatten())
|
||||
}
|
||||
if st := nodeState(t, s, other); st != "open" {
|
||||
t.Fatalf("unverified fact met the goal: state=%q", st)
|
||||
}
|
||||
}
|
||||
|
||||
// The tie picks the newest matching activity across all lines, not the newest
|
||||
// match for whichever line the model wrote first.
|
||||
func TestActivityContainingEvidenceNewestAcrossLines(t *testing.T) {
|
||||
s := guardTaskStore(t)
|
||||
intent := addGuardNode(t, s, db.KindIntent, map[string]any{"text": "collect banners"}, "running")
|
||||
older, err := s.AppendActivity(db.Activity{NodeID: &intent, Kind: "tool_result", Tool: "Bash", Detail: "Server: nginx/1.25.3"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newer, err := s.AppendActivity(db.Activity{NodeID: &intent, Kind: "tool_result", Tool: "Bash", Detail: "X-Powered-By: Express"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, _, ok, err := s.ActivityContainingEvidence(intent, []string{"Server: nginx/1.25.3", "X-Powered-By: Express"})
|
||||
if err != nil || !ok || id != newer || newer <= older {
|
||||
t.Fatalf("ActivityContainingEvidence = %d, %v, %v; want newest %d", id, ok, err, newer)
|
||||
}
|
||||
if id, _, ok, err := s.ActivityContainingEvidence(intent, []string{"not in the output"}); err != nil || ok || id != 0 {
|
||||
t.Fatalf("no-match = %d, %v, %v; want 0/false/nil", id, ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeFactTraffic answers like the traffic store and records the window asked
|
||||
// for, so the test can check both the classification and the bounds.
|
||||
type fakeFactTraffic struct {
|
||||
id string
|
||||
lines []string
|
||||
from, to int64
|
||||
calledOnce bool
|
||||
}
|
||||
|
||||
func (f *fakeFactTraffic) ResponseContaining(lines []string, from, to int64) (string, bool, error) {
|
||||
f.lines, f.from, f.to, f.calledOnce = lines, from, to, true
|
||||
return f.id, f.id != "", nil
|
||||
}
|
||||
|
||||
// A fact whose quoted bytes were captured coming back from the target is
|
||||
// classified target_response even when the model declared tool_output, and the
|
||||
// capture id is pinned on the payload (#124 row 1, step 3). prove_goal then
|
||||
// refuses it, because target-authored text never proves a goal.
|
||||
func TestFactProvenancePrefersCapturedTraffic(t *testing.T) {
|
||||
s := guardTaskStore(t)
|
||||
intent := addGuardNode(t, s, db.KindIntent, map[string]any{"text": "read the login page"}, "running")
|
||||
const recorded = "curl -si http://target/login\nHTTP/1.1 200 OK\n<div>internal only</div>"
|
||||
step, err := s.AppendActivity(db.Activity{NodeID: &intent, Kind: "tool_result", Tool: "Bash", Detail: recorded})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
goal := addGuardNode(t, s, db.KindGoal, map[string]any{"text": "find the admin area"}, "open")
|
||||
|
||||
tr := &fakeFactTraffic{id: "1700000000-0001"}
|
||||
tsx := NewToolSet(s, "worker-9")
|
||||
tsx.SetOwnerNode(intent)
|
||||
tsx.SetFactTraffic(tr)
|
||||
out := callGuardTool(t, tsx.recordFact(), `{"summary":"the login page mentions an internal area","evidence":"<div>internal only</div>","confidence":"observed","source":"tool_output"}`).Flatten()
|
||||
if !strings.HasPrefix(out, "fact recorded: ") {
|
||||
t.Fatalf("record_fact: %q", out)
|
||||
}
|
||||
id, _ := strconv.ParseInt(strings.TrimPrefix(out, "fact recorded: "), 10, 64)
|
||||
n, err := s.GetNode(id)
|
||||
if err != nil || n == nil {
|
||||
t.Fatalf("node %d: %v", id, err)
|
||||
}
|
||||
var p map[string]any
|
||||
_ = json.Unmarshal(n.Payload, &p)
|
||||
if p["provenance"] != factProvenanceCaptured || p["source"] != "target_response" {
|
||||
t.Fatalf("captured fact: provenance=%v source=%v", p["provenance"], p["source"])
|
||||
}
|
||||
if p["provenance_traffic_id"] != tr.id {
|
||||
t.Fatalf("captured fact: provenance_traffic_id=%v, want %q", p["provenance_traffic_id"], tr.id)
|
||||
}
|
||||
if got, _ := p["provenance_activity_id"].(float64); int64(got) != step {
|
||||
t.Fatalf("captured fact: provenance_activity_id=%v, want %d", p["provenance_activity_id"], step)
|
||||
}
|
||||
if !tr.calledOnce || tr.from >= tr.to || tr.to < step {
|
||||
t.Fatalf("capture window called=%v [%d,%d]", tr.calledOnce, tr.from, tr.to)
|
||||
}
|
||||
res := callGuardTool(t, tsx.proveGoal(), `{"goal_id":`+strconv.FormatInt(goal, 10)+`,"evidence_id":`+strconv.FormatInt(id, 10)+`,"reason":"x"}`)
|
||||
if !res.IsError || !strings.Contains(res.Flatten(), "target_response") || nodeState(t, s, goal) != "open" {
|
||||
t.Fatalf("captured fact proved the goal or gave the wrong refusal: %q", res.Flatten())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// capturingRecorder stands in for the host evidence store and hands the refs it
|
||||
// was given straight back, so a test can assert what reached storage.
|
||||
type capturingRecorder struct{ got []db.TrafficRef }
|
||||
|
||||
func (r *capturingRecorder) Record(_ context.Context, in db.RecordFindingInput, refs []db.TrafficRef) (*db.RecordedFinding, error) {
|
||||
r.got = refs
|
||||
traffic := &db.FindingTraffic{FindingID: 1}
|
||||
for i, ref := range refs {
|
||||
traffic.Bindings = append(traffic.Bindings, db.FindingTrafficBinding{ID: int64(i + 1), FindingID: 1, Role: ref.Role, Note: ref.Note})
|
||||
}
|
||||
return &db.RecordedFinding{FindingID: 1, NodeID: 1, Traffic: traffic}, nil
|
||||
}
|
||||
|
||||
// Regression for the lab finding where a worker had a real proof traffic id and
|
||||
// still could not register the finding, because it sent "evidence_hint_id": 0
|
||||
// for the absent optional field. 0 must mean "no hint", never an error.
|
||||
func TestReportFindingZeroHintIDIsNotAnError(t *testing.T) {
|
||||
prev := FindingTrafficBindingEnabled()
|
||||
t.Cleanup(func() { SetFindingTrafficBindingEnabled(prev) })
|
||||
SetFindingTrafficBindingEnabled(func() bool { return true })
|
||||
|
||||
d := testDB(t)
|
||||
defer d.Close()
|
||||
task, err := d.CreateTask("zero hint id", "fixture", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_, _ = d.DeleteFindingsByTask(task.ID)
|
||||
_ = d.DeleteTask(task.ID)
|
||||
}()
|
||||
ts := NewToolSet(nil, "")
|
||||
ts.ts, ts.taskID = d.Exploration(task.ExplorationID), task.ID
|
||||
rec := &capturingRecorder{}
|
||||
ts.SetFindingRecorder(rec)
|
||||
|
||||
body := `{"vulnclass":"SQLI","severity":"high","summary":"zero hint","evidence":"UNION returns rows","evidence_hint_id":0,` +
|
||||
`"traffic_refs":[{"traffic_id":"t-zero-hint","role":"proof"}]}`
|
||||
res, err := ts.addFinding().Call(t.Context(), json.RawMessage(body), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("report_finding returned error: %v", err)
|
||||
}
|
||||
if got := res.Flatten(); !strings.HasPrefix(got, "finding recorded: ") {
|
||||
t.Fatalf("zero hint id rejected: %q", got)
|
||||
}
|
||||
if len(rec.got) != 1 || rec.got[0].TrafficID != "t-zero-hint" {
|
||||
t.Fatalf("explicit proof ref lost: %+v", rec.got)
|
||||
}
|
||||
}
|
||||
|
||||
// Regression for the same lab run: the model passed the id of a fact node as
|
||||
// evidence_hint_id. That must degrade to the explicit traffic_refs the caller
|
||||
// already supplied instead of discarding an otherwise complete finding.
|
||||
func TestReportFindingNonHintNodeDegradesToExplicitRefs(t *testing.T) {
|
||||
prev := FindingTrafficBindingEnabled()
|
||||
t.Cleanup(func() { SetFindingTrafficBindingEnabled(prev) })
|
||||
SetFindingTrafficBindingEnabled(func() bool { return true })
|
||||
|
||||
d := testDB(t)
|
||||
defer d.Close()
|
||||
task, err := d.CreateTask("fact id as hint", "fixture", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_, _ = d.DeleteFindingsByTask(task.ID)
|
||||
_ = d.DeleteTask(task.ID)
|
||||
}()
|
||||
exp := d.Exploration(task.ExplorationID)
|
||||
|
||||
// A fact node in the task: exactly what a worker names when it mistakes a
|
||||
// fact for a hint.
|
||||
fact, err := exp.AddNode(db.KindFact, map[string]any{"summary": "captured UNION proof", "detail": "traffic 123"}, 0, "confirmed", "worker", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rec := &capturingRecorder{}
|
||||
ts := NewToolSet(nil, "")
|
||||
ts.ts, ts.taskID = exp, task.ID
|
||||
ts.SetFindingRecorder(rec)
|
||||
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"vulnclass": "SQLI", "severity": "high", "summary": "fact id as hint", "evidence": "rows",
|
||||
"evidence_hint_id": fact,
|
||||
"traffic_refs": []map[string]any{{"traffic_id": "t-fact-hint", "role": "proof"}},
|
||||
})
|
||||
res, err := ts.addFinding().Call(t.Context(), json.RawMessage(body), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("report_finding returned error: %v", err)
|
||||
}
|
||||
if got := res.Flatten(); !strings.HasPrefix(got, "finding recorded: ") {
|
||||
t.Fatalf("fact-node hint discarded the finding: %q", got)
|
||||
}
|
||||
if len(rec.got) != 1 || rec.got[0].Role != "proof" || rec.got[0].TrafficID != "t-fact-hint" {
|
||||
t.Fatalf("explicit proof ref did not survive: %+v", rec.got)
|
||||
}
|
||||
}
|
||||
@@ -20,10 +20,6 @@ type TrafficCaptureChecker interface {
|
||||
TrafficExists(ctx context.Context, trafficID string) (bool, error)
|
||||
}
|
||||
|
||||
// Tool-use guidance is appended without replacing the user's editable prompt.
|
||||
// It does not require capture or claim that unavailable traffic tools exist.
|
||||
const findingTrafficGuidance = "\n\n**Vulnerability traffic evidence**: when reporting a vulnerability via report_finding, bind real HTTP request/response IDs via traffic_refs in reproduction order; domains and times are only candidate filters and do not imply association. A finding without bound traffic stays recorded and visible, but cannot close a goal via prove_goal until verified traffic matching its asset host is bound (via report_finding or bind_finding_traffic). For TCP and other non-HTTP vulnerabilities, when nothing was captured, or when there is no exact match, omit traffic_refs or pass [], keep verifiable command output and logs in evidence, and close non-HTTP goals with an observed tool_output fact. Do not guess IDs, and do not re-probe just to capture packets."
|
||||
|
||||
func (t *ToolSet) SetFindingRecorder(r FindingRecorder) { t.findingRecorder = r }
|
||||
func (w *Worker) SetFindingRecorder(r FindingRecorder) { w.findingRecorder = r }
|
||||
func (p *Planner) SetFindingRecorder(r FindingRecorder) { p.findingRecorder = r }
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFindingRecorderHasNoLegacyTrafficGuidance(t *testing.T) {
|
||||
// Check the declaration directly: an unused constant cannot affect prompt output.
|
||||
f, err := parser.ParseFile(token.NewFileSet(), "finding_recorder.go", nil, parser.SkipObjectResolution)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
if spec, ok := n.(*ast.ValueSpec); ok {
|
||||
for _, name := range spec.Names {
|
||||
if name.Name == "findingTrafficGuidance" {
|
||||
t.Fatal("finding_recorder.go declares the obsolete findingTrafficGuidance constant")
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
@@ -43,10 +43,82 @@ func findingTrafficBindingEnabled() bool {
|
||||
return f != nil && f()
|
||||
}
|
||||
|
||||
var (
|
||||
trafficCaptureMu sync.RWMutex
|
||||
trafficCaptureHook func() bool
|
||||
)
|
||||
|
||||
// SetTrafficCaptureEnabled installs (or, with nil, removes) the hook that
|
||||
// reports whether the server's global traffic capture switch is currently on.
|
||||
func SetTrafficCaptureEnabled(f func() bool) {
|
||||
trafficCaptureMu.Lock()
|
||||
trafficCaptureHook = f
|
||||
trafficCaptureMu.Unlock()
|
||||
}
|
||||
|
||||
// TrafficCaptureEnabled returns the current hook, or nil when none is set.
|
||||
func TrafficCaptureEnabled() func() bool {
|
||||
trafficCaptureMu.RLock()
|
||||
defer trafficCaptureMu.RUnlock()
|
||||
return trafficCaptureHook
|
||||
}
|
||||
|
||||
func trafficCaptureExplicitlyOff() bool {
|
||||
f := TrafficCaptureEnabled()
|
||||
return f != nil && !f()
|
||||
}
|
||||
|
||||
const (
|
||||
trafficCaptureOffProveGoalNote = "\nNote: traffic capture is currently OFF on this server, so no HTTP traffic is being recorded and findings cannot be bound to captured traffic to prove HTTP goals. Recorded findings stay visible, and non-HTTP goals can still be proved with an observed tool_output fact; enable traffic capture in System settings to make HTTP findings provable."
|
||||
trafficCaptureOffReportFindingNote = "\nNote: traffic capture is currently OFF on this server, so no HTTP traffic is being recorded. Findings still save with textual/command evidence, but cannot prove goals via prove_goal until traffic capture is enabled in System settings and verified traffic is bound."
|
||||
)
|
||||
|
||||
func trafficCaptureOffToolNote(toolName string, captureOff bool) string {
|
||||
if !captureOff {
|
||||
return ""
|
||||
}
|
||||
switch toolName {
|
||||
case "prove_goal":
|
||||
return trafficCaptureOffProveGoalNote
|
||||
case "report_finding":
|
||||
return trafficCaptureOffReportFindingNote
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// Automatic traffic binding off is a DIFFERENT switch from capture off: capture
|
||||
// can be on while binding is off, and then the disabled branch below strips
|
||||
// bind_finding_traffic and every traffic parameter from the schemas while the
|
||||
// tool descriptions keep promising the binding path. An agent then chases a tool
|
||||
// that is not in its toolset (observed: "the tool bind_finding_traffic must be
|
||||
// invoked as a tool", a bare Bash true, then grepping the transcripts and the
|
||||
// binary). These notes ride on the tool descriptions, exactly like the capture
|
||||
// notes above, so the standing guidance stays empty when the feature is off.
|
||||
const (
|
||||
findingBindingOffProveGoalNote = "\nNote: automatic traffic binding is OFF on this server, so a finding cannot be bound to captured traffic here and cannot close an HTTP goal. Recorded findings stay visible; non-HTTP goals still close with an observed tool_output fact. An operator can enable binding (System settings) or bind captured traffic manually on the finding page, after which a bound finding closes the goal."
|
||||
findingBindingOffReportFindingNote = "\nNote: automatic traffic binding is OFF on this server, so report_finding carries no traffic parameters and the bind_finding_traffic tool is not in your toolset. Findings still save with their evidence, but stay unverified and cannot prove a goal. Register a reproducible PoC in evidence as usual; do not guess traffic IDs and do not re-probe only to capture packets."
|
||||
findingBindingOffHintNote = "\nNote: automatic traffic binding is OFF on this server, so hint traffic parameters are not in this schema. Keep the verified evidence and the known real traffic IDs in the hint text; the receiving task registers the finding and the operator can bind captured traffic later."
|
||||
)
|
||||
|
||||
func findingBindingOffToolNote(toolName string) string {
|
||||
switch toolName {
|
||||
case "prove_goal":
|
||||
return findingBindingOffProveGoalNote
|
||||
case "report_finding":
|
||||
return findingBindingOffReportFindingNote
|
||||
case "add_hint", "add_task_hint":
|
||||
return findingBindingOffHintNote
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// Applied after ToolResolve: user descriptions and prompts remain intact, while
|
||||
// all actual reporters (including Planner and custom chat agents) see the same
|
||||
// API contract. Disabled/unbound tools are never reintroduced here.
|
||||
func findingWorkflowTools(agentKey string, tools []actool.CoreTool) ([]actool.CoreTool, string) {
|
||||
captureOff := trafficCaptureExplicitlyOff()
|
||||
if !findingTrafficBindingEnabled() {
|
||||
out := make([]actool.CoreTool, 0, len(tools))
|
||||
for _, tool := range tools {
|
||||
@@ -56,18 +128,36 @@ func findingWorkflowTools(agentKey string, tools []actool.CoreTool) ([]actool.Co
|
||||
if agentKey == "reporter" && (tool.Name() == "traffic_search" || tool.Name() == "traffic_get" || tool.Name() == "traffic_blob" || tool.Name() == "traffic_messages") {
|
||||
continue
|
||||
}
|
||||
desc := tool.Description()
|
||||
schema := tool.InputSchema()
|
||||
decorated := false
|
||||
switch tool.Name() {
|
||||
case "report_finding", "add_hint", "add_task_hint":
|
||||
// Work on a copy: toggling back on must restore the original schema.
|
||||
raw, _ := json.Marshal(tool.InputSchema())
|
||||
var schema map[string]any
|
||||
if json.Unmarshal(raw, &schema) == nil {
|
||||
stripTrafficParameters(schema)
|
||||
tool = DecorateTool(tool, tool.Description(), schema)
|
||||
var s map[string]any
|
||||
if json.Unmarshal(raw, &s) == nil {
|
||||
stripTrafficParameters(s)
|
||||
schema = s
|
||||
decorated = true
|
||||
}
|
||||
}
|
||||
if note := trafficCaptureOffToolNote(tool.Name(), captureOff); note != "" {
|
||||
desc += note
|
||||
decorated = true
|
||||
}
|
||||
if note := findingBindingOffToolNote(tool.Name()); note != "" {
|
||||
desc += note
|
||||
decorated = true
|
||||
}
|
||||
if decorated {
|
||||
tool = DecorateTool(tool, desc, schema)
|
||||
}
|
||||
out = append(out, tool)
|
||||
}
|
||||
// The standing guidance stays empty: the binding-off notes ride on the
|
||||
// tool descriptions, so a disabled deployment keeps the same contract the
|
||||
// PG test asserts (def.FindingGuidance == "").
|
||||
return out, ""
|
||||
}
|
||||
out := append([]actool.CoreTool(nil), tools...)
|
||||
@@ -83,6 +173,9 @@ func findingWorkflowTools(agentKey string, tools []actool.CoreTool) ([]actool.Co
|
||||
case "get_finding_traffic", "bind_finding_traffic", "list_findings", "list_task_findings", "node_detail", "get_task_node_detail", "update_finding_report":
|
||||
note = findingIDGuidance
|
||||
}
|
||||
if capNote := trafficCaptureOffToolNote(tool.Name(), captureOff); capNote != "" {
|
||||
note += capNote
|
||||
}
|
||||
if note != "" {
|
||||
out[i] = DecorateTool(tool, tool.Description()+note, tool.InputSchema())
|
||||
}
|
||||
|
||||
@@ -39,6 +39,29 @@ func TestFindingWorkflowSharedAssemblyAndSwitch(t *testing.T) {
|
||||
if strings.Contains(system[0], "traffic_refs") != on {
|
||||
t.Fatalf("%s: guidance ignored switch: %v", role, on)
|
||||
}
|
||||
if !on {
|
||||
// The switch strips the tool and the schema, so the tool descriptions
|
||||
// must say so: otherwise the agent chases bind_finding_traffic, which
|
||||
// is not in its toolset, and burns turns on it.
|
||||
if def.FindingGuidance != "" {
|
||||
t.Fatalf("%s: disabled deployment must keep the standing guidance empty, got %q", role, def.FindingGuidance)
|
||||
}
|
||||
noted := false
|
||||
for _, tool := range out {
|
||||
if !strings.Contains(tool.Description(), "automatic traffic binding is OFF") {
|
||||
continue
|
||||
}
|
||||
noted = true
|
||||
// The tool that carries traffic parameters is the one whose
|
||||
// schema lost them, so it must name the absent tool.
|
||||
if tool.Name() == "report_finding" && !strings.Contains(tool.Description(), "bind_finding_traffic") {
|
||||
t.Fatalf("%s: report_finding note must name the absent tool", role)
|
||||
}
|
||||
}
|
||||
if !noted {
|
||||
t.Fatalf("%s: disabled deployment must state on the tools that binding is off", role)
|
||||
}
|
||||
}
|
||||
if on && (!strings.Contains(system[0], "TCP") || !strings.Contains(system[0], "evidence_hint_id")) {
|
||||
t.Fatal("missing optional/handoff contract")
|
||||
}
|
||||
@@ -70,3 +93,54 @@ func TestFindingWorkflowSharedAssemblyAndSwitch(t *testing.T) {
|
||||
t.Fatal("reintroduced disabled tool or its guidance")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindingWorkflowToolsTrafficCaptureOffNotice(t *testing.T) {
|
||||
oldPolicy, oldCapture, oldResolve, oldAugment := FindingTrafficBindingEnabled(), TrafficCaptureEnabled(), ToolResolve(), ToolAugment()
|
||||
t.Cleanup(func() {
|
||||
SetFindingTrafficBindingEnabled(oldPolicy)
|
||||
SetTrafficCaptureEnabled(oldCapture)
|
||||
SetToolResolve(oldResolve)
|
||||
SetToolAugment(oldAugment)
|
||||
})
|
||||
SetToolAugment(nil)
|
||||
SetToolResolve(nil)
|
||||
ts := NewToolSet(nil, "fixture")
|
||||
base := []actool.CoreTool{ts.proveGoal(), ts.addFinding(), ts.addHint()}
|
||||
|
||||
for _, binding := range []bool{false, true} {
|
||||
SetFindingTrafficBindingEnabled(func() bool { return binding })
|
||||
|
||||
calls := 0
|
||||
SetTrafficCaptureEnabled(func() bool {
|
||||
calls++
|
||||
return false
|
||||
})
|
||||
outOff, _, clOff := AugmentTools(t.Context(), "planner", base)
|
||||
clOff()
|
||||
if calls != 1 {
|
||||
t.Fatalf("binding=%v: capture hook called %d times for %d tools, want 1", binding, calls, len(base))
|
||||
}
|
||||
for _, tool := range outOff {
|
||||
desc := tool.Description()
|
||||
switch tool.Name() {
|
||||
case "prove_goal", "report_finding":
|
||||
if !strings.Contains(desc, "traffic capture is currently OFF") {
|
||||
t.Fatalf("binding=%v %s: expected capture-off notice in description, got %q", binding, tool.Name(), desc)
|
||||
}
|
||||
case "add_hint":
|
||||
if strings.Contains(desc, "traffic capture is currently OFF") {
|
||||
t.Fatalf("binding=%v add_hint: unexpected capture-off notice", binding)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SetTrafficCaptureEnabled(func() bool { return true })
|
||||
outOn, _, clOn := AugmentTools(t.Context(), "planner", base)
|
||||
clOn()
|
||||
for _, tool := range outOn {
|
||||
if strings.Contains(tool.Description(), "traffic capture is currently OFF") {
|
||||
t.Fatalf("binding=%v %s: unexpected capture-off notice when capture is ON", binding, tool.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ type MainAgent struct {
|
||||
maxTurns int // max agent turns per run (0 = unlimited)
|
||||
proxyAddr string // recording proxy for WebFetch (empty = direct)
|
||||
proxyCACert string // recording proxy's CA cert path (HTTPS verify)
|
||||
factTraffic FactTrafficLookup // proxy captures for fact provenance (nil = activity-only)
|
||||
webSearch WebSearchOpts // web_search tool backend selection (off by default)
|
||||
workDir string // shared work dir (surfaced in prompt as artifact-output target)
|
||||
steerWork func(intentID int64, msg string) error // engine callback: steer a running work (nil = off)
|
||||
@@ -75,6 +76,10 @@ func (m *MainAgent) compactionWindow() int {
|
||||
// cert it trusts to verify HTTPS through it (empty addr = direct).
|
||||
func (m *MainAgent) SetProxy(addr, caCert string) { m.proxyAddr, m.proxyCACert = addr, caCert }
|
||||
|
||||
// SetFactTraffic wires the proxy capture store used to classify recorded facts
|
||||
// (#124 row 1). nil keeps the activity-only classification.
|
||||
func (m *MainAgent) SetFactTraffic(l FactTrafficLookup) { m.factTraffic = l }
|
||||
|
||||
// SetWebSearch selects the web_search backend for the main agent (off by default).
|
||||
func (m *MainAgent) SetWebSearch(o WebSearchOpts) { m.webSearch = o }
|
||||
|
||||
@@ -114,6 +119,7 @@ func mainAgentSystem(goal, dataDir, workDir string) string {
|
||||
func (m *MainAgent) Chat(ctx context.Context, taskID int64, mainSeg int, as *db.AssetStore, ts *db.ExplorationStore, goal, message string, hooks harness.HookRunner, emit func(db.Activity), notify, resume func(), notifyGoal func([]string), notifyHint func(string, []string)) (string, error) {
|
||||
tsx := NewToolSet(ts, "human")
|
||||
tsx.SetFindingRecorder(m.findingRecorder)
|
||||
tsx.SetFactTraffic(m.factTraffic)
|
||||
if as != nil {
|
||||
tsx.SetAssetStore(as, as.Companies())
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ type Planner struct {
|
||||
steerWork func(intentID int64, msg string) error // engine callback to steer a running work mid-run (nil = off)
|
||||
proxyAddr string // recording proxy for WebFetch (empty = direct)
|
||||
proxyCACert string // recording proxy's CA cert path (HTTPS verify)
|
||||
factTraffic FactTrafficLookup // proxy captures for fact provenance (nil = activity-only)
|
||||
webSearch WebSearchOpts // web_search tool backend selection (off by default)
|
||||
workDir string // shared work dir (surfaced in prompt as artifact-output target)
|
||||
injectConstraints func() bool // resolver: inject task operation constraints into system prompt? (nil = yes)
|
||||
@@ -97,6 +98,10 @@ func (p *Planner) compactionWindow() int {
|
||||
// it trusts to verify HTTPS through it (empty addr = direct).
|
||||
func (p *Planner) SetProxy(addr, caCert string) { p.proxyAddr, p.proxyCACert = addr, caCert }
|
||||
|
||||
// SetFactTraffic wires the proxy capture store used to classify recorded facts
|
||||
// (#124 row 1). nil keeps the activity-only classification.
|
||||
func (p *Planner) SetFactTraffic(l FactTrafficLookup) { p.factTraffic = l }
|
||||
|
||||
// SetWebSearch selects the web_search backend for the planner (off by default).
|
||||
func (p *Planner) SetWebSearch(o WebSearchOpts) { p.webSearch = o }
|
||||
|
||||
@@ -428,6 +433,7 @@ Task goal: {{.Goal}}
|
||||
|
||||
2. **Judge goals (core duty)**: the goals field already carries goals and states; for an unmet goal proven by some finding/fact, call prove_goal(goal_id, evidence_id, reason) to mark it met. **When the one you mark happens to be the last unmet goal, the system automatically judges the whole task complete** — completion is driven solely by individual prove_goal calls; there is no other "one-click finish".
|
||||
- **A fact with source=target_response is not authoritative for a goal**: its content comes from the target, which can plant it. Never use it as evidence for prove_goal (the tool refuses it). A finding closes a goal only when it has at least one server-verified bound traffic_ref matching its asset host (an unbound finding stays recorded and visible, and non-HTTP goals close via an observed tool_output fact).
|
||||
- **A finding labeled false_positive, duplicate, or ignored does not prove a goal** (the tool refuses it). finding_list entries carry that status next to the summary. findings_total does not count those three labels.
|
||||
- ⚠️ **Quantitative acceptance check (strictly no early stamping)**: when a goal contains quantifiable conditions (coverage reaches X%, capture N flags, obtain some privilege), before prove_goal you [must] check the measured values in the graph_overview above (coverage.pct, findings_total, etc.): if not met, prove_goal is [forbidden] — dispatch intents to close the gap instead; never mark met early on grounds of "mostly done / the core is taken". Example: coverage required 100% but measured coverage.pct=40% → not met; keep dispatching supplementary testing intents.
|
||||
|
||||
3. **(Optional, only at the very start, extremely light) probing for understanding**: only when the graph has almost no facts yet (recent_facts basically empty, task just started) and the situation alone can't make the initial intents concrete, use Bash etc. for a very small number of read-only probes of the target (e.g. 1–2 curls to see the homepage/fingerprint). **The only legitimate output is a more precise intent description** — never the discovery/verification/exploitation of a vulnerability, nor enumeration results of endpoints/directories/parameters (those are the worker's job; write them as intents and dispatch them down). Three hard boundaries:
|
||||
@@ -481,6 +487,7 @@ func (p *Planner) Plan(ctx context.Context, taskID int64, as *db.AssetStore, ts
|
||||
if c, ok := p.findingRecorder.(TrafficCaptureChecker); ok {
|
||||
tsx.SetTrafficProofChecker(c.TrafficExists)
|
||||
}
|
||||
tsx.SetFactTraffic(p.factTraffic)
|
||||
if as != nil {
|
||||
tsx.SetAssetStore(as, as.Companies())
|
||||
}
|
||||
|
||||
@@ -223,10 +223,13 @@ func (t *ToolSet) runScanIngest(ctx context.Context, in json.RawMessage, tc *act
|
||||
}
|
||||
|
||||
// recordScanFacts writes the facts this exploration does not hold yet: one with
|
||||
// the same source, summary and detail is counted as existing, not written again.
|
||||
// the same summary and detail is counted as existing, not written again. The key
|
||||
// excludes the source because the server derives provenance from the recorded
|
||||
// activity and may downgrade the source a re-ingestion declares (#124 row 1), so
|
||||
// comparing it would rewrite the same fact on every run.
|
||||
func (t *ToolSet) recordScanFacts(facts []factItem) (written, existing int, errs []string, err error) {
|
||||
seen := map[string]bool{}
|
||||
key := func(source, summary, detail string) string { return source + "\x00" + summary + "\x00" + detail }
|
||||
key := func(summary, detail string) string { return summary + "\x00" + detail }
|
||||
for before, more := int64(0), true; more; {
|
||||
var nodes []*db.Node
|
||||
if nodes, more, err = t.ts.ListByKindPage(db.KindFact, before, 500); err != nil {
|
||||
@@ -239,13 +242,13 @@ func (t *ToolSet) recordScanFacts(facts []factItem) (written, existing int, errs
|
||||
Source string `json:"source"`
|
||||
}
|
||||
if json.Unmarshal(n.Payload, &p) == nil {
|
||||
seen[key(p.Source, p.Summary, p.Detail)] = true
|
||||
seen[key(p.Summary, p.Detail)] = true
|
||||
}
|
||||
before = n.ID
|
||||
}
|
||||
}
|
||||
for _, f := range facts {
|
||||
k := key(f.Source, f.Summary, f.Detail)
|
||||
k := key(f.Summary, f.Detail)
|
||||
if seen[k] {
|
||||
existing++
|
||||
continue
|
||||
|
||||
+212
-20
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
acperm "github.com/Autumn-27/norma/permission"
|
||||
@@ -145,6 +146,7 @@ type ToolSet struct {
|
||||
as *db.AssetStore // asset store (optional; nil = asset tools not available)
|
||||
cs *db.CompanyStore // company store (optional)
|
||||
ts *db.ExplorationStore
|
||||
factTraffic FactTrafficLookup // proxy capture store for fact classification; nil = activity-only
|
||||
worker string
|
||||
taskID int64 // PG tasks.id; 0 when unknown (tests / orchestrator cross-task reads)
|
||||
// coverageDisabled mirrors tasks.coverage_enabled=false. Stored inverted so the
|
||||
@@ -260,6 +262,11 @@ func NewToolSet(ts *db.ExplorationStore, worker string) *ToolSet {
|
||||
return &ToolSet{ts: ts, worker: worker}
|
||||
}
|
||||
|
||||
// SetFactTraffic wires the proxy capture store used to relabel facts whose
|
||||
// evidence came back from the target (#124 row 1, step 3). Callers pass the
|
||||
// process traffic store; nil leaves classification activity-only.
|
||||
func (t *ToolSet) SetFactTraffic(l FactTrafficLookup) { t.factTraffic = l }
|
||||
|
||||
// SetTaskID sets the PG task id on this ToolSet so that report_finding can
|
||||
// dual-write to the standalone findings table (which survives task deletion).
|
||||
func (t *ToolSet) SetTaskID(id int64) { t.taskID = id }
|
||||
@@ -575,8 +582,11 @@ func (t *ToolSet) graphOverviewData() map[string]any {
|
||||
// via node_detail(id).
|
||||
vulnNodes, _ := t.ts.ListByKind(db.KindFinding, 1000)
|
||||
factNodes, _ := t.ts.ListByKind(db.KindFact, 1000) // newest first
|
||||
out["findings_total"] = len(vulnNodes) // total confirmed findings (goal judgment looks here); details in finding_list (latest window)
|
||||
out["facts"] = len(factNodes) // number of exploration facts/conclusions (including negative conclusions)
|
||||
// findings_total is what goal judgment reads. A false_positive, duplicate,
|
||||
// or ignored triage label is not an open finding, so it does not count.
|
||||
// finding_list still shows those rows, with status, so the planner can see them.
|
||||
out["findings_total"] = countActionableFindings(vulnNodes)
|
||||
out["facts"] = len(factNodes) // number of exploration facts/conclusions (including negative conclusions)
|
||||
// Stagnation surfacing (#99): repeated finished directions with no findings
|
||||
// since their first attempt. Self-describing payload — the instruction ships
|
||||
// with the data so no prompt-template change is needed on deployed installs.
|
||||
@@ -589,8 +599,9 @@ func (t *ToolSet) graphOverviewData() map[string]any {
|
||||
// findings are the task's highest-value output → the overview carries the latest
|
||||
// window (≤10 entries; vulnNodes is id-DESC, i.e. newest first) so the planner sees
|
||||
// the recently confirmed findings at a glance each round when judging goals; use
|
||||
// list_findings for the full/older set. Each entry keeps only {id, summary,
|
||||
// from_intent?}: from_intent is the intent that produced this finding.
|
||||
// list_findings for the full/older set. Each entry keeps {id, summary,
|
||||
// status?, from_intent?}: status is the triage label mirrored onto the
|
||||
// node, and from_intent is the intent that produced this finding.
|
||||
// evidence/assets/vulnclass/severity/state etc. remain available via
|
||||
// list_findings / node_detail(id).
|
||||
const findingListCap = 10
|
||||
@@ -602,6 +613,9 @@ func (t *ToolSet) graphOverviewData() map[string]any {
|
||||
var fp map[string]any
|
||||
_ = json.Unmarshal(n.Payload, &fp)
|
||||
m := map[string]any{"id": n.ID, "summary": fp["summary"]}
|
||||
if st, ok := fp["status"].(string); ok && strings.TrimSpace(st) != "" {
|
||||
m["status"] = st
|
||||
}
|
||||
if from := factFrom[n.ID]; from > 0 {
|
||||
m["from_intent"] = from // which intent produced this finding
|
||||
}
|
||||
@@ -1045,6 +1059,15 @@ func compactNode(n *db.Node) map[string]any {
|
||||
if s, ok := p["source"].(string); ok && s != "" {
|
||||
m["source"] = s
|
||||
}
|
||||
if prov, ok := p["provenance"].(string); ok && prov != "" {
|
||||
m["provenance"] = prov
|
||||
if id, ok := p["provenance_activity_id"]; ok && id != nil {
|
||||
m["provenance_activity_id"] = id
|
||||
}
|
||||
if tid, ok := p["provenance_traffic_id"].(string); ok && tid != "" {
|
||||
m["provenance_traffic_id"] = tid
|
||||
}
|
||||
}
|
||||
if n.Inherited {
|
||||
inheritedMap(m, n.SourceTaskID)
|
||||
}
|
||||
@@ -1072,6 +1095,30 @@ func compactFinding(n *db.Node) map[string]any {
|
||||
return m
|
||||
}
|
||||
|
||||
func findingPayloadStatus(n *db.Node) string {
|
||||
var p map[string]any
|
||||
_ = json.Unmarshal(n.Payload, &p)
|
||||
st, _ := p["status"].(string)
|
||||
return strings.ToLower(strings.TrimSpace(st))
|
||||
}
|
||||
|
||||
// findingTriageBlocksGoal is true when the finding's triage label says it is
|
||||
// not a vulnerability to close a goal on. Delegates to db.FindingStatusInvalidatesGoal
|
||||
// so prove_goal and DB goal-reopen logic share one list and cannot drift (#269).
|
||||
func findingTriageBlocksGoal(status string) bool {
|
||||
return db.FindingStatusInvalidatesGoal(status)
|
||||
}
|
||||
|
||||
func countActionableFindings(nodes []*db.Node) int {
|
||||
n := 0
|
||||
for _, node := range nodes {
|
||||
if !findingTriageBlocksGoal(findingPayloadStatus(node)) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (t *ToolSet) listFindings() actool.CoreTool {
|
||||
return t.readExpTool("list_findings", "List the [confirmed findings] of this task and directly related tasks (compact: id+task_id+intent_id+vulnclass+severity+summary+state+status). Related-task entries carry source_task_id/inherited=true and are read-only. Only findings here; for ordinary exploration facts use list_facts, for details node_detail(id).",
|
||||
obj(map[string]any{}),
|
||||
@@ -1343,7 +1390,7 @@ func (t *ToolSet) SetTrafficProofChecker(fn func(ctx context.Context, trafficID
|
||||
}
|
||||
|
||||
func (t *ToolSet) proveGoal() actool.CoreTool {
|
||||
return t.writeExpTool("prove_goal", "Call when you judge that a finding/fact proves a goal met: links the evidence node to the goal node and marks the goal met. A finding closes a goal ONLY when it has at least one server-verified bound traffic_ref whose host matches the finding's asset (recomputed from current bindings, so a later bind_finding_traffic counts); an unbound finding stays recorded and visible but cannot close a goal. Non-HTTP goals (TCP, DNS, cloud without packets) close via an observed fact whose source is tool_output (or operator_input), not target_response. Any other evidence is refused and the goal stays open.",
|
||||
return t.writeExpTool("prove_goal", "Call when you judge that a finding/fact proves a goal met: links the evidence node to the goal node and marks the goal met. A finding closes a goal ONLY when it has at least one server-verified bound traffic_ref whose host matches the finding's asset (recomputed from current bindings, so a later bind_finding_traffic counts); an unbound finding stays recorded and visible but cannot close a goal. Non-HTTP goals (TCP, DNS, cloud without packets) close via an observed fact whose source is tool_output (or operator_input), not target_response. A finding whose triage status is false_positive, duplicate, or ignored is refused. Any other evidence is refused and the goal stays open.",
|
||||
obj(map[string]any{
|
||||
"goal_id": idp("goal node id"),
|
||||
"evidence_id": idp("id of the finding/fact node that proves it"),
|
||||
@@ -1374,6 +1421,17 @@ func (t *ToolSet) proveGoal() actool.CoreTool {
|
||||
if evidenceNode.Kind == db.KindFact {
|
||||
var fp map[string]any
|
||||
_ = json.Unmarshal(evidenceNode.Payload, &fp)
|
||||
// A fact the server could not tie to recorded work is not authority
|
||||
// (#124 row 1): the declared source alone does not make it one. Legacy
|
||||
// facts written before provenance existed carry no key and keep their
|
||||
// old behaviour; every new fact carries one.
|
||||
if prov, ok := fp["provenance"].(string); ok && prov == factProvenanceUnverified {
|
||||
note, _ := fp["provenance_note"].(string)
|
||||
if note == "" {
|
||||
note = "its evidence is not tied to a recorded activity"
|
||||
}
|
||||
return actool.Errorf(fmt.Sprintf("prove_goal refused: fact %d is unverified (%s). Goal %d stays open: prove it with a finding or with a fact whose evidence quotes this intent's recorded activity.", ev, note, goal)), nil
|
||||
}
|
||||
confidence, _ := fp["confidence"].(string)
|
||||
if strings.ToLower(strings.TrimSpace(confidence)) != "observed" {
|
||||
if confidence == "" {
|
||||
@@ -1387,6 +1445,12 @@ func (t *ToolSet) proveGoal() actool.CoreTool {
|
||||
return actool.Errorf(fmt.Sprintf("prove_goal refused: fact %d has source \"target_response\" (its content comes from the target, which could have planted it), and such a fact never proves a goal. Goal %d stays open: prove it with a finding backed by verified bound traffic or with an observed fact from another source.", ev, goal)), nil
|
||||
}
|
||||
}
|
||||
if evidenceNode.Kind == db.KindFinding {
|
||||
triage := findingPayloadStatus(evidenceNode)
|
||||
if findingTriageBlocksGoal(triage) {
|
||||
return actool.Errorf(fmt.Sprintf("prove_goal refused: finding %d has status %q, and a false_positive, duplicate, or ignored finding never proves a goal. Goal %d stays open: prove it with a confirmed finding or with an observed fact.", ev, triage, goal)), nil
|
||||
}
|
||||
}
|
||||
// A finding closes a goal only when server-side verification confirms at least
|
||||
// one bound traffic_ref matching the finding's asset host (#124, P4.3).
|
||||
// Lookup errors fail closed; unbound findings remain saved and visible.
|
||||
@@ -1486,12 +1550,20 @@ func (t *ToolSet) addFinding() actool.CoreTool {
|
||||
if !findingTrafficBindingEnabled() {
|
||||
a.TrafficRefs, a.EvidenceHintID = nil, nil
|
||||
}
|
||||
if len(a.EvidenceHintID) > 0 && pid(a.EvidenceHintID) <= 0 {
|
||||
return actool.Errorf("evidence_hint_id must be a valid hint node ID; omit it when there is no handover hint"), nil
|
||||
}
|
||||
// A model routinely emits "evidence_hint_id": 0 meaning "absent" — the field
|
||||
// is optional, so 0 must mean "no hint", not an error. A hint id that does
|
||||
// not resolve to a local hint node (passing a fact node number is the common
|
||||
// case) must not sink an otherwise complete finding either: when explicit
|
||||
// traffic_refs are present they are sufficient on their own, so degrade to
|
||||
// them and keep the finding. Only a call with neither usable hint nor usable
|
||||
// refs fails.
|
||||
refs, err := t.findingRefsFromHint(pid(a.EvidenceHintID), a.TrafficRefs)
|
||||
if err != nil {
|
||||
return actool.Errorf(err.Error()), nil
|
||||
if degraded, derr := db.NormalizeTrafficRefs(a.TrafficRefs); derr == nil && len(degraded) > 0 {
|
||||
refs = degraded
|
||||
} else {
|
||||
return actool.Errorf(err.Error()), nil
|
||||
}
|
||||
}
|
||||
input := db.RecordFindingInput{TaskID: t.taskID, ExplorationID: t.ts.ID(), IntentID: pid(a.IntentID), VulnClass: a.VulnClass, Name: a.Name, Severity: a.Severity, Summary: a.Summary, Evidence: a.Evidence, Worker: t.worker, AssetIDs: pidList(a.AssetIDs), Counterevidence: a.Counterevidence, ConfidenceRationale: a.ConfidenceRationale}
|
||||
var recorded *db.RecordedFinding
|
||||
@@ -1567,6 +1639,97 @@ const (
|
||||
|
||||
var factSources = []string{factSourceTargetResponse, factSourceToolOutput, factSourceThirdPartyIntel, factSourceOperatorInput, factSourceInference}
|
||||
|
||||
// Fact provenance (#124, row 1): the source above is declared by the agent, so
|
||||
// on its own it cannot distinguish an observation from a claim. The server ties
|
||||
// each fact to a recorded activity of its intent and stores the result beside
|
||||
// the payload; only a tie (or an explicit operator statement) is authoritative.
|
||||
const (
|
||||
factProvenanceRecorded = "recorded_activity" // evidence quoted from this intent's activity
|
||||
factProvenanceCaptured = "captured_traffic" // the quoted bytes came back from the target through the proxy
|
||||
factProvenanceOperator = "operator_input" // told to the agent by the human operator
|
||||
factProvenanceUnverified = "unverified" // no recorded activity carries the evidence
|
||||
)
|
||||
|
||||
// factCaptureWindowBefore/After bound the proxy captures considered part of one
|
||||
// recorded activity. The activity row is written when the tool call returns, so
|
||||
// the matching response lands just before it; the margins absorb slow calls and
|
||||
// small clock differences between the two stores.
|
||||
const (
|
||||
factCaptureWindowBefore = 10 * time.Minute
|
||||
factCaptureWindowAfter = 2 * time.Minute
|
||||
)
|
||||
|
||||
// FactTrafficLookup is the proxy capture store seen from the fact layer. Kept as
|
||||
// an interface so the agent does not depend on the traffic package. nil = no
|
||||
// capture available, and facts keep the activity-only classification.
|
||||
type FactTrafficLookup interface {
|
||||
ResponseContaining(lines []string, fromTS, toTS int64) (id string, ok bool, err error)
|
||||
}
|
||||
|
||||
// factEvidenceMinRunes keeps trivial fragments (a bare tool name, "200 OK")
|
||||
// from tying a fact to unrelated activity. A real evidence line quotes the
|
||||
// output; a paraphrase matches nothing, which fails safe (unverified, not
|
||||
// authoritative).
|
||||
const factEvidenceMinRunes = 12
|
||||
|
||||
// factEvidenceMaxLines bounds the lookup cost of one record_fact item.
|
||||
const factEvidenceMaxLines = 20
|
||||
|
||||
// factEvidenceLines splits a fact's one-line evidence into the substantive lines
|
||||
// the server can look for verbatim in the intent's recorded activity.
|
||||
func factEvidenceLines(evidence string) []string {
|
||||
out := make([]string, 0, 4)
|
||||
seen := map[string]bool{}
|
||||
for _, raw := range strings.Split(evidence, "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if len([]rune(line)) < factEvidenceMinRunes || seen[line] {
|
||||
continue
|
||||
}
|
||||
seen[line] = true
|
||||
out = append(out, line)
|
||||
if len(out) == factEvidenceMaxLines {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// factProvenance classifies one fact against the intent's recorded activity.
|
||||
// Beyond the activity tie it checks whether the quoted bytes were captured
|
||||
// coming back from the target through the proxy, in which case the target
|
||||
// authored them and the fact is classified target_response even when the model
|
||||
// wrote tool_output (#124 row 1, step 3). A database error keeps the fact
|
||||
// (writes must not fail on the check) but marks it unverified, so the
|
||||
// conservative outcome is the one that survives.
|
||||
func (t *ToolSet) factProvenance(source, evidence string, intent int64) (status string, activityID int64, trafficID string, note string) {
|
||||
if source == factSourceOperatorInput {
|
||||
return factProvenanceOperator, 0, "", ""
|
||||
}
|
||||
if intent <= 0 || t.ts == nil {
|
||||
return factProvenanceUnverified, 0, "", "no intent was recorded, so no activity could corroborate the evidence"
|
||||
}
|
||||
lines := factEvidenceLines(evidence)
|
||||
if len(lines) == 0 {
|
||||
return factProvenanceUnverified, 0, "", "evidence is missing or too short to quote a recorded activity; a fact proves a goal only when its evidence is tied to one"
|
||||
}
|
||||
id, at, ok, err := t.ts.ActivityContainingEvidence(intent, lines)
|
||||
if err != nil {
|
||||
return factProvenanceUnverified, 0, "", "the recorded activity could not be read, so the evidence is unverified"
|
||||
}
|
||||
if !ok {
|
||||
return factProvenanceUnverified, 0, "", "evidence does not quote any line of this intent's recorded activity; quote the output verbatim instead of paraphrasing it"
|
||||
}
|
||||
if t.factTraffic != nil && !at.IsZero() {
|
||||
from, to := at.Add(-factCaptureWindowBefore).Unix(), at.Add(factCaptureWindowAfter).Unix()
|
||||
// A capture miss or an unreadable capture store is not a write failure:
|
||||
// the activity tie stands and the fact stays classified as tool output.
|
||||
if tid, hit, _ := t.factTraffic.ResponseContaining(lines, from, to); hit && tid != "" {
|
||||
return factProvenanceCaptured, id, tid, ""
|
||||
}
|
||||
}
|
||||
return factProvenanceRecorded, id, "", ""
|
||||
}
|
||||
|
||||
// normalizeFactSource returns the stored source for a record_fact input: empty
|
||||
// means inference, a value outside factSources is an error.
|
||||
func normalizeFactSource(s string) (string, error) {
|
||||
@@ -1597,16 +1760,6 @@ func (t *ToolSet) recordOneFact(it factItem, defaultIntent int64) (int64, error)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
payload := map[string]any{"summary": it.Summary, "source": source}
|
||||
if it.Detail != "" {
|
||||
payload["detail"] = it.Detail
|
||||
}
|
||||
if e := strings.TrimSpace(it.Evidence); e != "" {
|
||||
payload["evidence"] = e
|
||||
}
|
||||
if c := strings.TrimSpace(it.Confidence); c != "" {
|
||||
payload["confidence"] = c
|
||||
}
|
||||
intent := pid(it.IntentID)
|
||||
if intent <= 0 {
|
||||
intent = defaultIntent
|
||||
@@ -1617,6 +1770,44 @@ func (t *ToolSet) recordOneFact(it factItem, defaultIntent int64) (int64, error)
|
||||
return 0, fmt.Errorf("intent_id must be an intent of this task (related-task intents are read-only)")
|
||||
}
|
||||
}
|
||||
// Tie the claim to recorded work before storing it (#124 row 1). A fact the
|
||||
// server cannot corroborate loses the authority its declared source and
|
||||
// confidence imply: it is stored as inference/inferred and marked unverified,
|
||||
// so prove_goal refuses it even if the model labelled it observed.
|
||||
confidence := strings.TrimSpace(it.Confidence)
|
||||
prov, provActivity, provTraffic, provNote := t.factProvenance(source, it.Evidence, intent)
|
||||
if prov == factProvenanceUnverified {
|
||||
if source == factSourceTargetResponse || source == factSourceToolOutput {
|
||||
source = factSourceInference
|
||||
}
|
||||
if strings.EqualFold(confidence, "observed") {
|
||||
confidence = "inferred"
|
||||
}
|
||||
}
|
||||
if prov == factProvenanceCaptured {
|
||||
// The matched bytes came back from the target, whichever tool carried
|
||||
// them: relabel from the capture rather than trusting the declaration.
|
||||
source = factSourceTargetResponse
|
||||
}
|
||||
payload := map[string]any{"summary": it.Summary, "source": source, "provenance": prov}
|
||||
if it.Detail != "" {
|
||||
payload["detail"] = it.Detail
|
||||
}
|
||||
if e := strings.TrimSpace(it.Evidence); e != "" {
|
||||
payload["evidence"] = e
|
||||
}
|
||||
if confidence != "" {
|
||||
payload["confidence"] = confidence
|
||||
}
|
||||
if provActivity > 0 {
|
||||
payload["provenance_activity_id"] = provActivity
|
||||
}
|
||||
if provTraffic != "" {
|
||||
payload["provenance_traffic_id"] = provTraffic
|
||||
}
|
||||
if provNote != "" {
|
||||
payload["provenance_note"] = provNote
|
||||
}
|
||||
// a fact is its OWN node kind (distinct from a vuln finding).
|
||||
id, err := t.ts.AddNode(db.KindFact, payload, 5, "confirmed", t.worker, anchors)
|
||||
if err != nil {
|
||||
@@ -1650,7 +1841,8 @@ func (t *ToolSet) recordFact() actool.CoreTool {
|
||||
"⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n"+
|
||||
" · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n"+
|
||||
" · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n"+
|
||||
" · source = where the content came from: target_response (content the target itself returned: page bodies, headers, error messages, banners, even when a tool fetched it) | tool_output (a tool's own result about the target: port states, scanner verdicts, status codes, timings) | third_party_intel (OSINT, CVE databases, public reports) | operator_input (given by the human operator) | inference (your own reasoning; the default when omitted). Any other value is refused.\n"+
|
||||
" · source = where the content came from: target_response (content the target itself returned: page bodies, headers, error messages, banners, even when a tool fetched it) | tool_output (a tool's own result about the target: port states, scanner verdicts, status codes, timings) | third_party_intel (OSINT, CVE databases, public reports) | operator_input (given by the human operator) | inference (your own reasoning; the default when omitted). Any other value is refused.\n"+
|
||||
" · **evidence is checked against this intent's recorded activity**: quote the output verbatim rather than paraphrasing it. A fact whose evidence matches no recorded activity is stored as provenance=unverified, its source is downgraded to inference, an observed confidence becomes inferred, and prove_goal refuses it. If the quoted bytes were also captured coming back from the target through the proxy, the server stores source=target_response (provenance=captured_traffic) regardless of the source you declared, because the target authored them.\n"+
|
||||
" · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen."+
|
||||
recordFactContractNote,
|
||||
recordFactSchema(),
|
||||
|
||||
@@ -52,3 +52,29 @@ func TestCompactFindingIncludesStatus(t *testing.T) {
|
||||
t.Fatalf("empty payload should omit status: %#v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGraphOverviewFindingListCarriesStatus(t *testing.T) {
|
||||
s := guardTaskStore(t)
|
||||
addGuardNode(t, s, db.KindFinding, map[string]any{"summary": "noise", "status": "false_positive"}, "confirmed")
|
||||
addGuardNode(t, s, db.KindFinding, map[string]any{"summary": "real", "status": "confirmed"}, "confirmed")
|
||||
ov := NewToolSet(s, "planner").graphOverviewData()
|
||||
if ov["findings_total"] != 1 {
|
||||
t.Fatalf("findings_total=%v, want 1 actionable finding", ov["findings_total"])
|
||||
}
|
||||
list, _ := ov["finding_list"].([]map[string]any)
|
||||
if len(list) != 2 {
|
||||
t.Fatalf("finding_list=%v, want both rows", list)
|
||||
}
|
||||
sawFP, sawReal := false, false
|
||||
for _, item := range list {
|
||||
switch item["status"] {
|
||||
case "false_positive":
|
||||
sawFP = true
|
||||
case "confirmed":
|
||||
sawReal = true
|
||||
}
|
||||
}
|
||||
if !sawFP || !sawReal {
|
||||
t.Fatalf("finding_list statuses: %#v", list)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,6 +67,7 @@ type Worker struct {
|
||||
workDir string
|
||||
proxyAddr string
|
||||
proxyCACert string // recording proxy's CA cert path (for WebFetch HTTPS verify)
|
||||
factTraffic FactTrafficLookup // proxy captures for fact provenance (nil = activity-only)
|
||||
webSearch WebSearchOpts // web_search tool backend selection (off by default)
|
||||
tx *transcript.Store // raw LLM conversation persistence (nil = off)
|
||||
window int // context window in tokens (for compaction)
|
||||
@@ -206,6 +207,10 @@ func (w *Worker) compactionWindow() int {
|
||||
// that MITM proxy. Empty addr disables the hint.
|
||||
func (w *Worker) SetProxy(addr, caCert string) { w.proxyAddr, w.proxyCACert = addr, caCert }
|
||||
|
||||
// SetFactTraffic wires the proxy capture store used to classify recorded facts
|
||||
// (#124 row 1). nil keeps the activity-only classification.
|
||||
func (w *Worker) SetFactTraffic(l FactTrafficLookup) { w.factTraffic = l }
|
||||
|
||||
// SetWebSearch selects the web_search backend for this worker (off by default).
|
||||
func (w *Worker) SetWebSearch(o WebSearchOpts) { w.webSearch = o }
|
||||
|
||||
@@ -527,6 +532,7 @@ func (w *Worker) ExecuteWithMessage(ctx context.Context, name string, taskID int
|
||||
func (w *Worker) execute(ctx context.Context, name string, taskID int64, as *db.AssetStore, ts *db.ExplorationStore, intent *db.Node, hooks harness.HookRunner, emit func(db.Activity), enr EnrichTrigger, notifyFinding func(int64, string), requestID, message string) (harness.TerminalReason, WriteCounts, error) {
|
||||
tsx := NewToolSet(ts, name)
|
||||
tsx.SetFindingRecorder(w.findingRecorder)
|
||||
tsx.SetFactTraffic(w.factTraffic)
|
||||
tsx.SetTaskID(taskID)
|
||||
coverageEnabled := coverageOn(as, taskID)
|
||||
tsx.SetCoverageEnabled(coverageEnabled)
|
||||
|
||||
+39
-2
@@ -157,7 +157,15 @@ func SkillDir() (string, error) {
|
||||
|
||||
// PostgresDSN resolves the connection string with precedence:
|
||||
//
|
||||
// env ARTEX_PG_DSN > config file (database.dsn, or assembled from fields)
|
||||
// env ARTEX_PG_DSN
|
||||
// > env ARTEX_PG_HOST / ARTEX_PG_PORT / ARTEX_PG_USER / ARTEX_PG_PASSWORD /
|
||||
// ARTEX_PG_DBNAME / ARTEX_PG_SSLMODE
|
||||
// > config file (database.dsn, or assembled from fields)
|
||||
//
|
||||
// The component variables are how Docker Compose passes a password that contains
|
||||
// URL-special characters: the password stays a raw environment value, and
|
||||
// buildDSN percent-encodes it. An explicit ARTEX_PG_DSN is returned unchanged,
|
||||
// so a value that was already assembled by something else is not rewritten.
|
||||
//
|
||||
// There is NO built-in fallback: when neither source supplies a database config,
|
||||
// it returns an error naming the config path it inspected, so startup fails loudly
|
||||
@@ -167,6 +175,9 @@ func PostgresDSN() (dsn, source string, err error) {
|
||||
if v := strings.TrimSpace(os.Getenv("ARTEX_PG_DSN")); v != "" {
|
||||
return v, "environment variable ARTEX_PG_DSN", nil
|
||||
}
|
||||
if d, ok, err := postgresDSNFromEnv(); ok || err != nil {
|
||||
return d, "environment variables ARTEX_PG_HOST/USER/PASSWORD/DBNAME", err
|
||||
}
|
||||
c, err := Load()
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
@@ -178,7 +189,33 @@ func PostgresDSN() (dsn, source string, err error) {
|
||||
if db.Host != "" || db.DBName != "" || db.User != "" {
|
||||
return db.buildDSN(), "config file " + Path() + " (database fields)", nil
|
||||
}
|
||||
return "", "", fmt.Errorf("no database config found: the environment variable ARTEX_PG_DSN is not set, and config file %s provides no database (dsn or host/user/dbname). Create the config file or set the environment variable and retry", Path())
|
||||
return "", "", fmt.Errorf("no database config found: ARTEX_PG_DSN is not set, ARTEX_PG_HOST/USER/PASSWORD/DBNAME are not set, and config file %s provides no database (dsn or host/user/dbname). Create the config file or set the environment variable and retry", Path())
|
||||
}
|
||||
|
||||
// postgresDSNFromEnv assembles a URL from the component environment variables.
|
||||
// ok is false when none of them is set. The password is not trimmed: a space is
|
||||
// a legal character and is the case docker-compose used to break.
|
||||
func postgresDSNFromEnv() (dsn string, ok bool, err error) {
|
||||
host := strings.TrimSpace(os.Getenv("ARTEX_PG_HOST"))
|
||||
user := strings.TrimSpace(os.Getenv("ARTEX_PG_USER"))
|
||||
password := os.Getenv("ARTEX_PG_PASSWORD")
|
||||
dbname := strings.TrimSpace(os.Getenv("ARTEX_PG_DBNAME"))
|
||||
portStr := strings.TrimSpace(os.Getenv("ARTEX_PG_PORT"))
|
||||
ssl := strings.TrimSpace(os.Getenv("ARTEX_PG_SSLMODE"))
|
||||
if host == "" && user == "" && password == "" && dbname == "" && portStr == "" && ssl == "" {
|
||||
return "", false, nil
|
||||
}
|
||||
port := 0
|
||||
if portStr != "" {
|
||||
n, convErr := strconv.Atoi(portStr)
|
||||
if convErr != nil || n <= 0 || n > 65535 {
|
||||
return "", true, fmt.Errorf("ARTEX_PG_PORT %q is not a TCP port", portStr)
|
||||
}
|
||||
port = n
|
||||
}
|
||||
return Database{
|
||||
Host: host, Port: port, User: user, Password: password, DBName: dbname, SSLMode: ssl,
|
||||
}.buildDSN(), true, nil
|
||||
}
|
||||
|
||||
func (d Database) buildDSN() string {
|
||||
|
||||
@@ -1,17 +1,29 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func clearPGComponentEnv(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, k := range []string{
|
||||
"ARTEX_PG_HOST", "ARTEX_PG_PORT", "ARTEX_PG_USER", "ARTEX_PG_PASSWORD",
|
||||
"ARTEX_PG_DBNAME", "ARTEX_PG_SSLMODE",
|
||||
} {
|
||||
t.Setenv(k, "")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresDSNPrecedence(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "config.json")
|
||||
os.WriteFile(cfgPath, []byte(`{"database":{"host":"10.1.2.3","port":6000,"user":"u","password":"p","dbname":"d","sslmode":"require"}}`), 0o644)
|
||||
t.Setenv("ARTEX_CONFIG", cfgPath)
|
||||
clearPGComponentEnv(t)
|
||||
|
||||
// no env DSN → built from config file fields
|
||||
t.Setenv("ARTEX_PG_DSN", "")
|
||||
@@ -42,6 +54,53 @@ func TestPostgresDSNPrecedence(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresDSNEncodesComponentPassword(t *testing.T) {
|
||||
clearPGComponentEnv(t)
|
||||
t.Setenv("ARTEX_PG_DSN", "")
|
||||
t.Setenv("ARTEX_CONFIG", filepath.Join(t.TempDir(), "nope.json"))
|
||||
const password = "p/ss w@rd?#%"
|
||||
t.Setenv("ARTEX_PG_HOST", "postgres")
|
||||
t.Setenv("ARTEX_PG_PORT", "5432")
|
||||
t.Setenv("ARTEX_PG_USER", "artex")
|
||||
t.Setenv("ARTEX_PG_PASSWORD", password)
|
||||
t.Setenv("ARTEX_PG_DBNAME", "artex")
|
||||
t.Setenv("ARTEX_PG_SSLMODE", "disable")
|
||||
|
||||
got, source, err := PostgresDSN()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(source, "ARTEX_PG_HOST") {
|
||||
t.Fatalf("source %q", source)
|
||||
}
|
||||
u, err := url.Parse(got)
|
||||
if err != nil {
|
||||
t.Fatalf("parse %q: %v", got, err)
|
||||
}
|
||||
pw, ok := u.User.Password()
|
||||
if !ok || pw != password || u.User.Username() != "artex" || u.Host != "postgres:5432" || u.Path != "/artex" {
|
||||
t.Fatalf("dsn %q decoded user=%q password=%q ok=%v host=%q path=%q", got, u.User.Username(), pw, ok, u.Host, u.Path)
|
||||
}
|
||||
if u.Query().Get("sslmode") != "disable" {
|
||||
t.Fatalf("sslmode %q", u.RawQuery)
|
||||
}
|
||||
if strings.Contains(got, password) {
|
||||
t.Fatalf("password left raw in %q", got)
|
||||
}
|
||||
|
||||
// An explicit URL is not rewritten, even when the component password is set.
|
||||
t.Setenv("ARTEX_PG_DSN", "postgres://already/set")
|
||||
if got, _, err = PostgresDSN(); err != nil || got != "postgres://already/set" {
|
||||
t.Fatalf("explicit DSN got %q err %v", got, err)
|
||||
}
|
||||
|
||||
t.Setenv("ARTEX_PG_DSN", "")
|
||||
t.Setenv("ARTEX_PG_PORT", "nope")
|
||||
if _, _, err = PostgresDSN(); err == nil || !strings.Contains(err.Error(), "ARTEX_PG_PORT") {
|
||||
t.Fatalf("bad port: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A config file that exists but is not valid JSON must fail loudly, not run with
|
||||
// defaults (or report "no database config") as if the file were absent.
|
||||
func TestRedactMasksEveryDSNPasswordForm(t *testing.T) {
|
||||
@@ -66,6 +125,7 @@ func TestMalformedConfigIsAnError(t *testing.T) {
|
||||
os.WriteFile(cfgPath, []byte(`{"database":{"host":"h","password":"a"b","dbname":"d"},"skill_dir":"/s"}`), 0o644)
|
||||
t.Setenv("ARTEX_CONFIG", cfgPath)
|
||||
t.Setenv("ARTEX_PG_DSN", "")
|
||||
clearPGComponentEnv(t)
|
||||
t.Setenv("ARTEX_SKILL_DIR", "")
|
||||
|
||||
if _, _, err := PostgresDSN(); err == nil || !strings.Contains(err.Error(), "parse config file") {
|
||||
|
||||
+20
-1
@@ -165,7 +165,7 @@ func calcCSegment(ipStr string) string {
|
||||
func normalizeURL(raw string) string {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return raw
|
||||
return stripURLUserinfo(raw)
|
||||
}
|
||||
u.Scheme = strings.ToLower(u.Scheme)
|
||||
u.Host = strings.ToLower(u.Host)
|
||||
@@ -177,6 +177,25 @@ func normalizeURL(raw string) string {
|
||||
return s
|
||||
}
|
||||
|
||||
// stripURLUserinfo removes scheme://userinfo@ when url.Parse cannot. The
|
||||
// userinfo is the span before the last @ that still sits in the authority.
|
||||
func stripURLUserinfo(raw string) string {
|
||||
i := strings.Index(raw, "://")
|
||||
if i < 0 {
|
||||
return raw
|
||||
}
|
||||
rest := raw[i+3:]
|
||||
authEnd := len(rest)
|
||||
if j := strings.IndexAny(rest, "/?#"); j >= 0 {
|
||||
authEnd = j
|
||||
}
|
||||
at := strings.LastIndex(rest[:authEnd], "@")
|
||||
if at < 0 {
|
||||
return raw
|
||||
}
|
||||
return raw[:i+3] + rest[at+1:]
|
||||
}
|
||||
|
||||
// parseURL extracts domain, port, service_name from a URL.
|
||||
func parseURL(raw string) (domain string, port int, serviceName string) {
|
||||
u, err := url.Parse(raw)
|
||||
|
||||
@@ -908,6 +908,7 @@ func TestNormalizeURL(t *testing.T) {
|
||||
{"https://user@example.com/", "https://example.com"},
|
||||
{"https://u:p@example.com:8443/users/@alice?x=1", "https://example.com:8443/users/@alice?x=1"},
|
||||
{"https://example.com/@me?r=http://a:b@x/", "https://example.com/@me?r=http://a:b@x/"},
|
||||
{"http://user:secret@ex ample.com/a", "http://ex ample.com/a"},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
got := normalizeURL(tc.raw)
|
||||
|
||||
@@ -102,8 +102,9 @@ func coordinateWithSchemaMigration(tx *sql.Tx) error {
|
||||
}
|
||||
|
||||
// DSN resolves the PostgreSQL connection string and reports where it came from.
|
||||
// Precedence: env ARTEX_PG_DSN > config file (config.json). There is no
|
||||
// built-in default — it errors if neither source is configured.
|
||||
// Precedence: env ARTEX_PG_DSN > ARTEX_PG_HOST/USER/PASSWORD/DBNAME > config
|
||||
// file (config.json). There is no built-in default — it errors if no source
|
||||
// is configured.
|
||||
func DSN() (dsn, source string, err error) {
|
||||
return config.PostgresDSN()
|
||||
}
|
||||
|
||||
@@ -1768,6 +1768,34 @@ FROM activity WHERE exploration_id=$1 AND id>$2 ORDER BY id LIMIT $3`, s.expID,
|
||||
return out, cursor, rows.Err()
|
||||
}
|
||||
|
||||
// ActivityContainingEvidence returns the newest activity recorded for nodeID
|
||||
// whose summary or full detail contains one of the given evidence lines (#124
|
||||
// row 1). ok is false when no recorded activity carries any of them. The match
|
||||
// is literal and case-sensitive: evidence has to quote the recorded output, not
|
||||
// paraphrase it. Callers pass only substantive lines, so a command name alone
|
||||
// cannot tie a claim to an activity. The activity's timestamp bounds the search
|
||||
// for a proxy capture of the same bytes.
|
||||
func (s *ExplorationStore) ActivityContainingEvidence(nodeID int64, lines []string) (id int64, createdAt time.Time, ok bool, err error) {
|
||||
if nodeID <= 0 || len(lines) == 0 {
|
||||
return 0, time.Time{}, false, nil
|
||||
}
|
||||
// One query across all lines: the newest matching activity wins, not the
|
||||
// newest match for whichever line came first.
|
||||
err = s.db.QueryRow(`SELECT id, created_at FROM activity a
|
||||
WHERE exploration_id=$1 AND node_id=$2
|
||||
AND EXISTS (SELECT 1 FROM unnest($3::text[]) AS line
|
||||
WHERE strpos(COALESCE(a.detail,''), line) > 0
|
||||
OR strpos(COALESCE(a.summary,''), line) > 0)
|
||||
ORDER BY id DESC LIMIT 1`, s.expID, nodeID, lines).Scan(&id, &createdAt)
|
||||
if err == sql.ErrNoRows {
|
||||
return 0, time.Time{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, time.Time{}, false, err
|
||||
}
|
||||
return id, createdAt, true, nil
|
||||
}
|
||||
|
||||
// ActivityListForTerminalIntent is the inherited, incremental-read boundary.
|
||||
// The intent state check and activity read happen in one statement so a source
|
||||
// intent reopened between API-level checks cannot expose its new in-flight trace.
|
||||
|
||||
@@ -282,7 +282,15 @@ func (d *DB) FinishFindingRetest(id int64, status, reason string) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
if finalStatus == "completed" {
|
||||
if err := syncLessonsForFindingTx(context.Background(), tx, findingID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DB) RecoverFindingRetests() error {
|
||||
|
||||
+30
-1
@@ -750,7 +750,36 @@ func (d *DB) DeleteFindingsByTask(taskID int64) (int64, error) {
|
||||
// do not care about notifications (parameter validation, retest flows) can
|
||||
// drive the status on their own.
|
||||
func (d *DB) SetFindingStatus(id int64, status string) (int64, error) {
|
||||
return d.setFindingCol(id, "status", "status", status)
|
||||
ctx := context.Background()
|
||||
tx, err := d.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var nodeID sql.NullInt64
|
||||
err = tx.QueryRowContext(ctx, `UPDATE findings SET status=$1 WHERE id=$2 RETURNING node_id`, status, id).Scan(&nodeID)
|
||||
if err == sql.ErrNoRows {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if nodeID.Valid {
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE exploration_nodes
|
||||
SET payload = jsonb_set(payload, '{status}', to_jsonb($1::text))
|
||||
WHERE id = $2`, status, nodeID.Int64); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if FindingStatusInvalidatesGoal(status) {
|
||||
if err := reopenGoalsProvedByNodeTx(ctx, tx, nodeID.Int64); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
// SetFindingReportState updates one finding's report stage and mirrors it onto
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFindingStatusReopensProvedGoals(t *testing.T) {
|
||||
d := notifyTestDB(t)
|
||||
ctx := context.Background()
|
||||
|
||||
newTaskWithProvedGoal := func(t *testing.T, label string) (taskID int64, es *ExplorationStore, findingID, findingNodeID, goalID int64) {
|
||||
t.Helper()
|
||||
task, err := d.CreateTaskWithOptions("reopen-"+label, "goal "+label, TaskCreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = d.DeleteTask(task.ID) })
|
||||
es = d.Exploration(task.ExplorationID)
|
||||
goalID, err = es.AddNode(KindGoal, map[string]any{"text": "prove " + label}, 0, "open", "goals", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec, err := es.RecordFinding(ctx, RecordFindingInput{
|
||||
TaskID: task.ID,
|
||||
ExplorationID: task.ExplorationID,
|
||||
VulnClass: "sqli",
|
||||
Name: "finding " + label,
|
||||
Severity: "high",
|
||||
Summary: "summary " + label,
|
||||
Worker: "worker-1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = d.Exec(`DELETE FROM rsi_lessons WHERE id IN (SELECT lesson_id FROM rsi_lesson_evidence WHERE source_kind='finding' AND source_id=$1)`, rec.FindingID)
|
||||
})
|
||||
if err := es.Link(rec.NodeID, RelProves, goalID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := es.SetNodeState(goalID, "met"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return task.ID, es, rec.FindingID, rec.NodeID, goalID
|
||||
}
|
||||
|
||||
assertGoalState := func(t *testing.T, es *ExplorationStore, goalID int64, want string) {
|
||||
t.Helper()
|
||||
n, err := es.GetNode(goalID)
|
||||
if err != nil || n == nil {
|
||||
t.Fatalf("GetNode(%d): %v", goalID, err)
|
||||
}
|
||||
if n.State != want {
|
||||
t.Fatalf("goal %d state = %q, want %q", goalID, n.State, want)
|
||||
}
|
||||
}
|
||||
|
||||
assertProvesEdgeKept := func(t *testing.T, fromID, toID int64) {
|
||||
t.Helper()
|
||||
var cnt int
|
||||
if err := d.QueryRow(`SELECT COUNT(*) FROM exploration_edges WHERE src_id=$1 AND dst_id=$2 AND rel=$3`, fromID, toID, RelProves).Scan(&cnt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cnt != 1 {
|
||||
t.Fatalf("proves edge (%d -> %d) count = %d, want 1 (must be preserved for history)", fromID, toID, cnt)
|
||||
}
|
||||
}
|
||||
|
||||
goalContentVersion := func(t *testing.T, goalID int64) int {
|
||||
t.Helper()
|
||||
var v int
|
||||
if err := d.QueryRow(`SELECT content_version FROM exploration_nodes WHERE id=$1`, goalID).Scan(&v); err != nil {
|
||||
t.Fatalf("SELECT content_version(%d): %v", goalID, err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
t.Run("single_status_path_reopens_for_fp_duplicate_ignored", func(t *testing.T) {
|
||||
for _, status := range []string{FindingFalsePositive, FindingDuplicate, FindingIgnored} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
_, es, fid, fnid, gid := newTaskWithProvedGoal(t, "single-"+status)
|
||||
vBefore := goalContentVersion(t, gid)
|
||||
_, found, _, err := d.SetFindingStatusWithNotify(ctx, fid, status)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("SetFindingStatusWithNotify(%s): found=%v err=%v", status, found, err)
|
||||
}
|
||||
assertGoalState(t, es, gid, "open")
|
||||
if vAfter := goalContentVersion(t, gid); vAfter != vBefore+1 {
|
||||
t.Fatalf("goal %d content_version = %d, want %d (bumped on reopen)", gid, vAfter, vBefore+1)
|
||||
}
|
||||
assertProvesEdgeKept(t, fnid, gid)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("bulk_status_path_reopens_all_proved_goals", func(t *testing.T) {
|
||||
_, es1, fid1, fnid1, gid1 := newTaskWithProvedGoal(t, "bulk-1")
|
||||
_, es2, fid2, fnid2, gid2 := newTaskWithProvedGoal(t, "bulk-2")
|
||||
res, err := d.SetFindingStatusesWithNotify(ctx, []int64{fid1, fid2}, FindingFalsePositive)
|
||||
if err != nil || len(res) != 2 {
|
||||
t.Fatalf("SetFindingStatusesWithNotify: res=%+v err=%v", res, err)
|
||||
}
|
||||
assertGoalState(t, es1, gid1, "open")
|
||||
assertGoalState(t, es2, gid2, "open")
|
||||
assertProvesEdgeKept(t, fnid1, gid1)
|
||||
assertProvesEdgeKept(t, fnid2, gid2)
|
||||
})
|
||||
|
||||
t.Run("retest_path_uses_same_tx_semantics", func(t *testing.T) {
|
||||
_, es, fid, fnid, gid := newTaskWithProvedGoal(t, "retest")
|
||||
// Retest concluding "fixed" calls SetFindingStatusTx(FindingFixed): goal stays met.
|
||||
rt, conv, _, err := d.CreateFindingRetest(ctx, fid, "test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = d.Exec(`DELETE FROM rsi_lessons WHERE id IN (SELECT lesson_id FROM rsi_lesson_evidence WHERE source_kind='retest' AND source_id=$1)`, rt.ID)
|
||||
_ = d.DeleteConversation(conv.ID)
|
||||
})
|
||||
if _, err := d.StartFindingRetest(ctx, rt.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.RecordFindingRetestResult(ctx, conv.ID, "fixed", "patched", "verified 403"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.FinishFindingRetest(rt.ID, "completed", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertGoalState(t, es, gid, "met")
|
||||
|
||||
// In-transaction rollback check: SetFindingStatusTx(FindingFalsePositive) reopens inside
|
||||
// the same tx, and rolling back the tx leaves the goal met.
|
||||
tx, err := d.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, found, changed, _, err := SetFindingStatusTx(ctx, tx, fid, FindingFalsePositive); err != nil || !found || !changed {
|
||||
tx.Rollback()
|
||||
t.Fatalf("SetFindingStatusTx: found=%v changed=%v err=%v", found, changed, err)
|
||||
}
|
||||
var stateInTx string
|
||||
if err := tx.QueryRowContext(ctx, `SELECT state FROM exploration_nodes WHERE id=$1`, gid).Scan(&stateInTx); err != nil {
|
||||
tx.Rollback()
|
||||
t.Fatal(err)
|
||||
}
|
||||
if stateInTx != "open" {
|
||||
tx.Rollback()
|
||||
t.Fatalf("goal state inside tx = %q, want open", stateInTx)
|
||||
}
|
||||
_ = tx.Rollback()
|
||||
assertGoalState(t, es, gid, "met")
|
||||
assertProvesEdgeKept(t, fnid, gid)
|
||||
})
|
||||
|
||||
t.Run("finding_that_never_proved_a_goal_does_not_touch_other_goals", func(t *testing.T) {
|
||||
taskID, es, _, fnid1, gid1 := newTaskWithProvedGoal(t, "unlinked")
|
||||
rec2, err := es.RecordFinding(ctx, RecordFindingInput{
|
||||
TaskID: taskID,
|
||||
ExplorationID: es.ID(),
|
||||
VulnClass: "xss",
|
||||
Name: "unlinked finding",
|
||||
Severity: "medium",
|
||||
Summary: "never proved any goal",
|
||||
Worker: "worker-1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = d.Exec(`DELETE FROM rsi_lessons WHERE id IN (SELECT lesson_id FROM rsi_lesson_evidence WHERE source_kind='finding' AND source_id=$1)`, rec2.FindingID)
|
||||
})
|
||||
if _, found, _, err := d.SetFindingStatusWithNotify(ctx, rec2.FindingID, FindingFalsePositive); err != nil || !found {
|
||||
t.Fatalf("SetFindingStatusWithNotify(unlinked): found=%v err=%v", found, err)
|
||||
}
|
||||
assertGoalState(t, es, gid1, "met")
|
||||
assertProvesEdgeKept(t, fnid1, gid1)
|
||||
})
|
||||
|
||||
t.Run("risk_accepted_keeps_proved_goal_met", func(t *testing.T) {
|
||||
_, es, fid, fnid, gid := newTaskWithProvedGoal(t, "risk-accepted")
|
||||
if _, found, _, err := d.SetFindingStatusWithNotify(ctx, fid, FindingRiskAccepted); err != nil || !found {
|
||||
t.Fatalf("SetFindingStatusWithNotify(risk_accepted): found=%v err=%v", found, err)
|
||||
}
|
||||
assertGoalState(t, es, gid, "met")
|
||||
assertProvesEdgeKept(t, fnid, gid)
|
||||
})
|
||||
|
||||
t.Run("stays_met_when_another_valid_finding_proves_goal", func(t *testing.T) {
|
||||
taskID, es, fidA, fnidA, gid := newTaskWithProvedGoal(t, "multi-finding-A")
|
||||
recB, err := es.RecordFinding(ctx, RecordFindingInput{
|
||||
TaskID: taskID,
|
||||
ExplorationID: es.ID(),
|
||||
VulnClass: "sqli",
|
||||
Name: "finding B",
|
||||
Severity: "high",
|
||||
Summary: "independent confirmed finding",
|
||||
Worker: "worker-2",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = d.Exec(`DELETE FROM rsi_lessons WHERE id IN (SELECT lesson_id FROM rsi_lesson_evidence WHERE source_kind='finding' AND source_id=$1)`, recB.FindingID)
|
||||
})
|
||||
if _, found, _, err := d.SetFindingStatusWithNotify(ctx, recB.FindingID, FindingConfirmed); err != nil || !found {
|
||||
t.Fatalf("confirm finding B: found=%v err=%v", found, err)
|
||||
}
|
||||
if err := es.Link(recB.NodeID, RelProves, gid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Invalidating A while confirmed B still proves the goal must keep the goal met.
|
||||
if _, found, _, err := d.SetFindingStatusWithNotify(ctx, fidA, FindingFalsePositive); err != nil || !found {
|
||||
t.Fatalf("invalidate finding A: found=%v err=%v", found, err)
|
||||
}
|
||||
assertGoalState(t, es, gid, "met")
|
||||
assertProvesEdgeKept(t, fnidA, gid)
|
||||
assertProvesEdgeKept(t, recB.NodeID, gid)
|
||||
|
||||
// Once B is also invalidated, no valid proof remains and the goal reopens.
|
||||
if _, found, _, err := d.SetFindingStatusWithNotify(ctx, recB.FindingID, FindingFalsePositive); err != nil || !found {
|
||||
t.Fatalf("invalidate finding B: found=%v err=%v", found, err)
|
||||
}
|
||||
assertGoalState(t, es, gid, "open")
|
||||
assertProvesEdgeKept(t, fnidA, gid)
|
||||
assertProvesEdgeKept(t, recB.NodeID, gid)
|
||||
})
|
||||
|
||||
t.Run("stays_met_when_observed_tool_output_fact_proves_goal", func(t *testing.T) {
|
||||
_, es, fidA, fnidA, gid := newTaskWithProvedGoal(t, "finding-and-fact")
|
||||
factID, err := es.AddNode(KindFact, map[string]any{
|
||||
"summary": "admin shell confirmed",
|
||||
"confidence": "observed",
|
||||
"source": "tool_output",
|
||||
"provenance": "activity_verified",
|
||||
}, 0, "confirmed", "worker-1", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := es.Link(factID, RelProves, gid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, found, _, err := d.SetFindingStatusWithNotify(ctx, fidA, FindingFalsePositive); err != nil || !found {
|
||||
t.Fatalf("invalidate finding A: found=%v err=%v", found, err)
|
||||
}
|
||||
assertGoalState(t, es, gid, "met")
|
||||
assertProvesEdgeKept(t, fnidA, gid)
|
||||
assertProvesEdgeKept(t, factID, gid)
|
||||
})
|
||||
|
||||
t.Run("reopens_when_remaining_fact_fails_prove_goal_gate", func(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
payload map[string]any
|
||||
}{
|
||||
{"inferred_fact", map[string]any{"summary": "guess", "confidence": "inferred", "source": "tool_output"}},
|
||||
{"target_response_fact", map[string]any{"summary": "banner", "confidence": "observed", "source": "target_response"}},
|
||||
{"unverified_fact", map[string]any{"summary": "paraphrase", "confidence": "observed", "source": "tool_output", "provenance": "unverified"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, es, fidA, fnidA, gid := newTaskWithProvedGoal(t, "invalid-fact-"+tc.name)
|
||||
factID, err := es.AddNode(KindFact, tc.payload, 0, "confirmed", "worker-1", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := es.Link(factID, RelProves, gid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, found, _, err := d.SetFindingStatusWithNotify(ctx, fidA, FindingFalsePositive); err != nil || !found {
|
||||
t.Fatalf("invalidate finding A: found=%v err=%v", found, err)
|
||||
}
|
||||
assertGoalState(t, es, gid, "open")
|
||||
assertProvesEdgeKept(t, fnidA, gid)
|
||||
assertProvesEdgeKept(t, factID, gid)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
+238
-68
@@ -6,8 +6,10 @@ import (
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// The RSI lesson library is deterministic in this phase: it turns closed-task
|
||||
@@ -61,9 +63,39 @@ func lessonDedupeKey(kind string, categoryID, companyID int64, cue, action strin
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func taskLessonScope(ctx context.Context, d *DB, taskID int64) (categoryID int64, companyIDs []int64, err error) {
|
||||
type sqlExecQuerier interface {
|
||||
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
|
||||
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
|
||||
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
|
||||
}
|
||||
|
||||
// lessonSyncTxTestHook is nil in production; tests set it to inject an
|
||||
// extraction failure inside a status or retest transaction.
|
||||
var lessonSyncTxTestHook func(ctx context.Context, tx *sql.Tx, taskID int64) error
|
||||
|
||||
func taskCompanyIDsQ(ctx context.Context, q sqlExecQuerier, taskID int64) ([]int64, error) {
|
||||
rows, err := q.QueryContext(ctx, `
|
||||
SELECT company_id FROM task_scope
|
||||
WHERE task_id=$1 AND kind='company' AND company_id IS NOT NULL
|
||||
ORDER BY id`, taskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, id)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func taskLessonScopeQ(ctx context.Context, q sqlExecQuerier, taskID int64) (categoryID int64, companyIDs []int64, err error) {
|
||||
var cat sql.NullInt64
|
||||
err = d.QueryRowContext(ctx,
|
||||
err = q.QueryRowContext(ctx,
|
||||
`SELECT category_id FROM tasks WHERE id=$1 AND deleted_at IS NULL`, taskID).
|
||||
Scan(&cat)
|
||||
if err != nil {
|
||||
@@ -72,7 +104,7 @@ func taskLessonScope(ctx context.Context, d *DB, taskID int64) (categoryID int64
|
||||
if cat.Valid {
|
||||
categoryID = cat.Int64
|
||||
}
|
||||
companyIDs, err = d.TaskCompanyIDsContext(ctx, taskID)
|
||||
companyIDs, err = taskCompanyIDsQ(ctx, q, taskID)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
@@ -82,9 +114,14 @@ func taskLessonScope(ctx context.Context, d *DB, taskID int64) (categoryID int64
|
||||
return categoryID, companyIDs, nil
|
||||
}
|
||||
|
||||
func taskLessonScope(ctx context.Context, d *DB, taskID int64) (categoryID int64, companyIDs []int64, err error) {
|
||||
return taskLessonScopeQ(ctx, d.DB, taskID)
|
||||
}
|
||||
|
||||
// lessonExtractCompanyID is the company written onto a new lesson. The row has
|
||||
// one company_id. Scope only when the task has exactly one company. Zero
|
||||
// companies → generic (NULL). ExtractLessonsForTask skips when there are several.
|
||||
// companies → generic (NULL). ExtractLessonsForTask still retracts when there
|
||||
// are several, and it does not write a new lesson in that case.
|
||||
func lessonExtractCompanyID(companyIDs []int64) int64 {
|
||||
if len(companyIDs) == 1 {
|
||||
return companyIDs[0]
|
||||
@@ -96,44 +133,83 @@ func lessonExtractCompanyID(companyIDs []int64) int64 {
|
||||
// retests. It is idempotent for the same disposition: the evidence primary key
|
||||
// means a second run adds no support. A later run after a reversed disposition
|
||||
// retracts that finding's evidence from the opposite-kind lesson, including
|
||||
// every retest of the finding, and retires the lesson when nothing else
|
||||
// every retest of the finding (a negative disposition also withdraws its
|
||||
// retest cautions), and retires the lesson when nothing else
|
||||
// supports it. Confirming again restores the evidence; a retired lesson is
|
||||
// un-retired only when it has no remaining evidence. Pending and in_progress
|
||||
// produce nothing and retract nothing. A task with more than one company is
|
||||
// skipped so a generic (NULL company_id) lesson cannot leak across companies.
|
||||
// Returns the number of distinct lessons written.
|
||||
// un-retired only when it has no remaining live evidence. Pending and in_progress
|
||||
// produce nothing and retract nothing. A task with more than one company does
|
||||
// not gain a lesson, but a reversed disposition still withdraws evidence written
|
||||
// while it had one company. Returns the number of distinct lessons written.
|
||||
func (d *DB) ExtractLessonsForTask(ctx context.Context, taskID int64) (int, error) {
|
||||
categoryID, companyIDs, err := taskLessonScope(ctx, d, taskID)
|
||||
return extractLessonsForTaskQ(ctx, d.DB, taskID)
|
||||
}
|
||||
|
||||
// ExtractLessonsForTaskTx runs lesson extraction/retraction inside the caller's
|
||||
// transaction so status changes and lesson updates commit or roll back together.
|
||||
func ExtractLessonsForTaskTx(ctx context.Context, tx *sql.Tx, taskID int64) (int, error) {
|
||||
if lessonSyncTxTestHook != nil {
|
||||
if err := lessonSyncTxTestHook(ctx, tx, taskID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return extractLessonsForTaskQ(ctx, tx, taskID)
|
||||
}
|
||||
|
||||
func extractLessonsForTaskQ(ctx context.Context, q sqlExecQuerier, taskID int64) (int, error) {
|
||||
categoryID, companyIDs, err := taskLessonScopeQ(ctx, q, taskID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(companyIDs) > 1 {
|
||||
return 0, nil
|
||||
}
|
||||
// Several companies: retract only. A new row cannot name one company, and a
|
||||
// NULL company_id is not a scope.
|
||||
write := len(companyIDs) <= 1
|
||||
companyID := lessonExtractCompanyID(companyIDs)
|
||||
rows, err := d.QueryContext(ctx, `
|
||||
// Only a completed retest with a verdict is an outcome. A pending, running,
|
||||
// failed or stopped retest can carry a staged verdict that never finished.
|
||||
// Buffer rows before writing so a single-connection *sql.Tx is never busy.
|
||||
type findingRow struct {
|
||||
findingID int64
|
||||
vulnclass string
|
||||
name string
|
||||
summary string
|
||||
severity string
|
||||
status string
|
||||
retestID int64
|
||||
verdict string
|
||||
}
|
||||
rows, err := q.QueryContext(ctx, `
|
||||
SELECT f.id, f.vulnclass, f.name, f.summary, f.severity, f.status,
|
||||
COALESCE(r.id,0), COALESCE(r.verdict,'')
|
||||
FROM findings f
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT id, verdict FROM finding_retests
|
||||
WHERE finding_id=f.id ORDER BY id DESC LIMIT 1
|
||||
WHERE finding_id=f.id AND status='completed' AND verdict<>''
|
||||
ORDER BY id DESC LIMIT 1
|
||||
) r ON true
|
||||
WHERE f.task_id=$1
|
||||
ORDER BY f.id`, taskID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
touched := map[int64]bool{}
|
||||
var findings []findingRow
|
||||
for rows.Next() {
|
||||
var findingID int64
|
||||
var vulnclass, name, summary, severity, status, verdict string
|
||||
var retestID int64
|
||||
if err := rows.Scan(&findingID, &vulnclass, &name, &summary, &severity, &status, &retestID, &verdict); err != nil {
|
||||
var fr findingRow
|
||||
if err := rows.Scan(&fr.findingID, &fr.vulnclass, &fr.name, &fr.summary, &fr.severity, &fr.status, &fr.retestID, &fr.verdict); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
findings = append(findings, fr)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
touched := map[int64]bool{}
|
||||
for _, fr := range findings {
|
||||
findingID, vulnclass, name, summary, severity, status, retestID, verdict :=
|
||||
fr.findingID, fr.vulnclass, fr.name, fr.summary, fr.severity, fr.status, fr.retestID, fr.verdict
|
||||
cue := lessonText(vulnclass, lessonCueMax)
|
||||
if cue == "" {
|
||||
cue = lessonText(name, lessonCueMax)
|
||||
@@ -155,14 +231,16 @@ func (d *DB) ExtractLessonsForTask(ctx context.Context, taskID int64) (int, erro
|
||||
if cue == "" || action == "" || sourceID == 0 {
|
||||
return nil // evidence gate: no write without cue + action + source
|
||||
}
|
||||
id, err := d.upsertLesson(ctx, lessonDedupeKey(kind, categoryID, companyID, cue, action),
|
||||
id, err := upsertLessonQ(ctx, q, lessonDedupeKey(kind, categoryID, companyID, cue, action),
|
||||
kind, cue, action, categoryID, companyID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := d.ExecContext(ctx, `
|
||||
if _, err := q.ExecContext(ctx, `
|
||||
INSERT INTO rsi_lesson_evidence(lesson_id,source_kind,source_id,detail)
|
||||
VALUES($1,$2,$3,$4) ON CONFLICT DO NOTHING`,
|
||||
VALUES($1,$2,$3,$4)
|
||||
ON CONFLICT (lesson_id, source_kind, source_id) WHERE withdrawn_at IS NULL
|
||||
DO UPDATE SET detail=EXCLUDED.detail`,
|
||||
id, sourceKind, sourceID, lessonText(detail, lessonDetailMax)); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -172,42 +250,67 @@ func (d *DB) ExtractLessonsForTask(ctx context.Context, taskID int64) (int, erro
|
||||
|
||||
switch status {
|
||||
case "confirmed", "resolved", "fixed":
|
||||
if err := add("positive", "Confirmed "+cue+": "+label, "finding", findingID, sourceDetail); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if retestID > 0 && verdict == "reproduced" {
|
||||
if err := add("positive", "Reproduced "+cue+" on retest: "+label, "retest", retestID, "verdict=reproduced"); err != nil {
|
||||
if write {
|
||||
if err := add("positive", "Confirmed "+cue+": "+label, "finding", findingID, sourceDetail); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if retestID > 0 && verdict == "reproduced" {
|
||||
if err := add("positive", "Reproduced "+cue+" on retest: "+label, "retest", retestID, "verdict=reproduced"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
// A caution needs an outcome too: pending and in_progress write
|
||||
// nothing, and a negative disposition withdraws it below.
|
||||
if retestID > 0 && (verdict == "fixed" || verdict == "inconclusive") {
|
||||
if err := add("caution", "Recheck before reporting "+cue+": a prior retest was "+verdict,
|
||||
"retest", retestID, "verdict="+verdict); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := d.retractLessonEvidence(ctx, "finding", findingID, "negative"); err != nil {
|
||||
if err := retractLessonEvidenceQ(ctx, q, "finding", findingID, "negative", ""); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
case "false_positive", "duplicate", "ignored", "risk_accepted":
|
||||
if err := add("negative", "Do not re-report without new mechanics: "+label+" (prior "+status+")",
|
||||
"finding", findingID, sourceDetail); err != nil {
|
||||
// The action text names the status, so a change inside this group is a
|
||||
// different lesson. Withdraw this finding from every negative lesson
|
||||
// with another action, and leave lessons with this action alone so an
|
||||
// adopted row is not retired and restored on a repeat run. The match is
|
||||
// on the action, not the dedupe key: the key carries the company, and a
|
||||
// task with several companies cannot rebuild the key it was written
|
||||
// under, though its disposition is unchanged.
|
||||
// Cut the label, not the status, at the action cap: without the
|
||||
// suffix, a long summary gives every negative status the same action
|
||||
// and the same dedupe key.
|
||||
prefix := "Do not re-report without new mechanics: "
|
||||
suffix := " (prior " + status + ")"
|
||||
room := lessonActionMax - utf8.RuneCountInString(prefix) - utf8.RuneCountInString(suffix)
|
||||
action := prefix + lessonText(label, room) + suffix
|
||||
if err := retractLessonEvidenceQ(ctx, q, "finding", findingID, "negative", lessonText(action, lessonActionMax)); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := d.retractLessonEvidence(ctx, "finding", findingID, "positive"); err != nil {
|
||||
if write {
|
||||
if err := add("negative", action, "finding", findingID, sourceDetail); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if err := retractLessonEvidenceQ(ctx, q, "finding", findingID, "positive", ""); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := d.retractFindingRetestEvidence(ctx, findingID, "positive"); err != nil {
|
||||
if err := retractFindingRetestEvidenceQ(ctx, q, findingID, "positive"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if retestID > 0 && (verdict == "fixed" || verdict == "inconclusive") {
|
||||
if err := add("caution", "Recheck before reporting "+cue+": a prior retest was "+verdict,
|
||||
"retest", retestID, "verdict="+verdict); err != nil {
|
||||
if err := retractFindingRetestEvidenceQ(ctx, q, findingID, "caution"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return len(touched), rows.Err()
|
||||
return len(touched), nil
|
||||
}
|
||||
|
||||
func (d *DB) upsertLesson(ctx context.Context, key, kind, cue, action string, categoryID, companyID int64) (int64, error) {
|
||||
func upsertLessonQ(ctx context.Context, q sqlExecQuerier, key, kind, cue, action string, categoryID, companyID int64) (int64, error) {
|
||||
var id int64
|
||||
err := d.QueryRowContext(ctx, `
|
||||
err := q.QueryRowContext(ctx, `
|
||||
INSERT INTO rsi_lessons(dedupe_key,kind,cue,action,category_id,company_id)
|
||||
VALUES($1,$2,$3,$4,NULLIF($5,0),NULLIF($6,0))
|
||||
ON CONFLICT (dedupe_key) DO UPDATE SET
|
||||
@@ -215,7 +318,8 @@ func (d *DB) upsertLesson(ctx context.Context, key, kind, cue, action string, ca
|
||||
status=CASE
|
||||
WHEN rsi_lessons.status='retired'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id
|
||||
SELECT 1 FROM rsi_lesson_evidence e
|
||||
WHERE e.lesson_id=rsi_lessons.id AND e.withdrawn_at IS NULL
|
||||
)
|
||||
THEN 'candidate'
|
||||
ELSE rsi_lessons.status
|
||||
@@ -224,18 +328,24 @@ func (d *DB) upsertLesson(ctx context.Context, key, kind, cue, action string, ca
|
||||
return id, err
|
||||
}
|
||||
|
||||
// retractLessonEvidence removes this source from lessons of fromKind and
|
||||
// retires any of those lessons that now have zero evidence. Shared lessons
|
||||
// that still have other evidence stay active.
|
||||
func (d *DB) retractLessonEvidence(ctx context.Context, sourceKind string, sourceID int64, fromKind string) error {
|
||||
rows, err := d.QueryContext(ctx, `
|
||||
DELETE FROM rsi_lesson_evidence e
|
||||
USING rsi_lessons l
|
||||
// retractLessonEvidenceQ withdraws this source from lessons of fromKind and
|
||||
// retires any of those lessons that now have zero live evidence. The row stays,
|
||||
// with withdrawn_at set, so the lesson still names what it rested on. Shared
|
||||
// lessons that still have other live evidence stay active. A non-empty
|
||||
// keepAction spares lessons with that action, compared without case as the
|
||||
// dedupe key does.
|
||||
func retractLessonEvidenceQ(ctx context.Context, q sqlExecQuerier, sourceKind string, sourceID int64, fromKind, keepAction string) error {
|
||||
rows, err := q.QueryContext(ctx, `
|
||||
UPDATE rsi_lesson_evidence e
|
||||
SET withdrawn_at=now()
|
||||
FROM rsi_lessons l
|
||||
WHERE e.lesson_id=l.id
|
||||
AND e.withdrawn_at IS NULL
|
||||
AND l.kind=$1
|
||||
AND e.source_kind=$2
|
||||
AND e.source_id=$3
|
||||
RETURNING e.lesson_id`, fromKind, sourceKind, sourceID)
|
||||
AND ($4 = '' OR lower(l.action) <> lower($4))
|
||||
RETURNING e.lesson_id`, fromKind, sourceKind, sourceID, keepAction)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -243,16 +353,18 @@ func (d *DB) retractLessonEvidence(ctx context.Context, sourceKind string, sourc
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return d.retireLessonsIfEmpty(ctx, ids)
|
||||
return retireLessonsIfEmptyQ(ctx, q, ids)
|
||||
}
|
||||
|
||||
// retractFindingRetestEvidence removes every retest of this finding from
|
||||
// retractFindingRetestEvidenceQ withdraws every retest of this finding from
|
||||
// lessons of fromKind, not only the latest row.
|
||||
func (d *DB) retractFindingRetestEvidence(ctx context.Context, findingID int64, fromKind string) error {
|
||||
rows, err := d.QueryContext(ctx, `
|
||||
DELETE FROM rsi_lesson_evidence e
|
||||
USING rsi_lessons l, finding_retests r
|
||||
func retractFindingRetestEvidenceQ(ctx context.Context, q sqlExecQuerier, findingID int64, fromKind string) error {
|
||||
rows, err := q.QueryContext(ctx, `
|
||||
UPDATE rsi_lesson_evidence e
|
||||
SET withdrawn_at=now()
|
||||
FROM rsi_lessons l, finding_retests r
|
||||
WHERE e.lesson_id=l.id
|
||||
AND e.withdrawn_at IS NULL
|
||||
AND l.kind=$1
|
||||
AND e.source_kind='retest'
|
||||
AND e.source_id=r.id
|
||||
@@ -265,7 +377,7 @@ func (d *DB) retractFindingRetestEvidence(ctx context.Context, findingID int64,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return d.retireLessonsIfEmpty(ctx, ids)
|
||||
return retireLessonsIfEmptyQ(ctx, q, ids)
|
||||
}
|
||||
|
||||
func collectLessonIDs(rows *sql.Rows) ([]int64, error) {
|
||||
@@ -288,21 +400,26 @@ func collectLessonIDs(rows *sql.Rows) ([]int64, error) {
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (d *DB) retireLessonsIfEmpty(ctx context.Context, ids []int64) error {
|
||||
func retireLessonsIfEmptyQ(ctx context.Context, q sqlExecQuerier, ids []int64) error {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
_, err := d.ExecContext(ctx, `
|
||||
_, err := q.ExecContext(ctx, `
|
||||
UPDATE rsi_lessons l
|
||||
SET status='retired', updated_at=now()
|
||||
WHERE l.id=ANY($1::bigint[])
|
||||
AND l.status<>'retired'
|
||||
AND NOT EXISTS (SELECT 1 FROM rsi_lesson_evidence e WHERE e.lesson_id=l.id)`, ids)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM rsi_lesson_evidence e
|
||||
WHERE e.lesson_id=l.id AND e.withdrawn_at IS NULL
|
||||
)`, ids)
|
||||
return err
|
||||
}
|
||||
|
||||
// TaskLessons selects lessons visible to one task: generic lessons plus those
|
||||
// scoped to the task's company and category. Scope matches rank first, then
|
||||
// TaskLessons selects lessons visible to one task. A lesson with no company is
|
||||
// visible only to a task that also has no company. A task with one or more
|
||||
// companies sees only lessons of those companies. Category NULL still matches
|
||||
// any category inside that company filter. Scope matches rank first, then
|
||||
// positive before negative before caution, then support.
|
||||
func (d *DB) TaskLessons(ctx context.Context, taskID int64, limit int) ([]Lesson, error) {
|
||||
categoryID, companyIDs, err := taskLessonScope(ctx, d, taskID)
|
||||
@@ -312,25 +429,33 @@ func (d *DB) TaskLessons(ctx context.Context, taskID int64, limit int) ([]Lesson
|
||||
if limit <= 0 {
|
||||
limit = 8
|
||||
}
|
||||
// A separate flag, not CARDINALITY of the array: an empty company list can
|
||||
// arrive as SQL NULL, and CARDINALITY(NULL) is NULL.
|
||||
unscoped := len(companyIDs) == 0
|
||||
rows, err := d.QueryContext(ctx, `
|
||||
SELECT l.id, l.kind, l.cue, l.action,
|
||||
COALESCE(l.category_id,0), COALESCE(l.company_id,0), l.status,
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=l.id) AS support,
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e
|
||||
WHERE e.lesson_id=l.id AND e.withdrawn_at IS NULL) AS support,
|
||||
COALESCE((
|
||||
SELECT jsonb_agg(e.source_kind || ':' || e.source_id::text ||
|
||||
CASE WHEN e.detail <> '' THEN ' (' || e.detail || ')' ELSE '' END
|
||||
ORDER BY e.source_kind, e.source_id)
|
||||
FROM rsi_lesson_evidence e WHERE e.lesson_id=l.id
|
||||
FROM rsi_lesson_evidence e
|
||||
WHERE e.lesson_id=l.id AND e.withdrawn_at IS NULL
|
||||
), '[]'::jsonb)
|
||||
FROM rsi_lessons l
|
||||
WHERE l.status <> 'retired'
|
||||
AND (l.company_id IS NULL OR l.company_id = ANY($1::bigint[]))
|
||||
AND (
|
||||
($3::bool AND l.company_id IS NULL)
|
||||
OR l.company_id = ANY($1::bigint[])
|
||||
)
|
||||
AND (l.category_id IS NULL OR l.category_id=$2)
|
||||
ORDER BY (l.company_id IS NOT NULL AND l.company_id = ANY($1::bigint[])) DESC,
|
||||
(l.category_id IS NOT NULL AND l.category_id=$2) DESC,
|
||||
CASE l.kind WHEN 'positive' THEN 0 WHEN 'negative' THEN 1 ELSE 2 END,
|
||||
support DESC, l.id DESC
|
||||
LIMIT $3`, companyIDs, categoryID, limit)
|
||||
LIMIT $4`, companyIDs, categoryID, unscoped, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -367,6 +492,49 @@ func (d *DB) LogLessonUsage(ctx context.Context, taskID int64, lessonIDs []int64
|
||||
return nil
|
||||
}
|
||||
|
||||
// syncLessonsForFindingTx re-runs extraction for the finding's task inside the
|
||||
// caller's transaction so a failure rolls back the status or retest change and
|
||||
// never leaves a stale lesson selectable (#269).
|
||||
func syncLessonsForFindingTx(ctx context.Context, tx *sql.Tx, findingID int64) error {
|
||||
var taskID sql.NullInt64
|
||||
err := tx.QueryRowContext(ctx, `SELECT task_id FROM findings WHERE id=$1`, findingID).Scan(&taskID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !taskID.Valid || taskID.Int64 == 0 {
|
||||
return nil
|
||||
}
|
||||
_, err = ExtractLessonsForTaskTx(ctx, tx, taskID.Int64)
|
||||
return err
|
||||
}
|
||||
|
||||
// syncLessonsForFindingsTx re-runs extraction once per distinct task for all
|
||||
// changed findings inside the caller's transaction.
|
||||
func syncLessonsForFindingsTx(ctx context.Context, tx *sql.Tx, findingIDs []int64) error {
|
||||
seen := map[int64]bool{}
|
||||
for _, fid := range findingIDs {
|
||||
var taskID sql.NullInt64
|
||||
err := tx.QueryRowContext(ctx, `SELECT task_id FROM findings WHERE id=$1`, fid).Scan(&taskID)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !taskID.Valid || taskID.Int64 == 0 || seen[taskID.Int64] {
|
||||
continue
|
||||
}
|
||||
seen[taskID.Int64] = true
|
||||
if _, err := ExtractLessonsForTaskTx(ctx, tx, taskID.Int64); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RenderLessons builds the prompt block. The text is data, not instructions:
|
||||
// lesson fields come from task evidence and can contain target-controlled
|
||||
// strings, so they are flattened to one line and length-capped here.
|
||||
@@ -391,7 +559,9 @@ func RenderLessons(lessons []Lesson, maxChars int) (block string, renderedIDs []
|
||||
entry += " (evidence: " + strings.Join(shown, "; ") + ")"
|
||||
}
|
||||
entry += "\n"
|
||||
if maxChars > 0 && b.Len()+len(entry)+1 > maxChars {
|
||||
// Cap in runes. Cue and action are already cut in runes; a byte cap
|
||||
// drops a full CJK lesson that still fits in maxChars characters.
|
||||
if maxChars > 0 && utf8.RuneCountInString(b.String())+utf8.RuneCountInString(entry)+1 > maxChars {
|
||||
break
|
||||
}
|
||||
b.WriteString(entry)
|
||||
|
||||
+793
-8
@@ -1,6 +1,9 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -73,6 +76,23 @@ func TestRenderLessonsUsageOmitsTruncated(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderLessonsCapsRunesNotBytes(t *testing.T) {
|
||||
// 160 + 320 four-byte runes are 1920 bytes before the wrapper. A byte cap of
|
||||
// 1800 renders nothing. A rune cap still fits one lesson.
|
||||
lesson := Lesson{
|
||||
ID: 1, Kind: "positive", Support: 1,
|
||||
Cue: strings.Repeat("𠀋", 160),
|
||||
Action: strings.Repeat("𠀋", 320),
|
||||
}
|
||||
block, ids := RenderLessons([]Lesson{lesson}, 1800)
|
||||
if block == "" || len(ids) != 1 || ids[0] != 1 {
|
||||
t.Fatalf("rune cap dropped a lesson that fits in characters: bytes=%d ids=%v", len(block), ids)
|
||||
}
|
||||
if !utf8.ValidString(block) {
|
||||
t.Fatal("rendered block is not valid UTF-8")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskLessonsUsesTaskScopeAndRanksCompanyFirst(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
@@ -153,13 +173,37 @@ func TestTaskLessonsUsesTaskScopeAndRanksCompanyFirst(t *testing.T) {
|
||||
sawB = true
|
||||
}
|
||||
}
|
||||
if !sawA || !sawGeneric {
|
||||
t.Fatalf("task A should see its company lesson and generic, sawA=%v sawGeneric=%v", sawA, sawGeneric)
|
||||
if !sawA || sawGeneric {
|
||||
t.Fatalf("task A should see its company lesson only, sawA=%v sawGeneric=%v", sawA, sawGeneric)
|
||||
}
|
||||
if sawB {
|
||||
t.Fatal("task A saw company B's lesson")
|
||||
}
|
||||
|
||||
bare, err := d.CreateTaskWithOptions("lesson scope none", "goal", TaskCreateOptions{
|
||||
Name: fmt.Sprintf("lesson-none-%d", stamp),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "task none", d.DeleteTask(bare.ID)) })
|
||||
gotNone, err := d.TaskLessons(t.Context(), bare.ID, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sawGeneric = false
|
||||
for _, l := range gotNone {
|
||||
switch l.ID {
|
||||
case genericID:
|
||||
sawGeneric = true
|
||||
case aID, bID:
|
||||
t.Fatalf("company-less task saw company lesson %d", l.ID)
|
||||
}
|
||||
}
|
||||
if !sawGeneric {
|
||||
t.Fatal("company-less task should see the generic lesson")
|
||||
}
|
||||
|
||||
gotB, err := d.TaskLessons(t.Context(), taskB.ID, 8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -516,13 +560,21 @@ func TestExtractLessonsRetractsReversedDisposition(t *testing.T) {
|
||||
var posStatus string
|
||||
var posSupport int
|
||||
if err := d.QueryRow(`SELECT status,
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id)
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id AND e.withdrawn_at IS NULL)
|
||||
FROM rsi_lessons WHERE id=$1`, posID).Scan(&posStatus, &posSupport); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if posStatus != "retired" || posSupport != 0 {
|
||||
t.Fatalf("positive after FP: status=%s support=%d", posStatus, posSupport)
|
||||
}
|
||||
var withdrawn int
|
||||
if err := d.QueryRow(`SELECT COUNT(*) FROM rsi_lesson_evidence
|
||||
WHERE lesson_id=$1 AND withdrawn_at IS NOT NULL`, posID).Scan(&withdrawn); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if withdrawn != 1 {
|
||||
t.Fatalf("withdrawn evidence rows=%d, want the finding kept on the lesson", withdrawn)
|
||||
}
|
||||
|
||||
afterFP, err := d.TaskLessons(t.Context(), task.ID, 8)
|
||||
if err != nil {
|
||||
@@ -550,7 +602,7 @@ func TestExtractLessonsRetractsReversedDisposition(t *testing.T) {
|
||||
}
|
||||
|
||||
if err := d.QueryRow(`SELECT status,
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id)
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id AND e.withdrawn_at IS NULL)
|
||||
FROM rsi_lessons WHERE id=$1`, posID).Scan(&posStatus, &posSupport); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -561,7 +613,7 @@ func TestExtractLessonsRetractsReversedDisposition(t *testing.T) {
|
||||
var negStatus string
|
||||
var negSupport int
|
||||
if err := d.QueryRow(`SELECT status,
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id)
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id AND e.withdrawn_at IS NULL)
|
||||
FROM rsi_lessons WHERE id=$1`, neg.ID).Scan(&negStatus, &negSupport); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -634,7 +686,7 @@ func TestExtractLessonsRetireOnlyWhenLastEvidence(t *testing.T) {
|
||||
var status string
|
||||
var support int
|
||||
if err := d.QueryRow(`SELECT status,
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id)
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id AND e.withdrawn_at IS NULL)
|
||||
FROM rsi_lessons WHERE id=$1`, pos.ID).Scan(&status, &support); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -870,6 +922,412 @@ func TestExtractLessonsReopenRetractsNothing(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispositionChangeExtractsLessons(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
task, err := d.CreateTask(fmt.Sprintf("lesson on patch %d", stamp), "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "task", d.DeleteTask(task.ID)) })
|
||||
cue := fmt.Sprintf("xss-patch-%d", stamp)
|
||||
label := fmt.Sprintf("reflected in search %d", stamp)
|
||||
var fid int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS',$3,'high','confirmed')
|
||||
RETURNING id`, task.ID, cue, label).Scan(&fid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pos := findLessonByAction(t, d, task.ID, "positive", "Confirmed", label)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, pos.ID) })
|
||||
|
||||
if _, _, _, err := d.SetFindingStatusWithNotify(context.Background(), fid, FindingFalsePositive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
status, support := lessonStatusSupport(t, d, pos.ID)
|
||||
if status != "retired" || support != 0 {
|
||||
t.Fatalf("status change did not retract: status=%s support=%d", status, support)
|
||||
}
|
||||
neg := findLessonByAction(t, d, task.ID, "negative", "Do not re-report", label)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, neg.ID) })
|
||||
if !strings.Contains(neg.Action, "false_positive") {
|
||||
t.Fatalf("negative action=%q", neg.Action)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNegativeStatusReplacesPriorNegativeLesson(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
task, err := d.CreateTask(fmt.Sprintf("lesson neg fork %d", stamp), "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "task", d.DeleteTask(task.ID)) })
|
||||
cue := fmt.Sprintf("xss-neg-%d", stamp)
|
||||
label := fmt.Sprintf("reflected in search %d", stamp)
|
||||
var fid int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS',$3,'high','false_positive')
|
||||
RETURNING id`, task.ID, cue, label).Scan(&fid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := findLessonByAction(t, d, task.ID, "negative", "false_positive", label)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, first.ID) })
|
||||
|
||||
if _, err := d.SetFindingStatus(fid, FindingDuplicate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
status, support := lessonStatusSupport(t, d, first.ID)
|
||||
if status != "retired" || support != 0 {
|
||||
t.Fatalf("prior negative lesson: status=%s support=%d", status, support)
|
||||
}
|
||||
second := findLessonByAction(t, d, task.ID, "negative", "duplicate", label)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, second.ID) })
|
||||
if second.ID == first.ID {
|
||||
t.Fatal("status change inside the negative group reused the old lesson")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiCompanyExtractStillRetracts(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
cs := d.Companies()
|
||||
companyA, _, err := cs.UpsertCompany(fmt.Sprintf("LessonRetractA %d", stamp), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupCompany(d, companyA) })
|
||||
companyB, _, err := cs.UpsertCompany(fmt.Sprintf("LessonRetractB %d", stamp), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupCompany(d, companyB) })
|
||||
task, err := d.CreateTaskWithOptions("extract then multi", "goal", TaskCreateOptions{
|
||||
Name: fmt.Sprintf("lesson-retract-multi-%d", stamp), CompanyIDs: []int64{companyA},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "task", d.DeleteTask(task.ID)) })
|
||||
cue := fmt.Sprintf("xss-multi-retract-%d", stamp)
|
||||
label := fmt.Sprintf("reflected in search %d", stamp)
|
||||
var fid int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS',$3,'high','confirmed')
|
||||
RETURNING id`, task.ID, cue, label).Scan(&fid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pos := findLessonByAction(t, d, task.ID, "positive", "Confirmed", label)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE id=$1`, pos.ID) })
|
||||
|
||||
if _, err := d.Exec(`INSERT INTO task_scope(task_id, kind, company_id, source) VALUES ($1,'company',$2,'manual')`, task.ID, companyB); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.SetFindingStatus(fid, FindingFalsePositive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, err := d.ExtractLessonsForTask(t.Context(), task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Fatalf("multi-company extract wrote %d lessons", n)
|
||||
}
|
||||
status, support := lessonStatusSupport(t, d, pos.ID)
|
||||
if status != "retired" || support != 0 {
|
||||
t.Fatalf("lesson after the task gained a second company: status=%s support=%d", status, support)
|
||||
}
|
||||
}
|
||||
|
||||
// A second company stops new writes, but the disposition did not change: the
|
||||
// negative lesson written while the task had one company keeps its evidence.
|
||||
func TestMultiCompanyExtractKeepsUnchangedNegative(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
SkipPGUnavailable(t, err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
cs := d.Companies()
|
||||
companyA, _, err := cs.UpsertCompany(fmt.Sprintf("LessonKeepNegA %d", stamp), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupCompany(d, companyA) })
|
||||
companyB, _, err := cs.UpsertCompany(fmt.Sprintf("LessonKeepNegB %d", stamp), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupCompany(d, companyB) })
|
||||
task, err := d.CreateTaskWithOptions("negative then multi", "goal", TaskCreateOptions{
|
||||
Name: fmt.Sprintf("lesson-keep-neg-%d", stamp), CompanyIDs: []int64{companyA},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "task", d.DeleteTask(task.ID)) })
|
||||
cue := fmt.Sprintf("xss-keep-neg-%d", stamp)
|
||||
label := fmt.Sprintf("reflected in search %d", stamp)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE cue=$1`, cue) })
|
||||
var fid int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS',$3,'high','false_positive')
|
||||
RETURNING id`, task.ID, cue, label).Scan(&fid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
neg := findLessonByAction(t, d, task.ID, "negative", "false_positive", label)
|
||||
if _, err := d.Exec(`UPDATE rsi_lessons SET status='adopted' WHERE id=$1`, neg.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := d.Exec(`INSERT INTO task_scope(task_id, kind, company_id, source) VALUES ($1,'company',$2,'manual')`, task.ID, companyB); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, err := d.ExtractLessonsForTask(t.Context(), task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Fatalf("multi-company extract wrote %d lessons", n)
|
||||
}
|
||||
status, support := lessonStatusSupport(t, d, neg.ID)
|
||||
if status != "adopted" || support != 1 {
|
||||
t.Fatalf("unchanged negative after a second company: status=%s support=%d, want adopted 1", status, support)
|
||||
}
|
||||
|
||||
// A status change inside the negative group still withdraws it.
|
||||
if _, err := d.SetFindingStatus(fid, FindingDuplicate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
status, support = lessonStatusSupport(t, d, neg.ID)
|
||||
if status != "retired" || support != 0 {
|
||||
t.Fatalf("negative after false_positive -> duplicate: status=%s support=%d, want retired 0", status, support)
|
||||
}
|
||||
}
|
||||
|
||||
// liveNegativeLesson returns the one negative lesson that holds this finding as
|
||||
// live evidence.
|
||||
func liveNegativeLesson(t *testing.T, d *DB, findingID int64) (id int64, key, action string) {
|
||||
t.Helper()
|
||||
if err := d.QueryRow(`
|
||||
SELECT l.id, l.dedupe_key, l.action FROM rsi_lessons l
|
||||
JOIN rsi_lesson_evidence e ON e.lesson_id=l.id
|
||||
WHERE l.kind='negative' AND e.source_kind='finding' AND e.source_id=$1
|
||||
AND e.withdrawn_at IS NULL`, findingID).Scan(&id, &key, &action); err != nil {
|
||||
t.Fatalf("live negative lesson of finding %d: %v", findingID, err)
|
||||
}
|
||||
return id, key, action
|
||||
}
|
||||
|
||||
// A summary long enough to fill the action cap must not cut the status off:
|
||||
// false_positive and duplicate would then share one action and one key.
|
||||
func TestNegativeStatusSurvivesActionCap(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
SkipPGUnavailable(t, err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
task, err := d.CreateTask(fmt.Sprintf("lesson long summary %d", stamp), "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "task", d.DeleteTask(task.ID)) })
|
||||
cue := fmt.Sprintf("xss-long-%d", stamp)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE cue=$1`, cue) })
|
||||
summary := fmt.Sprintf("long-%d-", stamp)
|
||||
summary += strings.Repeat("x", 300-utf8.RuneCountInString(summary))
|
||||
if n := utf8.RuneCountInString(summary); n != 300 {
|
||||
t.Fatalf("summary has %d runes, want 300", n)
|
||||
}
|
||||
var fid int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS',$3,'high','false_positive')
|
||||
RETURNING id`, task.ID, cue, summary).Scan(&fid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
firstID, firstKey, firstAction := liveNegativeLesson(t, d, fid)
|
||||
|
||||
if _, err := d.SetFindingStatus(fid, FindingDuplicate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, err := d.ExtractLessonsForTask(t.Context(), task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("duplicate extract wrote %d lessons, want 1", n)
|
||||
}
|
||||
secondID, secondKey, secondAction := liveNegativeLesson(t, d, fid)
|
||||
if secondID == firstID || secondKey == firstKey {
|
||||
t.Fatalf("false_positive and duplicate share lesson %d / key %s", secondID, secondKey)
|
||||
}
|
||||
if !strings.HasSuffix(firstAction, "(prior false_positive)") {
|
||||
t.Fatalf("false_positive action lost its status: %q", firstAction)
|
||||
}
|
||||
if !strings.HasSuffix(secondAction, "(prior duplicate)") {
|
||||
t.Fatalf("duplicate action lost its status: %q", secondAction)
|
||||
}
|
||||
if n := utf8.RuneCountInString(secondAction); n > lessonActionMax {
|
||||
t.Fatalf("action has %d runes, cap is %d", n, lessonActionMax)
|
||||
}
|
||||
status, support := lessonStatusSupport(t, d, firstID)
|
||||
if status != "retired" || support != 0 {
|
||||
t.Fatalf("false_positive lesson after duplicate: status=%s support=%d, want retired 0", status, support)
|
||||
}
|
||||
}
|
||||
|
||||
// A pending or in-progress finding has no outcome yet, so even a completed
|
||||
// retest on it writes no caution.
|
||||
func TestCautionNeedsFindingOutcome(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
SkipPGUnavailable(t, err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
task, err := d.CreateTask(fmt.Sprintf("lesson caution open %d", stamp), "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "task", d.DeleteTask(task.ID)) })
|
||||
cue := fmt.Sprintf("xss-caution-open-%d", stamp)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE cue=$1`, cue) })
|
||||
|
||||
for _, status := range []string{FindingPending, FindingInProgress} {
|
||||
var fid int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS','reflected in search','high',$3)
|
||||
RETURNING id`, task.ID, cue, status).Scan(&fid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
for _, verdict := range []string{"fixed", "inconclusive"} {
|
||||
rid := insertLessonRetestStatus(t, d, fid, "completed", verdict)
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := retestEvidence(t, d, rid); n != 0 {
|
||||
t.Fatalf("finding status=%s, completed %s retest wrote %d evidence rows", status, verdict, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
var lessons int
|
||||
if err := d.QueryRow(`SELECT COUNT(*) FROM rsi_lessons WHERE cue=$1`, cue).Scan(&lessons); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if lessons != 0 {
|
||||
t.Fatalf("open findings produced %d lessons", lessons)
|
||||
}
|
||||
}
|
||||
|
||||
// A reversed disposition withdraws the finding's evidence, and a caution from
|
||||
// its retest is part of that evidence. Confirming again restores it.
|
||||
func TestFalsePositiveWithdrawsRetestCaution(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
SkipPGUnavailable(t, err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
task, err := d.CreateTask(fmt.Sprintf("lesson caution fp %d", stamp), "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "task", d.DeleteTask(task.ID)) })
|
||||
cue := fmt.Sprintf("xss-caution-fp-%d", stamp)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE cue=$1`, cue) })
|
||||
var fid int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS','reflected in search','high','confirmed')
|
||||
RETURNING id`, task.ID, cue).Scan(&fid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
insertLessonRetestStatus(t, d, fid, "completed", "inconclusive")
|
||||
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
caution := findLessonByAction(t, d, task.ID, "caution", "Recheck", cue)
|
||||
if status, support := lessonStatusSupport(t, d, caution.ID); status != "candidate" || support != 1 {
|
||||
t.Fatalf("caution after confirmed: status=%s support=%d, want candidate 1", status, support)
|
||||
}
|
||||
|
||||
if _, err := d.SetFindingStatus(fid, FindingFalsePositive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status, support := lessonStatusSupport(t, d, caution.ID); status != "retired" || support != 0 {
|
||||
t.Fatalf("caution after false_positive: status=%s support=%d, want retired 0", status, support)
|
||||
}
|
||||
|
||||
if _, err := d.SetFindingStatus(fid, FindingConfirmed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status, support := lessonStatusSupport(t, d, caution.ID); status != "candidate" || support != 1 {
|
||||
t.Fatalf("caution after re-confirm: status=%s support=%d, want candidate 1", status, support)
|
||||
}
|
||||
}
|
||||
|
||||
func insertLessonRetest(t *testing.T, d *DB, findingID int64, verdict string) int64 {
|
||||
t.Helper()
|
||||
var id int64
|
||||
@@ -883,6 +1341,112 @@ func insertLessonRetest(t *testing.T, d *DB, findingID int64, verdict string) in
|
||||
return id
|
||||
}
|
||||
|
||||
func insertLessonRetestStatus(t *testing.T, d *DB, findingID int64, status, verdict string) int64 {
|
||||
t.Helper()
|
||||
var id int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO finding_retests(finding_id,status,verdict,snapshot)
|
||||
VALUES ($1,$2,$3,'{}'::jsonb)
|
||||
RETURNING id`, findingID, status, verdict).Scan(&id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM finding_retests WHERE id=$1`, id) })
|
||||
return id
|
||||
}
|
||||
|
||||
// retestEvidence counts every evidence row of this retest, live or withdrawn,
|
||||
// in lessons of any kind.
|
||||
func retestEvidence(t *testing.T, d *DB, retestID int64) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := d.QueryRow(`
|
||||
SELECT COUNT(*) FROM rsi_lesson_evidence
|
||||
WHERE source_kind='retest' AND source_id=$1`, retestID).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// A staged verdict on a retest that did not complete is not an outcome:
|
||||
// FinishFindingRetest turns completed-without-verdict into failed, and a
|
||||
// stopped or failed retest keeps whatever verdict the agent staged.
|
||||
func TestExtractLessonsIgnoresUnfinishedRetests(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
SkipPGUnavailable(t, err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
stamp := time.Now().UnixNano()
|
||||
task, err := d.CreateTask(fmt.Sprintf("lesson unfinished retest %d", stamp), "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupErr(t, "task", d.DeleteTask(task.ID)) })
|
||||
cue := fmt.Sprintf("xss-unfinished-%d", stamp)
|
||||
label := fmt.Sprintf("reflected in search %d", stamp)
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM rsi_lessons WHERE cue=$1`, cue) })
|
||||
var fid int64
|
||||
if err := d.QueryRow(`
|
||||
INSERT INTO findings(task_id,vulnclass,name,summary,severity,status)
|
||||
VALUES ($1,$2,'Reflected XSS',$3,'high','confirmed')
|
||||
RETURNING id`, task.ID, cue, label).Scan(&fid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { cleanupExec(t, d, `DELETE FROM findings WHERE id=$1`, fid) })
|
||||
|
||||
for _, status := range []string{"pending", "running", "failed", "stopped"} {
|
||||
for _, verdict := range []string{"reproduced", "fixed", "inconclusive"} {
|
||||
rid := insertLessonRetestStatus(t, d, fid, status, verdict)
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := retestEvidence(t, d, rid); n != 0 {
|
||||
t.Fatalf("retest status=%s verdict=%s wrote %d evidence rows", status, verdict, n)
|
||||
}
|
||||
if status == "pending" || status == "running" {
|
||||
// idx_finding_retests_active allows one active retest per finding.
|
||||
cleanupExec(t, d, `DELETE FROM finding_retests WHERE id=$1`, rid)
|
||||
}
|
||||
}
|
||||
}
|
||||
var retestLessons int
|
||||
if err := d.QueryRow(`SELECT COUNT(*) FROM rsi_lessons
|
||||
WHERE cue=$1 AND (kind='caution' OR action LIKE 'Reproduced %')`, cue).Scan(&retestLessons); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if retestLessons != 0 {
|
||||
t.Fatalf("unfinished retests produced %d lessons", retestLessons)
|
||||
}
|
||||
|
||||
done := insertLessonRetestStatus(t, d, fid, "completed", "reproduced")
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := positiveRetestEvidence(t, d, done); n != 1 {
|
||||
t.Fatalf("completed reproduced retest: positive evidence=%d, want 1", n)
|
||||
}
|
||||
findLessonByAction(t, d, task.ID, "positive", "Reproduced", label)
|
||||
|
||||
// A later unfinished retest does not hide the completed one.
|
||||
insertLessonRetestStatus(t, d, fid, "running", "fixed")
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := positiveRetestEvidence(t, d, done); n != 1 {
|
||||
t.Fatalf("completed retest after a running one: positive evidence=%d, want 1", n)
|
||||
}
|
||||
|
||||
inconclusive := insertLessonRetestStatus(t, d, fid, "completed", "inconclusive")
|
||||
if _, err := d.ExtractLessonsForTask(t.Context(), task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := retestEvidence(t, d, inconclusive); n != 1 {
|
||||
t.Fatalf("completed inconclusive retest: evidence=%d, want 1", n)
|
||||
}
|
||||
findLessonByAction(t, d, task.ID, "caution", "Recheck", cue)
|
||||
}
|
||||
|
||||
func findLessonByAction(t *testing.T, d *DB, taskID int64, kind, actionPrefix, label string) *Lesson {
|
||||
t.Helper()
|
||||
lessons, err := d.TaskLessons(t.Context(), taskID, 8)
|
||||
@@ -904,7 +1468,8 @@ func positiveRetestEvidence(t *testing.T, d *DB, retestID int64) int {
|
||||
if err := d.QueryRow(`
|
||||
SELECT COUNT(*) FROM rsi_lesson_evidence e
|
||||
JOIN rsi_lessons l ON l.id=e.lesson_id
|
||||
WHERE e.source_kind='retest' AND e.source_id=$1 AND l.kind='positive'`, retestID).Scan(&n); err != nil {
|
||||
WHERE e.source_kind='retest' AND e.source_id=$1 AND l.kind='positive'
|
||||
AND e.withdrawn_at IS NULL`, retestID).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
@@ -913,9 +1478,229 @@ func positiveRetestEvidence(t *testing.T, d *DB, retestID int64) int {
|
||||
func lessonStatusSupport(t *testing.T, d *DB, lessonID int64) (status string, support int) {
|
||||
t.Helper()
|
||||
if err := d.QueryRow(`SELECT status,
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id)
|
||||
(SELECT COUNT(*) FROM rsi_lesson_evidence e WHERE e.lesson_id=rsi_lessons.id AND e.withdrawn_at IS NULL)
|
||||
FROM rsi_lessons WHERE id=$1`, lessonID).Scan(&status, &support); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return status, support
|
||||
}
|
||||
|
||||
// Re-confirming a previously withdrawn finding must NOT clear withdrawn_at on
|
||||
// the old evidence row: the withdrawal row stays for history and re-confirmation
|
||||
// inserts a new live row (#269).
|
||||
func TestReconfirmingFindingPreservesWithdrawalHistory(t *testing.T) {
|
||||
d := notifyTestDB(t)
|
||||
ctx := context.Background()
|
||||
|
||||
task, err := d.CreateTaskWithOptions("withdraw-history", "test withdrawal history", TaskCreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = d.DeleteTask(task.ID) })
|
||||
es := d.Exploration(task.ExplorationID)
|
||||
rec, err := es.RecordFinding(ctx, RecordFindingInput{
|
||||
TaskID: task.ID,
|
||||
ExplorationID: task.ExplorationID,
|
||||
VulnClass: "ssrf",
|
||||
Name: "SSRF on webhook",
|
||||
Severity: "high",
|
||||
Summary: "SSRF on webhook endpoint",
|
||||
Worker: "worker-1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fid := rec.FindingID
|
||||
t.Cleanup(func() {
|
||||
_, _ = d.Exec(`DELETE FROM rsi_lessons WHERE id IN (SELECT lesson_id FROM rsi_lesson_evidence WHERE source_kind='finding' AND source_id=$1)`, fid)
|
||||
})
|
||||
|
||||
// 1. Confirm -> positive lesson created with 1 live evidence row.
|
||||
if _, _, _, err := d.SetFindingStatusWithNotify(ctx, fid, FindingConfirmed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var lessonID, firstRowID int64
|
||||
if err := d.QueryRow(`SELECT e.lesson_id, e.id
|
||||
FROM rsi_lesson_evidence e JOIN rsi_lessons l ON l.id=e.lesson_id
|
||||
WHERE l.kind='positive' AND e.source_kind='finding' AND e.source_id=$1 AND e.withdrawn_at IS NULL`,
|
||||
fid).Scan(&lessonID, &firstRowID); err != nil {
|
||||
t.Fatalf("positive evidence row: %v", err)
|
||||
}
|
||||
|
||||
// 2. Flip to false_positive -> firstRowID gets withdrawn_at set.
|
||||
if _, _, _, err := d.SetFindingStatusWithNotify(ctx, fid, FindingFalsePositive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var withdrawnAt1 sql.NullTime
|
||||
if err := d.QueryRow(`SELECT withdrawn_at FROM rsi_lesson_evidence WHERE id=$1`, firstRowID).Scan(&withdrawnAt1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !withdrawnAt1.Valid {
|
||||
t.Fatalf("expected firstRowID=%d to have withdrawn_at set after false_positive", firstRowID)
|
||||
}
|
||||
|
||||
// 3. Re-confirm -> old withdrawal row stays untouched, new live row is inserted.
|
||||
if _, _, _, err := d.SetFindingStatusWithNotify(ctx, fid, FindingConfirmed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var withdrawnAt2 sql.NullTime
|
||||
if err := d.QueryRow(`SELECT withdrawn_at FROM rsi_lesson_evidence WHERE id=$1`, firstRowID).Scan(&withdrawnAt2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !withdrawnAt2.Valid || !withdrawnAt2.Time.Equal(withdrawnAt1.Time) {
|
||||
t.Fatalf("old withdrawal row %d was altered: got %v, want %v", firstRowID, withdrawnAt2, withdrawnAt1)
|
||||
}
|
||||
|
||||
var secondRowID int64
|
||||
if err := d.QueryRow(`SELECT id FROM rsi_lesson_evidence
|
||||
WHERE lesson_id=$1 AND source_kind='finding' AND source_id=$2 AND withdrawn_at IS NULL`,
|
||||
lessonID, fid).Scan(&secondRowID); err != nil {
|
||||
t.Fatalf("new live evidence row missing after re-confirm: %v", err)
|
||||
}
|
||||
if secondRowID == firstRowID {
|
||||
t.Fatalf("re-confirm reused old row %d instead of inserting a new live row", firstRowID)
|
||||
}
|
||||
|
||||
// 4. Repeat extraction is idempotent: still 1 live + 1 withdrawn row.
|
||||
if _, err := d.ExtractLessonsForTask(ctx, task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var totalRows, liveRows, withdrawnRows int
|
||||
if err := d.QueryRow(`SELECT COUNT(*), COUNT(*) FILTER (WHERE withdrawn_at IS NULL), COUNT(*) FILTER (WHERE withdrawn_at IS NOT NULL)
|
||||
FROM rsi_lesson_evidence WHERE lesson_id=$1 AND source_kind='finding' AND source_id=$2`,
|
||||
lessonID, fid).Scan(&totalRows, &liveRows, &withdrawnRows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if totalRows != 2 || liveRows != 1 || withdrawnRows != 1 {
|
||||
t.Fatalf("evidence rows = total %d live %d withdrawn %d, want 2 / 1 / 1", totalRows, liveRows, withdrawnRows)
|
||||
}
|
||||
if status, support := lessonStatusSupport(t, d, lessonID); status != "candidate" || support != 1 {
|
||||
t.Fatalf("lesson status=%s support=%d, want candidate / 1", status, support)
|
||||
}
|
||||
}
|
||||
|
||||
// Lesson extraction now runs inside the same transaction as status changes and
|
||||
// retest completion (#269). If extraction fails, the status / retest write
|
||||
// rolls back atomically so the DB never ends up with a flipped finding status
|
||||
// and a stale selectable lesson.
|
||||
func TestLessonExtractionFailureRollsBackStatusAndRetest(t *testing.T) {
|
||||
d := notifyTestDB(t)
|
||||
ctx := context.Background()
|
||||
|
||||
task, err := d.CreateTaskWithOptions("lesson-tx-rollback", "test lesson extraction tx rollback", TaskCreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = d.DeleteTask(task.ID) })
|
||||
es := d.Exploration(task.ExplorationID)
|
||||
rec, err := es.RecordFinding(ctx, RecordFindingInput{
|
||||
TaskID: task.ID,
|
||||
ExplorationID: task.ExplorationID,
|
||||
VulnClass: "sqli",
|
||||
Name: "SQLi in filter",
|
||||
Severity: "high",
|
||||
Summary: "Boolean SQLi in filter param",
|
||||
Worker: "worker-1",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fid := rec.FindingID
|
||||
t.Cleanup(func() {
|
||||
_, _ = d.Exec(`DELETE FROM rsi_lessons WHERE id IN (SELECT lesson_id FROM rsi_lesson_evidence WHERE source_kind='finding' AND source_id=$1)`, fid)
|
||||
})
|
||||
|
||||
// Seed a confirmed finding and its positive lesson.
|
||||
if _, _, _, err := d.SetFindingStatusWithNotify(ctx, fid, FindingConfirmed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var posLessonID int64
|
||||
if err := d.QueryRow(`SELECT e.lesson_id
|
||||
FROM rsi_lesson_evidence e JOIN rsi_lessons l ON l.id=e.lesson_id
|
||||
WHERE l.kind='positive' AND e.source_kind='finding' AND e.source_id=$1 AND e.withdrawn_at IS NULL`,
|
||||
fid).Scan(&posLessonID); err != nil {
|
||||
t.Fatalf("positive lesson missing: %v", err)
|
||||
}
|
||||
|
||||
injected := errors.New("injected lesson extraction failure")
|
||||
prevHook := lessonSyncTxTestHook
|
||||
t.Cleanup(func() { lessonSyncTxTestHook = prevHook })
|
||||
|
||||
// 1. Single status path: failure during lesson extraction rolls back status.
|
||||
lessonSyncTxTestHook = func(ctx context.Context, tx *sql.Tx, taskID int64) error { return injected }
|
||||
if _, _, _, err := d.SetFindingStatusWithNotify(ctx, fid, FindingFalsePositive); !errors.Is(err, injected) {
|
||||
t.Fatalf("single status err=%v, want %v", err, injected)
|
||||
}
|
||||
f, err := d.GetFinding(fid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.Status != FindingConfirmed {
|
||||
t.Fatalf("single status leaked %q after rollback, want %q", f.Status, FindingConfirmed)
|
||||
}
|
||||
if status, support := lessonStatusSupport(t, d, posLessonID); status != "candidate" || support != 1 {
|
||||
t.Fatalf("positive lesson after single rollback: status=%s support=%d, want candidate / 1", status, support)
|
||||
}
|
||||
|
||||
// 2. Bulk status path: failure during lesson extraction rolls back status.
|
||||
if _, err := d.SetFindingStatusesWithNotify(ctx, []int64{fid}, FindingFalsePositive); !errors.Is(err, injected) {
|
||||
t.Fatalf("bulk status err=%v, want %v", err, injected)
|
||||
}
|
||||
f, err = d.GetFinding(fid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.Status != FindingConfirmed {
|
||||
t.Fatalf("bulk status leaked %q after rollback, want %q", f.Status, FindingConfirmed)
|
||||
}
|
||||
|
||||
// 3. Retest path: failure during lesson extraction rolls back both retest
|
||||
// completion and the finding status transition.
|
||||
lessonSyncTxTestHook = nil
|
||||
retest, conv, _, err := d.CreateFindingRetest(ctx, fid, "verify fix")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = d.DeleteConversation(conv.ID) })
|
||||
if _, err := d.StartFindingRetest(ctx, retest.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.RecordFindingRetestResult(ctx, conv.ID, "fixed", "patched", "verified 400 on payload"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
lessonSyncTxTestHook = func(ctx context.Context, tx *sql.Tx, taskID int64) error { return injected }
|
||||
if err := d.FinishFindingRetest(retest.ID, "completed", ""); !errors.Is(err, injected) {
|
||||
t.Fatalf("FinishFindingRetest err=%v, want %v", err, injected)
|
||||
}
|
||||
f, err = d.GetFinding(fid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.Status != FindingConfirmed {
|
||||
t.Fatalf("retest status leaked %q after rollback, want %q", f.Status, FindingConfirmed)
|
||||
}
|
||||
var retestStatus string
|
||||
if err := d.QueryRow(`SELECT status FROM finding_retests WHERE id=$1`, retest.ID).Scan(&retestStatus); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if retestStatus == "completed" {
|
||||
t.Fatalf("retest status committed %q despite lesson extraction rollback", retestStatus)
|
||||
}
|
||||
|
||||
// 4. Once extraction succeeds, the status flip and lesson retirement commit together.
|
||||
lessonSyncTxTestHook = nil
|
||||
if _, _, _, err := d.SetFindingStatusWithNotify(ctx, fid, FindingFalsePositive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err = d.GetFinding(fid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.Status != FindingFalsePositive {
|
||||
t.Fatalf("status=%q after retry, want %q", f.Status, FindingFalsePositive)
|
||||
}
|
||||
if status, support := lessonStatusSupport(t, d, posLessonID); status != "retired" || support != 0 {
|
||||
t.Fatalf("positive lesson after FP commit: status=%s support=%d, want retired / 0", status, support)
|
||||
}
|
||||
}
|
||||
|
||||
+95
-2
@@ -139,9 +139,10 @@ func assertEmbeddedApplied(t *testing.T, d *DB) []appliedMigration {
|
||||
}{{1, llmRecordsMigration}, {2, llmUsageMigration}, {3, "0003_task_scope_icp_index.sql"}, {4, "0004_llm_usage_ts_index.sql"},
|
||||
{5, "0005_traffic_rewrite_rules.sql"}, {6, "0006_activity_outbox.sql"}, {7, "0007_finding_report_state.sql"},
|
||||
{8, "0008_traffic_evidence_rewrite_flag.sql"}, {9, "0009_llm_context.sql"}, {10, "0010_dashboard_usage_indexes.sql"},
|
||||
{11, "0011_traffic_tamper_hooks.sql"}}
|
||||
{11, "0011_traffic_tamper_hooks.sql"}, {12, "0012_lesson_withdrawn.sql"},
|
||||
{13, "0013_lesson_evidence_history.sql"}}
|
||||
if len(applied) != len(want) {
|
||||
t.Fatalf("schema_migrations = %+v, want versions 1 to 11", applied)
|
||||
t.Fatalf("schema_migrations = %+v, want versions 1 to 13", applied)
|
||||
}
|
||||
for i, w := range want {
|
||||
content, err := fs.ReadFile(migrationsFS(), w.name)
|
||||
@@ -416,3 +417,95 @@ func TestMigrationsRefuseOutOfOrderVersion(t *testing.T) {
|
||||
t.Error("llm_usage exists: the refused 0002 migration ran")
|
||||
}
|
||||
}
|
||||
|
||||
// A database from before withdrawn_at has lesson evidence and no 0012 row.
|
||||
// Open adds the column, keeps every row live, and a second Open is a no-op.
|
||||
func TestLessonWithdrawnMigrationKeepsExistingEvidence(t *testing.T) {
|
||||
dsn := scratchDSN(t, "lesson_withdrawn")
|
||||
d, err := Open(dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
const migrationName = "0012_lesson_withdrawn.sql"
|
||||
migrationSQL, err := fs.ReadFile(migrationsFS(), migrationName)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The migration file alone adds the column to a table without it.
|
||||
for _, q := range []string{
|
||||
`DROP INDEX IF EXISTS uq_rsi_lesson_evidence_live`,
|
||||
`ALTER TABLE rsi_lesson_evidence DROP COLUMN withdrawn_at`,
|
||||
string(migrationSQL),
|
||||
`ALTER TABLE rsi_lesson_evidence DROP COLUMN withdrawn_at`,
|
||||
`DELETE FROM schema_migrations WHERE version IN (12, 13)`,
|
||||
`INSERT INTO rsi_lessons(dedupe_key,kind,cue,action,status)
|
||||
VALUES ('legacy-lesson','positive','xss','Confirmed xss: legacy','adopted')`,
|
||||
`INSERT INTO rsi_lesson_evidence(lesson_id,source_kind,source_id,detail)
|
||||
SELECT id,'finding',101,'legacy' FROM rsi_lessons WHERE dedupe_key='legacy-lesson'
|
||||
UNION ALL
|
||||
SELECT id,'retest',202,'legacy' FROM rsi_lessons WHERE dedupe_key='legacy-lesson'`,
|
||||
} {
|
||||
if _, err := d.Exec(q); err != nil {
|
||||
t.Fatalf("%s: %v", q, err)
|
||||
}
|
||||
}
|
||||
d.Close()
|
||||
|
||||
check := func(d *DB) {
|
||||
t.Helper()
|
||||
var nullable, dataType string
|
||||
var hasDefault bool
|
||||
if err := d.QueryRow(`SELECT is_nullable, data_type, column_default IS NOT NULL
|
||||
FROM information_schema.columns
|
||||
WHERE table_name='rsi_lesson_evidence' AND column_name='withdrawn_at'`).Scan(&nullable, &dataType, &hasDefault); err != nil {
|
||||
t.Fatalf("withdrawn_at column: %v", err)
|
||||
}
|
||||
if nullable != "YES" || dataType != "timestamp with time zone" || hasDefault {
|
||||
t.Fatalf("withdrawn_at nullable=%s type=%s default=%v, want YES, timestamp with time zone, none", nullable, dataType, hasDefault)
|
||||
}
|
||||
var status string
|
||||
var rows, live int
|
||||
if err := d.QueryRow(`SELECT l.status, COUNT(e.*), COUNT(e.*) FILTER (WHERE e.withdrawn_at IS NULL)
|
||||
FROM rsi_lessons l JOIN rsi_lesson_evidence e ON e.lesson_id=l.id
|
||||
WHERE l.dedupe_key='legacy-lesson' GROUP BY l.status`).Scan(&status, &rows, &live); err != nil {
|
||||
t.Fatalf("legacy lesson: %v", err)
|
||||
}
|
||||
if status != "adopted" || rows != 2 || live != 2 {
|
||||
t.Fatalf("legacy lesson status=%s evidence=%d live=%d, want adopted 2 2", status, rows, live)
|
||||
}
|
||||
}
|
||||
|
||||
d, err = Open(dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("Open on the pre-0012 database: %v", err)
|
||||
}
|
||||
check(d)
|
||||
before := readApplied(t, d)
|
||||
has12 := false
|
||||
for _, m := range before {
|
||||
if m.version == 12 && m.name == migrationName {
|
||||
has12 = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !has12 {
|
||||
t.Fatalf("applied migrations = %+v, missing 0012", before)
|
||||
}
|
||||
d.Close()
|
||||
|
||||
d, err = Open(dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("second Open: %v", err)
|
||||
}
|
||||
defer d.Close()
|
||||
check(d)
|
||||
after := readApplied(t, d)
|
||||
if len(after) != len(before) {
|
||||
t.Fatalf("second Open changed schema_migrations: before %+v, after %+v", before, after)
|
||||
}
|
||||
for i := range before {
|
||||
if !after[i].appliedAt.Equal(before[i].appliedAt) || after[i].checksum != before[i].checksum {
|
||||
t.Errorf("second Open changed migration %04d: before %+v, after %+v", before[i].version, before[i], after[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
-- RSI lesson evidence keeps a retracted source on its row: withdrawn_at is set
|
||||
-- instead of deleting it. Support and selection count only rows where it is
|
||||
-- NULL, so existing evidence stays live. schema.sql carries the same column for
|
||||
-- a fresh install.
|
||||
ALTER TABLE rsi_lesson_evidence ADD COLUMN IF NOT EXISTS withdrawn_at TIMESTAMPTZ;
|
||||
@@ -0,0 +1,22 @@
|
||||
-- Keep withdrawal history on rsi_lesson_evidence (#269): replace the composite
|
||||
-- primary key (lesson_id, source_kind, source_id) with a surrogate id and a
|
||||
-- partial unique index over live rows (withdrawn_at IS NULL). Re-confirming a
|
||||
-- previously withdrawn finding inserts a new live row while preserving the old
|
||||
-- withdrawal row and its withdrawn_at timestamp.
|
||||
ALTER TABLE rsi_lesson_evidence ADD COLUMN IF NOT EXISTS id BIGSERIAL;
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_constraint
|
||||
WHERE conrelid = 'rsi_lesson_evidence'::regclass
|
||||
AND contype = 'p'
|
||||
AND conname = 'rsi_lesson_evidence_pkey'
|
||||
AND array_length(conkey, 1) > 1
|
||||
) THEN
|
||||
ALTER TABLE rsi_lesson_evidence DROP CONSTRAINT rsi_lesson_evidence_pkey;
|
||||
ALTER TABLE rsi_lesson_evidence ADD PRIMARY KEY (id);
|
||||
END IF;
|
||||
END $$;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_rsi_lesson_evidence_live
|
||||
ON rsi_lesson_evidence(lesson_id, source_kind, source_id)
|
||||
WHERE withdrawn_at IS NULL;
|
||||
+92
-4
@@ -502,11 +502,21 @@ func assetDisplayName(typ, domain, ip, url, appName, bundleID string) string {
|
||||
// already succeeded and the caller must not report an error for it.
|
||||
func (d *DB) SetFindingStatusWithNotify(ctx context.Context, id int64, status string) (from string, found bool, notified bool, err error) {
|
||||
err = d.WithEvidenceTx(ctx, func(tx *sql.Tx) error {
|
||||
var changed bool
|
||||
var txErr error
|
||||
from, found, _, notified, txErr = SetFindingStatusTx(ctx, tx, id, status)
|
||||
return txErr
|
||||
from, found, changed, notified, txErr = SetFindingStatusTx(ctx, tx, id, status)
|
||||
if txErr != nil {
|
||||
return txErr
|
||||
}
|
||||
if found && changed {
|
||||
return syncLessonsForFindingTx(ctx, tx, id)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return from, found, notified, err
|
||||
if err != nil {
|
||||
return from, found, false, err
|
||||
}
|
||||
return from, found, notified, nil
|
||||
}
|
||||
|
||||
// FindingStatusBatchResult is one id's outcome in a bulk status update.
|
||||
@@ -526,14 +536,18 @@ type FindingStatusBatchResult struct {
|
||||
func (d *DB) SetFindingStatusesWithNotify(ctx context.Context, ids []int64, status string) ([]FindingStatusBatchResult, error) {
|
||||
out := make([]FindingStatusBatchResult, 0, len(ids))
|
||||
err := d.WithEvidenceTx(ctx, func(tx *sql.Tx) error {
|
||||
changedIDs := make([]int64, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
from, found, changed, notified, err := SetFindingStatusTx(ctx, tx, id, status)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if changed {
|
||||
changedIDs = append(changedIDs, id)
|
||||
}
|
||||
out = append(out, FindingStatusBatchResult{ID: id, From: from, Found: found, Changed: changed, Notified: notified})
|
||||
}
|
||||
return nil
|
||||
return syncLessonsForFindingsTx(ctx, tx, changedIDs)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -541,6 +555,75 @@ func (d *DB) SetFindingStatusesWithNotify(ctx context.Context, ids []int64, stat
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// GoalInvalidatingFindingStatuses is the single source of truth for finding
|
||||
// triage statuses that mean "not a valid vulnerability" and therefore cannot
|
||||
// prove or keep a goal met (#269). Both db goal-reopen logic and agent
|
||||
// prove_goal checks use this list so they cannot drift.
|
||||
//
|
||||
// FindingRiskAccepted is deliberately excluded: it represents a real finding
|
||||
// whose risk is accepted, not an invalid one, so it may prove a goal and must
|
||||
// never reopen one.
|
||||
var GoalInvalidatingFindingStatuses = []string{
|
||||
FindingFalsePositive,
|
||||
FindingDuplicate,
|
||||
FindingIgnored,
|
||||
}
|
||||
|
||||
// FindingStatusInvalidatesGoal reports whether a finding disposition means the
|
||||
// finding is not a valid vulnerability and therefore cannot prove or keep a
|
||||
// goal met.
|
||||
func FindingStatusInvalidatesGoal(status string) bool {
|
||||
s := strings.ToLower(strings.TrimSpace(status))
|
||||
for _, inv := range GoalInvalidatingFindingStatuses {
|
||||
if s == inv {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// reopenGoalsProvedByNodeTx resets any goal node proved by nodeID back to open
|
||||
// inside the caller's transaction, unless another valid proof still exists for
|
||||
// that goal (a finding whose status is not in GoalInvalidatingFindingStatuses,
|
||||
// or an observed non-target_response non-unverified fact matching prove_goal's
|
||||
// gate). All proves edges are preserved for history; only the goal state changes.
|
||||
func reopenGoalsProvedByNodeTx(ctx context.Context, tx *sql.Tx, nodeID int64) error {
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
UPDATE exploration_nodes g
|
||||
SET state = 'open',
|
||||
blocked_reason = NULL,
|
||||
content_version = content_version + 1
|
||||
FROM exploration_edges e
|
||||
WHERE e.src_id = $1
|
||||
AND e.rel = $2
|
||||
AND g.id = e.dst_id
|
||||
AND g.kind = $3
|
||||
AND g.state = 'met'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM exploration_edges pe
|
||||
JOIN exploration_nodes src ON src.id = pe.src_id
|
||||
WHERE pe.dst_id = g.id
|
||||
AND pe.rel = $2
|
||||
AND src.id <> $1
|
||||
AND (
|
||||
(src.kind = $4 AND EXISTS (
|
||||
SELECT 1
|
||||
FROM findings f
|
||||
WHERE f.node_id = src.id
|
||||
AND NOT (LOWER(BTRIM(f.status)) = ANY($6::text[]))
|
||||
))
|
||||
OR
|
||||
(src.kind = $5
|
||||
AND LOWER(BTRIM(COALESCE(src.payload->>'confidence', ''))) = 'observed'
|
||||
AND LOWER(BTRIM(COALESCE(src.payload->>'source', ''))) <> 'target_response'
|
||||
AND LOWER(BTRIM(COALESCE(src.payload->>'provenance', ''))) <> 'unverified')
|
||||
)
|
||||
)`,
|
||||
nodeID, RelProves, KindGoal, KindFinding, KindFact, GoalInvalidatingFindingStatuses)
|
||||
return err
|
||||
}
|
||||
|
||||
// SetFindingStatusTx updates a finding's status and registers a status-change delivery
|
||||
// event inside the **caller's transaction**.
|
||||
//
|
||||
@@ -584,6 +667,11 @@ FROM findings WHERE id=$1 FOR UPDATE`, id).
|
||||
WHERE id = $2`, status, nodeID.Int64); err != nil {
|
||||
return from, true, false, false, err
|
||||
}
|
||||
if FindingStatusInvalidatesGoal(status) {
|
||||
if err := reopenGoalsProvedByNodeTx(ctx, tx, nodeID.Int64); err != nil {
|
||||
return from, true, false, false, err
|
||||
}
|
||||
}
|
||||
}
|
||||
var assets []int64
|
||||
_ = json.Unmarshal(assetIDs, &assets)
|
||||
|
||||
+13
-7
@@ -1560,15 +1560,21 @@ CREATE TABLE IF NOT EXISTS rsi_lessons (
|
||||
CREATE INDEX IF NOT EXISTS idx_rsi_lessons_scope ON rsi_lessons(company_id, category_id, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_rsi_lessons_kind ON rsi_lessons(kind, updated_at DESC);
|
||||
|
||||
-- One row per (lesson, source). The primary key makes extraction idempotent:
|
||||
-- a second run for the same task adds nothing and cannot inflate support.
|
||||
-- Live evidence is unique per (lesson, source) via uq_rsi_lesson_evidence_live,
|
||||
-- making repeated extraction idempotent without inflating support. Withdrawn
|
||||
-- rows keep withdrawn_at set for history; re-confirming inserts a new live row.
|
||||
CREATE TABLE IF NOT EXISTS rsi_lesson_evidence (
|
||||
lesson_id BIGINT NOT NULL REFERENCES rsi_lessons(id) ON DELETE CASCADE,
|
||||
source_kind TEXT NOT NULL,
|
||||
source_id BIGINT NOT NULL,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (lesson_id, source_kind, source_id)
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
lesson_id BIGINT NOT NULL REFERENCES rsi_lessons(id) ON DELETE CASCADE,
|
||||
source_kind TEXT NOT NULL,
|
||||
source_id BIGINT NOT NULL,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
withdrawn_at TIMESTAMPTZ
|
||||
);
|
||||
ALTER TABLE rsi_lesson_evidence ADD COLUMN IF NOT EXISTS withdrawn_at TIMESTAMPTZ;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_rsi_lesson_evidence_live
|
||||
ON rsi_lesson_evidence(lesson_id, source_kind, source_id)
|
||||
WHERE withdrawn_at IS NULL;
|
||||
|
||||
-- What was shown to which task, and in which mode (shadow = logged but not
|
||||
-- injected; inject = appended to the planner/worker system prompt).
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
# ARTEX from the published GHCR image, with the same PostgreSQL service as
|
||||
# docker-compose.yml. Use this file to run and update without a git checkout:
|
||||
#
|
||||
# docker compose -f docker-compose.ghcr.yml pull
|
||||
# docker compose -f docker-compose.ghcr.yml up -d
|
||||
#
|
||||
# Set POSTGRES_PASSWORD in .env. Pin the image with ARTEX_IMAGE_TAG=v0.7.1 in
|
||||
# .env; unset uses latest. The image carries the built-in /app/skills; this file
|
||||
# does not mount a host skills/ directory. To use custom skills, copy them into a
|
||||
# host directory and add `- ./skills:/app/skills` to the artex volumes.
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:-artex}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Please set POSTGRES_PASSWORD in .env}
|
||||
POSTGRES_DB: ${POSTGRES_DB:-artex}
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-artex} -d ${POSTGRES_DB:-artex}"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
restart: unless-stopped
|
||||
|
||||
artex:
|
||||
image: ghcr.io/baarai/artex:${ARTEX_IMAGE_TAG:-latest}
|
||||
# Agent processes run as an unprivileged executor (-exec-uid): no setuid program
|
||||
# or file capability may raise them back to root.
|
||||
security_opt:
|
||||
- "no-new-privileges:true"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
# Parts, not a pasted URL. PostgresDSN percent-encodes the password, so a
|
||||
# value with / ? % @ : or a space still connects.
|
||||
ARTEX_PG_HOST: postgres
|
||||
ARTEX_PG_PORT: "5432"
|
||||
ARTEX_PG_USER: ${POSTGRES_USER:-artex}
|
||||
ARTEX_PG_PASSWORD: ${POSTGRES_PASSWORD:?Please set POSTGRES_PASSWORD in .env}
|
||||
ARTEX_PG_DBNAME: ${POSTGRES_DB:-artex}
|
||||
ARTEX_PG_SSLMODE: disable
|
||||
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
||||
OPENAI_API_KEY: ${OPENAI_API_KEY:-}
|
||||
ARTEX_LLM_PROVIDER: ${ARTEX_LLM_PROVIDER:-}
|
||||
ARTEX_LLM_MODEL: ${ARTEX_LLM_MODEL:-}
|
||||
ARTEX_LLM_BASE_URL: ${ARTEX_LLM_BASE_URL:-}
|
||||
ARTEX_LLM_PROXY: ${ARTEX_LLM_PROXY:-}
|
||||
# Exact host:port list the model-list fetch may reach on private addresses
|
||||
# (e.g. host.docker.internal:11434 for Ollama).
|
||||
ARTEX_LLM_PRIVATE_ENDPOINTS: ${ARTEX_LLM_PRIVATE_ENDPOINTS:-}
|
||||
ARTEX_JWT_KEY_DIR: /app/keys
|
||||
# Optional: key for secrets at rest; empty = keys/secret.key (generated at
|
||||
# first start).
|
||||
ARTEX_SECRET_KEY: ${ARTEX_SECRET_KEY:-}
|
||||
ARTEX_TRUSTED_PROXIES: ${ARTEX_TRUSTED_PROXIES:-}
|
||||
# Untrusted-output boundary: on by default (empty = on); set 0 to disable.
|
||||
ARTEX_UNTRUSTED_BOUNDARY: ${ARTEX_UNTRUSTED_BOUNDARY:-}
|
||||
ARTEX_UPDATE_CHECK: ${ARTEX_UPDATE_CHECK:-}
|
||||
# Noise profile for target requests: off (default) | quiet | balanced | aggressive.
|
||||
ARTEX_NOISE_PROFILE: ${ARTEX_NOISE_PROFILE:-}
|
||||
ARTEX_TARGET_USER_AGENT: ${ARTEX_TARGET_USER_AGENT:-}
|
||||
# Hierarchical run budgets: unset or 0 = unlimited. See README.
|
||||
ARTEX_TASK_BUDGET_TOKENS: ${ARTEX_TASK_BUDGET_TOKENS:-}
|
||||
ARTEX_TASK_BUDGET_TURNS: ${ARTEX_TASK_BUDGET_TURNS:-}
|
||||
ARTEX_INTENT_BUDGET_TOKENS: ${ARTEX_INTENT_BUDGET_TOKENS:-}
|
||||
ARTEX_INTENT_BUDGET_TURNS: ${ARTEX_INTENT_BUDGET_TURNS:-}
|
||||
ports:
|
||||
- "${ARTEX_BIND_ADDR:-127.0.0.1}:8787:8787"
|
||||
- "127.0.0.1:8788:8788"
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
- ./keys:/app/keys
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
+9
-1
@@ -31,7 +31,15 @@ services:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
ARTEX_PG_DSN: postgres://${POSTGRES_USER:-artex}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-artex}?sslmode=disable
|
||||
# Parts, not a pasted URL. PostgresDSN percent-encodes the password, so a
|
||||
# value with / ? % @ : or a space still connects. ARTEX_PG_DSN, when set,
|
||||
# would override these and is responsible for its own escaping.
|
||||
ARTEX_PG_HOST: postgres
|
||||
ARTEX_PG_PORT: "5432"
|
||||
ARTEX_PG_USER: ${POSTGRES_USER:-artex}
|
||||
ARTEX_PG_PASSWORD: ${POSTGRES_PASSWORD:?Please set POSTGRES_PASSWORD in .env}
|
||||
ARTEX_PG_DBNAME: ${POSTGRES_DB:-artex}
|
||||
ARTEX_PG_SSLMODE: disable
|
||||
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
||||
OPENAI_API_KEY: ${OPENAI_API_KEY:-}
|
||||
ARTEX_LLM_PROVIDER: ${ARTEX_LLM_PROVIDER:-}
|
||||
|
||||
+1
-1
@@ -32,7 +32,7 @@ boundary and trusted proxies. No password or token value is ever logged there.
|
||||
| Setting | Resolution order | Code |
|
||||
| --- | --- | --- |
|
||||
| Config file | `ARTEX_CONFIG` → `./config.json` → `config.json` next to the binary | `config.Path` |
|
||||
| PostgreSQL DSN | `ARTEX_PG_DSN` → `database.dsn` → assembled `host/port/user/password/dbname` | `config.PostgresDSN` |
|
||||
| PostgreSQL DSN | `ARTEX_PG_DSN` → `ARTEX_PG_HOST` / `ARTEX_PG_PORT` / `ARTEX_PG_USER` / `ARTEX_PG_PASSWORD` / `ARTEX_PG_DBNAME` / `ARTEX_PG_SSLMODE` → `database.dsn` → assembled `host/port/user/password/dbname` | `config.PostgresDSN` |
|
||||
| Skill directory | `ARTEX_SKILL_DIR` → `skill_dir` → `<binary dir>/skills` | `config.SkillDir` |
|
||||
| JWT / secret key directory | `ARTEX_JWT_KEY_DIR` → `<binary dir>` (never the browsable data dir) | `config.KeyDir` |
|
||||
| Secret encryption key | `ARTEX_SECRET_KEY` → `<key dir>/secret.key` → newly generated key file | `db.InitSecrets` |
|
||||
|
||||
+31
-13
@@ -35,11 +35,19 @@ POST /api/tasks/{id}/lessons/extract
|
||||
It is idempotent for the same disposition. `rsi_lesson_evidence` is keyed by
|
||||
`(lesson, source_kind, source_id)`, so a second run for the same finding adds
|
||||
no evidence and cannot inflate support. A later extract after a reversed
|
||||
disposition retracts that finding's evidence from the opposite-kind lesson,
|
||||
including every retest of the finding, and retires the lesson when its
|
||||
support hits zero. Confirming again restores the evidence; a retired lesson
|
||||
is un-retired only when it has no remaining evidence. Extraction does not
|
||||
run on status PATCH; call the endpoint after triage. Reopening a finding
|
||||
disposition withdraws that finding's evidence from the opposite-kind lesson,
|
||||
including every retest of the finding, and retires the lesson when its live
|
||||
support hits zero. The evidence row stays, with `withdrawn_at` set. Confirming
|
||||
again clears that timestamp and restores the evidence; a retired lesson is
|
||||
un-retired only when it has no remaining live evidence. A negative disposition
|
||||
also withdraws the cautions written from the finding's retests. A change inside
|
||||
the negative group (`false_positive`, `duplicate`, `ignored`, `risk_accepted`)
|
||||
withdraws the finding from the previous negative lesson before writing the
|
||||
new action text.
|
||||
|
||||
Extraction also runs after a disposition change (`SetFindingStatusWithNotify`,
|
||||
including bulk) and after a completed retest, in addition to terminal status
|
||||
and `POST /api/tasks/{id}/lessons/extract`. Reopening a finding
|
||||
(`pending` / `in_progress`) produces nothing and retracts nothing.
|
||||
|
||||
Rules:
|
||||
@@ -49,30 +57,40 @@ Rules:
|
||||
| finding `confirmed` / `resolved` / `fixed` | positive | `Confirmed <vulnclass>: <summary>` |
|
||||
| retest `reproduced` on such a finding | positive | `Reproduced <vulnclass> on retest: <summary>` |
|
||||
| finding `false_positive` / `duplicate` / `ignored` / `risk_accepted` | negative | `Do not re-report without new mechanics: <summary> (prior <status>)` |
|
||||
| retest `fixed` / `inconclusive` | caution | `Recheck before reporting <vulnclass>: a prior retest was <verdict>` |
|
||||
| retest `fixed` / `inconclusive` on a `confirmed` / `resolved` / `fixed` finding | caution | `Recheck before reporting <vulnclass>: a prior retest was <verdict>` |
|
||||
|
||||
Pending and in-progress findings produce nothing: there is no outcome yet.
|
||||
Empty cue, action, or source is refused by the gate.
|
||||
|
||||
A task with more than one company in `task_scope` is skipped. A generic
|
||||
(`company_id` NULL) lesson would be visible to every later task, including
|
||||
those of the other companies on this one. Per-finding company attribution is
|
||||
a later change.
|
||||
An action is capped at 320 runes. In a negative action the summary is cut, not
|
||||
the `(prior <status>)` suffix, so each negative status keeps its own action and
|
||||
dedupe key.
|
||||
|
||||
A retest counts only when it is `completed` with a verdict; extraction reads
|
||||
the latest such retest. A verdict staged on a pending, running, failed, or
|
||||
stopped retest produces nothing.
|
||||
|
||||
A task with more than one company in `task_scope` does not gain a new lesson.
|
||||
Retraction still runs, so a lesson written while the task had one company can
|
||||
be withdrawn after a second company is added. It withdraws only on a changed
|
||||
disposition: a finding whose status did not move keeps its evidence. Per-finding
|
||||
company attribution is a later change.
|
||||
|
||||
## Selection
|
||||
|
||||
`GET /api/tasks/{id}/lessons` returns the lessons visible to a task plus the
|
||||
rendered block. Selection is scoped:
|
||||
|
||||
- generic lessons (no company, no category) are visible to every task;
|
||||
- a lesson with no company is visible only to a task that also has no company;
|
||||
- company scope comes from `task_scope`, not `tasks.company_id`; a lesson is
|
||||
company-scoped only when the source task had exactly one company, and a
|
||||
task with several companies is not extracted;
|
||||
task with several companies is not given a new lesson;
|
||||
- lessons from the same company or the same task category rank first
|
||||
(company/category matches sort ahead of generic rows);
|
||||
- within a scope: positive before negative before caution, then support, then
|
||||
recency;
|
||||
- at most eight lessons and 1800 characters are rendered.
|
||||
- at most eight lessons are rendered, and the block stops at 1800 characters
|
||||
(runes, not bytes).
|
||||
|
||||
Duplicates are merged at extraction time by a dedupe key over kind, scope, cue,
|
||||
and action, so the same lesson from several tasks becomes one row with several
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
# Task archive storage
|
||||
|
||||
Task archives use local disk by default. The optional S3-compatible backend moves
|
||||
cold task packages to object storage while retaining Archive, Restore and Delete.
|
||||
Active workspaces and PostgreSQL remain local. This is task cold storage, not a
|
||||
full-installation backup; a recovery plan must preserve the database, data
|
||||
directory, bucket objects, and the credentials or roles needed to read them.
|
||||
|
||||
## Configuration
|
||||
|
||||
Use a private bucket and a separate prefix for each ARTEX installation. Configure
|
||||
the bucket's default encryption to match the selected mode below, and enforce
|
||||
that mode/key with a bucket policy. ARTEX does not provision or inspect bucket
|
||||
encryption settings.
|
||||
|
||||
Set the following server environment and restart ARTEX:
|
||||
|
||||
```dotenv
|
||||
ARTEX_ARCHIVE_STORAGE=s3
|
||||
ARTEX_ARCHIVE_S3_ENDPOINT=https://s3.us-east-1.amazonaws.com
|
||||
ARTEX_ARCHIVE_S3_REGION=us-east-1
|
||||
ARTEX_ARCHIVE_S3_BUCKET=your-artex-archives
|
||||
ARTEX_ARCHIVE_S3_PREFIX=artex/tasks
|
||||
ARTEX_ARCHIVE_S3_SSE=AES256
|
||||
```
|
||||
|
||||
With an EC2 instance role, ECS task role or other SDK-supported credential source,
|
||||
no archive-specific access keys are needed.
|
||||
|
||||
| Variable | Meaning |
|
||||
| --- | --- |
|
||||
| `ARTEX_ARCHIVE_STORAGE` | `local` (default) or `s3`. |
|
||||
| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local tests. |
|
||||
| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. |
|
||||
| `ARTEX_ARCHIVE_S3_BUCKET` | Required bucket name. |
|
||||
| `ARTEX_ARCHIVE_S3_PREFIX` | Object prefix; defaults to `artex/tasks`. |
|
||||
| `ARTEX_ARCHIVE_S3_SSE` | `AES256` (default, SSE-S3) or `aws:kms` (SSE-KMS). There is no unencrypted mode. |
|
||||
| `ARTEX_ARCHIVE_S3_KMS_KEY_ID` | Optional full KMS key ARN (not an alias or bare key ID); valid only with `aws:kms`. Omit to use the provider's managed KMS key. |
|
||||
| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Optional dedicated credential override; must be supplied with the secret key. |
|
||||
| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Secret for the dedicated override. |
|
||||
| `ARTEX_ARCHIVE_S3_SESSION_TOKEN` | Optional session token for dedicated temporary credentials. |
|
||||
| `ARTEX_ARCHIVE_S3_PATH_STYLE` | `true` for path-style servers; otherwise virtual-hosted addressing is used. |
|
||||
|
||||
An invalid mode or partial credential override fails configuration instead of
|
||||
silently falling back to local storage or ambient credentials. Preserve the
|
||||
bucket, endpoint, prefix and read/delete access while remote archives remain.
|
||||
Older local archives remain usable; they are not uploaded automatically.
|
||||
|
||||
## Encryption and permissions
|
||||
|
||||
Every single-object upload and multipart initiation explicitly requests the
|
||||
configured server-side encryption. Upload verification also requires HEAD to
|
||||
report the requested mode and a nonempty KMS key identifier for SSE-KMS. When a
|
||||
key ARN is configured, HEAD must return that exact ARN before ARTEX accepts
|
||||
the object. Size and full SHA-256 readback remain mandatory; encryption metadata
|
||||
does not replace content verification.
|
||||
|
||||
For SSE-KMS, set both the bucket's default encryption and ARTEX to `aws:kms`;
|
||||
set `ARTEX_ARCHIVE_S3_KMS_KEY_ID` to the intended key when using a customer-managed
|
||||
key. An explicit `AES256` request takes precedence over a bucket's KMS default,
|
||||
so do not leave ARTEX on SSE-S3 when the bucket is intended to use KMS.
|
||||
|
||||
The principal needs object upload, read (HEAD and ranged GET), delete and
|
||||
multipart upload/abort permissions. For versioned buckets, grant
|
||||
`s3:DeleteObjectVersion` as well: failed verification deletes the version just
|
||||
uploaded, rather than creating a delete marker. KMS uploads/readback also need the applicable
|
||||
key-policy and IAM permissions for `kms:GenerateDataKey` and `kms:Decrypt`.
|
||||
Retain old KMS keys and decrypt permissions when changing the key or mode; restore
|
||||
does not require an older object to use the current upload policy.
|
||||
|
||||
Compatible providers must support the requested SSE mode and its HEAD metadata,
|
||||
plus conditional ranged reads. A provider that ignores the encryption request
|
||||
or cannot confirm it is rejected; the original task is retained. Local source and
|
||||
temporary files keep their existing local-disk protection and restricted modes.
|
||||
|
||||
See the official [PutObject encryption and permission contract](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html)
|
||||
and [HEAD encryption response rules](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html).
|
||||
|
||||
## Credential sources
|
||||
|
||||
Without the archive-specific override, ARTEX uses the AWS SDK default credential
|
||||
chain, including standard AWS environment variables, shared profiles, web
|
||||
identity, ECS task credentials and EC2 instance credentials through IMDS. Role
|
||||
credentials retain the SDK's automatic refresh behavior. A named shared profile
|
||||
can be selected with `AWS_PROFILE`.
|
||||
|
||||
For an external S3-compatible provider, dedicated keys can be supplied through
|
||||
the archive-specific variables. Both key fields are required; a session token
|
||||
is supported for temporary credentials. This explicit override takes precedence
|
||||
over ambient AWS credentials. Operator-supplied temporary credentials must be
|
||||
rotated by the operator; they are not an IAM role provider.
|
||||
|
||||
At server startup, the archive client captures its credential configuration before
|
||||
ARTEX removes archive-specific keys and supported ambient AWS credential variables
|
||||
from the process environment. Agent shells, tools and MCP stdio servers therefore
|
||||
do not inherit those values. Role providers retain the configuration needed for
|
||||
refresh. This environment cleanup does not restrict access to shared credential
|
||||
files, token files or instance metadata; those require their own access controls.
|
||||
|
||||
Both credential modes load shared AWS configuration, including `AWS_CA_BUNDLE`
|
||||
for private certificate authorities. HTTP redirects are rejected for all archive
|
||||
requests, including 307/308; configure the final endpoint directly.
|
||||
|
||||
Keep secrets in a protected service environment file outside the repository and
|
||||
frontend. A systemd `EnvironmentFile=` can point to a root-owned mode-0600 file.
|
||||
See the [AWS SDK credential-chain documentation](https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-gosdk.html).
|
||||
|
||||
## Lifecycle
|
||||
|
||||
1. Pause or finish a task and use its **Archive** action.
|
||||
2. ARTEX stages the source files and builds a local `.tar.zst` package. Large
|
||||
packages use multipart upload, initially 8 MiB parts with two upload workers;
|
||||
the SDK increases the part size for exceptionally large seekable files.
|
||||
3. ARTEX confirms encryption metadata and reads the complete object back,
|
||||
checking size and SHA-256. ETags and uploaded checksum metadata alone are
|
||||
insufficient.
|
||||
4. **Only after verification and a successful PostgreSQL commit** does ARTEX
|
||||
remove the local package and staged source files. The committed archive
|
||||
location is an `s3://` URI; there is no schema or archive-format migration.
|
||||
5. **Restore** downloads and verifies before installing files and database rows.
|
||||
The remote package is consumed only after restoration commits.
|
||||
6. **Delete** commits removal of the archive stub, then cleans up the object.
|
||||
A durable journal retains failed cleanup work.
|
||||
|
||||
The archive list identifies **Object storage** or **Local storage** only once a
|
||||
stored location exists; queued/uploading archives do not default to a local
|
||||
label. Upload/download verification phases remain visible. Reads use bounded
|
||||
8 MiB ranges pinned to the object's ETag. Each range has at most five HTTP
|
||||
attempts total, including transient service errors and interrupted bodies;
|
||||
SDK retries are disabled inside that loop. Each attempt has a two-minute limit
|
||||
and cancellation interrupts both requests and retry delays.
|
||||
|
||||
## Interruption recovery
|
||||
|
||||
Source staging and restore journals live with task archives. Remote cleanup
|
||||
journals live under `data/archives/tasks/.remote`; they contain locations, not
|
||||
credentials. Keep this directory with PostgreSQL and the installation data.
|
||||
|
||||
| Interruption | Recovery |
|
||||
| --- | --- |
|
||||
| During staging/compression, before an upload journal exists | Roll staged source files back. Remove any temporary package, including a partially written `.partial` file, only after rollback succeeds. Keep the staging journal if cleanup fails so recovery can retry. The task remains in PostgreSQL. |
|
||||
| After the upload journal, before successful verification | Roll source files back and clean up any uncommitted object/package. Failed cleanup retains its journal for retry. |
|
||||
| After verification, before the database commit | Treat the object as uncommitted: retain the original task and clean up the orphan object. Verification alone never authorizes local deletion. |
|
||||
| After the database commit, before package/staging cleanup | Keep the committed object and finish local cleanup. The persisted location and checksum establish commit, even if Restore or Delete has since been queued. |
|
||||
| Download/extraction before restored files are installed | Remove temporary extraction data; keep the remote object and archive stub for another attempt. |
|
||||
| Restored files installed, before database restoration commits | Roll back installed files using the restore journal. Retain the remote package. |
|
||||
| Database restoration committed, before remote cleanup | Recognize the already-restored task and finish cleanup without replaying database rows. |
|
||||
| Object consumed, before the restored archive stub is removed | Skip download/replay; repeat idempotent object deletion and remove the stub. |
|
||||
| Delete journal written, before database deletion commits | Keep the object while its archive stub exists. The queued Delete operation can retry. |
|
||||
| Database deletion committed, before object deletion succeeds | The durable outbox retains the URI and retries object deletion. |
|
||||
| An old failed upload survives a retry using local storage | Remove the abandoned remote object, but retain a local package referenced by the committed archive row. |
|
||||
|
||||
The staging recovery rule also applies to local archives. Dedicated tests cover
|
||||
both storage modes with ready, restore-queued, delete-queued and uncommitted
|
||||
states. Additional PostgreSQL/HTTP-S3 tests reconstruct interrupted operations at
|
||||
14 persisted boundaries and repeat recovery to check idempotence.
|
||||
|
||||
Handled multipart-upload failures, including cancellation, trigger an abort with
|
||||
a separate 30-second cleanup context. A failed encryption/content verification
|
||||
similarly deletes the newly uploaded object (its exact version when available)
|
||||
while retaining the source. Cleanup failures are returned together with the
|
||||
original error. Abrupt process death may still leave unfinished multipart uploads
|
||||
or object versions; configure bucket lifecycle expiration for incomplete uploads
|
||||
and a version-retention policy. The URI-based recovery journal does not retain
|
||||
version IDs, so lifecycle policy remains necessary after a crash or failed
|
||||
version cleanup.
|
||||
|
||||
## Disk and transfer requirements
|
||||
|
||||
Archive creation still requires temporary local space for the compressed
|
||||
package. Restore requires space for the download and extracted files. Full
|
||||
upload readback and restores incur storage reads and applicable transfer charges.
|
||||
Failed remote cleanup does not prevent unrelated local archive jobs from being
|
||||
processed once the failed cleanup request returns.
|
||||
|
||||
## Optional connectivity check
|
||||
|
||||
Run `go run ./cmd/artex-archive-check` with the archive environment to upload,
|
||||
verify, download and delete a small unique object. It performs real bucket
|
||||
operations but does not touch task data, and reports cleanup failures.
|
||||
|
||||
Automated tests use HTTP fixtures for S3 and metadata credentials, plus isolated
|
||||
PostgreSQL. They do not establish compatibility with a live cloud bucket.
|
||||
+50
-2
@@ -29,8 +29,10 @@ ensure_docker(){
|
||||
case "$(uname -s)" in
|
||||
Linux)
|
||||
if [ "$(ask 'Install Docker automatically? (y/n)' y)" = y ]; then
|
||||
curl -fsSL https://get.docker.com | sh
|
||||
install_docker_linux
|
||||
sudo usermod -aG docker "$USER" || true
|
||||
docker compose version >/dev/null 2>&1 || sudo docker compose version >/dev/null 2>&1 \
|
||||
|| die "docker installed but 'docker compose' is unavailable; install the compose v2 plugin and retry"
|
||||
ok "Docker installed (re-login required for the group change to take effect)"
|
||||
else
|
||||
die "please install docker yourself and retry"
|
||||
@@ -40,6 +42,52 @@ ensure_docker(){
|
||||
esac
|
||||
}
|
||||
|
||||
# install_docker_linux uses get.docker.com only on the distros it supports. That
|
||||
# script keys its repo URL off /etc/os-release ID + codename, so on derivatives
|
||||
# such as Kali it writes a repo for a nonexistent release (debian kali-rolling)
|
||||
# and leaves apt broken. Other apt-based distros get the distro's own packages.
|
||||
install_docker_linux(){
|
||||
local id="" id_like=""
|
||||
if [ -r /etc/os-release ]; then
|
||||
id="$(. /etc/os-release && echo "${ID:-}")"
|
||||
id_like="$(. /etc/os-release && echo "${ID_LIKE:-}")"
|
||||
fi
|
||||
case "$id" in
|
||||
ubuntu|debian|raspbian|centos|fedora|rhel|sles)
|
||||
curl -fsSL https://get.docker.com | sh
|
||||
return ;;
|
||||
esac
|
||||
command -v apt-get >/dev/null 2>&1 \
|
||||
|| die "automatic Docker install is not supported on '${id:-unknown}'; please install docker + the compose plugin yourself and retry"
|
||||
info "'${id:-unknown}' (like: ${id_like:-n/a}) is not supported by get.docker.com; installing from the distro's apt repos"
|
||||
# Drop a broken repo left by an earlier get.docker.com run (it fails every apt update).
|
||||
# Match the URI and the distro id anywhere in the file: in a one-line .list entry
|
||||
# they share a line, but in a deb822 .sources file the URIs and Suites fields are
|
||||
# on separate lines. -w keeps "kali" matching inside "kali-rolling".
|
||||
local f
|
||||
for f in /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/docker.sources; do
|
||||
if [ -n "$id" ] && [ -f "$f" ] \
|
||||
&& grep -qF "download.docker.com/linux/debian" "$f" 2>/dev/null \
|
||||
&& grep -qwF "$id" "$f" 2>/dev/null; then
|
||||
warn "removing broken Docker repo $f"
|
||||
sudo rm -f "$f"
|
||||
fi
|
||||
done
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y docker.io
|
||||
# The compose v2 package name varies across Debian derivatives; take the first that exists.
|
||||
local p
|
||||
for p in docker-compose-plugin docker-compose-v2 docker-compose; do
|
||||
if apt-cache madison "$p" 2>/dev/null | grep -q .; then
|
||||
sudo apt-get install -y "$p" && break
|
||||
fi
|
||||
done
|
||||
# Kali (and some others) don't start services on package install.
|
||||
if command -v systemctl >/dev/null 2>&1; then
|
||||
sudo systemctl enable --now docker || true
|
||||
fi
|
||||
}
|
||||
|
||||
# ── 1) All-Docker ───────────────────────────────
|
||||
install_docker(){
|
||||
ensure_docker
|
||||
@@ -107,7 +155,7 @@ JSON
|
||||
if command -v npm >/dev/null 2>&1; then
|
||||
info "building frontend static assets..."
|
||||
( cd web && npm ci --ignore-scripts && npm run build:static )
|
||||
rm -rf server/webui/dist && cp -r web/out server/webui/dist
|
||||
rm -rf server/webui/dist && mkdir -p server/webui && cp -r web/out server/webui/dist
|
||||
info "compiling embedded single binary..."
|
||||
CGO_ENABLED=0 go build -tags embedui -trimpath -o artex ./cmd/artex
|
||||
else
|
||||
|
||||
@@ -104,6 +104,10 @@ var notifyTransport = func() *http.Transport {
|
||||
return &http.Transport{}
|
||||
}
|
||||
clone := t.Clone()
|
||||
// Clone keeps ProxyFromEnvironment. The dial would then be to the proxy, so
|
||||
// blockInternalDial would see the proxy address and a link-local target
|
||||
// could still be reached through it. Notifications go direct.
|
||||
clone.Proxy = nil
|
||||
clone.DialContext = (&net.Dialer{Timeout: 10 * time.Second, Control: blockInternalDial}).DialContext
|
||||
return clone
|
||||
}()
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
package notify
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNotifyTransportIgnoresEnvironmentProxy(t *testing.T) {
|
||||
if notifyTransport.Proxy != nil {
|
||||
t.Fatal("notification transport must dial the target, not an environment proxy")
|
||||
}
|
||||
}
|
||||
+66
-9
@@ -9,8 +9,11 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
// Rendering for the findings page "Export": renders a batch of findings table rows
|
||||
@@ -122,21 +125,75 @@ func SingleFindingMarkdown(f *db.DBFinding, generatedAt time.Time) string {
|
||||
|
||||
var unsafeFilenameChars = regexp.MustCompile(`[^\p{Han}\p{L}\p{N}._-]+`)
|
||||
|
||||
const maxFindingFilenameBaseBytes = 120
|
||||
|
||||
// stripCombiningAndFormat drops uncombined non-spacing marks (e.g. Arabic
|
||||
// diacritics after NFC composition) and format controls (e.g. BiDi RLO/LRO)
|
||||
// so they are omitted rather than turned into underscores between letters.
|
||||
func stripCombiningAndFormat(s string) string {
|
||||
s = norm.NFC.String(s)
|
||||
return strings.Map(func(r rune) rune {
|
||||
if unicode.Is(unicode.Mn, r) || unicode.Is(unicode.Cf, r) {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, s)
|
||||
}
|
||||
|
||||
func truncateUTF8Bytes(s string, maxBytes int) string {
|
||||
if maxBytes <= 0 {
|
||||
return ""
|
||||
}
|
||||
if len(s) <= maxBytes {
|
||||
return s
|
||||
}
|
||||
n := 0
|
||||
for i := range s {
|
||||
if i > maxBytes {
|
||||
break
|
||||
}
|
||||
n = i
|
||||
}
|
||||
if n == 0 && len(s) > 0 {
|
||||
_, size := utf8.DecodeRuneInString(s)
|
||||
if size <= maxBytes {
|
||||
return s[:size]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return s[:n]
|
||||
}
|
||||
|
||||
// FindingFilename generates a safe .md filename for "one file per vulnerability", e.g.
|
||||
// `critical_SQL_injection_#123.md`. Strips path separators and control characters to avoid illegal paths inside the zip.
|
||||
// `critical_SQL_injection_123.md`. Strips path separators, combining diacritics, and
|
||||
// control characters to avoid illegal paths inside the zip while preserving the ID suffix.
|
||||
func FindingFilename(f *db.DBFinding) string {
|
||||
sev := nz(f.Severity, "info")
|
||||
title := findingTitle(f)
|
||||
name := fmt.Sprintf("%s_%s_#%d", sev, title, f.ID)
|
||||
name = unsafeFilenameChars.ReplaceAllString(name, "_")
|
||||
name = strings.Trim(name, "._")
|
||||
if name == "" {
|
||||
sev := unsafeFilenameChars.ReplaceAllString(stripCombiningAndFormat(nz(f.Severity, "info")), "_")
|
||||
sev = strings.Trim(sev, "._")
|
||||
if sev == "" {
|
||||
sev = "info"
|
||||
}
|
||||
title := unsafeFilenameChars.ReplaceAllString(stripCombiningAndFormat(findingTitle(f)), "_")
|
||||
title = strings.Trim(title, "._")
|
||||
suffix := fmt.Sprintf("_%d", f.ID)
|
||||
prefix := sev
|
||||
if title != "" {
|
||||
prefix = sev + "_" + title
|
||||
}
|
||||
budget := maxFindingFilenameBaseBytes - len(suffix)
|
||||
if len(prefix) > budget {
|
||||
prefix = strings.TrimRight(truncateUTF8Bytes(prefix, budget), "._")
|
||||
}
|
||||
var name string
|
||||
if prefix == "" {
|
||||
name = fmt.Sprintf("finding_%d", f.ID)
|
||||
} else {
|
||||
name = prefix + suffix
|
||||
}
|
||||
// Defensive: strip one more path layer to rule out zip slip.
|
||||
name = path.Base(name)
|
||||
if len(name) > 120 {
|
||||
name = strings.ToValidUTF8(name[:120], "")
|
||||
if name == "" || name == "." || name == ".." {
|
||||
name = fmt.Sprintf("finding_%d", f.ID)
|
||||
}
|
||||
return name + ".md"
|
||||
}
|
||||
|
||||
@@ -355,3 +355,49 @@ func TestFindingsCSVNeutralizesFormulas(t *testing.T) {
|
||||
t.Errorf("plain severity %q should not be prefixed", row[3])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindingFilenameUnicodeAndIDSuffix(t *testing.T) {
|
||||
// Vocalized Arabic should strip combining diacritics cleanly rather than inserting underscores.
|
||||
arabic := FindingFilename(&db.DBFinding{
|
||||
ID: 42,
|
||||
Severity: "critical",
|
||||
Name: "ثَغْرَة حَقْن SQL",
|
||||
})
|
||||
if arabic != "critical_ثغرة_حقن_SQL_42.md" {
|
||||
t.Fatalf("arabic filename = %q, want %q", arabic, "critical_ثغرة_حقن_SQL_42.md")
|
||||
}
|
||||
|
||||
// Composed Latin/Turkish letters stay intact after NFC normalization.
|
||||
turkish := FindingFilename(&db.DBFinding{
|
||||
ID: 9,
|
||||
Severity: "medium",
|
||||
Name: "Gu\u0308venlik Ac\u0327ıg\u0306ı",
|
||||
})
|
||||
if turkish != "medium_Güvenlik_Açığı_9.md" {
|
||||
t.Fatalf("turkish filename = %q, want %q", turkish, "medium_Güvenlik_Açığı_9.md")
|
||||
}
|
||||
|
||||
// BiDi RLO control characters are stripped without breaking words.
|
||||
bidi := FindingFilename(&db.DBFinding{
|
||||
ID: 15,
|
||||
Severity: "high",
|
||||
Name: "upload\u202Egpj.exe",
|
||||
})
|
||||
if strings.ContainsRune(bidi, '\u202E') || bidi != "high_uploadgpj.exe_15.md" {
|
||||
t.Fatalf("bidi filename = %q", bidi)
|
||||
}
|
||||
|
||||
// Long multi-byte titles stay valid UTF-8, within 120 bytes before .md, and keep distinct ID suffixes.
|
||||
longPrefix := strings.Repeat("اختراق_صلاحيات_", 10)
|
||||
f1 := FindingFilename(&db.DBFinding{ID: 101, Severity: "high", Name: longPrefix + "أ"})
|
||||
f2 := FindingFilename(&db.DBFinding{ID: 102, Severity: "high", Name: longPrefix + "ب"})
|
||||
for _, name := range []string{f1, f2} {
|
||||
base := strings.TrimSuffix(name, ".md")
|
||||
if len(base) > maxFindingFilenameBaseBytes {
|
||||
t.Fatalf("filename base %q exceeds %d bytes (%d)", base, maxFindingFilenameBaseBytes, len(base))
|
||||
}
|
||||
}
|
||||
if !strings.HasSuffix(f1, "_101.md") || !strings.HasSuffix(f2, "_102.md") || f1 == f2 {
|
||||
t.Fatalf("expected preserved distinct ID suffixes: f1=%q f2=%q", f1, f2)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,6 +223,7 @@ func seedPrompts(pg *db.DB) {
|
||||
// and default params get injected. MCP/skill/host tools have no row and pass through.
|
||||
func wireTools(pg *db.DB, domainReg map[string]actool.CoreTool) {
|
||||
agent.SetFindingTrafficBindingEnabled(func() bool { return pg.GetBool(settingAgentTrafficBinding, false) })
|
||||
agent.SetTrafficCaptureEnabled(func() bool { return pg.GetBool(settingTrafficCapture, false) })
|
||||
// Seed the built-in domain tools (first-insert only; DO NOTHING preserves edits).
|
||||
// No startup prune: rows we didn't seed are left alone so future user-defined
|
||||
// custom tools (system=false, added via the UI) survive restarts.
|
||||
|
||||
+31
-8
@@ -21,6 +21,10 @@ const (
|
||||
// Allow multipart headers and boundaries in addition to the file data.
|
||||
chatUploadMultipartOverhead = 1 << 20 // 1 MiB
|
||||
chatUploadLimitMessage = "files exceed the 512 MiB upload limit"
|
||||
// maxUploadNameTries bounds the name-1, name-2, … search. A Stat error that
|
||||
// is not "not exist" (a component the filesystem rejects) must stop the
|
||||
// search instead of spinning.
|
||||
maxUploadNameTries = 10000
|
||||
)
|
||||
|
||||
// safeChatID guards the {id} path segment against traversal — task ids are numeric,
|
||||
@@ -123,7 +127,11 @@ func (s *Server) chatUpload(w http.ResponseWriter, r *http.Request) {
|
||||
if name == "" || name == "." || name == ".." || strings.ContainsAny(name, `/\`) {
|
||||
continue
|
||||
}
|
||||
dest := uniqueUploadPath(dir, name)
|
||||
dest, err := uniqueUploadPath(dir, name)
|
||||
if err != nil {
|
||||
writeErr(w, 500, "Save failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
if err := saveUpload(hdr, dest); err != nil {
|
||||
writeErr(w, 500, "Save failed: "+err.Error())
|
||||
return
|
||||
@@ -136,19 +144,34 @@ func (s *Server) chatUpload(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// uniqueUploadPath returns dir/name, or dir/name-1, dir/name-2… when it already exists,
|
||||
// so re-uploading the same filename never clobbers a prior attachment.
|
||||
func uniqueUploadPath(dir, name string) string {
|
||||
dest := filepath.Join(dir, name)
|
||||
if _, err := os.Stat(dest); os.IsNotExist(err) {
|
||||
return dest
|
||||
func uniqueUploadPath(dir, name string) (string, error) {
|
||||
if dest, ok, err := uploadPathFree(filepath.Join(dir, name)); err != nil || ok {
|
||||
return dest, err
|
||||
}
|
||||
ext := filepath.Ext(name)
|
||||
stem := strings.TrimSuffix(name, ext)
|
||||
for i := 1; ; i++ {
|
||||
for i := 1; i <= maxUploadNameTries; i++ {
|
||||
cand := filepath.Join(dir, fmt.Sprintf("%s-%d%s", stem, i, ext))
|
||||
if _, err := os.Stat(cand); os.IsNotExist(err) {
|
||||
return cand
|
||||
dest, ok, err := uploadPathFree(cand)
|
||||
if err != nil || ok {
|
||||
return dest, err
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("no free name for upload %q", name)
|
||||
}
|
||||
|
||||
// uploadPathFree reports whether path can be created. ok is false when a file
|
||||
// or a symlink (including a dangling one) is already there. Any other Lstat
|
||||
// error is returned so the caller stops.
|
||||
func uploadPathFree(path string) (dest string, ok bool, err error) {
|
||||
_, err = os.Lstat(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return path, true, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// composeAgentMessage appends an attachment manifest to the user's message so the agent
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUniqueUploadPathSuffixesCollisions(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "note.txt"), []byte("a"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "note-1.txt"), []byte("b"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := uniqueUploadPath(dir, "note.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if filepath.Base(got) != "note-2.txt" {
|
||||
t.Fatalf("got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUniqueUploadPathStopsOnUnstorableName(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
name := strings.Repeat("a", 300) + ".txt"
|
||||
if _, err := uniqueUploadPath(dir, name); err == nil {
|
||||
t.Fatal("an overlong file name must not be searched forever")
|
||||
}
|
||||
}
|
||||
+4
-2
@@ -3,7 +3,9 @@ package server
|
||||
import "os"
|
||||
|
||||
// secretEnv lists the credentials read from the environment only at startup:
|
||||
// ARTEX_PG_DSN and ARTEX_SECRET_KEY by NewManager, the API keys by agent.FromEnv inside New.
|
||||
// ARTEX_PG_DSN and ARTEX_PG_PASSWORD by NewManager (the password is a separate
|
||||
// variable so Docker Compose does not paste it into a URL), ARTEX_SECRET_KEY,
|
||||
// and the API keys by agent.FromEnv inside New.
|
||||
// agent.FromEnv also reads ARTEX_LLM_PROXY and ARTEX_LLM_BASE_URL: both are URLs that
|
||||
// can carry credentials in their userinfo (http://user:pass@proxy:3128). The archive
|
||||
// store also captures ARTEX_ARCHIVE_S3_* credentials during New, before the scrub,
|
||||
@@ -16,7 +18,7 @@ import "os"
|
||||
// credentials or config file, or from the instance metadata service, are not in
|
||||
// the environment: the scrub does not hide them from child processes.
|
||||
var secretEnv = []string{
|
||||
"ARTEX_PG_DSN", "ARTEX_SECRET_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY",
|
||||
"ARTEX_PG_DSN", "ARTEX_PG_PASSWORD", "ARTEX_SECRET_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY",
|
||||
"ARTEX_LLM_PROXY", "ARTEX_LLM_BASE_URL",
|
||||
"ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_SESSION_TOKEN",
|
||||
"AWS_ACCESS_KEY_ID", "AWS_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_SESSION_TOKEN",
|
||||
|
||||
+3
-2
@@ -13,8 +13,9 @@ import (
|
||||
|
||||
// Sealing the server environment. ScrubSecretEnv removes the secrets from the Go
|
||||
// environment, but the kernel keeps the environment block the process was executed
|
||||
// with: /proc/<pid>/environ still shows ARTEX_PG_DSN, ARTEX_SECRET_KEY and the API
|
||||
// keys to any process that can read it. SealProcessEnv re-executes the server once,
|
||||
// with: /proc/<pid>/environ still shows ARTEX_PG_DSN, ARTEX_PG_PASSWORD,
|
||||
// ARTEX_SECRET_KEY and the API keys to any process that can read it.
|
||||
// SealProcessEnv re-executes the server once,
|
||||
// with those variables left out of the block; the values travel to the new image
|
||||
// through an inherited pipe and are put back into the Go environment there, so the
|
||||
// readers that run next see the same configuration.
|
||||
|
||||
@@ -51,6 +51,7 @@ func TestSealedEnvironmentHandoff(t *testing.T) {
|
||||
}
|
||||
want := map[string]string{
|
||||
"ARTEX_PG_DSN": "postgres://artex:p@ss@db.internal/artex?sslmode=disable\nsecond line\twith tab",
|
||||
"ARTEX_PG_PASSWORD": "p/ss w@rd?#%",
|
||||
"ARTEX_SECRET_KEY": "",
|
||||
"ANTHROPIC_API_KEY": "sk-ant-test",
|
||||
"OPENAI_API_KEY": "=leading-equals",
|
||||
|
||||
+8
-9
@@ -152,9 +152,15 @@ func (s *logSinkT) add(l LogLine) {
|
||||
s.ring = s.ring[len(s.ring)-s.cap:]
|
||||
}
|
||||
ch := s.dbCh
|
||||
subs := make([]chan LogLine, 0, len(s.subs))
|
||||
// Fan-out stays under the lock. subscribe's cancel removes the channel and
|
||||
// then closes it; a send on a closed channel panics even inside select, so
|
||||
// the send and the close must not interleave. The send does not block: a
|
||||
// full buffer drops the line.
|
||||
for sub := range s.subs {
|
||||
subs = append(subs, sub)
|
||||
select {
|
||||
case sub <- l:
|
||||
default:
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
@@ -165,13 +171,6 @@ func (s *logSinkT) add(l LogLine) {
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
for _, sub := range subs {
|
||||
select {
|
||||
case sub <- l:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// recent returns ring lines with Seq > since (capped to limit), and the latest seq.
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"io"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLogSinkUnsubscribeDuringAddDoesNotPanic(t *testing.T) {
|
||||
s := &logSinkT{cap: 8, subs: map[chan LogLine]struct{}{}, out: io.Discard}
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 32; i++ {
|
||||
_, unsub := s.subscribe()
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
unsub()
|
||||
}()
|
||||
s.add(LogLine{Text: "line"})
|
||||
}
|
||||
wg.Wait()
|
||||
s.add(LogLine{Text: "after"})
|
||||
}
|
||||
@@ -807,6 +807,7 @@ func (s *Server) agentsForTask(t *Task) *taskAgentBundle {
|
||||
wk.SetNoaEnabled(s.m.NoaCompactionEnabled) // experimental: noa context compaction (platform-level switch, read every run)
|
||||
wk.SetRunTimeout(time.Duration(s.agentRunSeconds("worker")) * time.Second)
|
||||
wk.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert())
|
||||
wk.SetFactTraffic(s.m.traffic) // fact provenance: relabel bytes captured from the target (#124 row 1)
|
||||
wk.SetWebSearch(s.webSearchFor("worker"))
|
||||
wk.SetConstraintInject(s.constraintInjectWorker) // operation-constraint injection into the worker (configurable, on by default)
|
||||
pl := agent.NewPlanner(plannerRuntime, "task-router", s.m.dir, tx, plannerRuntime.CompactionWindow(), s.agentMaxTurns("planner"))
|
||||
@@ -818,6 +819,7 @@ func (s *Server) agentsForTask(t *Task) *taskAgentBundle {
|
||||
pl.SetKillWork(s.engine.KillWork)
|
||||
pl.SetSteerWork(s.engine.SteerWork)
|
||||
pl.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert())
|
||||
pl.SetFactTraffic(s.m.traffic)
|
||||
pl.SetWebSearch(s.webSearchFor("planner"))
|
||||
pl.SetConstraintInject(s.constraintInjectPlanner) // operation-constraint injection into the planner (configurable, on by default)
|
||||
// cold-digest §7: background compaction of cold nodes. This per-task planner set
|
||||
@@ -834,6 +836,7 @@ func (s *Server) agentsForTask(t *Task) *taskAgentBundle {
|
||||
main.SetMaxTokens(mainRuntime.maxTokens)
|
||||
main.SetNoaEnabled(s.m.NoaCompactionEnabled) // experimental: noa context compaction (platform-level switch, read every run)
|
||||
main.SetProxy(s.m.ProxyAddr(), s.m.ProxyCACert())
|
||||
main.SetFactTraffic(s.m.traffic)
|
||||
main.SetWebSearch(s.webSearchFor("mainagent"))
|
||||
main.SetSteerWork(s.engine.SteerWork) // steer_work: real-time course correction on running work by a human
|
||||
bundle := &taskAgentBundle{
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
//go:build !windows
|
||||
|
||||
package server
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// openUploadFileNoFollow opens dest for writing without following a symlink at
|
||||
// the final path component, closing the TOCTOU window after refuseUploadSymlink.
|
||||
func openUploadFileNoFollow(dest string) (*os.File, error) {
|
||||
return os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC|syscall.O_NOFOLLOW, 0o644)
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
//go:build windows
|
||||
|
||||
package server
|
||||
|
||||
import "os"
|
||||
|
||||
func openUploadFileNoFollow(dest string) (*os.File, error) {
|
||||
return os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRefuseUploadSymlink(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
target := filepath.Join(dir, "secret")
|
||||
if err := os.WriteFile(target, []byte("keep"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
link := filepath.Join(dir, "upload")
|
||||
if err := os.Symlink(target, link); err != nil {
|
||||
t.Skipf("symlink: %v", err)
|
||||
}
|
||||
if err := refuseUploadSymlink(link); err == nil {
|
||||
t.Fatal("symlink destination was accepted")
|
||||
}
|
||||
body, err := os.ReadFile(target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(body) != "keep" {
|
||||
t.Fatalf("target changed to %q", body)
|
||||
}
|
||||
missing := filepath.Join(dir, "fresh")
|
||||
if err := refuseUploadSymlink(missing); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenUploadFileNoFollowRejectsSymlink(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("O_NOFOLLOW is Unix-only")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
target := filepath.Join(dir, "secret")
|
||||
if err := os.WriteFile(target, []byte("keep"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
link := filepath.Join(dir, "link")
|
||||
if err := os.Symlink(target, link); err != nil {
|
||||
t.Skipf("symlink: %v", err)
|
||||
}
|
||||
if f, err := openUploadFileNoFollow(link); err == nil {
|
||||
_ = f.Close()
|
||||
t.Fatal("openUploadFileNoFollow accepted a symlink")
|
||||
}
|
||||
body, err := os.ReadFile(target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(body) != "keep" {
|
||||
t.Fatalf("target changed to %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUniqueUploadPathSkipsDanglingSymlink(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dangling := filepath.Join(dir, "note.txt")
|
||||
if err := os.Symlink(filepath.Join(dir, "nonexistent"), dangling); err != nil {
|
||||
t.Skipf("symlink: %v", err)
|
||||
}
|
||||
got, err := uniqueUploadPath(dir, "note.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if filepath.Base(got) != "note-1.txt" {
|
||||
t.Fatalf("got %s, want note-1.txt", got)
|
||||
}
|
||||
}
|
||||
+23
-1
@@ -2,6 +2,7 @@ package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
@@ -340,12 +341,15 @@ func (s *Server) wsUpload(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func saveUpload(hdr *multipart.FileHeader, dest string) error {
|
||||
if err := refuseUploadSymlink(dest); err != nil {
|
||||
return err
|
||||
}
|
||||
src, err := hdr.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
out, err := os.Create(dest)
|
||||
out, err := openUploadFileNoFollow(dest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -354,6 +358,24 @@ func saveUpload(hdr *multipart.FileHeader, dest string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// refuseUploadSymlink rejects a destination that is already a symlink. os.Create
|
||||
// follows one, including a dangling link that Stat reports as missing, so an
|
||||
// agent that can write the uploads directory could aim the server's write at
|
||||
// another path. Lstat sees the link itself.
|
||||
func refuseUploadSymlink(dest string) error {
|
||||
fi, err := os.Lstat(dest)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if fi.Mode()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("refusing to write through a symlink")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sanitizeFilename strips characters unsafe for a Content-Disposition filename token.
|
||||
func sanitizeFilename(name string) string {
|
||||
name = strings.ReplaceAll(name, "\"", "")
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSpillBlobIsOwnerOnly(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tr := &Traffic{dir: dir}
|
||||
body := bytes.Repeat([]byte("a"), maxInlineBody+1)
|
||||
got := tr.spill(body, "text/plain")
|
||||
if got.hash == "" {
|
||||
t.Fatal("expected a spilled blob")
|
||||
}
|
||||
blobDir := filepath.Join(dir, "_blobs", "sha256", got.hash[:2])
|
||||
blobPath := filepath.Join(blobDir, got.hash+".bin")
|
||||
info, err := os.Stat(blobPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dirInfo, err := os.Stat(blobDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if runtime.GOOS == "windows" {
|
||||
return
|
||||
}
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("blob mode %o, want 0600", info.Mode().Perm())
|
||||
}
|
||||
if dirInfo.Mode().Perm() != 0o700 {
|
||||
t.Fatalf("blob dir mode %o, want 0700", dirInfo.Mode().Perm())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenTightensExistingBlobPerms(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("POSIX file permissions not enforced on Windows")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
blobsRoot := filepath.Join(dir, "_blobs")
|
||||
shaDir := filepath.Join(blobsRoot, "sha256")
|
||||
bucket := filepath.Join(shaDir, "ab")
|
||||
if err := os.MkdirAll(bucket, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, d := range []string{blobsRoot, shaDir, bucket} {
|
||||
if err := os.Chmod(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
blob := filepath.Join(bucket, "abcdef.bin")
|
||||
if err := os.WriteFile(blob, []byte("secret-body"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chmod(blob, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tr, err := Open(dir, "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
for _, d := range []string{blobsRoot, shaDir, bucket} {
|
||||
info, err := os.Stat(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Mode().Perm() != 0o700 {
|
||||
t.Fatalf("%s mode %o, want 0700", d, info.Mode().Perm())
|
||||
}
|
||||
}
|
||||
info, err := os.Stat(blob)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("%s mode %o, want 0600", blob, info.Mode().Perm())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ResponseContaining matches only the response side, inside the time window:
|
||||
// a line that appears only in a request, or a capture outside the window, must
|
||||
// not classify a fact as target-authored (#124 row 1, step 3).
|
||||
func TestResponseContaining(t *testing.T) {
|
||||
tr, _ := openTraffic(t)
|
||||
tr.record(newFlow("target.example", "GET", "/login",
|
||||
[]byte("request-only-marker-abc"),
|
||||
[]byte(`<html><body>welcome, internal only</body></html>`)))
|
||||
id := onlyExchangeID(t, tr)
|
||||
|
||||
now := time.Now().Unix()
|
||||
got, ok, err := tr.ResponseContaining([]string{"internal only"}, now-60, now+60)
|
||||
if err != nil || !ok || got != id {
|
||||
t.Fatalf("response match = %q, %v, %v; want %q, true", got, ok, err, id)
|
||||
}
|
||||
if _, ok, err := tr.ResponseContaining([]string{"request-only-marker-abc"}, now-60, now+60); err != nil || ok {
|
||||
t.Fatalf("request-side match: ok=%v err=%v, want false", ok, err)
|
||||
}
|
||||
if _, ok, err := tr.ResponseContaining([]string{"internal only"}, now+600, now+900); err != nil || ok {
|
||||
t.Fatalf("out-of-window match: ok=%v err=%v, want false", ok, err)
|
||||
}
|
||||
if _, ok, err := tr.ResponseContaining([]string{"never recorded anywhere"}, now-60, now+60); err != nil || ok {
|
||||
t.Fatalf("absent match: ok=%v err=%v, want false", ok, err)
|
||||
}
|
||||
var nilStore *Traffic
|
||||
if _, ok, err := nilStore.ResponseContaining([]string{"internal only"}, now-60, now+60); err != nil || ok {
|
||||
t.Fatalf("nil store: ok=%v err=%v, want false", ok, err)
|
||||
}
|
||||
}
|
||||
@@ -330,24 +330,30 @@ func (e *vm) execRange(s *ast.RangeStmt, sc *env) (flowSignal, any) {
|
||||
var items []kvPair
|
||||
switch x := e.eval(s.X, sc).(type) {
|
||||
case []byte:
|
||||
e.alloc(int64(len(x)) * 32)
|
||||
for i, v := range x {
|
||||
items = append(items, kvPair{int64(i), int64(v)})
|
||||
}
|
||||
case []any:
|
||||
e.alloc(int64(len(x)) * 32)
|
||||
for i, v := range x {
|
||||
items = append(items, kvPair{int64(i), v})
|
||||
}
|
||||
case []string:
|
||||
e.alloc(int64(len(x)) * 32)
|
||||
for i, v := range x {
|
||||
items = append(items, kvPair{int64(i), v})
|
||||
}
|
||||
case string:
|
||||
e.alloc(int64(len(x)) * 32)
|
||||
for i, r := range x {
|
||||
items = append(items, kvPair{int64(i), int64(r)})
|
||||
}
|
||||
case map[string]any:
|
||||
e.alloc(int64(len(x)) * 32)
|
||||
items = sortedMapPairs(x)
|
||||
case map[string][]string:
|
||||
e.alloc(int64(len(x)) * 32)
|
||||
items = sortedMapPairs(x)
|
||||
case nil:
|
||||
return flowNone, nil
|
||||
|
||||
@@ -204,6 +204,25 @@ func OnResponse(req *Request, resp *Response) error {
|
||||
}
|
||||
|
||||
func TestResourceBoundsMemoryStepTimeoutPanicAndRegexp(t *testing.T) {
|
||||
t.Run("range_over_body_stopped_by_memory_budget", func(t *testing.T) {
|
||||
src := `package hook
|
||||
func OnRequest(req *Request) error {
|
||||
for _, b := range req.Body {
|
||||
_ = b
|
||||
}
|
||||
return nil
|
||||
}`
|
||||
prog, err := Compile("request", src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := make([]byte, 256<<10)
|
||||
err = prog.RunRequest(context.Background(), &Request{Body: body}, RunOptions{MaxMemory: 64 << 10})
|
||||
if !errors.Is(err, ErrMemoryLimit) {
|
||||
t.Fatalf("err = %v, want ErrMemoryLimit", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("make_100MiB_stopped_by_memory_budget", func(t *testing.T) {
|
||||
src := `package hook
|
||||
func OnRequest(req *Request) error {
|
||||
|
||||
+2
-2
@@ -63,7 +63,7 @@ func (t *Traffic) spillRawDir(data []byte, previewCap int) (preview []byte, blob
|
||||
sum := sha256.Sum256(data)
|
||||
h := hex.EncodeToString(sum[:])
|
||||
blobDir := filepath.Join(t.dir, "_blobs", "sha256", h[:2])
|
||||
if err := os.MkdirAll(blobDir, 0o755); err != nil {
|
||||
if err := os.MkdirAll(blobDir, 0o700); err != nil {
|
||||
log.Printf("[traffic/socks5] failed to create blob directory: %v", err)
|
||||
return append([]byte(nil), data[:previewCap]...), ""
|
||||
}
|
||||
@@ -96,7 +96,7 @@ func writeBlobAtomic(blobDir, blobPath, hash string, data []byte) error {
|
||||
_ = tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Chmod(0o644); err != nil {
|
||||
if err := tmp.Chmod(0o600); err != nil {
|
||||
_ = tmp.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
+90
-3
@@ -295,11 +295,16 @@ type Traffic struct {
|
||||
|
||||
// Open initializes the traffic tree, blob store and SQLite index under dir.
|
||||
func Open(dir, addr string) (*Traffic, error) {
|
||||
for _, d := range []string{dir, filepath.Join(dir, "_index"), filepath.Join(dir, "_blobs"), filepath.Join(dir, "_ca")} {
|
||||
for _, d := range []string{dir, filepath.Join(dir, "_index"), filepath.Join(dir, "_ca")} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
blobsRoot := filepath.Join(dir, "_blobs")
|
||||
if err := os.MkdirAll(blobsRoot, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tightenBlobPerms(blobsRoot)
|
||||
// busy_timeout is a per-connection setting, so it belongs in the DSN rather
|
||||
// than in a one-off Exec: the pool opens connections on demand, and an Exec
|
||||
// only configures whichever one happened to serve it — leaving every other
|
||||
@@ -847,6 +852,38 @@ type storedBody struct {
|
||||
// reader can identify them without fetching the blob. Either way, text bodies
|
||||
// are handed to the full-text index in full (up to maxIndexBody) — indexing is
|
||||
// independent of where the bytes end up.
|
||||
// tightenBlobPerms restricts existing spilled blob directories to 0700 and
|
||||
// regular blob files to 0600 on startup so pre-existing 0755/0644 artifacts
|
||||
// from earlier versions do not stay world-readable.
|
||||
func tightenBlobPerms(root string) {
|
||||
_ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if d.Type()&os.ModeSymlink != 0 {
|
||||
return nil
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() {
|
||||
if info.Mode().Perm() != 0o700 {
|
||||
if err := os.Chmod(path, 0o700); err != nil {
|
||||
log.Printf("[traffic] failed to restrict blob directory %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if info.Mode().IsRegular() && info.Mode().Perm() != 0o600 {
|
||||
if err := os.Chmod(path, 0o600); err != nil {
|
||||
log.Printf("[traffic] failed to restrict blob %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Traffic) spill(body []byte, contentType string) storedBody {
|
||||
return t.spillAt(body, contentType, maxInlineBody)
|
||||
}
|
||||
@@ -873,16 +910,21 @@ func (t *Traffic) spillAt(body []byte, contentType string, maxInline int) stored
|
||||
// One bucket level (256 buckets) is enough to keep any single directory small;
|
||||
// the store only ever holds bodies above maxInlineBody, deduplicated by hash.
|
||||
blobDir := filepath.Join(t.dir, "_blobs", "sha256", h[:2])
|
||||
if err := os.MkdirAll(blobDir, 0o755); err != nil {
|
||||
if err := os.MkdirAll(blobDir, 0o700); err != nil {
|
||||
log.Printf("[traffic] failed to create blob directory: %v", err)
|
||||
return storedBody{inline: clipBytes(body, blobPreview), index: indexText()}
|
||||
}
|
||||
if err := os.Chmod(blobDir, 0o700); err != nil {
|
||||
log.Printf("[traffic] failed to restrict blob directory: %v", err)
|
||||
}
|
||||
blobPath := filepath.Join(blobDir, h+".bin")
|
||||
if _, err := os.Stat(blobPath); os.IsNotExist(err) {
|
||||
if err := os.WriteFile(blobPath, body, 0o644); err != nil {
|
||||
if err := os.WriteFile(blobPath, body, 0o600); err != nil {
|
||||
log.Printf("[traffic] failed to write blob %s: %v", h, err)
|
||||
return storedBody{inline: clipBytes(body, blobPreview), index: indexText()}
|
||||
}
|
||||
} else if err := os.Chmod(blobPath, 0o600); err != nil {
|
||||
log.Printf("[traffic] failed to restrict blob %s: %v", h, err)
|
||||
}
|
||||
sb := storedBody{hash: h, index: indexText()}
|
||||
if text {
|
||||
@@ -1156,6 +1198,51 @@ func (t *Traffic) Search(host string, page, size int) ([]ExchangeMeta, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ResponseContaining reports the newest capture recorded in [fromTS,toTS]
|
||||
// (unix seconds) whose response side contains one of lines. Request-side matches
|
||||
// are ignored after the index lookup: what classifies a fact is what came back
|
||||
// from the target. ok is false when no capture matches, when this build has no
|
||||
// full-text index, or when the store is unavailable; callers keep the
|
||||
// activity-only classification in that case.
|
||||
func (t *Traffic) ResponseContaining(lines []string, fromTS, toTS int64) (id string, ok bool, err error) {
|
||||
if t == nil || len(lines) == 0 {
|
||||
return "", false, nil
|
||||
}
|
||||
for _, line := range lines {
|
||||
cond, arg, hasFTS := t.ftsFilter(line)
|
||||
if !hasFTS {
|
||||
return "", false, nil
|
||||
}
|
||||
rows, qerr := t.db.Query(`SELECT id FROM exchanges
|
||||
WHERE ts>=? AND ts<=? AND `+cond+`
|
||||
ORDER BY ts DESC LIMIT 8`, fromTS, toTS, arg)
|
||||
if qerr != nil {
|
||||
return "", false, qerr
|
||||
}
|
||||
ids := make([]string, 0, 8)
|
||||
for rows.Next() {
|
||||
var cand string
|
||||
if serr := rows.Scan(&cand); serr != nil {
|
||||
rows.Close()
|
||||
return "", false, serr
|
||||
}
|
||||
ids = append(ids, cand)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
for _, cand := range ids {
|
||||
_, resp, gerr := t.Get(cand)
|
||||
if gerr != nil || !strings.Contains(resp, line) {
|
||||
continue
|
||||
}
|
||||
return cand, true, nil
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// ftsFilter builds the SQL condition restricting rows to those whose indexed
|
||||
// text matches term. ok is false when full-text search cannot serve the term —
|
||||
// no FTS index on this build, or fewer than three characters, which the trigram
|
||||
|
||||
@@ -57,7 +57,7 @@ update_local(){
|
||||
if command -v npm >/dev/null 2>&1; then
|
||||
info "Rebuilding frontend static assets…"
|
||||
( cd web && npm ci --ignore-scripts && npm run build:static )
|
||||
rm -rf server/webui/dist && cp -r web/out server/webui/dist
|
||||
rm -rf server/webui/dist && mkdir -p server/webui && cp -r web/out server/webui/dist
|
||||
info "Recompiling the embedded single binary…"
|
||||
CGO_ENABLED=0 go build -tags embedui -trimpath -o artex ./cmd/artex
|
||||
else
|
||||
|
||||
Generated
+73
-7
@@ -3924,6 +3924,72 @@
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/core": {
|
||||
"version": "1.11.1",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"@emnapi/wasi-threads": "1.2.2",
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/runtime": {
|
||||
"version": "1.11.1",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/wasi-threads": {
|
||||
"version": "1.2.2",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": {
|
||||
"version": "1.1.4",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"@tybys/wasm-util": "^0.10.1"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@emnapi/core": "^1.7.1",
|
||||
"@emnapi/runtime": "^1.7.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@tybys/wasm-util": {
|
||||
"version": "0.10.2",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/tslib": {
|
||||
"version": "2.8.1",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "0BSD",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-win32-arm64-msvc": {
|
||||
"version": "4.3.2",
|
||||
"resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.3.2.tgz",
|
||||
@@ -4569,9 +4635,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/baseline-browser-mapping": {
|
||||
"version": "2.10.40",
|
||||
"resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.40.tgz",
|
||||
"integrity": "sha512-BSSLZ9/Cjjv7Gtj5B68ZzXcXUg8iOf3fme+FCuh8rC/Go+Kmh8cox7M3A8dolou16s64QjLPOSdngh7GxXvkSw==",
|
||||
"version": "2.11.39",
|
||||
"resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.39.tgz",
|
||||
"integrity": "sha512-Q9buOYhr2ggrEZpImICecZTJHCCd8yXkRUokFlUEQ8uiD6oP82F4W4jWnhOHZYV+wlvrC51V7Wy0ULyWjil5Wg==",
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"baseline-browser-mapping": "dist/cli.cjs"
|
||||
@@ -10160,12 +10226,12 @@
|
||||
}
|
||||
},
|
||||
"node_modules/source-map-js": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz",
|
||||
"integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==",
|
||||
"version": "1.2.4",
|
||||
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.4.tgz",
|
||||
"integrity": "sha512-J7ZjMvHd9O1fcT/5V09QWaaSl2BbWx3F6O08spde82Namf9UB0GxRVIPiNbVsdKkoizHiSpTodo3vp2bAy91MA==",
|
||||
"license": "BSD-3-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
"node": ">=6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/space-separated-tokens": {
|
||||
|
||||
@@ -139,6 +139,9 @@ export default function SetupPage() {
|
||||
>
|
||||
{loading ? t("Saving...") : t("Set password and sign in")}
|
||||
</Button>
|
||||
<p className="rounded-md border bg-muted/40 px-3 py-2 text-xs text-muted-foreground">
|
||||
{t("Traffic capture is off by default. Enable it in System settings after setup if you need provable HTTP findings — captured traffic is required for findings to prove goals.")}
|
||||
</p>
|
||||
</form>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -572,7 +572,9 @@ export default function DashboardPage() {
|
||||
<span>{t("Recording")}</span>
|
||||
</>
|
||||
) : (
|
||||
<span>{t("Capture is off")}</span>
|
||||
<Link href="/system/settings" className="text-amber-500 hover:underline">
|
||||
{t("Capture is off · required for provable findings")}
|
||||
</Link>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
@@ -1164,19 +1166,29 @@ export default function DashboardPage() {
|
||||
</Badge>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between rounded-lg border bg-muted/20 px-3 py-2">
|
||||
<div className="text-[10px] text-muted-foreground">{t("Traffic capture")}</div>
|
||||
<Badge
|
||||
variant="outline"
|
||||
className={cn(
|
||||
"text-[10px]",
|
||||
settings?.traffic_capture
|
||||
? "border-violet-500/30 bg-violet-500/10 text-violet-400"
|
||||
: "text-muted-foreground",
|
||||
)}
|
||||
>
|
||||
{settings?.traffic_capture ? t("On") : t("Off")}
|
||||
</Badge>
|
||||
<div className="rounded-lg border bg-muted/20 px-3 py-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="text-[10px] text-muted-foreground">{t("Traffic capture")}</div>
|
||||
<Badge
|
||||
variant="outline"
|
||||
className={cn(
|
||||
"text-[10px]",
|
||||
settings?.traffic_capture
|
||||
? "border-violet-500/30 bg-violet-500/10 text-violet-400"
|
||||
: "border-amber-500/30 bg-amber-500/10 text-amber-500",
|
||||
)}
|
||||
>
|
||||
{settings?.traffic_capture ? t("On") : t("Off")}
|
||||
</Badge>
|
||||
</div>
|
||||
{settings && !settings.traffic_capture && (
|
||||
<p className="mt-1.5 text-[10px] leading-relaxed text-amber-500">
|
||||
{t("Traffic capture is off: findings stay recorded, but captured HTTP traffic is required for provable findings.")}{" "}
|
||||
<Link href="/system/settings" className="underline">
|
||||
{t("Enable in settings")}
|
||||
</Link>
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{activeProfile && (
|
||||
|
||||
@@ -1555,7 +1555,7 @@ function ArchiveConfirmDialog({
|
||||
<AlertDialogTitle>{count === 1 ? t("Archive task") : t("Archive {count} tasks", { count })}</AlertDialogTitle>
|
||||
<AlertDialogDescription className="[overflow-wrap:anywhere]">
|
||||
{t(
|
||||
"Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into local cold storage. Once archived, a task can be restored from \"Archived\".",
|
||||
"Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into the configured archive storage. Once archived, a task can be restored from \"Archived\". Restoring requires free space on this server.",
|
||||
)}
|
||||
</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
@@ -1999,6 +1999,11 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {
|
||||
title={t("Before compression: {size}", { size: formatArchiveBytes(archive.original_size) })}
|
||||
>
|
||||
{archiveCompressionLabel(archive)}
|
||||
{(archive.storage === "s3" || archive.storage === "local") && (
|
||||
<div className="text-muted-foreground">
|
||||
{archive.storage === "s3" ? t("Object storage") : t("Local storage")}
|
||||
</div>
|
||||
)}
|
||||
</TableCell>
|
||||
<TableCell className="text-xs tabular-nums">{archiveDataTotal(archive).toLocaleString(intlLocale)}</TableCell>
|
||||
<TableCell>
|
||||
@@ -2017,7 +2022,11 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {
|
||||
</p>
|
||||
))}
|
||||
<span className="text-muted-foreground text-xs">
|
||||
{t(archive.phase)} · {archive.progress}%
|
||||
{archive.phase === "upload_and_verify"
|
||||
? t("Uploading and verifying")
|
||||
: archive.phase === "download_and_verify"
|
||||
? t("Downloading and verifying")
|
||||
: t(archive.phase)} · {archive.progress}%
|
||||
</span>
|
||||
</div>
|
||||
</TableCell>
|
||||
|
||||
@@ -320,16 +320,23 @@ export default function SystemSettingsPage() {
|
||||
)}
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="flex items-center justify-between gap-4">
|
||||
<Label htmlFor="traffic-capture" className="text-sm font-normal text-muted-foreground">
|
||||
{trafficCapture ? t("On · recording traffic and injecting proxy") : t("Off · no recording, no proxy injection")}
|
||||
</Label>
|
||||
<Switch
|
||||
id="traffic-capture"
|
||||
checked={trafficCapture}
|
||||
disabled={!loaded || saving}
|
||||
onCheckedChange={toggleTraffic}
|
||||
/>
|
||||
<CardContent className="space-y-2.5">
|
||||
<div className="flex items-center justify-between gap-4">
|
||||
<Label htmlFor="traffic-capture" className="text-sm font-normal text-muted-foreground">
|
||||
{trafficCapture ? t("On · recording traffic and injecting proxy") : t("Off · no recording, no proxy injection")}
|
||||
</Label>
|
||||
<Switch
|
||||
id="traffic-capture"
|
||||
checked={trafficCapture}
|
||||
disabled={!loaded || saving}
|
||||
onCheckedChange={toggleTraffic}
|
||||
/>
|
||||
</div>
|
||||
{loaded && !trafficCapture && (
|
||||
<p className="rounded-md border border-amber-500/30 bg-amber-500/10 px-3 py-2 text-xs text-amber-600 dark:text-amber-400">
|
||||
{t("Traffic capture is off: findings stay recorded, but captured HTTP traffic is required for provable findings.")}
|
||||
</p>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
|
||||
@@ -5,6 +5,10 @@
|
||||
"-1 = no rerun; the intent is marked blocked outright": "-1 = yeniden çalıştırma yok; niyet doğrudan engellendi olarak işaretlenir",
|
||||
"-1 = no retries at all; the failure bubbles up immediately": "-1 = hiç yeniden deneme yok; hata hemen yukarı iletilir",
|
||||
"-1 = transient failures never trip (deterministic failures still do)": "-1 = geçici hatalar asla devre kesmez (kesin hatalar yine keser)",
|
||||
"Downloading and verifying": "İndiriliyor ve doğrulanıyor",
|
||||
"Local storage": "Yerel depolama",
|
||||
"Object storage": "Nesne depolama",
|
||||
"Uploading and verifying": "Karşıya yükleniyor ve doğrulanıyor",
|
||||
"— (task deleted)": "— (görev silinmiş)",
|
||||
"— agent generates —": "— ajan oluşturur —",
|
||||
"… {count} errors in total": "… toplam {count} hata",
|
||||
@@ -307,7 +311,7 @@
|
||||
"Archived": "Arşivlenenler",
|
||||
"Archived at": "Arşivlenme zamanı",
|
||||
"Archiving": "Arşivleniyor",
|
||||
"Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into local cold storage. Once archived, a task can be restored from \"Archived\".": "Arşivleme, görev zamanlamasını durdurur ve grafiği, LLM geçmişini, dosyaları, göreve özel varlıkları ve trafiği yerel soğuk depolamaya sıkıştırır. Arşivlenen görev \"Arşivlenenler\" sekmesinden geri yüklenebilir.",
|
||||
"Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into the configured archive storage. Once archived, a task can be restored from \"Archived\". Restoring requires free space on this server.": "Arşivleme, görev zamanlamasını durdurur ve grafiği, LLM geçmişini, dosyaları, özel varlıkları ve trafiği yapılandırılmış arşiv depolama alanına sıkıştırır. Arşivlenen bir görev, \"Arşivlenenler\" bölümünden geri yüklenebilir. Geri yükleme, bu sunucuda boş alan gerektirir.",
|
||||
"Are you sure you want to delete asset interception rule \"{pattern}\"?": "\"{pattern}\" varlık yakalama kuralını silmek istediğinizden emin misiniz?",
|
||||
"Arguments": "Argümanlar",
|
||||
"Arguments (space-separated)": "Argümanlar (boşlukla ayrılmış)",
|
||||
@@ -2705,5 +2709,9 @@
|
||||
"Hashes": "Özetler",
|
||||
"Masked (HMAC)": "Maskeli (HMAC)",
|
||||
"Resulting request": "Ortaya çıkan istek",
|
||||
"Resulting response": "Ortaya çıkan yanıt"
|
||||
"Resulting response": "Ortaya çıkan yanıt",
|
||||
"Traffic capture is off by default. Enable it in System settings after setup if you need provable HTTP findings — captured traffic is required for findings to prove goals.": "Trafik yakalama varsayılan olarak kapalıdır. Kanıtlanabilir HTTP bulguları gerekiyorsa kurulumdan sonra Sistem ayarlarından etkinleştirin — bulguların hedefleri kanıtlaması için yakalanmış trafik gereklidir.",
|
||||
"Capture is off · required for provable findings": "Yakalama kapalı · kanıtlanabilir bulgular için gerekli",
|
||||
"Traffic capture is off: findings stay recorded, but captured HTTP traffic is required for provable findings.": "Trafik yakalama kapalı: bulgular kaydedilmeye devam eder ancak kanıtlanabilir bulgular için yakalanmış HTTP trafiği gereklidir.",
|
||||
"Enable in settings": "Ayarlardan etkinleştir"
|
||||
}
|
||||
|
||||
@@ -5,6 +5,10 @@
|
||||
"-1 = no rerun; the intent is marked blocked outright": "-1 = 不重跑,该意图直接判为 blocked",
|
||||
"-1 = no retries at all; the failure bubbles up immediately": "-1 = 一次都不重试,失败立刻上抛",
|
||||
"-1 = transient failures never trip (deterministic failures still do)": "-1 = 瞬时失败永不熔断(确定性失败仍然熔断)",
|
||||
"Downloading and verifying": "正在下载并校验",
|
||||
"Local storage": "本地存储",
|
||||
"Object storage": "对象存储",
|
||||
"Uploading and verifying": "正在上传并校验",
|
||||
"— (task deleted)": "—(任务已删除)",
|
||||
"— agent generates —": "— 由智能体生成 —",
|
||||
"… {count} errors in total": "…共 {count} 个错误",
|
||||
@@ -307,7 +311,7 @@
|
||||
"Archived": "已归档",
|
||||
"Archived at": "归档时间",
|
||||
"Archiving": "归档中",
|
||||
"Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into local cold storage. Once archived, a task can be restored from \"Archived\".": "归档会停止任务调度,将图谱、LLM 历史、文件以及独占资产和流量压缩到本地冷存储。归档完成后可从“已归档”中还原。",
|
||||
"Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into the configured archive storage. Once archived, a task can be restored from \"Archived\". Restoring requires free space on this server.": "归档会停止任务调度,将图谱、LLM 历史、文件以及独占资产和流量压缩到已配置的归档存储中。归档完成后可从“已归档”中还原。还原需要此服务器有足够的可用空间。",
|
||||
"Are you sure you want to delete asset interception rule \"{pattern}\"?": "确定删除资产拦截规则“{pattern}”?",
|
||||
"Arguments": "参数",
|
||||
"Arguments (space-separated)": "参数(空格分隔)",
|
||||
@@ -2705,5 +2709,9 @@
|
||||
"Hashes": "哈希",
|
||||
"Masked (HMAC)": "已脱敏 (HMAC)",
|
||||
"Resulting request": "结果请求",
|
||||
"Resulting response": "结果响应"
|
||||
"Resulting response": "结果响应",
|
||||
"Traffic capture is off by default. Enable it in System settings after setup if you need provable HTTP findings — captured traffic is required for findings to prove goals.": "流量捕获默认关闭。如需可证明的 HTTP 漏洞发现,请在初始化后前往系统设置开启——漏洞证明目标需要已捕获的 HTTP 流量。",
|
||||
"Capture is off · required for provable findings": "捕获已关闭 · 可证明漏洞所需",
|
||||
"Traffic capture is off: findings stay recorded, but captured HTTP traffic is required for provable findings.": "流量捕获已关闭:漏洞仍会被记录,但证明目标需要已捕获的 HTTP 流量。",
|
||||
"Enable in settings": "前往设置开启"
|
||||
}
|
||||
|
||||
@@ -101,6 +101,7 @@ export interface TaskArchiveTokenStats {
|
||||
}
|
||||
|
||||
export interface TaskArchive {
|
||||
storage?: "local" | "s3";
|
||||
id: number;
|
||||
task_id: number;
|
||||
state: TaskArchiveState;
|
||||
|
||||
Reference in New Issue
Block a user