mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-20 23:20:53 +08:00
Compare commits
11
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
61d567a771 | ||
|
|
74b9843761 | ||
|
|
ab40834567 | ||
|
|
79c5daf076 | ||
|
|
64b90a746a | ||
|
|
dbbc287cd8 | ||
|
|
72fe3c3da1 | ||
|
|
15ed120758 | ||
|
|
221ad67549 | ||
|
|
dbf5cc3186 | ||
|
|
32ebb38e0f |
+2
-1
@@ -23,8 +23,9 @@ All notable changes to this project are documented in this file. The format is b
|
||||
- **`ask_operator` target warning**: a request whose question names a host or URL outside the goal targets is tagged as containing target-derived text (#197).
|
||||
- **Opt-in update banner**: `ARTEX_UPDATE_CHECK` makes the backend check the latest GitHub release at most once per 24 hours and the UI show a banner; off by default (#202).
|
||||
- **Release provenance**: release archives carry GitHub build provenance attestations and a CycloneDX SBOM (`gh attestation verify <zip> --repo BaaRai/ARTEX`); release notes are the matching `CHANGELOG.md` section (#219, #201).
|
||||
- **HAR 1.2 traffic import and export**: `POST /api/traffic/import/har` imports raw HAR 1.2 JSON captures (up to 32 MiB / 2,000 entries / 32 MiB decoded bodies) with deterministic `har-<sha256>` IDs (and `_artexExchangeId` preservation for ARTEX exports so distinct same-second exchanges round-trip without collapsing), base64 decoding, `postData.params` and `cookies` array fallbacks, URL userinfo stripping, blob spilling above 256 KiB, and full-text indexing, without applying capture rewrite rules (`exchange_rewrites` stays empty); per-entry errors are reported alongside `imported`/`skipped`/`failed` counts. `GET /api/traffic/export/har` exports filtered traffic (reusing the `GET /api/traffic` filters) as a compliant HAR 1.2 document (with explicit unknown placeholders for unrecorded timings and header sizes) in stable `(ts, id)` order, encoding UTF-8 bodies as text and binary bodies as base64, capped at 1,000 entries / 20 MiB raw headers+bodies (31 MiB JSON backstop, guaranteed under the 32 MiB import limit) with explicit truncation reporting via `X-ARTEX-HAR-*` headers and `_truncated`/`_truncatedReason`/`_totalMatched` fields. The Traffic page adds **Import HAR** and **Export HAR** controls. See `docs/traffic-har.md`.
|
||||
- **Capture-proxy rewrite rules, storage and API**: `GET`, `POST`, `PUT` and `DELETE /api/traffic/rewrite-rules` and `POST /api/traffic/rewrite-rules/{id}/toggle` manage rules that match a request or a response (method, host, URL, headers, body, status) and describe a rewrite (method, path and query, headers, body, status). Rules are validated on save, limited to 100 of which 50 enabled, created disabled, and stored in the new table `traffic_rewrite_rules` (migration `0005_traffic_rewrite_rules.sql`) with `matchers` and `actions` encrypted at rest. The value of an action marked `sensitive` is never returned, and an update sent without it keeps the stored one. The server compiles the enabled rules at startup and after every change and hands the snapshot to the proxy. The proxy applies them from the next entry. See `docs/traffic-rewrite-rules.md`.
|
||||
- **Capture-proxy rewrite rules applied**: the capture proxy applies enabled rules to each HTTP request before it is sent to the target and to each response before it reaches the client. Each action that ran is recorded in the new table `exchange_rewrites` in the same transaction as the exchange, `traffic_get` shows a `REWRITTEN` line, and `traffic_search` never finds a value a rule wrote. SSE status and headers are rewritten at the start of the stream. **WebSocket handshakes are not rewritten**: the capture library runs its hook after the upgrade, so the status and headers cannot be changed, and frames pass unchanged. **Responses of 5 MiB or more** are forwarded unchanged and not recorded in full, because the library does not call the response hook for them. Snapshots of original requests are capped at 1024 entries and 64 MiB. Replays go through the proxy, so the rules apply to them. Archives, a per-replay rule toggle, the web UI and the proof flag come later.
|
||||
- **Capture-proxy rewrite rules applied**: the capture proxy applies enabled rules to each HTTP request before it is sent to the target and to each response before it reaches the client. Each action that ran is recorded in the new table `exchange_rewrites` in the same transaction as the exchange, `traffic_get` shows a `REWRITTEN` line, and `traffic_search` never finds a value a rule wrote. SSE status and headers are rewritten at the start of the stream. **WebSocket handshakes are not rewritten**: the capture library runs its hook after the upgrade, so the status and headers cannot be changed, and frames pass unchanged. **Responses of 5 MiB or more** are forwarded unchanged and not recorded in full, because the library does not call the response hook for them. Snapshots of original requests are capped at 1024 entries and 64 MiB. Replays (`POST /api/traffic/replay`) bypass capture rewrite rules by default via an internal `X-ARTEX-Replay-Nonce` header stripped unconditionally by the proxy, and apply the rules when opted in with `apply_rules: true` (the **Apply capture rewrite rules** checkbox in the Replay panel). `GET /api/traffic` exposes the proxy's `rewrite` counters (`large_responses`, `capacity_refusals`, `tombstone_drops`, `conversions`, `sweeps`), flags each exchange row with `rewritten`, and filters with `rewritten=all|rewritten|original` (also accepting `true`/`false`), surfaced with a filter control and row badge on the Traffic page. Traffic archives export version 2 with per-exchange `rewritten` and sorted distinct applied `rewrite_rule_ids` while remaining deterministic and backward-compatible with version 1 archives. The rule-management web UI comes later.
|
||||
- **Bulk finding status** (#298): select findings in the findings table and apply one disposition to all of them, up to 500 per call, in one transaction; each row still goes through the single-decision notification semantics and the change is mirrored onto its exploration node (#289).
|
||||
- **Stream message direction filter** (#379): `GET /api/traffic/stream` and the `traffic_messages` agent tool take an optional `direction` (`c2s` or `s2c`) that lists only one side of a WebSocket or SSE conversation. `total` and `offset` apply to the filtered list, each row keeps its `seq` in the whole stream, and the `meta` row (messages dropped past the recorder limits) stays in a filtered page. Any other value is a 400 (API) or a tool error.
|
||||
- **Traffic replay** (#266): replay a recorded request from the traffic detail with its method, URL, headers and stored body; replay is disabled when only the body preview is loaded.
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
[Unit]
|
||||
Description=ARTEX application
|
||||
After=network.target
|
||||
StartLimitIntervalSec=60
|
||||
StartLimitBurst=5
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=artex
|
||||
Group=artex
|
||||
WorkingDirectory=/opt/artex
|
||||
Environment=ARTEX_JWT_KEY_DIR=/opt/artex/keys
|
||||
ExecStart=/opt/artex/artex -addr 127.0.0.1:8787 -proxy 127.0.0.1:8788
|
||||
Restart=on-failure
|
||||
RestartSec=5s
|
||||
TimeoutStopSec=30s
|
||||
KillSignal=SIGTERM
|
||||
KillMode=control-group
|
||||
UMask=0002
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=artex
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,178 @@
|
||||
# Local ARTEX systemd deployment
|
||||
|
||||
This guide describes how to deploy ARTEX locally under `systemd` using the example unit at `docs/examples/artex.service`.
|
||||
|
||||
## Placeholders and defaults
|
||||
|
||||
The example unit uses these defaults; replace them if your host uses different paths or names:
|
||||
|
||||
- Install directory (`<install-dir>`): `/opt/artex` (executable at `/opt/artex/artex`).
|
||||
- System user and group (`<artex-user>`): `artex` (non-root).
|
||||
- PostgreSQL service unit (`<pg-unit>`): distribution-specific — for example `postgresql.service` on RHEL/Fedora/Arch or `postgresql@<pg-version>-<pg-cluster>.service` on Debian/Ubuntu (where `<pg-version>/<pg-cluster>` comes from `pg_lsclusters`, such as `18/main`).
|
||||
- Configuration: `<install-dir>/config.json`, ignored by Git, owner-only mode `0600`.
|
||||
- Web UI: `http://127.0.0.1:8787`; recording proxy: `127.0.0.1:8788`.
|
||||
- Example unit file: `docs/examples/artex.service`; installed at `/etc/systemd/system/artex.service`.
|
||||
|
||||
## System user and directory permissions
|
||||
|
||||
Create a dedicated non-login system user and group before installing the binary:
|
||||
|
||||
```bash
|
||||
sudo useradd --system --user-group --home-dir /opt/artex --shell /usr/sbin/nologin artex
|
||||
```
|
||||
|
||||
Create the install, runtime data, and key directories with explicit ownership and modes:
|
||||
|
||||
```bash
|
||||
sudo install -d -o root -g artex -m 0755 /opt/artex
|
||||
sudo install -d -o artex -g artex -m 0770 /opt/artex/data
|
||||
sudo install -d -o artex -g artex -m 0700 /opt/artex/keys
|
||||
```
|
||||
|
||||
Directory ownership and modes:
|
||||
|
||||
- `<install-dir>` (`/opt/artex`): `root:<artex-user>`, mode `0755` (executable `/opt/artex/artex` mode `0755`).
|
||||
- `<install-dir>/data` (`/opt/artex/data`): `<artex-user>:<artex-user>`, mode `0770` so the service user (and, in a root-run deployment using an executor group, that group) can write task run directories while blocking world access.
|
||||
- `<install-dir>/keys` (`/opt/artex/keys`): `<artex-user>:<artex-user>`, mode `0700`. Holds `jwt.key` and `secret.key` (mode `0600`) outside the browsable `data/` directory. `docs/examples/artex.service` sets `Environment=ARTEX_JWT_KEY_DIR=/opt/artex/keys` so the non-root service writes key files here rather than into `root`-owned `/opt/artex` (if `ARTEX_JWT_KEY_DIR` were left unset, ARTEX would fall back to `<install-dir>` and fail to create `jwt.key` and `secret.key` when `<install-dir>` is `root:<artex-user>` `0755`).
|
||||
- `<install-dir>/config.json` (`/opt/artex/config.json`): `<artex-user>:<artex-user>`, mode `0600`.
|
||||
|
||||
## Database and credentials
|
||||
|
||||
Install PostgreSQL on the host (or use an existing reachable PostgreSQL instance) and create a dedicated non-superuser login role and database named `artex` with no role-creation or database-creation privileges.
|
||||
|
||||
On Debian/Ubuntu, confirm the cluster name (`<pg-version>/<pg-cluster>`) with `pg_lsclusters`, start `<pg-unit>`, and create the role and database if absent:
|
||||
|
||||
```bash
|
||||
sudo systemctl start <pg-unit>
|
||||
sudo -u postgres createuser --pwprompt artex
|
||||
sudo -u postgres createdb --owner=artex artex
|
||||
```
|
||||
|
||||
Enter a unique password at the prompt. In `<install-dir>/config.json`, set `database.host` to `127.0.0.1`, `port` to `5432`, `user` and `dbname` to `artex`, `password` to that secret, and `sslmode` to `disable` (disabling TLS is appropriate only for loopback traffic). Preserve other settings and restrict access:
|
||||
|
||||
```bash
|
||||
sudo chown artex:artex /opt/artex/config.json
|
||||
sudo chmod 0600 /opt/artex/config.json
|
||||
```
|
||||
|
||||
`ARTEX_PG_DSN`, if set in a systemd drop-in override, takes precedence over `config.json`. Never commit configuration, credentials, encryption/signing keys, or startup logs.
|
||||
|
||||
## Build and install
|
||||
|
||||
The binary embeds the UI when built with `-tags embedui`. To rebuild the UI from a source checkout, run `npm ci` and `npm run build:static` in `web/`, create the parent directory with `mkdir -p server/webui`, and replace `server/webui/dist` with `web/out`. Build the binary with:
|
||||
|
||||
```bash
|
||||
CGO_ENABLED=0 go build -tags embedui -trimpath -o artex ./cmd/artex
|
||||
```
|
||||
|
||||
Install the binary into `<install-dir>`:
|
||||
|
||||
```bash
|
||||
sudo install -o root -g artex -m 0755 artex /opt/artex/artex
|
||||
```
|
||||
|
||||
For a running deployment, build into a separate staging path and stop the service before replacing `/opt/artex/artex` or generated assets. Avoid rerunning `install.sh` without reviewing its prompts: it regenerates `config.json` and starts a separate process.
|
||||
|
||||
Review `User`, `Group`, `WorkingDirectory`, `Environment=ARTEX_JWT_KEY_DIR=...`, and `ExecStart` in `docs/examples/artex.service` before installing it on your host:
|
||||
|
||||
```bash
|
||||
systemd-analyze verify docs/examples/artex.service
|
||||
sudo install -o root -g root -m 0644 docs/examples/artex.service /etc/systemd/system/artex.service
|
||||
sudo systemctl daemon-reload
|
||||
```
|
||||
|
||||
### Database dependency and environment overrides
|
||||
|
||||
`docs/examples/artex.service` intentionally declares only `After=network.target` and does not hardcode a distribution-specific PostgreSQL unit (such as Debian's `postgresql@<pg-version>-<pg-cluster>.service`), so it works across Linux distributions and with remote PostgreSQL servers.
|
||||
|
||||
When PostgreSQL runs locally on the same host, add a systemd drop-in (`sudo systemctl edit artex.service`) so ARTEX starts after and depends on your local PostgreSQL unit:
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Requires=<pg-unit>
|
||||
After=<pg-unit>
|
||||
```
|
||||
|
||||
For example, set `<pg-unit>` to `postgresql.service` on RHEL/Fedora/Arch, or to `postgresql@<pg-version>-<pg-cluster>.service` on Debian/Ubuntu (and verify `/etc/postgresql/<pg-version>/<pg-cluster>/start.conf` is `auto`, not `disabled`). `ARTEX_JWT_KEY_DIR=/opt/artex/keys` is already set in `docs/examples/artex.service`; override it in the drop-in only if you install ARTEX under a different `<install-dir>`. Run `sudo systemctl daemon-reload` after editing the drop-in.
|
||||
|
||||
### Enable and start
|
||||
|
||||
If ARTEX is running outside `systemd`, identify its PID/executable and send that verified process `SIGTERM`. Wait for shutdown and release of ports `8787` and `8788` before enabling the service:
|
||||
|
||||
```bash
|
||||
ss -ltnp '( sport = :8787 or sport = :8788 )'
|
||||
sudo systemctl enable --now artex.service
|
||||
systemctl status artex.service --no-pager
|
||||
curl --fail --silent --show-error http://127.0.0.1:8787/api/health
|
||||
```
|
||||
|
||||
The unit runs `/opt/artex/artex` directly without an extra supervisor script. It starts after `network.target`, restarts on failure after five seconds, allows 30 seconds for graceful shutdown, logs to the journal, and uses `UMask=0002` (rather than `0077`, which would block group writes in a root-run deployment where executor privilege drop sets umask `002` for the executor group; sensitive files such as `config.json`, `keys/jwt.key`, and `keys/secret.key` are protected explicitly with `0600`/`0700` permissions). Five starts within 60 seconds trigger the start limit.
|
||||
|
||||
### Agent isolation
|
||||
|
||||
Because `docs/examples/artex.service` runs the server as the non-root `<artex-user>` (`User=artex`), the `-exec-uid`/`-exec-gid` privilege drop cannot be applied: `ResolveExecPrivilege` disables the drop when the server is not root (and errors if those flags are passed explicitly). The example unit deliberately omits `-exec-uid`/`-exec-gid`, so agents and spawned tools run under the `<artex-user>` service identity rather than a separate executor account. The root-owned run-directory layout and executor-group containment exist only when the server runs as root with explicit `-exec-uid`/`-exec-gid`; if you require unprivileged-executor containment, use the Docker deployment, which is the supported path for that isolation model.
|
||||
|
||||
## Web setup
|
||||
|
||||
On a new database, visit `http://127.0.0.1:8787/setup`, provide the one-time setup token from the private startup log or journal (`journalctl -u artex.service`), and choose a unique password of at least 12 characters. The username is `ARTEX`. Store the password in your password manager rather than in a plaintext file on disk.
|
||||
|
||||
## Reboot persistence verification (example)
|
||||
|
||||
Pre-reboot checks confirm persistent ARTEX enablement, active PostgreSQL and ARTEX services, HTTP 200 health, and initialized authentication, but do not by themselves prove survival across a reboot.
|
||||
|
||||
To verify reboot persistence on a host, record the current boot ID and timestamp in an ignored runtime file such as `<install-dir>/data/reboot-verification-baseline.json` before rebooting, and do not overwrite that baseline until the test is complete:
|
||||
|
||||
```bash
|
||||
sudo -u artex python3 -c 'import json, time; from pathlib import Path; Path("/opt/artex/data/reboot-verification-baseline.json").write_text(json.dumps({"boot_id": Path("/proc/sys/kernel/random/boot_id").read_text().strip(), "ts": time.time()}))'
|
||||
```
|
||||
|
||||
Save other work before rebooting; this disconnects local and remote sessions:
|
||||
|
||||
```bash
|
||||
sudo systemctl reboot
|
||||
```
|
||||
|
||||
After reconnecting, do not manually start ARTEX before checking automatic startup. Compare the current boot ID with the saved baseline and check the services:
|
||||
|
||||
```bash
|
||||
cd /opt/artex
|
||||
python3 -c 'import json; from pathlib import Path; old=json.loads(Path("data/reboot-verification-baseline.json").read_text())["boot_id"]; new=Path("/proc/sys/kernel/random/boot_id").read_text().strip(); print("PASS: a reboot occurred" if old != new else "FAIL: no reboot occurred"); raise SystemExit(0 if old != new else 1)'
|
||||
systemctl is-enabled artex.service
|
||||
systemctl is-active <pg-unit> artex.service
|
||||
curl --fail --silent --show-error http://127.0.0.1:8787/api/health
|
||||
python3 -c 'import json, urllib.request; r=json.load(urllib.request.urlopen("http://127.0.0.1:8787/api/auth/status", timeout=5)); print("Authentication initialized:", r.get("initialized")); raise SystemExit(0 if r.get("initialized") else 1)'
|
||||
systemctl show artex.service --no-pager -p ActiveState -p SubState -p MainPID -p ActiveEnterTimestamp
|
||||
```
|
||||
|
||||
Success requires a changed boot ID, enabled ARTEX, both services active without manual startup, HTTP 200, and authentication still initialized. A new PID alone is not proof of reboot. If the system has just booted, allow startup time and repeat health checks. Refresh the baseline before running a future independent reboot test.
|
||||
|
||||
### Example reboot verification record
|
||||
|
||||
Example of a completed post-reboot verification (recorded from the original Debian/PostgreSQL 18 reference deployment on 2026-11-17, without manually starting or restarting either service):
|
||||
|
||||
- Current boot ID differed from the saved pre-reboot baseline, confirming an actual reboot.
|
||||
- `artex.service` remained enabled and was `active (running)` under its non-root service user with `NRestarts=0`.
|
||||
- The local PostgreSQL cluster service was `active`.
|
||||
- `/api/health` returned HTTP 200.
|
||||
- `/api/auth/status` confirmed initialized authentication was preserved.
|
||||
|
||||
Observed timestamps, PIDs, and restart counts are host-specific examples, not fixed configuration values.
|
||||
|
||||
## Operations and troubleshooting
|
||||
|
||||
```bash
|
||||
sudo systemctl restart artex.service
|
||||
sudo systemctl stop artex.service
|
||||
sudo systemctl start artex.service
|
||||
journalctl -b -u artex.service --no-pager
|
||||
journalctl -u artex.service -f
|
||||
```
|
||||
|
||||
Treat journal output as potentially sensitive; never share unredacted setup tokens or credentials. If startup fails, check PostgreSQL status, database settings and permissions on `<install-dir>/config.json`, `<install-dir>/data`, and `<install-dir>/keys`, occupied ports (`8787`/`8788`), and the journal. After a PostgreSQL dependency failure, restart ARTEX explicitly once the database is healthy. To recover from the start limit:
|
||||
|
||||
```bash
|
||||
sudo systemctl reset-failed artex.service
|
||||
sudo systemctl start artex.service
|
||||
```
|
||||
|
||||
Back up PostgreSQL, `<install-dir>/config.json`, `<install-dir>/data`, and `<install-dir>/keys` (`jwt.key` and `secret.key`) securely before upgrades. The application stays loopback-only; remote access requires a separately reviewed tunnel or reverse proxy.
|
||||
@@ -0,0 +1,130 @@
|
||||
# Traffic HAR 1.2 Import and Export
|
||||
|
||||
ARTEX supports importing and exporting captured HTTP traffic using the **HAR 1.2** JSON format so operators can bring captures from browser DevTools or external proxies into the Traffic store and export filtered subsets out of ARTEX.
|
||||
|
||||
Both routes require the standard administrator JWT (`Authorization: Bearer <token>`) like the rest of `/api/traffic`, and return `503` when the traffic store is unavailable.
|
||||
|
||||
---
|
||||
|
||||
## Import: `POST /api/traffic/import/har`
|
||||
|
||||
### Request format
|
||||
|
||||
Send a raw HAR 1.2 JSON document directly as the HTTP request body (`Content-Type: application/json`), capped at the standard API JSON body limit (**32 MiB**; oversized bodies return `413 Request Entity Too Large`).
|
||||
|
||||
```bash
|
||||
curl -X POST \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data-binary @capture.har \
|
||||
http://127.0.0.1:8787/api/traffic/import/har
|
||||
```
|
||||
|
||||
The Traffic page also provides an **Import HAR** file picker (with the same 32 MiB limit) that reads a `.har` / `.json` file, posts it to this endpoint, displays loading/error state and a count summary, and refreshes the table.
|
||||
|
||||
### Validation & error handling
|
||||
|
||||
The request is rejected with `400 Bad Request` naming the reason when:
|
||||
|
||||
- the body is empty or malformed JSON (including trailing content after the root object),
|
||||
- the `log` object is missing,
|
||||
- `log.version` is not `"1.2"`,
|
||||
- `log.entries` is missing or `null`,
|
||||
- `log.entries` exceeds the per-import entry cap (`2000` entries).
|
||||
|
||||
Individual malformed entries (invalid `startedDateTime`, non-HTTP(S) URL, invalid HTTP method, header/cookie names or values containing CR/LF/NUL, out-of-range status, invalid base64 or unsupported body encoding) do **not** fail the whole import: they are counted in `failed` and reported in `errors` (up to 50 error details) while valid entries continue importing. HTTP/2 pseudo-headers starting with `:` (such as `:authority` or `:method` from browser DevTools) are skipped cleanly.
|
||||
|
||||
### Response
|
||||
|
||||
```json
|
||||
{
|
||||
"imported": 12,
|
||||
"skipped": 3,
|
||||
"failed": 1,
|
||||
"truncated": false,
|
||||
"errors": [
|
||||
{ "index": 4, "url": "notaurl", "reason": "invalid request url \"notaurl\"" }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Deterministic IDs, round-trip fidelity, & idempotence
|
||||
|
||||
- **ARTEX exports (`_artexExchangeId`)**: Every entry exported by ARTEX carries the HAR 1.2 custom field `_artexExchangeId` set to the exchange's stored ID. When a valid `_artexExchangeId` (`[A-Za-z0-9._:-]`, 1–128 chars) is present on import, ARTEX preserves it. This guarantees that two distinct ARTEX exchanges recorded within the same second with identical method, URL, status, and bodies do **not** collapse into one row when imported into a fresh store, and that re-importing an export into the source store stays idempotent (`skipped`).
|
||||
- **External HARs (no `_artexExchangeId`)**: Each entry receives a deterministic content-based ID:
|
||||
|
||||
```text
|
||||
har-<hex(sha256(ts_unix_seconds + NUL + METHOD + NUL + canonical_url + NUL + status + NUL + sha256(req_body) + NUL + sha256(resp_body)))>
|
||||
```
|
||||
|
||||
`ts_unix_seconds` is `startedDateTime` parsed as RFC 3339 and truncated to UTC Unix seconds (matching `exchanges.ts`). Re-importing the same external HAR skips existing IDs (`INSERT OR IGNORE`, hot rows always win). Identical entries in the same external HAR sharing `(ts_unix_seconds, method, canonical_url, status, req_body, resp_body)` are deduplicated (the first is imported, duplicates are counted in `skipped`).
|
||||
|
||||
### Storage, bodies, cookies, `postData.params`, and userinfo
|
||||
|
||||
- **URL userinfo**: Any `user:pass@` userinfo in `request.url` is stripped before storing, indexing, listing, exporting, or reporting errors (matching ARTEX's asset store).
|
||||
- `host` is derived from `URL.Hostname()` and `url_template` from `db.TemplatePath(URL.EscapedPath())`; `path` stays empty.
|
||||
- **Bodies & base64**: `response.content.text` and `request.postData.text` are stored as raw UTF-8 bytes when `encoding` is empty, and decoded via standard base64 when `encoding` is `"base64"`. When `request.postData.text` is empty and `request.postData.params` is populated, ARTEX reconstructs the form body as `application/x-www-form-urlencoded` (preserving parameter order).
|
||||
- **Cookies arrays**: When `request.headers` does not already include a `Cookie` header and `request.cookies` is non-empty, ARTEX synthesizes a `Cookie` header from `request.cookies`. Likewise, when `response.headers` does not already include a `Set-Cookie` header and `response.cookies` is non-empty, ARTEX synthesizes `Set-Cookie` headers from `response.cookies`. When the headers are already present, they are kept as-is without duplication.
|
||||
- `req_len` and `resp_len` record the actual decoded byte lengths.
|
||||
- `content_type` comes from `response.content.mimeType` (falling back to the response `Content-Type` header).
|
||||
- Bodies larger than the 256 KiB inline threshold spill to the content-addressed blob store (`_blobs/sha256/...`) with `blob_refs` rows, and searchable text bodies are added to the `ex_fts` full-text index — identical to proxy capture and `ImportArchive`.
|
||||
|
||||
### Capture rewrite rules are NOT applied
|
||||
|
||||
HAR import writes directly to the traffic store without passing through the MITM capture proxy. **Capture rewrite rules are never applied to imported exchanges**, no `exchange_rewrites` rows are created, and imported rows always have `rewritten: false`.
|
||||
|
||||
### Import limits
|
||||
|
||||
| Limit | Default | Behavior when exceeded |
|
||||
|---|---|---|
|
||||
| Raw HTTP request body | 32 MiB | `413 Request Entity Too Large` |
|
||||
| Entries per HAR | 2,000 | `400 Bad Request` |
|
||||
| Total decoded request + response body bytes | 32 MiB | Stops importing remaining entries, sets `"truncated": true`, counts remaining entries in `skipped`, and appends an explanatory item in `errors`. |
|
||||
|
||||
---
|
||||
|
||||
## Export: `GET /api/traffic/export/har`
|
||||
|
||||
### Request & filters
|
||||
|
||||
Returns an `application/json` HAR 1.2 document (`Content-Disposition: attachment; filename="artex-traffic.har"`) ordered deterministically by `(ts ASC, id ASC)`.
|
||||
|
||||
It reuses the same filter parser and SQL predicate builder as `GET /api/traffic`, supporting:
|
||||
|
||||
- `host`, `method`, `path`, `status`, `q`, `body`, `rewritten` (`all` | `rewritten` | `original` | `true` | `false`), `resp_min`, `resp_max`.
|
||||
|
||||
### HAR 1.2 compliance & unknown placeholders
|
||||
|
||||
- `log.version` is `"1.2"` and `log.creator` is `{"name": "ARTEX", "version": "<build-version>"}`.
|
||||
- `startedDateTime` is formatted from `exchanges.ts` as UTC RFC 3339, and `_artexExchangeId` carries the exchange's stored ID.
|
||||
- Request and response headers are parsed from the stored raw heads (`req_head` / `resp_head`). ARTEX does not invent `serverIPAddress` or headers that are not present in the store.
|
||||
- Required HAR 1.2 fields whose values are not recorded by ARTEX are emitted with explicit **unknown placeholders** rather than fabricated measurements:
|
||||
- `entry.time`: `0`
|
||||
- `entry.cache`: `{}`
|
||||
- `entry.timings`: `{"send": -1, "wait": -1, "receive": -1}` (unknown/unavailable per HAR 1.2)
|
||||
- `request.cookies`, `request.queryString`, `response.cookies`: `[]`
|
||||
- `request.headersSize`, `response.headersSize`: `-1` (unknown per HAR 1.2)
|
||||
- `response.redirectURL`: `""`
|
||||
- Bodies are read in full from inline columns or spilled blobs: valid NUL-free UTF-8 bodies are emitted as plain `text`, and binary / non-UTF-8 bodies are emitted with `encoding: "base64"`.
|
||||
|
||||
### Export limits, import-compatibility invariant, & truncation reporting
|
||||
|
||||
| Limit | Default | `truncatedReason` |
|
||||
|---|---|---|
|
||||
| Exported entries | 1,000 | `max_entries` |
|
||||
| Total raw request + response headers and bodies | 20 MiB | `max_bytes` |
|
||||
| Marshaled JSON size backstop | 31 MiB | `max_bytes` |
|
||||
|
||||
**Cap invariant**: Every HAR produced by `GET /api/traffic/export/har` fits inside `POST /api/traffic/import/har`'s **32 MiB** HTTP body limit (and the Traffic page's 32 MiB file limit). Raw headers and bodies are capped at **20 MiB** so worst-case base64 expansion (`4/3x ≈ 26.7 MiB`) plus JSON structure stays comfortably under 32 MiB, backed by an explicit **31 MiB** marshaled-JSON cap so even heavily JSON-escaped payloads cannot trigger a `413` on re-import.
|
||||
|
||||
Truncation is never silent. Every export reports it in both HTTP response headers and HAR 1.2 custom (`_`-prefixed) `log` fields:
|
||||
|
||||
- Headers:
|
||||
- `X-ARTEX-HAR-Truncated: true|false`
|
||||
- `X-ARTEX-HAR-Truncated-Reason: max_entries|max_bytes` (when truncated)
|
||||
- `X-ARTEX-HAR-Exported: <count>`
|
||||
- `X-ARTEX-HAR-Matched: <total matching rows>`
|
||||
- JSON (`log` object, omitted when not truncated):
|
||||
- `_truncated: true`
|
||||
- `_truncatedReason: "max_entries" | "max_bytes"`
|
||||
- `_totalMatched: <total matching rows>`
|
||||
@@ -4,9 +4,11 @@ A rewrite rule matches a request or a response that goes through the capture pro
|
||||
how to change it: the method, the path and query, headers, the body, the status code.
|
||||
|
||||
**Status: the capture proxy applies enabled rules to HTTP traffic (request and response phases),
|
||||
and records what it did.** Not yet in this change: archive format, a per-replay rule toggle, the web UI,
|
||||
and the proof flag. See [Application in the capture proxy](#application-in-the-capture-proxy) for
|
||||
what is covered and what is not.
|
||||
records what it did, tags bound evidence snapshots, supports per-replay rule opt-in, carries
|
||||
rewrite provenance in traffic archive v2, exposes rewrite counters and a rewritten filter/flag on
|
||||
`GET /api/traffic`, and shows the filter and badge on the Traffic page.** Not yet in this change:
|
||||
the rule-management web UI. See [Application in the capture proxy](#application-in-the-capture-proxy)
|
||||
for what is covered and what is not.
|
||||
|
||||
Rules are a convenience for the operator, not a security control: a rule that fails leaves the
|
||||
message unchanged (fail open).
|
||||
@@ -207,5 +209,52 @@ the slot may be converted into a tombstone if a new reservation needs the room.
|
||||
|
||||
### Replay
|
||||
|
||||
Replay goes through the recording proxy like any other traffic, so the enabled rules apply to a
|
||||
replay and the resulting exchange is traced like a captured one. There is no per-replay toggle yet.
|
||||
`POST /api/traffic/replay` sends the edited request through the recording proxy so the exchange is
|
||||
captured in Traffic, but **bypasses capture rewrite rules by default** (`apply_rules: false`). To
|
||||
run enabled capture rules against a replay and trace them like captured traffic, opt in with
|
||||
`"apply_rules": true` (or check **Apply capture rewrite rules** in the Replay panel).
|
||||
|
||||
When `apply_rules` is false, `traffic.Replay` registers a single-use random 32-byte hex nonce in an
|
||||
in-memory registry, attaches the internal header `X-ARTEX-Replay-Nonce: <nonce>`, and removes the
|
||||
nonce when the call returns. The capture proxy strips `X-ARTEX-Replay-Nonce` unconditionally in the
|
||||
request hook so the target never sees it; only a value matching a registered nonce bypasses both
|
||||
request and response rule phases, while any forged header is stripped and ignored.
|
||||
|
||||
### Stats
|
||||
|
||||
`GET /api/traffic` includes a `rewrite` object exposing the proxy's `RewriteStats` counters since
|
||||
start: `large_responses`, `capacity_refusals`, `tombstone_drops`, `conversions`, and `sweeps`.
|
||||
|
||||
### Traffic list
|
||||
|
||||
Each exchange returned by `GET /api/traffic` carries `"rewritten": true|false`, computed with an
|
||||
`EXISTS` check against `exchange_rewrites` for rows with `outcome = 'applied'` (skipped or failed
|
||||
rules do not mark an exchange as rewritten).
|
||||
|
||||
`GET /api/traffic` accepts an optional `rewritten` query parameter:
|
||||
|
||||
- `all` (default when omitted or unrecognized): returns all exchanges.
|
||||
- `rewritten` (alias `true`): returns only exchanges with at least one applied rewrite row.
|
||||
- `original` (alias `false`): returns only exchanges with no applied rewrite row.
|
||||
|
||||
The filter combines with `host`, `method`, `path`, `status`, `q`, `body`, `resp_min`, and
|
||||
`resp_max` using `AND`, and `total` and pagination reflect the filtered set. The Traffic page shows
|
||||
the filter next to the existing filters and renders a `Rewritten` badge on rewritten rows.
|
||||
|
||||
### Task archives (v2)
|
||||
|
||||
Task archives export captured traffic into `traffic/traffic.json` using **format version 2**:
|
||||
|
||||
- Each `ArchiveExchange` carries optional `rewritten` (`bool`) and `rewrite_rule_ids` (`[]int64`),
|
||||
matching `RewriteInfo` (applied rows only, sorted ascending, distinct). Exchanges with no applied
|
||||
rewrite omit both fields.
|
||||
- Output stays deterministic: exchanges are ordered by `(ts, id)`, `rewrite_rule_ids` are sorted
|
||||
and distinct, and `blobs` are sorted so re-exporting the same data produces a byte-identical
|
||||
`traffic.json`.
|
||||
- `ImportArchive` accepts versions 1 and 2 and refuses any other version. Version 1 entries import
|
||||
as unrewritten with no rule IDs. Version 2 entries marked `rewritten: true` reconstruct minimal
|
||||
applied rows in `exchange_rewrites` (`phase = 'archived'`, `outcome = 'applied'`,
|
||||
`reason = 'archived'`, `rule_snapshot = NULL`) for each distinct rule ID (or a single `NULL`
|
||||
`rule_id` row when `rewrite_rule_ids` is empty), preserving `RewriteInfo`, `traffic_get`'s
|
||||
`=== REWRITTEN ===` header, evidence-binding provenance, and the traffic list filter without
|
||||
fabricating a rule snapshot or request/response phase. Existing hot rows always win on import.
|
||||
|
||||
@@ -168,6 +168,7 @@ type TrafficExchangeDTO struct {
|
||||
Status int `json:"status"`
|
||||
ContentType string `json:"content_type"`
|
||||
RespLen int `json:"resp_len"`
|
||||
Rewritten bool `json:"rewritten"`
|
||||
}
|
||||
|
||||
func trafficDTOs(ex []traffic.ExchangeMeta) []TrafficExchangeDTO {
|
||||
@@ -182,6 +183,7 @@ func trafficDTOs(ex []traffic.ExchangeMeta) []TrafficExchangeDTO {
|
||||
Status: e.Status,
|
||||
ContentType: e.ContentType,
|
||||
RespLen: e.RespLen,
|
||||
Rewritten: e.Rewritten,
|
||||
})
|
||||
}
|
||||
return out
|
||||
|
||||
+13
-14
@@ -898,6 +898,8 @@ func (s *Server) Handler() http.Handler {
|
||||
mux.HandleFunc("GET /api/traffic/blob", s.getTrafficBlob)
|
||||
mux.HandleFunc("POST /api/traffic/ingest", s.ingestTraffic)
|
||||
mux.HandleFunc("POST /api/traffic/replay", s.replayTraffic)
|
||||
mux.HandleFunc("POST /api/traffic/import/har", s.importTrafficHAR)
|
||||
mux.HandleFunc("GET /api/traffic/export/har", s.exportTrafficHAR)
|
||||
mux.HandleFunc("GET /api/commands", s.pgListCommands)
|
||||
mux.HandleFunc("GET /api/commands/stats", s.pgToolStats) // aggregate call counts per tool
|
||||
mux.HandleFunc("GET /api/llm/records", s.pgListLLMRecords)
|
||||
@@ -3535,24 +3537,13 @@ func taskActivityDetail(t *Task, seq int64) (string, error) {
|
||||
func (s *Server) getTraffic(w http.ResponseWriter, r *http.Request) {
|
||||
tr := s.m.Traffic()
|
||||
if tr == nil {
|
||||
writeJSON(w, 200, map[string]any{"enabled": false, "exchanges": []any{}})
|
||||
writeJSON(w, 200, map[string]any{"enabled": false, "exchanges": []any{}, "rewrite": traffic.RewriteStats{}})
|
||||
return
|
||||
}
|
||||
q := r.URL.Query()
|
||||
page := atoiDefault(q.Get("page"), 0)
|
||||
size := atoiDefault(q.Get("size"), 100)
|
||||
ex, matched, _ := tr.Page(traffic.PageQuery{
|
||||
Host: q.Get("host"),
|
||||
Method: q.Get("method"),
|
||||
Query: q.Get("q"),
|
||||
Body: q.Get("body"),
|
||||
Path: q.Get("path"),
|
||||
Status: q.Get("status"),
|
||||
RespMin: int64(atoiDefault(q.Get("resp_min"), -1)),
|
||||
RespMax: int64(atoiDefault(q.Get("resp_max"), -1)),
|
||||
Sort: q.Get("sort"),
|
||||
Order: q.Get("order"),
|
||||
}, page, size)
|
||||
ex, matched, _ := tr.Page(parseTrafficPageQuery(q), page, size)
|
||||
count, _ := tr.Count() // global total, for the stat card
|
||||
writeJSON(w, 200, map[string]any{
|
||||
"enabled": s.m.TrafficEnabled(), // reflect the capture toggle
|
||||
@@ -3562,6 +3553,7 @@ func (s *Server) getTraffic(w http.ResponseWriter, r *http.Request) {
|
||||
"page": page,
|
||||
"size": size,
|
||||
"exchanges": trafficDTOs(ex),
|
||||
"rewrite": tr.RewriteStats(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3740,6 +3732,7 @@ func (s *Server) replayTraffic(w http.ResponseWriter, r *http.Request) {
|
||||
Headers []traffic.ReplayHeader `json:"headers"`
|
||||
Body string `json:"body"`
|
||||
BodyBase64 bool `json:"body_base64"`
|
||||
ApplyRules bool `json:"apply_rules"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
var tooLarge *http.MaxBytesError
|
||||
@@ -3761,7 +3754,13 @@ func (s *Server) replayTraffic(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
|
||||
defer cancel()
|
||||
res, err := tr.Replay(ctx, traffic.ReplayRequest{Method: body.Method, URL: body.URL, Headers: body.Headers, Body: payload})
|
||||
res, err := tr.Replay(ctx, traffic.ReplayRequest{
|
||||
Method: body.Method,
|
||||
URL: body.URL,
|
||||
Headers: body.Headers,
|
||||
Body: payload,
|
||||
ApplyRules: body.ApplyRules,
|
||||
})
|
||||
switch {
|
||||
case errors.Is(err, traffic.ErrReplayTooLarge):
|
||||
writeErr(w, 413, err.Error())
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
|
||||
"github.com/Autumn-27/artex/traffic"
|
||||
)
|
||||
|
||||
func parseTrafficPageQuery(q url.Values) traffic.PageQuery {
|
||||
return traffic.PageQuery{
|
||||
Host: q.Get("host"),
|
||||
Method: q.Get("method"),
|
||||
Query: q.Get("q"),
|
||||
Body: q.Get("body"),
|
||||
Path: q.Get("path"),
|
||||
Status: q.Get("status"),
|
||||
Rewritten: q.Get("rewritten"),
|
||||
RespMin: int64(atoiDefault(q.Get("resp_min"), -1)),
|
||||
RespMax: int64(atoiDefault(q.Get("resp_max"), -1)),
|
||||
Sort: q.Get("sort"),
|
||||
Order: q.Get("order"),
|
||||
}
|
||||
}
|
||||
|
||||
// importTrafficHAR accepts a raw HAR 1.2 JSON document (capped at decodeMaxBytes,
|
||||
// 32 MiB) and imports valid entries into the traffic store with deterministic
|
||||
// IDs. Imported traffic never passes through capture rewrite rules.
|
||||
func (s *Server) importTrafficHAR(w http.ResponseWriter, r *http.Request) {
|
||||
tr := s.m.Traffic()
|
||||
if tr == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "traffic store is not available")
|
||||
return
|
||||
}
|
||||
body := r.Body
|
||||
if _, already := body.(*limitedBody); !already {
|
||||
body = http.MaxBytesReader(w, body, decodeMaxBytes)
|
||||
}
|
||||
raw, err := io.ReadAll(body)
|
||||
if err != nil {
|
||||
var tooLarge *http.MaxBytesError
|
||||
if errors.As(err, &tooLarge) {
|
||||
writeErr(w, http.StatusRequestEntityTooLarge, "HAR payload exceeds 32 MiB limit")
|
||||
return
|
||||
}
|
||||
writeErr(w, http.StatusBadRequest, "failed to read HAR body: "+err.Error())
|
||||
return
|
||||
}
|
||||
res, err := tr.ImportHAR(raw, traffic.HARImportOptions{})
|
||||
if err != nil {
|
||||
if errors.Is(err, traffic.ErrInvalidHAR) {
|
||||
writeErr(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeErr(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, res)
|
||||
}
|
||||
|
||||
// exportTrafficHAR exports recorded traffic matching the query filters as an
|
||||
// application/json HAR 1.2 document in stable chronological order. Entry and
|
||||
// total-byte caps are enforced and reported via both X-ARTEX-HAR-* headers and
|
||||
// HAR 1.2 custom fields (_truncated, _truncatedReason, _totalMatched).
|
||||
func (s *Server) exportTrafficHAR(w http.ResponseWriter, r *http.Request) {
|
||||
tr := s.m.Traffic()
|
||||
if tr == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "traffic store is not available")
|
||||
return
|
||||
}
|
||||
pq := parseTrafficPageQuery(r.URL.Query())
|
||||
doc, meta, err := tr.ExportHAR(pq, traffic.HARExportOptions{CreatorVersion: BuildVersion})
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
payload, err := json.Marshal(doc)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="artex-traffic.har"`)
|
||||
w.Header().Set("X-ARTEX-HAR-Truncated", strconv.FormatBool(meta.Truncated))
|
||||
w.Header().Set("X-ARTEX-HAR-Exported", strconv.Itoa(meta.Exported))
|
||||
w.Header().Set("X-ARTEX-HAR-Matched", strconv.Itoa(meta.TotalMatched))
|
||||
if meta.Truncated {
|
||||
w.Header().Set("X-ARTEX-HAR-Truncated-Reason", meta.TruncatedReason)
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write(payload)
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/traffic"
|
||||
)
|
||||
|
||||
func TestTrafficHAREndpointsRoundTripFilterAndErrors(t *testing.T) {
|
||||
s, request := ingestServer(t)
|
||||
|
||||
binPayload := []byte{0x00, 0x01, 0x89, 0x50, 0xfe, 0xff}
|
||||
harIn := traffic.HAR{
|
||||
Log: &traffic.HARLog{
|
||||
Version: "1.2",
|
||||
Creator: traffic.HARCreator{Name: "TestSuite", Version: "1.0"},
|
||||
Entries: []traffic.HAREntry{
|
||||
{
|
||||
StartedDateTime: "2026-06-01T08:00:00Z",
|
||||
Request: traffic.HARRequest{
|
||||
Method: "POST",
|
||||
URL: "https://har-a.example/api/login",
|
||||
Headers: []traffic.HARHeader{{Name: "Content-Type", Value: "application/json"}},
|
||||
PostData: &traffic.HARPostData{
|
||||
MimeType: "application/json",
|
||||
Text: `{"user":"alice"}`,
|
||||
},
|
||||
},
|
||||
Response: traffic.HARResponse{
|
||||
Status: 200,
|
||||
Headers: []traffic.HARHeader{{Name: "Content-Type", Value: "application/json"}},
|
||||
Content: traffic.HARContent{
|
||||
MimeType: "application/json",
|
||||
Text: `{"ok":true}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
StartedDateTime: "2026-06-01T08:00:10Z",
|
||||
Request: traffic.HARRequest{
|
||||
Method: "GET",
|
||||
URL: "https://har-b.example/blob.bin",
|
||||
Headers: []traffic.HARHeader{},
|
||||
},
|
||||
Response: traffic.HARResponse{
|
||||
Status: 200,
|
||||
Headers: []traffic.HARHeader{{Name: "Content-Type", Value: "application/octet-stream"}},
|
||||
Content: traffic.HARContent{
|
||||
MimeType: "application/octet-stream",
|
||||
Text: base64.StdEncoding.EncodeToString(binPayload),
|
||||
Encoding: "base64",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
rawIn, err := json.Marshal(harIn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Import via HTTP endpoint.
|
||||
w := request(http.MethodPost, "/api/traffic/import/har", string(rawIn))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("POST /api/traffic/import/har status=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
var importRes traffic.HARImportResult
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &importRes); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if importRes.Imported != 2 || importRes.Skipped != 0 || importRes.Failed != 0 {
|
||||
t.Fatalf("importRes = %+v, want 2/0/0", importRes)
|
||||
}
|
||||
|
||||
// Re-import is idempotent.
|
||||
w2 := request(http.MethodPost, "/api/traffic/import/har", string(rawIn))
|
||||
if w2.Code != http.StatusOK {
|
||||
t.Fatalf("re-import status=%d body=%s", w2.Code, w2.Body.String())
|
||||
}
|
||||
var importRes2 traffic.HARImportResult
|
||||
if err := json.Unmarshal(w2.Body.Bytes(), &importRes2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if importRes2.Imported != 0 || importRes2.Skipped != 2 || importRes2.Failed != 0 {
|
||||
t.Fatalf("re-import res = %+v, want 0/2/0", importRes2)
|
||||
}
|
||||
|
||||
// Imported rows are listed on GET /api/traffic with rewritten=false and zero exchange_rewrites rows.
|
||||
wList := request(http.MethodGet, "/api/traffic?host=har-", "")
|
||||
if wList.Code != http.StatusOK {
|
||||
t.Fatalf("GET /api/traffic status=%d body=%s", wList.Code, wList.Body.String())
|
||||
}
|
||||
var listResp struct {
|
||||
Total int `json:"total"`
|
||||
Exchanges []TrafficExchangeDTO `json:"exchanges"`
|
||||
}
|
||||
if err := json.Unmarshal(wList.Body.Bytes(), &listResp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if listResp.Total != 2 || len(listResp.Exchanges) != 2 {
|
||||
t.Fatalf("listResp = %+v, want 2 exchanges", listResp)
|
||||
}
|
||||
for _, row := range listResp.Exchanges {
|
||||
if row.Rewritten {
|
||||
t.Fatalf("imported row %s marked rewritten", row.ID)
|
||||
}
|
||||
}
|
||||
var rwCount int
|
||||
if err := s.m.Traffic().DB().QueryRow(`SELECT COUNT(*) FROM exchange_rewrites`).Scan(&rwCount); err != nil || rwCount != 0 {
|
||||
t.Fatalf("exchange_rewrites count after HTTP HAR import = %d err=%v, want 0", rwCount, err)
|
||||
}
|
||||
|
||||
// Filtered export by host returns only har-b.example with binary payload preserved as base64.
|
||||
wExp := request(http.MethodGet, "/api/traffic/export/har?host=har-b.example", "")
|
||||
if wExp.Code != http.StatusOK {
|
||||
t.Fatalf("GET /api/traffic/export/har status=%d body=%s", wExp.Code, wExp.Body.String())
|
||||
}
|
||||
if ct := wExp.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/json") {
|
||||
t.Fatalf("export Content-Type = %q, want application/json", ct)
|
||||
}
|
||||
if wExp.Header().Get("X-ARTEX-HAR-Truncated") != "false" || wExp.Header().Get("X-ARTEX-HAR-Exported") != "1" || wExp.Header().Get("X-ARTEX-HAR-Matched") != "1" {
|
||||
t.Fatalf("unexpected export headers: %+v", wExp.Header())
|
||||
}
|
||||
var exported traffic.HAR
|
||||
if err := json.Unmarshal(wExp.Body.Bytes(), &exported); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if exported.Log == nil || exported.Log.Version != "1.2" || len(exported.Log.Entries) != 1 {
|
||||
t.Fatalf("filtered export = %+v", exported.Log)
|
||||
}
|
||||
entry := exported.Log.Entries[0]
|
||||
if entry.Request.URL != "https://har-b.example/blob.bin" || entry.Response.Content.Encoding != "base64" {
|
||||
t.Fatalf("exported binary entry = %+v", entry)
|
||||
}
|
||||
decoded, err := base64.StdEncoding.DecodeString(entry.Response.Content.Text)
|
||||
if err != nil || !bytes.Equal(decoded, binPayload) {
|
||||
t.Fatalf("exported binary bytes mismatch: got=%x err=%v", decoded, err)
|
||||
}
|
||||
|
||||
// Re-importing the exported HAR via HTTP is idempotent.
|
||||
wReimp := request(http.MethodPost, "/api/traffic/import/har", wExp.Body.String())
|
||||
if wReimp.Code != http.StatusOK || !strings.Contains(wReimp.Body.String(), `"imported":0`) || !strings.Contains(wReimp.Body.String(), `"skipped":1`) {
|
||||
t.Fatalf("re-import exported HAR status=%d body=%s", wReimp.Code, wReimp.Body.String())
|
||||
}
|
||||
|
||||
// Error cases -> 400 naming the reason.
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body string
|
||||
want string
|
||||
}{
|
||||
{"malformed", `{"log":`, "malformed JSON"},
|
||||
{"missing entries", `{"log":{"version":"1.2"}}`, "missing log.entries"},
|
||||
{"unsupported version", `{"log":{"version":"2.0","entries":[]}}`, "unsupported HAR version"},
|
||||
} {
|
||||
wErr := request(http.MethodPost, "/api/traffic/import/har", tc.body)
|
||||
if wErr.Code != http.StatusBadRequest || !strings.Contains(wErr.Body.String(), tc.want) {
|
||||
t.Fatalf("%s: status=%d body=%s, want 400 containing %q", tc.name, wErr.Code, wErr.Body.String(), tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
// Traffic store unavailable -> 503 on both import and export.
|
||||
sOff := &Server{m: &Manager{}}
|
||||
wOffImp := httptest.NewRecorder()
|
||||
sOff.importTrafficHAR(wOffImp, httptest.NewRequest(http.MethodPost, "/api/traffic/import/har", strings.NewReader(`{}`)))
|
||||
if wOffImp.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("import with traffic off status=%d, want 503", wOffImp.Code)
|
||||
}
|
||||
wOffExp := httptest.NewRecorder()
|
||||
sOff.exportTrafficHAR(wOffExp, httptest.NewRequest(http.MethodGet, "/api/traffic/export/har", nil))
|
||||
if wOffExp.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("export with traffic off status=%d, want 503", wOffExp.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrafficHAREndpointsRequireAuth(t *testing.T) {
|
||||
s, _ := ingestServer(t)
|
||||
|
||||
rImp := httptest.NewRequest(http.MethodPost, "/api/traffic/import/har", strings.NewReader(`{"log":{"version":"1.2","entries":[]}}`))
|
||||
wImp := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(wImp, rImp)
|
||||
if wImp.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("unauthenticated HAR import status=%d, want 401", wImp.Code)
|
||||
}
|
||||
|
||||
rExp := httptest.NewRequest(http.MethodGet, "/api/traffic/export/har", nil)
|
||||
wExp := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(wExp, rExp)
|
||||
if wExp.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("unauthenticated HAR export status=%d, want 401", wExp.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrafficHARExportNearCapReimportsThroughHTTPHandler(t *testing.T) {
|
||||
s, request := ingestServer(t)
|
||||
tr := s.m.Traffic()
|
||||
|
||||
// Seed a binary exchange near the 20 MiB export raw cap (~19.5 MiB).
|
||||
const bodySize = 19*1024*1024 + 512*1024
|
||||
binBody := make([]byte, bodySize)
|
||||
for i := range binBody {
|
||||
binBody[i] = byte((i*17 + 0x81) & 0xff)
|
||||
}
|
||||
binBody[0] = 0x00
|
||||
|
||||
rawSeed, err := json.Marshal(traffic.HAR{
|
||||
Log: &traffic.HARLog{
|
||||
Version: "1.2",
|
||||
Entries: []traffic.HAREntry{{
|
||||
StartedDateTime: "2026-06-02T00:00:00Z",
|
||||
Request: traffic.HARRequest{Method: "GET", URL: "https://nearcap.example/bin", Headers: []traffic.HARHeader{}},
|
||||
Response: traffic.HARResponse{
|
||||
Status: 200,
|
||||
Headers: []traffic.HARHeader{{Name: "Content-Type", Value: "application/octet-stream"}},
|
||||
Content: traffic.HARContent{
|
||||
Size: len(binBody),
|
||||
MimeType: "application/octet-stream",
|
||||
Text: base64.StdEncoding.EncodeToString(binBody),
|
||||
Encoding: "base64",
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tr.ImportHAR(rawSeed, traffic.HARImportOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
wExp := request(http.MethodGet, "/api/traffic/export/har?host=nearcap.example", "")
|
||||
if wExp.Code != http.StatusOK {
|
||||
t.Fatalf("export status=%d", wExp.Code)
|
||||
}
|
||||
if wExp.Header().Get("X-ARTEX-HAR-Truncated") != "false" {
|
||||
t.Fatalf("expected non-truncated export, headers=%+v", wExp.Header())
|
||||
}
|
||||
if wExp.Body.Len() >= decodeMaxBytes {
|
||||
t.Fatalf("exported HAR size=%d exceeds decodeMaxBytes=%d", wExp.Body.Len(), decodeMaxBytes)
|
||||
}
|
||||
|
||||
// Re-import the exact exported bytes through the HTTP handler (including
|
||||
// limitRequestBody 32 MiB middleware) — must answer 200, never 413.
|
||||
wImp := request(http.MethodPost, "/api/traffic/import/har", wExp.Body.String())
|
||||
if wImp.Code != http.StatusOK {
|
||||
t.Fatalf("re-import near-cap export via HTTP status=%d body=%s, want 200", wImp.Code, wImp.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// apply_rules is a boolean in the API contract: a string or a number is a 400,
|
||||
// not a silent false or true.
|
||||
func TestAdversarialReplayRouteRejectsNonBoolApplyRules(t *testing.T) {
|
||||
_, replay := replayServer(t, true, false)
|
||||
for _, body := range []string{
|
||||
`{"method":"GET","url":"http://example.com/","apply_rules":"yes"}`,
|
||||
`{"method":"GET","url":"http://example.com/","apply_rules":1}`,
|
||||
} {
|
||||
w := replay(body)
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s = %d %s, want 400", body, w.Code, w.Body)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/traffic"
|
||||
"github.com/Autumn-27/artex/traffic/rewrite"
|
||||
)
|
||||
|
||||
// replayServer wires a Server like ingestServer. With traffic off,
|
||||
@@ -161,3 +162,80 @@ func TestTrafficReplayRouteEndToEnd(t *testing.T) {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetTrafficExposesRewriteStats(t *testing.T) {
|
||||
// Traffic disabled: rewrite object still present with zero counters.
|
||||
sOff := &Server{m: &Manager{}}
|
||||
wOff := httptest.NewRecorder()
|
||||
sOff.getTraffic(wOff, httptest.NewRequest(http.MethodGet, "/api/traffic", nil))
|
||||
if wOff.Code != http.StatusOK {
|
||||
t.Fatalf("getTraffic(off) status = %d", wOff.Code)
|
||||
}
|
||||
var bodyOff map[string]any
|
||||
if err := json.Unmarshal(wOff.Body.Bytes(), &bodyOff); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rwOff, ok := bodyOff["rewrite"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("getTraffic(off) missing rewrite object: %s", wOff.Body.String())
|
||||
}
|
||||
for _, k := range []string{"large_responses", "capacity_refusals", "tombstone_drops", "conversions", "sweeps"} {
|
||||
if _, exists := rwOff[k]; !exists {
|
||||
t.Fatalf("getTraffic(off) rewrite object missing key %q: %v", k, rwOff)
|
||||
}
|
||||
}
|
||||
|
||||
// Traffic store open: rewrite counters mirror tr.RewriteStats().
|
||||
tr, err := traffic.Open(t.TempDir(), ":0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { tr.Close() })
|
||||
sOn := &Server{m: &Manager{traffic: tr}}
|
||||
wOn := httptest.NewRecorder()
|
||||
sOn.getTraffic(wOn, httptest.NewRequest(http.MethodGet, "/api/traffic", nil))
|
||||
if wOn.Code != http.StatusOK {
|
||||
t.Fatalf("getTraffic(on) status = %d", wOn.Code)
|
||||
}
|
||||
var bodyOn struct {
|
||||
Rewrite traffic.RewriteStats `json:"rewrite"`
|
||||
}
|
||||
if err := json.Unmarshal(wOn.Body.Bytes(), &bodyOn); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bodyOn.Rewrite != tr.RewriteStats() {
|
||||
t.Fatalf("getTraffic rewrite = %+v, want %+v", bodyOn.Rewrite, tr.RewriteStats())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrafficReplayRouteApplyRulesToggle(t *testing.T) {
|
||||
s, replay := replayServer(t, true, true)
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Seen-Rule", r.Header.Get("X-Rule-Tag"))
|
||||
_, _ = io.WriteString(w, "ok")
|
||||
}))
|
||||
t.Cleanup(target.Close)
|
||||
|
||||
set := rewrite.Compile([]rewrite.Rule{{
|
||||
ID: 7, Name: "tag", Enabled: true, Phase: rewrite.PhaseRequest,
|
||||
Actions: []rewrite.Action{{Type: rewrite.ActionSetHeader, Name: "X-Rule-Tag", Value: "on"}},
|
||||
}}, rewrite.Options{})
|
||||
if len(set.LoadErrors()) != 0 {
|
||||
t.Fatalf("load errors: %v", set.LoadErrors())
|
||||
}
|
||||
s.m.Traffic().SetRewriteRules(set)
|
||||
|
||||
// Default (apply_rules absent -> false): bypasses rule.
|
||||
reqOff, _ := json.Marshal(map[string]any{"method": "GET", "url": target.URL + "/toggle-off"})
|
||||
wOff := replay(string(reqOff))
|
||||
if wOff.Code != 200 || strings.Contains(wOff.Body.String(), `"X-Seen-Rule","value":"on"`) {
|
||||
t.Fatalf("default replay applied rewrite rule: %d %s", wOff.Code, wOff.Body.String())
|
||||
}
|
||||
|
||||
// Opt-in (apply_rules: true): applies rule.
|
||||
reqOn, _ := json.Marshal(map[string]any{"method": "GET", "url": target.URL + "/toggle-on", "apply_rules": true})
|
||||
wOn := replay(string(reqOn))
|
||||
if wOn.Code != 200 || !strings.Contains(wOn.Body.String(), `"value":"on"`) {
|
||||
t.Fatalf("apply_rules=true did not apply rewrite rule: %d %s", wOn.Code, wOn.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,363 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/traffic"
|
||||
)
|
||||
|
||||
func TestGetTrafficRewrittenFlagAndFilter(t *testing.T) {
|
||||
tr, err := traffic.Open(t.TempDir(), ":0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { tr.Close() })
|
||||
|
||||
insertEx := func(id string, ts int64, host string, status int) {
|
||||
t.Helper()
|
||||
if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?)`, id, ts, host, "GET", "/"+id, "https://"+host+"/"+id, status, "application/json", 0, 12, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
insertEx("rw-alpha-200", 10, "alpha.example", 200)
|
||||
insertEx("rw-beta-404", 20, "beta.example", 404)
|
||||
insertEx("orig-alpha-200", 30, "alpha.example", 200)
|
||||
insertEx("skip-alpha-404", 40, "alpha.example", 404)
|
||||
|
||||
if _, err := tr.DB().Exec(`INSERT INTO exchange_rewrites(exchange_id,seq,phase,rule_id,outcome) VALUES
|
||||
('rw-alpha-200',0,'request',1,'applied'),
|
||||
('rw-beta-404',0,'response',2,'applied'),
|
||||
('skip-alpha-404',0,'request',3,'skipped')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s := &Server{m: &Manager{traffic: tr}}
|
||||
|
||||
type respPayload struct {
|
||||
Count int `json:"count"`
|
||||
Total int `json:"total"`
|
||||
Exchanges []TrafficExchangeDTO `json:"exchanges"`
|
||||
}
|
||||
fetch := func(query string) (respPayload, []map[string]any) {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
s.getTraffic(w, httptest.NewRequest(http.MethodGet, "/api/traffic"+query, nil))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("GET /api/traffic%s status=%d body=%s", query, w.Code, w.Body.String())
|
||||
}
|
||||
var out respPayload
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var raw struct {
|
||||
Exchanges []map[string]any `json:"exchanges"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out, raw.Exchanges
|
||||
}
|
||||
|
||||
// Default (absent) and rewritten=all return all 4 exchanges and each row carries explicit "rewritten" bool.
|
||||
for _, q := range []string{"", "?rewritten=all"} {
|
||||
res, rawRows := fetch(q)
|
||||
if res.Count != 4 || res.Total != 4 || len(res.Exchanges) != 4 {
|
||||
t.Fatalf("query %q count=%d total=%d len=%d, want 4/4/4", q, res.Count, res.Total, len(res.Exchanges))
|
||||
}
|
||||
flags := map[string]bool{}
|
||||
for i, row := range res.Exchanges {
|
||||
if _, ok := rawRows[i]["rewritten"].(bool); !ok {
|
||||
t.Fatalf("query %q row %s missing boolean rewritten field: %v", q, row.ID, rawRows[i])
|
||||
}
|
||||
flags[row.ID] = row.Rewritten
|
||||
}
|
||||
if !flags["rw-alpha-200"] || !flags["rw-beta-404"] || flags["orig-alpha-200"] || flags["skip-alpha-404"] {
|
||||
t.Fatalf("query %q flags=%v", q, flags)
|
||||
}
|
||||
}
|
||||
|
||||
// Filter rewritten (true and canonical "rewritten").
|
||||
for _, q := range []string{"?rewritten=true", "?rewritten=rewritten"} {
|
||||
res, _ := fetch(q)
|
||||
if res.Count != 4 || res.Total != 2 || len(res.Exchanges) != 2 {
|
||||
t.Fatalf("query %q count=%d total=%d len=%d, want 4/2/2", q, res.Count, res.Total, len(res.Exchanges))
|
||||
}
|
||||
ids := []string{res.Exchanges[0].ID, res.Exchanges[1].ID}
|
||||
slices.Sort(ids)
|
||||
if !slices.Equal(ids, []string{"rw-alpha-200", "rw-beta-404"}) {
|
||||
t.Fatalf("query %q ids=%v", q, ids)
|
||||
}
|
||||
for _, row := range res.Exchanges {
|
||||
if !row.Rewritten {
|
||||
t.Fatalf("query %q returned non-rewritten row %s", q, row.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Filter original (false and canonical "original"): finds plain + skipped-only, excludes rewritten.
|
||||
for _, q := range []string{"?rewritten=false", "?rewritten=original"} {
|
||||
res, _ := fetch(q)
|
||||
if res.Count != 4 || res.Total != 2 || len(res.Exchanges) != 2 {
|
||||
t.Fatalf("query %q count=%d total=%d len=%d, want 4/2/2", q, res.Count, res.Total, len(res.Exchanges))
|
||||
}
|
||||
ids := []string{res.Exchanges[0].ID, res.Exchanges[1].ID}
|
||||
slices.Sort(ids)
|
||||
if !slices.Equal(ids, []string{"orig-alpha-200", "skip-alpha-404"}) {
|
||||
t.Fatalf("query %q ids=%v", q, ids)
|
||||
}
|
||||
for _, row := range res.Exchanges {
|
||||
if row.Rewritten {
|
||||
t.Fatalf("query %q returned rewritten row %s", q, row.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Combined with host and status.
|
||||
resHostRW, _ := fetch("?rewritten=true&host=alpha.example&status=200")
|
||||
if resHostRW.Total != 1 || len(resHostRW.Exchanges) != 1 || resHostRW.Exchanges[0].ID != "rw-alpha-200" {
|
||||
t.Fatalf("combined rewritten+host+status = %+v", resHostRW)
|
||||
}
|
||||
resHostOrig, _ := fetch("?rewritten=false&host=alpha.example&status=4xx")
|
||||
if resHostOrig.Total != 1 || len(resHostOrig.Exchanges) != 1 || resHostOrig.Exchanges[0].ID != "skip-alpha-404" {
|
||||
t.Fatalf("combined original+host+4xx = %+v", resHostOrig)
|
||||
}
|
||||
|
||||
// Pagination stays consistent with total under rewritten filter.
|
||||
p0, _ := fetch("?rewritten=rewritten&page=0&size=1")
|
||||
p1, _ := fetch("?rewritten=rewritten&page=1&size=1")
|
||||
if p0.Total != 2 || p1.Total != 2 || len(p0.Exchanges) != 1 || len(p1.Exchanges) != 1 || p0.Exchanges[0].ID == p1.Exchanges[0].ID {
|
||||
t.Fatalf("paginated rewritten p0=%+v p1=%+v", p0, p1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskArchiveRestorePreservesRewriteProvenanceAndV1DoesNot(t *testing.T) {
|
||||
s, f, req := trafficEvidenceServer(t)
|
||||
ctx := context.Background()
|
||||
finding, err := s.m.pg.GetFinding(f.FindingID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tid := *finding.TaskID
|
||||
s.ctx = ctx
|
||||
|
||||
seedHostFlow := func(host, id, path string) {
|
||||
t.Helper()
|
||||
if _, err := s.m.traffic.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES(?,100,?,'GET',?,?,200,'text/plain',0,4,'')`, id, host, path, "https://"+host+path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.m.traffic.DB().Exec(`INSERT INTO exchange_bodies(id,req_head,resp_head,req_body,resp_body)
|
||||
VALUES(?,?, 'HTTP/1.1 200 OK', ?, ?)`, id, "GET "+path+" HTTP/1.1\nHost: "+host, []byte{}, []byte("body")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
callTrafficGet := func(id string) string {
|
||||
t.Helper()
|
||||
for _, tool := range s.m.traffic.Tools() {
|
||||
if tool.Name() != "traffic_get" {
|
||||
continue
|
||||
}
|
||||
res, err := tool.Call(ctx, json.RawMessage(fmt.Sprintf(`{"id":%q}`, id)), nil)
|
||||
if err != nil || res.IsError {
|
||||
t.Fatalf("traffic_get(%s) err=%v res=%s", id, err, res.Flatten())
|
||||
}
|
||||
return res.Flatten()
|
||||
}
|
||||
t.Fatal("traffic_get tool not found")
|
||||
return ""
|
||||
}
|
||||
|
||||
bindAndFetchViewer := func(findingID int64, trafficID string) (rewritten bool, ruleIDs []int64) {
|
||||
t.Helper()
|
||||
list, err := s.evidenceStore().Bind(ctx, findingID, []db.TrafficRef{{TrafficID: trafficID, Role: "proof"}})
|
||||
if err != nil {
|
||||
t.Fatalf("Bind(%s): %v", trafficID, err)
|
||||
}
|
||||
var bindingID int64
|
||||
for _, b := range list.Bindings {
|
||||
if b.Snapshot.SourceTrafficID == trafficID {
|
||||
bindingID = b.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
if bindingID == 0 {
|
||||
t.Fatalf("binding for %s not found in %+v", trafficID, list.Bindings)
|
||||
}
|
||||
w := req(http.MethodGet, fmt.Sprintf("/api/exploration/findings/%d/traffic/%d", findingID, bindingID), "")
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("evidence viewer GET status=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
var detail struct {
|
||||
Binding db.FindingTrafficBinding `json:"binding"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &detail); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return detail.Binding.Snapshot.Rewritten, detail.Binding.Snapshot.RewriteRuleIDs
|
||||
}
|
||||
|
||||
// ---- Part 1: v2 rewritten archive restored into task ----
|
||||
const v2Host = "v2-archive-provenance.local"
|
||||
var v2AssetID int64
|
||||
if err := s.m.pg.QueryRow(`INSERT INTO assets(type,domain,root_domain,task_ids) VALUES('subdomain',$1,$1,ARRAY[$2]::bigint[]) RETURNING id`, v2Host, tid).Scan(&v2AssetID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = s.m.pg.Exec(`DELETE FROM assets WHERE id=$1`, v2AssetID) })
|
||||
|
||||
seedHostFlow(v2Host, "ex-v2-rw", "/rw")
|
||||
seedHostFlow(v2Host, "ex-v2-orig", "/orig")
|
||||
if _, err := s.m.traffic.DB().Exec(`INSERT INTO exchange_rewrites(exchange_id,seq,phase,rule_id,outcome) VALUES
|
||||
('ex-v2-rw',0,'request',7,'applied'),
|
||||
('ex-v2-rw',1,'response',2,'applied'),
|
||||
('ex-v2-rw',2,'response',9,'skipped')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := s.m.SetTaskPaused(strconv.FormatInt(tid, 10), true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job, err := s.m.pg.QueueTaskArchive(tid)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.runOneTaskArchiveJob(); err != nil {
|
||||
t.Fatalf("archive v2 task: %v", err)
|
||||
}
|
||||
// Exclusive host traffic must have been compacted out of the hot SQLite store.
|
||||
if _, _, err := s.m.traffic.Get("ex-v2-rw"); err == nil {
|
||||
t.Fatal("expected hot traffic to be compacted by archive")
|
||||
}
|
||||
|
||||
if _, err := s.m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.runOneTaskArchiveJob(); err != nil {
|
||||
t.Fatalf("restore v2 task: %v", err)
|
||||
}
|
||||
|
||||
if out := callTrafficGet("ex-v2-rw"); !strings.HasPrefix(out, "=== REWRITTEN === rewrite rules [2 7] ") {
|
||||
t.Fatalf("restored v2 traffic_get(ex-v2-rw) = %q, want REWRITTEN line", out)
|
||||
}
|
||||
if out := callTrafficGet("ex-v2-orig"); strings.Contains(out, "REWRITTEN") {
|
||||
t.Fatalf("restored v2 traffic_get(ex-v2-orig) unexpectedly had REWRITTEN: %q", out)
|
||||
}
|
||||
if rw, ids := bindAndFetchViewer(f.FindingID, "ex-v2-rw"); !rw || !slices.Equal(ids, []int64{2, 7}) {
|
||||
t.Fatalf("restored v2 evidence viewer(ex-v2-rw) = (%v, %v), want (true, [2 7])", rw, ids)
|
||||
}
|
||||
if rw, ids := bindAndFetchViewer(f.FindingID, "ex-v2-orig"); rw || len(ids) != 0 {
|
||||
t.Fatalf("restored v2 evidence viewer(ex-v2-orig) = (%v, %v), want (false, nil)", rw, ids)
|
||||
}
|
||||
|
||||
// ---- Part 2: v1 traffic archive restored into a task must NOT gain a REWRITTEN line ----
|
||||
taskV1, err := s.m.CreateTask("v1 traffic archive", "verify v1 restore", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tidV1, _ := strconv.ParseInt(taskV1.ID, 10, 64)
|
||||
t.Cleanup(func() {
|
||||
_, _ = s.m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, tidV1)
|
||||
_ = s.m.pg.DeleteTask(tidV1)
|
||||
})
|
||||
fV1, err := s.evidenceStore().Record(ctx, db.RecordFindingInput{
|
||||
TaskID: tidV1, ExplorationID: taskV1.ExpID, Worker: "test",
|
||||
VulnClass: "TEST", Summary: "v1 evidence", Name: "V1 Finding", Severity: "low",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
const v1Host = "v1-archive-legacy.local"
|
||||
var v1AssetID int64
|
||||
if err := s.m.pg.QueryRow(`INSERT INTO assets(type,domain,root_domain,task_ids) VALUES('subdomain',$1,$1,ARRAY[$2]::bigint[]) RETURNING id`, v1Host, tidV1).Scan(&v1AssetID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = s.m.pg.Exec(`DELETE FROM assets WHERE id=$1`, v1AssetID) })
|
||||
|
||||
seedHostFlow(v1Host, "ex-v1-legacy", "/legacy")
|
||||
if err := s.m.SetTaskPaused(taskV1.ID, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jobV1, err := s.m.pg.QueueTaskArchive(tidV1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.runOneTaskArchiveJob(); err != nil {
|
||||
t.Fatalf("archive v1 fixture task: %v", err)
|
||||
}
|
||||
|
||||
// Repackage the task archive with a version-1 traffic.json (no rewrite provenance fields).
|
||||
archivedV1, err := s.m.pg.GetTaskArchive(jobV1.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repackDir := t.TempDir()
|
||||
if err := extractTaskArchivePackage(archivedV1.ArchivePath, archivedV1.SHA256, repackDir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifestRaw, err := os.ReadFile(filepath.Join(repackDir, "manifest.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var taskSnap db.TaskArchiveSnapshot
|
||||
if err := json.Unmarshal(manifestRaw, &taskSnap); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
trafficPath := filepath.Join(repackDir, "traffic", "traffic.json")
|
||||
rawTraffic, err := os.ReadFile(trafficPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var trafficSnap traffic.ArchiveSnapshot
|
||||
if err := json.Unmarshal(rawTraffic, &trafficSnap); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
trafficSnap.Version = 1
|
||||
for i := range trafficSnap.Exchanges {
|
||||
trafficSnap.Exchanges[i].Rewritten = false
|
||||
trafficSnap.Exchanges[i].RewriteRuleIDs = nil
|
||||
}
|
||||
v1Raw, err := json.Marshal(trafficSnap)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(trafficPath, v1Raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(archivedV1.ArchivePath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, newSHA, err := writeTaskArchivePackage(archivedV1.ArchivePath, repackDir, &taskSnap)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.m.pg.Exec(`UPDATE task_archives SET sha256=$1 WHERE id=$2`, newSHA, jobV1.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := s.m.pg.QueueTaskArchiveRestore(jobV1.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.runOneTaskArchiveJob(); err != nil {
|
||||
t.Fatalf("restore v1 task: %v", err)
|
||||
}
|
||||
if out := callTrafficGet("ex-v1-legacy"); strings.Contains(out, "REWRITTEN") {
|
||||
t.Fatalf("restored v1 traffic_get(ex-v1-legacy) unexpectedly gained REWRITTEN: %q", out)
|
||||
}
|
||||
if rw, ids := bindAndFetchViewer(fV1.FindingID, "ex-v1-legacy"); rw || len(ids) != 0 {
|
||||
t.Fatalf("restored v1 evidence viewer(ex-v1-legacy) = (%v, %v), want (false, nil)", rw, ids)
|
||||
}
|
||||
}
|
||||
+98
-28
@@ -9,10 +9,16 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
archiveVersionV1 = 1
|
||||
archiveVersionV2 = 2
|
||||
)
|
||||
|
||||
// ArchiveSnapshot is the portable traffic subset embedded in one task archive.
|
||||
// Large content-addressed bodies are copied alongside this manifest rather than
|
||||
// base64-encoded, preserving deduplication and allowing the tar writer to stream.
|
||||
@@ -23,22 +29,24 @@ type ArchiveSnapshot struct {
|
||||
}
|
||||
|
||||
type ArchiveExchange struct {
|
||||
ID string `json:"id"`
|
||||
TS int64 `json:"ts"`
|
||||
Host string `json:"host"`
|
||||
Method string `json:"method"`
|
||||
URLTemplate string `json:"url_template"`
|
||||
URL string `json:"url"`
|
||||
Status int `json:"status"`
|
||||
ContentType string `json:"content_type"`
|
||||
ReqLen int `json:"req_len"`
|
||||
RespLen int `json:"resp_len"`
|
||||
ReqHead string `json:"req_head"`
|
||||
ReqBody []byte `json:"req_body,omitempty"`
|
||||
ReqBlob string `json:"req_blob,omitempty"`
|
||||
RespHead string `json:"resp_head"`
|
||||
RespBody []byte `json:"resp_body,omitempty"`
|
||||
RespBlob string `json:"resp_blob,omitempty"`
|
||||
ID string `json:"id"`
|
||||
TS int64 `json:"ts"`
|
||||
Host string `json:"host"`
|
||||
Method string `json:"method"`
|
||||
URLTemplate string `json:"url_template"`
|
||||
URL string `json:"url"`
|
||||
Status int `json:"status"`
|
||||
ContentType string `json:"content_type"`
|
||||
ReqLen int `json:"req_len"`
|
||||
RespLen int `json:"resp_len"`
|
||||
ReqHead string `json:"req_head"`
|
||||
ReqBody []byte `json:"req_body,omitempty"`
|
||||
ReqBlob string `json:"req_blob,omitempty"`
|
||||
RespHead string `json:"resp_head"`
|
||||
RespBody []byte `json:"resp_body,omitempty"`
|
||||
RespBlob string `json:"resp_blob,omitempty"`
|
||||
Rewritten bool `json:"rewritten,omitempty"`
|
||||
RewriteRuleIDs []int64 `json:"rewrite_rule_ids,omitempty"`
|
||||
}
|
||||
|
||||
// ExportHosts writes an exact-host snapshot to dir. It holds the traffic writer
|
||||
@@ -66,7 +74,7 @@ FROM exchanges WHERE host IN (`+strings.Join(placeholders, ",")+`) ORDER BY ts,i
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
snapshot := ArchiveSnapshot{Version: 1}
|
||||
snapshot := ArchiveSnapshot{Version: archiveVersionV2}
|
||||
blobs := map[string]struct{}{}
|
||||
for rows.Next() {
|
||||
var item ArchiveExchange
|
||||
@@ -102,21 +110,37 @@ FROM exchange_bodies WHERE id=?`, item.ID).Scan(&item.ReqHead, &item.ReqBody, &r
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for hash := range blobs {
|
||||
source, err := t.blobPath(hash)
|
||||
_ = rows.Close()
|
||||
for i := range snapshot.Exchanges {
|
||||
rewritten, ruleIDs, err := t.RewriteInfo(snapshot.Exchanges[i].ID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
data, err := os.ReadFile(source)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
if rewritten {
|
||||
snapshot.Exchanges[i].Rewritten = true
|
||||
snapshot.Exchanges[i].RewriteRuleIDs = ruleIDs
|
||||
}
|
||||
}
|
||||
if len(blobs) > 0 {
|
||||
snapshot.Blobs = make([]string, 0, len(blobs))
|
||||
for hash := range blobs {
|
||||
snapshot.Blobs = append(snapshot.Blobs, hash)
|
||||
}
|
||||
sort.Strings(snapshot.Blobs)
|
||||
for _, hash := range snapshot.Blobs {
|
||||
source, err := t.blobPath(hash)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
data, err := os.ReadFile(source)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "blobs", hash+".bin"), data, 0o600); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "blobs", hash+".bin"), data, 0o600); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
snapshot.Blobs = append(snapshot.Blobs, hash)
|
||||
}
|
||||
sort.Strings(snapshot.Blobs)
|
||||
raw, err := json.Marshal(snapshot)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -129,6 +153,11 @@ FROM exchange_bodies WHERE id=?`, item.ID).Scan(&item.ReqHead, &item.ReqBody, &r
|
||||
|
||||
// ImportArchive imports only missing exchange IDs. Current hot rows always win,
|
||||
// and repeated restore attempts are safe after a partial external failure.
|
||||
// Version 1 archives carry no rewrite provenance (imported as unrewritten).
|
||||
// Version 2 archives carry per-exchange rewritten + rewrite_rule_ids and
|
||||
// reconstruct minimal applied rows in exchange_rewrites (phase='archived',
|
||||
// outcome='applied', rule_snapshot=NULL) so RewriteInfo and the rewritten list
|
||||
// filter return the same provenance without inventing a rule snapshot.
|
||||
func (t *Traffic) ImportArchive(dir string) (int64, error) {
|
||||
if t == nil {
|
||||
return 0, nil
|
||||
@@ -144,7 +173,7 @@ func (t *Traffic) ImportArchive(dir string) (int64, error) {
|
||||
if err := json.Unmarshal(raw, &snapshot); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if snapshot.Version != 1 {
|
||||
if snapshot.Version != archiveVersionV1 && snapshot.Version != archiveVersionV2 {
|
||||
return 0, fmt.Errorf("unsupported traffic archive version %d", snapshot.Version)
|
||||
}
|
||||
t.wmu.Lock()
|
||||
@@ -200,6 +229,11 @@ VALUES(?,?,?,?,?,?,?)`, item.ID, item.ReqHead, item.ReqBody, nullIfEmpty(item.Re
|
||||
}
|
||||
}
|
||||
}
|
||||
if snapshot.Version >= archiveVersionV2 && item.Rewritten {
|
||||
if err := insertArchivedRewriteProvenance(tx, item.ID, item.RewriteRuleIDs); err != nil {
|
||||
return imported, err
|
||||
}
|
||||
}
|
||||
if t.fts {
|
||||
var rowID int64
|
||||
if err := tx.QueryRow(`SELECT rowid FROM exchanges WHERE id=?`, item.ID).Scan(&rowID); err != nil {
|
||||
@@ -218,6 +252,42 @@ VALUES(?,?,?,?,?,?,?)`, item.ID, item.ReqHead, item.ReqBody, nullIfEmpty(item.Re
|
||||
return imported, nil
|
||||
}
|
||||
|
||||
// insertArchivedRewriteProvenance writes minimal applied rows so RewriteInfo
|
||||
// and the traffic list's rewritten flag/filter return the exact archived
|
||||
// provenance without fabricating a rule snapshot or request/response phase.
|
||||
func insertArchivedRewriteProvenance(tx *sql.Tx, exchangeID string, ruleIDs []int64) error {
|
||||
ids := normalizeArchiveRuleIDs(ruleIDs)
|
||||
if len(ids) == 0 {
|
||||
_, err := tx.Exec(`INSERT INTO exchange_rewrites(exchange_id,seq,phase,rule_id,outcome,reason)
|
||||
VALUES(?,0,'archived',NULL,'applied','archived')`, exchangeID)
|
||||
return err
|
||||
}
|
||||
for seq, rid := range ids {
|
||||
if _, err := tx.Exec(`INSERT INTO exchange_rewrites(exchange_id,seq,phase,rule_id,outcome,reason)
|
||||
VALUES(?,?,'archived',?,'applied','archived')`, exchangeID, seq, rid); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeArchiveRuleIDs(ids []int64) []int64 {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]int64, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
if id > 0 {
|
||||
out = append(out, id)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
slices.Sort(out)
|
||||
return slices.Compact(out)
|
||||
}
|
||||
|
||||
func uniqueArchiveHosts(hosts []string) []string {
|
||||
seen := make(map[string]struct{}, len(hosts))
|
||||
out := make([]string, 0, len(hosts))
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -156,3 +161,291 @@ func simulateTrafficStageProcessExit(t *testing.T, stage *HostDeleteStage) {
|
||||
stage.done = true
|
||||
stage.traffic.wmu.Unlock()
|
||||
}
|
||||
|
||||
func callTrafficGetTool(t *testing.T, tr *Traffic, id string) string {
|
||||
t.Helper()
|
||||
for _, tool := range tr.Tools() {
|
||||
if tool.Name() != "traffic_get" {
|
||||
continue
|
||||
}
|
||||
res, err := tool.Call(context.Background(), json.RawMessage(`{"id":`+mustJSON(t, id)+`}`), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("traffic_get(%s) error: %v", id, err)
|
||||
}
|
||||
if res.IsError {
|
||||
t.Fatalf("traffic_get(%s) tool error: %s", id, res.Flatten())
|
||||
}
|
||||
return res.Flatten()
|
||||
}
|
||||
t.Fatal("traffic_get tool not found")
|
||||
return ""
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, v any) string {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func TestTrafficArchiveV2RewriteProvenanceRoundTripAndByteIdentical(t *testing.T) {
|
||||
src, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
const host = "v2-provenance.example"
|
||||
insertEx := func(id string, ts int64, urlPath string) {
|
||||
t.Helper()
|
||||
if _, err := src.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?)`, id, ts, host, "GET", urlPath, "https://"+host+urlPath, 200, "text/plain", 0, 2, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := src.DB().Exec(`INSERT INTO exchange_bodies(id,req_head,req_body,resp_head,resp_body)
|
||||
VALUES(?,?,?,?,?)`, id, "GET "+urlPath+" HTTP/1.1\nHost: "+host, []byte{}, "HTTP/1.1 200 OK", []byte("ok")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Rewritten exchange with out-of-order + duplicate applied rule IDs plus skipped/error rows.
|
||||
insertEx("ex-rewritten", 10, "/rewritten")
|
||||
for seq, row := range []struct {
|
||||
phase string
|
||||
ruleID any
|
||||
outcome string
|
||||
}{
|
||||
{"request", 7, "applied"},
|
||||
{"request", 99, "skipped"},
|
||||
{"response", 2, "applied"},
|
||||
{"response", 7, "applied"},
|
||||
{"response", 88, "error"},
|
||||
} {
|
||||
if _, err := src.DB().Exec(`INSERT INTO exchange_rewrites(exchange_id,seq,phase,rule_id,rule_snapshot,outcome,reason)
|
||||
VALUES(?,?,?,?,?,?,?)`, "ex-rewritten", seq, row.phase, row.ruleID, `{"id":1}`, row.outcome, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Rewritten exchange with applied row carrying NULL rule_id.
|
||||
insertEx("ex-rewritten-null-rule", 20, "/rewritten-null")
|
||||
if _, err := src.DB().Exec(`INSERT INTO exchange_rewrites(exchange_id,seq,phase,rule_id,outcome,reason)
|
||||
VALUES('ex-rewritten-null-rule',0,'request',NULL,'applied','')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 3. Exchange with only skipped/error rows -> NOT rewritten.
|
||||
insertEx("ex-skipped-only", 30, "/skipped")
|
||||
if _, err := src.DB().Exec(`INSERT INTO exchange_rewrites(exchange_id,seq,phase,rule_id,outcome,reason)
|
||||
VALUES('ex-skipped-only',0,'request',5,'skipped','not rewritten: binary'),('ex-skipped-only',1,'response',6,'error','boom')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 4. Plain original exchange with no rewrite rows.
|
||||
insertEx("ex-plain", 40, "/plain")
|
||||
|
||||
export1Dir := filepath.Join(t.TempDir(), "export1")
|
||||
count, err := src.ExportHosts([]string{host}, export1Dir)
|
||||
if err != nil || count != 4 {
|
||||
t.Fatalf("ExportHosts = (%d, %v), want (4, nil)", count, err)
|
||||
}
|
||||
raw1, err := os.ReadFile(filepath.Join(export1Dir, "traffic.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var snap ArchiveSnapshot
|
||||
if err := json.Unmarshal(raw1, &snap); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if snap.Version != 2 {
|
||||
t.Fatalf("archive version = %d, want 2", snap.Version)
|
||||
}
|
||||
if len(snap.Exchanges) != 4 {
|
||||
t.Fatalf("archive exchanges = %d, want 4", len(snap.Exchanges))
|
||||
}
|
||||
if !snap.Exchanges[0].Rewritten || !slices.Equal(snap.Exchanges[0].RewriteRuleIDs, []int64{2, 7}) {
|
||||
t.Fatalf("ex-rewritten exported = (%v, %v), want (true, [2 7])", snap.Exchanges[0].Rewritten, snap.Exchanges[0].RewriteRuleIDs)
|
||||
}
|
||||
if !snap.Exchanges[1].Rewritten || len(snap.Exchanges[1].RewriteRuleIDs) != 0 {
|
||||
t.Fatalf("ex-rewritten-null-rule exported = (%v, %v), want (true, nil)", snap.Exchanges[1].Rewritten, snap.Exchanges[1].RewriteRuleIDs)
|
||||
}
|
||||
if snap.Exchanges[2].Rewritten || len(snap.Exchanges[2].RewriteRuleIDs) != 0 {
|
||||
t.Fatalf("ex-skipped-only exported = (%v, %v), want (false, nil)", snap.Exchanges[2].Rewritten, snap.Exchanges[2].RewriteRuleIDs)
|
||||
}
|
||||
if snap.Exchanges[3].Rewritten || len(snap.Exchanges[3].RewriteRuleIDs) != 0 {
|
||||
t.Fatalf("ex-plain exported = (%v, %v), want (false, nil)", snap.Exchanges[3].Rewritten, snap.Exchanges[3].RewriteRuleIDs)
|
||||
}
|
||||
|
||||
// Import into a fresh store and verify RewriteInfo, reconstructed rows, traffic_get, and Page.
|
||||
dst, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
imported, err := dst.ImportArchive(export1Dir)
|
||||
if err != nil || imported != 4 {
|
||||
t.Fatalf("ImportArchive = (%d, %v), want (4, nil)", imported, err)
|
||||
}
|
||||
|
||||
if rw, ids, err := dst.RewriteInfo("ex-rewritten"); err != nil || !rw || !slices.Equal(ids, []int64{2, 7}) {
|
||||
t.Fatalf("dst RewriteInfo(ex-rewritten) = (%v, %v, %v), want (true, [2 7], nil)", rw, ids, err)
|
||||
}
|
||||
if rw, ids, err := dst.RewriteInfo("ex-rewritten-null-rule"); err != nil || !rw || len(ids) != 0 {
|
||||
t.Fatalf("dst RewriteInfo(ex-rewritten-null-rule) = (%v, %v, %v), want (true, nil, nil)", rw, ids, err)
|
||||
}
|
||||
for _, id := range []string{"ex-skipped-only", "ex-plain"} {
|
||||
if rw, ids, err := dst.RewriteInfo(id); err != nil || rw || len(ids) != 0 {
|
||||
t.Fatalf("dst RewriteInfo(%s) = (%v, %v, %v), want (false, nil, nil)", id, rw, ids, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Reconstructed rows must not fabricate a rule_snapshot.
|
||||
var snapCount int
|
||||
if err := dst.DB().QueryRow(`SELECT COUNT(*) FROM exchange_rewrites WHERE exchange_id='ex-rewritten' AND rule_snapshot IS NOT NULL`).Scan(&snapCount); err != nil || snapCount != 0 {
|
||||
t.Fatalf("reconstructed rows should have NULL rule_snapshot, got count=%d err=%v", snapCount, err)
|
||||
}
|
||||
var phase, outcome string
|
||||
var rid sql.NullInt64
|
||||
if err := dst.DB().QueryRow(`SELECT phase, outcome, rule_id FROM exchange_rewrites WHERE exchange_id='ex-rewritten' AND seq=0`).Scan(&phase, &outcome, &rid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if phase != "archived" || outcome != "applied" || !rid.Valid || rid.Int64 != 2 {
|
||||
t.Fatalf("reconstructed row 0 = (%q, %q, %+v), want ('archived', 'applied', 2)", phase, outcome, rid)
|
||||
}
|
||||
|
||||
// traffic_get shows === REWRITTEN === for rewritten exchange and not for plain/skipped-only.
|
||||
if out := callTrafficGetTool(t, dst, "ex-rewritten"); !strings.HasPrefix(out, "=== REWRITTEN === rewrite rules [2 7] ") {
|
||||
t.Fatalf("traffic_get(ex-rewritten) = %q, want REWRITTEN header", out)
|
||||
}
|
||||
if out := callTrafficGetTool(t, dst, "ex-plain"); strings.Contains(out, "REWRITTEN") {
|
||||
t.Fatalf("traffic_get(ex-plain) unexpectedly contained REWRITTEN: %q", out)
|
||||
}
|
||||
if out := callTrafficGetTool(t, dst, "ex-skipped-only"); strings.Contains(out, "REWRITTEN") {
|
||||
t.Fatalf("traffic_get(ex-skipped-only) unexpectedly contained REWRITTEN: %q", out)
|
||||
}
|
||||
|
||||
// Re-export from the imported store must be byte-identical.
|
||||
export2Dir := filepath.Join(t.TempDir(), "export2")
|
||||
if count2, err := dst.ExportHosts([]string{host}, export2Dir); err != nil || count2 != 4 {
|
||||
t.Fatalf("re-export ExportHosts = (%d, %v), want (4, nil)", count2, err)
|
||||
}
|
||||
raw2, err := os.ReadFile(filepath.Join(export2Dir, "traffic.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(raw1, raw2) {
|
||||
t.Fatalf("re-exported traffic.json differs:\nfirst: %s\nsecond: %s", string(raw1), string(raw2))
|
||||
}
|
||||
|
||||
// Re-importing over the original hot store must preserve the original live trace rows.
|
||||
if importedAgain, err := src.ImportArchive(export1Dir); err != nil || importedAgain != 0 {
|
||||
t.Fatalf("ImportArchive over hot store = (%d, %v), want (0, nil)", importedAgain, err)
|
||||
}
|
||||
var liveTraceRows int
|
||||
if err := src.DB().QueryRow(`SELECT COUNT(*) FROM exchange_rewrites WHERE exchange_id='ex-rewritten'`).Scan(&liveTraceRows); err != nil || liveTraceRows != 5 {
|
||||
t.Fatalf("hot store live trace rows = %d err=%v, want 5", liveTraceRows, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrafficArchiveV1FixtureImportsWithoutRewritten(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "traffic")
|
||||
if err := os.MkdirAll(filepath.Join(dir, "blobs"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// v1 fixture: first entry is standard v1 without rewrite fields; second entry
|
||||
// tries to smuggle rewrite fields under version 1 and must still import as unrewritten.
|
||||
v1JSON := `{
|
||||
"version": 1,
|
||||
"exchanges": [
|
||||
{
|
||||
"id": "v1-plain",
|
||||
"ts": 100,
|
||||
"host": "legacy.example",
|
||||
"method": "GET",
|
||||
"url_template": "/legacy",
|
||||
"url": "https://legacy.example/legacy",
|
||||
"status": 200,
|
||||
"content_type": "text/plain",
|
||||
"req_len": 0,
|
||||
"resp_len": 2,
|
||||
"req_head": "GET /legacy HTTP/1.1\nHost: legacy.example",
|
||||
"resp_head": "HTTP/1.1 200 OK",
|
||||
"resp_body": "b2s="
|
||||
},
|
||||
{
|
||||
"id": "v1-ignored-fields",
|
||||
"ts": 101,
|
||||
"host": "legacy.example",
|
||||
"method": "GET",
|
||||
"url_template": "/legacy2",
|
||||
"url": "https://legacy.example/legacy2",
|
||||
"status": 200,
|
||||
"content_type": "text/plain",
|
||||
"req_len": 0,
|
||||
"resp_len": 2,
|
||||
"req_head": "GET /legacy2 HTTP/1.1\nHost: legacy.example",
|
||||
"resp_head": "HTTP/1.1 200 OK",
|
||||
"resp_body": "b2s=",
|
||||
"rewritten": true,
|
||||
"rewrite_rule_ids": [3, 9]
|
||||
}
|
||||
],
|
||||
"blobs": []
|
||||
}`
|
||||
if err := os.WriteFile(filepath.Join(dir, "traffic.json"), []byte(v1JSON), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
imported, err := tr.ImportArchive(dir)
|
||||
if err != nil || imported != 2 {
|
||||
t.Fatalf("ImportArchive(v1) = (%d, %v), want (2, nil)", imported, err)
|
||||
}
|
||||
for _, id := range []string{"v1-plain", "v1-ignored-fields"} {
|
||||
if rw, ids, err := tr.RewriteInfo(id); err != nil || rw || len(ids) != 0 {
|
||||
t.Fatalf("v1 RewriteInfo(%s) = (%v, %v, %v), want (false, nil, nil)", id, rw, ids, err)
|
||||
}
|
||||
if out := callTrafficGetTool(t, tr, id); strings.Contains(out, "REWRITTEN") {
|
||||
t.Fatalf("v1 traffic_get(%s) unexpectedly has REWRITTEN: %q", id, out)
|
||||
}
|
||||
}
|
||||
rows, total, err := tr.Page(PageQuery{RespMin: -1, RespMax: -1}, 0, 10)
|
||||
if err != nil || total != 2 || len(rows) != 2 {
|
||||
t.Fatalf("Page after v1 import = (%d, %d, %v), want (2, 2, nil)", len(rows), total, err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if r.Rewritten {
|
||||
t.Fatalf("v1 row %s marked rewritten", r.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrafficArchiveUnknownVersionRefused(t *testing.T) {
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
for _, ver := range []int{-1, 0, 3, 99} {
|
||||
dir := filepath.Join(t.TempDir(), "traffic")
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload := mustJSON(t, ArchiveSnapshot{Version: ver})
|
||||
if err := os.WriteFile(filepath.Join(dir, "traffic.json"), []byte(payload), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tr.ImportArchive(dir); err == nil || !strings.Contains(err.Error(), "unsupported traffic archive version") {
|
||||
t.Fatalf("ImportArchive(version=%d) err = %v, want unsupported version error", ver, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+966
@@ -0,0 +1,966 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
const (
|
||||
// HARVersion is the HAR specification version produced and accepted.
|
||||
HARVersion = "1.2"
|
||||
|
||||
// DefaultHARMaxImportEntries caps how many entries a single HAR import accepts.
|
||||
DefaultHARMaxImportEntries = 2000
|
||||
// DefaultHARMaxImportBytes caps total decoded request+response body bytes
|
||||
// imported in a single HAR document.
|
||||
DefaultHARMaxImportBytes int64 = 32 << 20
|
||||
|
||||
// DefaultHARMaxExportEntries caps how many exchanges a single HAR export emits.
|
||||
DefaultHARMaxExportEntries = 1000
|
||||
// DefaultHARMaxExportBytes caps total raw header + body bytes emitted in a
|
||||
// single HAR export. At 20 MiB raw, even pure binary payloads expand via
|
||||
// base64 (4/3x ≈ 26.7 MiB) plus HAR JSON overhead safely below the 32 MiB
|
||||
// HTTP import limit.
|
||||
DefaultHARMaxExportBytes int64 = 20 << 20
|
||||
// DefaultHARMaxExportJSONBytes backstops the marshaled JSON size of an export
|
||||
// (31 MiB, leaving 1 MiB margin below the 32 MiB import HTTP body cap) so
|
||||
// even pathological JSON-escaped control-character payloads cannot exceed
|
||||
// the import limit.
|
||||
DefaultHARMaxExportJSONBytes int64 = 31 << 20
|
||||
|
||||
maxHARImportErrors = 50
|
||||
maxArtexIDLen = 128
|
||||
)
|
||||
|
||||
// ErrInvalidHAR marks a malformed or unsupported HAR document (mapped to HTTP 400).
|
||||
var ErrInvalidHAR = errors.New("invalid HAR")
|
||||
|
||||
// HAR is a HAR 1.2 envelope. Required HAR 1.2 fields that the traffic store
|
||||
// does not record use explicit unknown placeholders (time: 0, cache: {},
|
||||
// timings: {send:-1, wait:-1, receive:-1}, cookies: [], queryString: [],
|
||||
// headersSize: -1, redirectURL: "") rather than fabricated measurements.
|
||||
type HAR struct {
|
||||
Log *HARLog `json:"log"`
|
||||
}
|
||||
|
||||
type HARLog struct {
|
||||
Version string `json:"version"`
|
||||
Creator HARCreator `json:"creator"`
|
||||
Entries []HAREntry `json:"entries"`
|
||||
Truncated bool `json:"_truncated,omitempty"`
|
||||
TruncatedReason string `json:"_truncatedReason,omitempty"`
|
||||
TotalMatched int `json:"_totalMatched,omitempty"`
|
||||
}
|
||||
|
||||
type HARCreator struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
type HARCache struct{}
|
||||
|
||||
type HARTimings struct {
|
||||
Send float64 `json:"send"`
|
||||
Wait float64 `json:"wait"`
|
||||
Receive float64 `json:"receive"`
|
||||
}
|
||||
|
||||
type HAREntry struct {
|
||||
StartedDateTime string `json:"startedDateTime"`
|
||||
Time float64 `json:"time"`
|
||||
Request HARRequest `json:"request"`
|
||||
Response HARResponse `json:"response"`
|
||||
Cache HARCache `json:"cache"`
|
||||
Timings HARTimings `json:"timings"`
|
||||
ArtexExchangeID string `json:"_artexExchangeId,omitempty"`
|
||||
}
|
||||
|
||||
type HARRequest struct {
|
||||
Method string `json:"method"`
|
||||
URL string `json:"url"`
|
||||
HTTPVersion string `json:"httpVersion"`
|
||||
Cookies []HARCookie `json:"cookies"`
|
||||
Headers []HARHeader `json:"headers"`
|
||||
QueryString []HARParam `json:"queryString"`
|
||||
PostData *HARPostData `json:"postData,omitempty"`
|
||||
HeadersSize int `json:"headersSize"`
|
||||
BodySize int `json:"bodySize"`
|
||||
}
|
||||
|
||||
type HARPostData struct {
|
||||
MimeType string `json:"mimeType,omitempty"`
|
||||
Params []HARParam `json:"params,omitempty"`
|
||||
Text string `json:"text,omitempty"`
|
||||
Encoding string `json:"encoding,omitempty"`
|
||||
}
|
||||
|
||||
type HARParam struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
ContentType string `json:"contentType,omitempty"`
|
||||
}
|
||||
|
||||
type HARCookie struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Domain string `json:"domain,omitempty"`
|
||||
Expires string `json:"expires,omitempty"`
|
||||
HTTPOnly bool `json:"httpOnly,omitempty"`
|
||||
Secure bool `json:"secure,omitempty"`
|
||||
}
|
||||
|
||||
type HARResponse struct {
|
||||
Status int `json:"status"`
|
||||
StatusText string `json:"statusText"`
|
||||
HTTPVersion string `json:"httpVersion"`
|
||||
Cookies []HARCookie `json:"cookies"`
|
||||
Headers []HARHeader `json:"headers"`
|
||||
Content HARContent `json:"content"`
|
||||
RedirectURL string `json:"redirectURL"`
|
||||
HeadersSize int `json:"headersSize"`
|
||||
BodySize int `json:"bodySize"`
|
||||
}
|
||||
|
||||
type HARContent struct {
|
||||
Size int `json:"size"`
|
||||
MimeType string `json:"mimeType"`
|
||||
Text string `json:"text,omitempty"`
|
||||
Encoding string `json:"encoding,omitempty"`
|
||||
}
|
||||
|
||||
type HARHeader struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
// HARImportOptions configures safety caps for ImportHAR. Zero values use the
|
||||
// package defaults.
|
||||
type HARImportOptions struct {
|
||||
MaxEntries int
|
||||
MaxTotalBytes int64
|
||||
}
|
||||
|
||||
// HARImportError describes one entry that could not be imported.
|
||||
type HARImportError struct {
|
||||
Index int `json:"index"`
|
||||
URL string `json:"url,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
// HARImportResult summarizes one ImportHAR run.
|
||||
type HARImportResult struct {
|
||||
Imported int `json:"imported"`
|
||||
Skipped int `json:"skipped"`
|
||||
Failed int `json:"failed"`
|
||||
Truncated bool `json:"truncated,omitempty"`
|
||||
Errors []HARImportError `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
// HARExportOptions configures ExportHAR. Zero caps use package defaults.
|
||||
type HARExportOptions struct {
|
||||
CreatorVersion string
|
||||
MaxEntries int
|
||||
MaxTotalBytes int64 // raw header + body bytes cap
|
||||
MaxJSONBytes int64 // marshaled JSON cap (defaults to DefaultHARMaxExportJSONBytes)
|
||||
}
|
||||
|
||||
// HARExportMeta reports export counts and whether a cap stopped the export.
|
||||
type HARExportMeta struct {
|
||||
Exported int
|
||||
TotalMatched int
|
||||
TotalBytes int64 // total raw header + body bytes exported
|
||||
Truncated bool
|
||||
TruncatedReason string // "" | "max_entries" | "max_bytes"
|
||||
}
|
||||
|
||||
// ImportHAR parses a HAR 1.2 JSON document and inserts valid entries into the
|
||||
// traffic store. ARTEX-exported entries carrying a valid _artexExchangeId
|
||||
// preserve that ID so distinct exchanges within the same second round-trip
|
||||
// cleanly; external HAR entries receive a deterministic content-based har- ID
|
||||
// so re-importing the same HAR is idempotent and skips existing rows. URL
|
||||
// userinfo is stripped before storing, indexing, or reporting errors. Imported
|
||||
// traffic never passes through capture rewrite rules and never gains
|
||||
// exchange_rewrites rows.
|
||||
func (t *Traffic) ImportHAR(raw []byte, opts HARImportOptions) (HARImportResult, error) {
|
||||
if t == nil {
|
||||
return HARImportResult{}, errors.New("traffic store is not available")
|
||||
}
|
||||
if opts.MaxEntries <= 0 {
|
||||
opts.MaxEntries = DefaultHARMaxImportEntries
|
||||
}
|
||||
if opts.MaxTotalBytes <= 0 {
|
||||
opts.MaxTotalBytes = DefaultHARMaxImportBytes
|
||||
}
|
||||
|
||||
doc, err := decodeHARDocument(raw)
|
||||
if err != nil {
|
||||
return HARImportResult{}, err
|
||||
}
|
||||
if len(doc.Log.Entries) > opts.MaxEntries {
|
||||
return HARImportResult{}, fmt.Errorf("%w: HAR contains %d entries, exceeding limit of %d", ErrInvalidHAR, len(doc.Log.Entries), opts.MaxEntries)
|
||||
}
|
||||
|
||||
var res HARImportResult
|
||||
addErr := func(idx int, rawURL, reason string) {
|
||||
if len(res.Errors) < maxHARImportErrors {
|
||||
res.Errors = append(res.Errors, HARImportError{
|
||||
Index: idx,
|
||||
URL: redactURLUserinfo(rawURL),
|
||||
Reason: reason,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var totalBytes int64
|
||||
for i, entry := range doc.Log.Entries {
|
||||
parsed, err := parseHAREntry(entry)
|
||||
if err != nil {
|
||||
res.Failed++
|
||||
addErr(i, entry.Request.URL, err.Error())
|
||||
continue
|
||||
}
|
||||
entryBytes := int64(len(parsed.reqBody) + len(parsed.respBody))
|
||||
if totalBytes+entryBytes > opts.MaxTotalBytes {
|
||||
remaining := len(doc.Log.Entries) - i
|
||||
res.Truncated = true
|
||||
res.Skipped += remaining
|
||||
addErr(i, entry.Request.URL, fmt.Sprintf("decoded body byte cap (%d bytes) reached; %d remaining entries skipped", opts.MaxTotalBytes, remaining))
|
||||
break
|
||||
}
|
||||
|
||||
inserted, err := t.insertHARExchange(parsed)
|
||||
if err != nil {
|
||||
res.Failed++
|
||||
addErr(i, entry.Request.URL, err.Error())
|
||||
continue
|
||||
}
|
||||
if !inserted {
|
||||
res.Skipped++
|
||||
continue
|
||||
}
|
||||
totalBytes += entryBytes
|
||||
res.Imported++
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func decodeHARDocument(raw []byte) (HAR, error) {
|
||||
if len(bytes.TrimSpace(raw)) == 0 {
|
||||
return HAR{}, fmt.Errorf("%w: empty HAR document", ErrInvalidHAR)
|
||||
}
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
var doc HAR
|
||||
if err := dec.Decode(&doc); err != nil {
|
||||
return HAR{}, fmt.Errorf("%w: malformed JSON: %v", ErrInvalidHAR, err)
|
||||
}
|
||||
// Reject trailing non-whitespace tokens after the root object.
|
||||
if err := dec.Decode(&struct{}{}); err != io.EOF {
|
||||
return HAR{}, fmt.Errorf("%w: malformed JSON: trailing content after HAR object", ErrInvalidHAR)
|
||||
}
|
||||
if doc.Log == nil {
|
||||
return HAR{}, fmt.Errorf("%w: missing log object", ErrInvalidHAR)
|
||||
}
|
||||
if strings.TrimSpace(doc.Log.Version) != HARVersion {
|
||||
return HAR{}, fmt.Errorf("%w: unsupported HAR version %q (expected %s)", ErrInvalidHAR, doc.Log.Version, HARVersion)
|
||||
}
|
||||
if doc.Log.Entries == nil {
|
||||
return HAR{}, fmt.Errorf("%w: missing log.entries", ErrInvalidHAR)
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
type parsedHARExchange struct {
|
||||
id string
|
||||
ts int64
|
||||
host string
|
||||
method string
|
||||
urlTemplate string
|
||||
rawURL string
|
||||
status int
|
||||
reqCT string
|
||||
respCT string
|
||||
reqHead string
|
||||
reqBody []byte
|
||||
respHead string
|
||||
respBody []byte
|
||||
}
|
||||
|
||||
func parseHAREntry(entry HAREntry) (parsedHARExchange, error) {
|
||||
tsStr := strings.TrimSpace(entry.StartedDateTime)
|
||||
if tsStr == "" {
|
||||
return parsedHARExchange{}, errors.New("missing startedDateTime")
|
||||
}
|
||||
parsedTS, err := time.Parse(time.RFC3339Nano, tsStr)
|
||||
if err != nil {
|
||||
return parsedHARExchange{}, fmt.Errorf("invalid startedDateTime %q", entry.StartedDateTime)
|
||||
}
|
||||
ts := parsedTS.UTC().Unix()
|
||||
|
||||
rawURL := strings.TrimSpace(entry.Request.URL)
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil || u.Hostname() == "" || (u.Scheme != "http" && u.Scheme != "https") {
|
||||
return parsedHARExchange{}, fmt.Errorf("invalid request url %q", redactURLUserinfo(entry.Request.URL))
|
||||
}
|
||||
// Strip URL userinfo (user:pass@) before storing, indexing, or exporting,
|
||||
// matching the asset store.
|
||||
u.User = nil
|
||||
|
||||
method := strings.ToUpper(strings.TrimSpace(entry.Request.Method))
|
||||
if method == "" || !validHeaderName(method) {
|
||||
return parsedHARExchange{}, fmt.Errorf("invalid request method %q", entry.Request.Method)
|
||||
}
|
||||
if entry.Response.Status < 0 || entry.Response.Status > 999 {
|
||||
return parsedHARExchange{}, fmt.Errorf("invalid response status %d", entry.Response.Status)
|
||||
}
|
||||
|
||||
reqHeaderText, reqCT, hasReqCookie, err := formatHARHeaders(entry.Request.Headers, "Cookie")
|
||||
if err != nil {
|
||||
return parsedHARExchange{}, fmt.Errorf("invalid request headers: %w", err)
|
||||
}
|
||||
if !hasReqCookie && len(entry.Request.Cookies) > 0 {
|
||||
cookieHeader, err := formatHARRequestCookies(entry.Request.Cookies)
|
||||
if err != nil {
|
||||
return parsedHARExchange{}, fmt.Errorf("invalid request cookies: %w", err)
|
||||
}
|
||||
reqHeaderText += cookieHeader
|
||||
}
|
||||
|
||||
respHeaderText, respHeaderCT, hasRespSetCookie, err := formatHARHeaders(entry.Response.Headers, "Set-Cookie")
|
||||
if err != nil {
|
||||
return parsedHARExchange{}, fmt.Errorf("invalid response headers: %w", err)
|
||||
}
|
||||
if !hasRespSetCookie && len(entry.Response.Cookies) > 0 {
|
||||
setCookies, err := formatHARResponseCookies(entry.Response.Cookies)
|
||||
if err != nil {
|
||||
return parsedHARExchange{}, fmt.Errorf("invalid response cookies: %w", err)
|
||||
}
|
||||
respHeaderText += setCookies
|
||||
}
|
||||
|
||||
var reqBody []byte
|
||||
if entry.Request.PostData != nil {
|
||||
if entry.Request.PostData.Text != "" || len(entry.Request.PostData.Params) == 0 {
|
||||
reqBody, err = decodeHARBody(entry.Request.PostData.Text, entry.Request.PostData.Encoding)
|
||||
if err != nil {
|
||||
return parsedHARExchange{}, fmt.Errorf("invalid request postData: %w", err)
|
||||
}
|
||||
if reqCT == "" {
|
||||
reqCT = strings.TrimSpace(entry.Request.PostData.MimeType)
|
||||
}
|
||||
} else {
|
||||
reqBody = encodeHARFormParams(entry.Request.PostData.Params)
|
||||
if reqCT == "" {
|
||||
reqCT = strings.TrimSpace(entry.Request.PostData.MimeType)
|
||||
if reqCT == "" {
|
||||
reqCT = "application/x-www-form-urlencoded"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
respBody, err := decodeHARBody(entry.Response.Content.Text, entry.Response.Content.Encoding)
|
||||
if err != nil {
|
||||
return parsedHARExchange{}, fmt.Errorf("invalid response content: %w", err)
|
||||
}
|
||||
|
||||
respCT := strings.TrimSpace(entry.Response.Content.MimeType)
|
||||
if respCT == "" {
|
||||
respCT = respHeaderCT
|
||||
}
|
||||
|
||||
reqProto := sanitizeHARProto(entry.Request.HTTPVersion)
|
||||
respProto := sanitizeHARProto(entry.Response.HTTPVersion)
|
||||
reqHead := fmt.Sprintf("%s %s %s\n%s", method, u.RequestURI(), reqProto, reqHeaderText)
|
||||
|
||||
statusText := strings.TrimSpace(entry.Response.StatusText)
|
||||
if strings.ContainsAny(statusText, "\r\n\x00") {
|
||||
return parsedHARExchange{}, errors.New("response statusText contains control characters")
|
||||
}
|
||||
respHead := fmt.Sprintf("%s %d\n%s", respProto, entry.Response.Status, respHeaderText)
|
||||
if statusText != "" {
|
||||
respHead = fmt.Sprintf("%s %d %s\n%s", respProto, entry.Response.Status, statusText, respHeaderText)
|
||||
}
|
||||
|
||||
canonicalURL := u.String()
|
||||
id := harExchangeID(ts, method, canonicalURL, entry.Response.Status, reqBody, respBody)
|
||||
if preserved := strings.TrimSpace(entry.ArtexExchangeID); validArtexExchangeID(preserved) {
|
||||
id = preserved
|
||||
}
|
||||
return parsedHARExchange{
|
||||
id: id,
|
||||
ts: ts,
|
||||
host: u.Hostname(),
|
||||
method: method,
|
||||
urlTemplate: db.TemplatePath(u.EscapedPath()),
|
||||
rawURL: canonicalURL,
|
||||
status: entry.Response.Status,
|
||||
reqCT: reqCT,
|
||||
respCT: respCT,
|
||||
reqHead: reqHead,
|
||||
reqBody: reqBody,
|
||||
respHead: respHead,
|
||||
respBody: respBody,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// harExchangeID derives a deterministic ID from the exchange's timestamp (Unix
|
||||
// seconds), method, canonical URL, status code, and SHA-256 digests of both
|
||||
// bodies. Using Unix seconds matches exchanges.ts so export -> re-import
|
||||
// produces the exact same ID for external HARs.
|
||||
func harExchangeID(ts int64, method, rawURL string, status int, reqBody, respBody []byte) string {
|
||||
reqSum := sha256.Sum256(reqBody)
|
||||
respSum := sha256.Sum256(respBody)
|
||||
h := sha256.New()
|
||||
fmt.Fprintf(h, "%d\x00%s\x00%s\x00%d\x00%x\x00%x", ts, method, rawURL, status, reqSum, respSum)
|
||||
return "har-" + hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
func validArtexExchangeID(id string) bool {
|
||||
if id == "" || len(id) > maxArtexIDLen {
|
||||
return false
|
||||
}
|
||||
for i := 0; i < len(id); i++ {
|
||||
c := id[i]
|
||||
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ||
|
||||
c == '-' || c == '_' || c == '.' || c == ':' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// redactURLUserinfo strips user:pass@ from both parseable and malformed URLs so
|
||||
// credentials never leak into stored URLs or import error details.
|
||||
func redactURLUserinfo(raw string) string {
|
||||
s := strings.TrimSpace(raw)
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
if u, err := url.Parse(s); err == nil && u.User != nil {
|
||||
u.User = nil
|
||||
return u.String()
|
||||
}
|
||||
schemeEnd := strings.Index(s, "://")
|
||||
if schemeEnd < 0 {
|
||||
return s
|
||||
}
|
||||
rest := s[schemeEnd+3:]
|
||||
authEnd := strings.IndexAny(rest, "/?#")
|
||||
auth := rest
|
||||
tail := ""
|
||||
if authEnd >= 0 {
|
||||
auth = rest[:authEnd]
|
||||
tail = rest[authEnd:]
|
||||
}
|
||||
if at := strings.LastIndexByte(auth, '@'); at >= 0 {
|
||||
return s[:schemeEnd+3] + auth[at+1:] + tail
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func formatHARHeaders(headers []HARHeader, checkHeader string) (formatted string, contentType string, hasChecked bool, err error) {
|
||||
var b strings.Builder
|
||||
for _, h := range headers {
|
||||
name := strings.TrimSpace(h.Name)
|
||||
if name == "" {
|
||||
return "", "", false, errors.New("empty header name")
|
||||
}
|
||||
// Skip HTTP/2 pseudo-headers (:method, :authority, :scheme, :path, :status)
|
||||
// emitted by browser DevTools HAR exports; their semantics live in the
|
||||
// request/status line.
|
||||
if strings.HasPrefix(name, ":") {
|
||||
continue
|
||||
}
|
||||
if !validHeaderName(name) {
|
||||
return "", "", false, fmt.Errorf("invalid header name %q", h.Name)
|
||||
}
|
||||
if strings.ContainsAny(h.Value, "\r\n\x00") {
|
||||
return "", "", false, fmt.Errorf("header %s contains a line break or NUL byte", name)
|
||||
}
|
||||
if contentType == "" && strings.EqualFold(name, "Content-Type") {
|
||||
contentType = strings.TrimSpace(h.Value)
|
||||
}
|
||||
if checkHeader != "" && strings.EqualFold(name, checkHeader) {
|
||||
hasChecked = true
|
||||
}
|
||||
b.WriteString(name)
|
||||
b.WriteString(": ")
|
||||
b.WriteString(h.Value)
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
return b.String(), contentType, hasChecked, nil
|
||||
}
|
||||
|
||||
func formatHARRequestCookies(cookies []HARCookie) (string, error) {
|
||||
parts := make([]string, 0, len(cookies))
|
||||
for _, c := range cookies {
|
||||
name := strings.TrimSpace(c.Name)
|
||||
if name == "" {
|
||||
return "", errors.New("empty cookie name")
|
||||
}
|
||||
if strings.ContainsAny(name, " \t;=\r\n\x00") || strings.ContainsAny(c.Value, "\r\n\x00") {
|
||||
return "", fmt.Errorf("invalid cookie %q", name)
|
||||
}
|
||||
parts = append(parts, name+"="+c.Value)
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return "Cookie: " + strings.Join(parts, "; ") + "\n", nil
|
||||
}
|
||||
|
||||
func formatHARResponseCookies(cookies []HARCookie) (string, error) {
|
||||
var b strings.Builder
|
||||
for _, c := range cookies {
|
||||
name := strings.TrimSpace(c.Name)
|
||||
if name == "" {
|
||||
return "", errors.New("empty cookie name")
|
||||
}
|
||||
if strings.ContainsAny(name, " \t;=\r\n\x00") ||
|
||||
strings.ContainsAny(c.Value, "\r\n\x00") ||
|
||||
strings.ContainsAny(c.Path, ";\r\n\x00") ||
|
||||
strings.ContainsAny(c.Domain, ";\r\n\x00") ||
|
||||
strings.ContainsAny(c.Expires, ";\r\n\x00") {
|
||||
return "", fmt.Errorf("invalid response cookie %q", name)
|
||||
}
|
||||
b.WriteString("Set-Cookie: ")
|
||||
b.WriteString(name)
|
||||
b.WriteByte('=')
|
||||
b.WriteString(c.Value)
|
||||
if c.Path != "" {
|
||||
b.WriteString("; Path=")
|
||||
b.WriteString(c.Path)
|
||||
}
|
||||
if c.Domain != "" {
|
||||
b.WriteString("; Domain=")
|
||||
b.WriteString(c.Domain)
|
||||
}
|
||||
if c.Expires != "" {
|
||||
b.WriteString("; Expires=")
|
||||
b.WriteString(c.Expires)
|
||||
}
|
||||
if c.Secure {
|
||||
b.WriteString("; Secure")
|
||||
}
|
||||
if c.HTTPOnly {
|
||||
b.WriteString("; HttpOnly")
|
||||
}
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
func encodeHARFormParams(params []HARParam) []byte {
|
||||
parts := make([]string, 0, len(params))
|
||||
for _, p := range params {
|
||||
parts = append(parts, url.QueryEscape(p.Name)+"="+url.QueryEscape(p.Value))
|
||||
}
|
||||
return []byte(strings.Join(parts, "&"))
|
||||
}
|
||||
|
||||
func decodeHARBody(text, encoding string) ([]byte, error) {
|
||||
enc := strings.ToLower(strings.TrimSpace(encoding))
|
||||
switch enc {
|
||||
case "":
|
||||
if text == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return []byte(text), nil
|
||||
case "base64":
|
||||
if text == "" {
|
||||
return nil, nil
|
||||
}
|
||||
decoded, err := base64.StdEncoding.DecodeString(text)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid base64 body: %w", err)
|
||||
}
|
||||
return decoded, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported body encoding %q", encoding)
|
||||
}
|
||||
}
|
||||
|
||||
func sanitizeHARProto(v string) string {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" || strings.ContainsAny(v, " \t\r\n\x00") || !strings.HasPrefix(strings.ToUpper(v), "HTTP/") {
|
||||
return "HTTP/1.1"
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// insertHARExchange inserts one HAR exchange idempotently (INSERT OR IGNORE)
|
||||
// through the same spill/blob_refs/FTS path as the recorder and ImportArchive.
|
||||
// It never writes to exchange_rewrites.
|
||||
func (t *Traffic) insertHARExchange(ex parsedHARExchange) (bool, error) {
|
||||
t.wmu.Lock()
|
||||
defer t.wmu.Unlock()
|
||||
|
||||
var exists int
|
||||
if err := t.db.QueryRow(`SELECT 1 FROM exchanges WHERE id=?`, ex.id).Scan(&exists); err == nil {
|
||||
return false, nil
|
||||
} else if !errors.Is(err, sql.ErrNoRows) {
|
||||
return false, err
|
||||
}
|
||||
|
||||
reqB := t.spill(ex.reqBody, ex.reqCT)
|
||||
respB := t.spill(ex.respBody, ex.respCT)
|
||||
|
||||
tx, err := t.db.Begin()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("beginning transaction: %w", err)
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck
|
||||
|
||||
res, err := tx.Exec(`INSERT OR IGNORE INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,'')`,
|
||||
ex.id, ex.ts, ex.host, ex.method, ex.urlTemplate, ex.rawURL, ex.status, ex.respCT, len(ex.reqBody), len(ex.respBody))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("writing index: %w", err)
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
if n == 0 {
|
||||
return false, nil
|
||||
}
|
||||
var rowid int64
|
||||
if err := tx.QueryRow(`SELECT rowid FROM exchanges WHERE id=?`, ex.id).Scan(&rowid); err != nil {
|
||||
return false, fmt.Errorf("fetching rowid: %w", err)
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO exchange_bodies(id,req_head,req_body,req_blob,resp_head,resp_body,resp_blob)
|
||||
VALUES(?,?,?,?,?,?,?)`,
|
||||
ex.id, ex.reqHead, reqB.inline, nullIfEmpty(reqB.hash), ex.respHead, respB.inline, nullIfEmpty(respB.hash)); err != nil {
|
||||
return false, fmt.Errorf("writing body: %w", err)
|
||||
}
|
||||
for _, h := range []string{reqB.hash, respB.hash} {
|
||||
if h == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT OR IGNORE INTO blob_refs(hash,exchange_id) VALUES(?,?)`, h, ex.id); err != nil {
|
||||
return false, fmt.Errorf("registering blob reference: %w", err)
|
||||
}
|
||||
}
|
||||
if t.fts {
|
||||
idx := strings.Join([]string{ex.rawURL, ex.reqHead, reqB.index, ex.respHead, respB.index}, "\n")
|
||||
if _, err := tx.Exec(`INSERT INTO ex_fts(rowid,content) VALUES(?,?)`, rowid, idx); err != nil {
|
||||
return false, fmt.Errorf("writing full-text index: %w", err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return false, fmt.Errorf("committing: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// ExportHAR exports exchanges matching f as a valid HAR 1.2 document in stable
|
||||
// chronological order (ts ASC, id ASC), capped by opts.MaxEntries,
|
||||
// opts.MaxTotalBytes (raw headers + bodies), and opts.MaxJSONBytes (marshaled
|
||||
// JSON size). Truncation is reported in both HARExportMeta and the HAR 1.2
|
||||
// custom fields _truncated / _truncatedReason / _totalMatched.
|
||||
func (t *Traffic) ExportHAR(f PageQuery, opts HARExportOptions) (HAR, HARExportMeta, error) {
|
||||
if t == nil {
|
||||
return HAR{}, HARExportMeta{}, errors.New("traffic store is not available")
|
||||
}
|
||||
if opts.MaxEntries <= 0 {
|
||||
opts.MaxEntries = DefaultHARMaxExportEntries
|
||||
}
|
||||
if opts.MaxTotalBytes <= 0 {
|
||||
opts.MaxTotalBytes = DefaultHARMaxExportBytes
|
||||
}
|
||||
if opts.MaxJSONBytes <= 0 {
|
||||
opts.MaxJSONBytes = DefaultHARMaxExportJSONBytes
|
||||
}
|
||||
creatorVer := strings.TrimSpace(opts.CreatorVersion)
|
||||
if creatorVer == "" {
|
||||
creatorVer = "dev"
|
||||
}
|
||||
|
||||
where, args := t.pageWhere(f)
|
||||
var totalMatched int
|
||||
if err := t.db.QueryRow(`SELECT COUNT(*) FROM exchanges`+where, args...).Scan(&totalMatched); err != nil {
|
||||
return HAR{}, HARExportMeta{}, err
|
||||
}
|
||||
|
||||
type exRow struct {
|
||||
id string
|
||||
ts int64
|
||||
host string
|
||||
method string
|
||||
rawURL string
|
||||
status int
|
||||
contentType string
|
||||
legacyPath string
|
||||
}
|
||||
|
||||
// Fetch at most MaxEntries+1 rows so we know whether the entry cap fired.
|
||||
sel := `SELECT id,ts,host,method,url,status,content_type,path FROM exchanges` +
|
||||
where + ` ORDER BY ts ASC, id ASC LIMIT ?`
|
||||
qargs := append(append([]any{}, args...), opts.MaxEntries+1)
|
||||
rows, err := t.db.Query(sel, qargs...)
|
||||
if err != nil {
|
||||
return HAR{}, HARExportMeta{}, err
|
||||
}
|
||||
var matched []exRow
|
||||
for rows.Next() {
|
||||
var r exRow
|
||||
if err := rows.Scan(&r.id, &r.ts, &r.host, &r.method, &r.rawURL, &r.status, &r.contentType, &r.legacyPath); err != nil {
|
||||
rows.Close()
|
||||
return HAR{}, HARExportMeta{}, err
|
||||
}
|
||||
matched = append(matched, r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return HAR{}, HARExportMeta{}, err
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
meta := HARExportMeta{TotalMatched: totalMatched}
|
||||
hitEntryCap := len(matched) > opts.MaxEntries
|
||||
if hitEntryCap {
|
||||
matched = matched[:opts.MaxEntries]
|
||||
}
|
||||
|
||||
// Reserve envelope overhead (~512 bytes) for the outer log object + metadata.
|
||||
const harEnvelopeMargin int64 = 512
|
||||
var encodedJSONBytes int64 = harEnvelopeMargin
|
||||
|
||||
entries := make([]HAREntry, 0, len(matched))
|
||||
for _, r := range matched {
|
||||
reqHead, reqBytes, respHead, respBytes, err := t.loadFullExchange(r.id, r.legacyPath)
|
||||
if err != nil {
|
||||
return HAR{}, HARExportMeta{}, err
|
||||
}
|
||||
entryRawBytes := int64(len(reqHead) + len(reqBytes) + len(respHead) + len(respBytes))
|
||||
if meta.TotalBytes+entryRawBytes > opts.MaxTotalBytes {
|
||||
meta.Truncated = true
|
||||
meta.TruncatedReason = "max_bytes"
|
||||
break
|
||||
}
|
||||
|
||||
reqProto, reqHeaders := parseStoredRequestHead(reqHead)
|
||||
respProto, respStatusText, respHeaders := parseStoredResponseHead(respHead)
|
||||
|
||||
harReq := HARRequest{
|
||||
Method: r.method,
|
||||
URL: redactURLUserinfo(r.rawURL),
|
||||
HTTPVersion: reqProto,
|
||||
Cookies: []HARCookie{},
|
||||
Headers: reqHeaders,
|
||||
QueryString: []HARParam{},
|
||||
HeadersSize: -1,
|
||||
BodySize: len(reqBytes),
|
||||
}
|
||||
if len(reqBytes) > 0 {
|
||||
text, enc := encodeHARBody(reqBytes)
|
||||
harReq.PostData = &HARPostData{
|
||||
MimeType: findHeaderValue(reqHeaders, "Content-Type"),
|
||||
Text: text,
|
||||
Encoding: enc,
|
||||
}
|
||||
}
|
||||
|
||||
harContent := HARContent{
|
||||
Size: len(respBytes),
|
||||
MimeType: r.contentType,
|
||||
}
|
||||
if len(respBytes) > 0 {
|
||||
harContent.Text, harContent.Encoding = encodeHARBody(respBytes)
|
||||
}
|
||||
|
||||
harResp := HARResponse{
|
||||
Status: r.status,
|
||||
StatusText: respStatusText,
|
||||
HTTPVersion: respProto,
|
||||
Cookies: []HARCookie{},
|
||||
Headers: respHeaders,
|
||||
Content: harContent,
|
||||
RedirectURL: "",
|
||||
HeadersSize: -1,
|
||||
BodySize: len(respBytes),
|
||||
}
|
||||
|
||||
entry := HAREntry{
|
||||
StartedDateTime: time.Unix(r.ts, 0).UTC().Format(time.RFC3339),
|
||||
Time: 0,
|
||||
Request: harReq,
|
||||
Response: harResp,
|
||||
Cache: HARCache{},
|
||||
Timings: HARTimings{Send: -1, Wait: -1, Receive: -1},
|
||||
ArtexExchangeID: r.id,
|
||||
}
|
||||
|
||||
entryJSON, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
return HAR{}, HARExportMeta{}, err
|
||||
}
|
||||
// +1 for the separating comma inside the JSON array.
|
||||
if encodedJSONBytes+int64(len(entryJSON))+1 > opts.MaxJSONBytes {
|
||||
meta.Truncated = true
|
||||
meta.TruncatedReason = "max_bytes"
|
||||
break
|
||||
}
|
||||
encodedJSONBytes += int64(len(entryJSON)) + 1
|
||||
meta.TotalBytes += entryRawBytes
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
|
||||
if !meta.Truncated && hitEntryCap {
|
||||
meta.Truncated = true
|
||||
meta.TruncatedReason = "max_entries"
|
||||
}
|
||||
meta.Exported = len(entries)
|
||||
|
||||
doc := HAR{
|
||||
Log: &HARLog{
|
||||
Version: HARVersion,
|
||||
Creator: HARCreator{Name: "ARTEX", Version: creatorVer},
|
||||
Entries: entries,
|
||||
Truncated: meta.Truncated,
|
||||
TruncatedReason: meta.TruncatedReason,
|
||||
},
|
||||
}
|
||||
if meta.Truncated {
|
||||
doc.Log.TotalMatched = meta.TotalMatched
|
||||
}
|
||||
return doc, meta, nil
|
||||
}
|
||||
|
||||
func (t *Traffic) loadFullExchange(id, legacyPath string) (reqHead string, reqBody []byte, respHead string, respBody []byte, err error) {
|
||||
var reqBlob, respBlob sql.NullString
|
||||
err = t.db.QueryRow(`SELECT req_head,req_body,req_blob,resp_head,resp_body,resp_blob
|
||||
FROM exchange_bodies WHERE id=?`, id).Scan(&reqHead, &reqBody, &reqBlob, &respHead, &respBody, &respBlob)
|
||||
if errors.Is(err, sql.ErrNoRows) && strings.TrimSpace(legacyPath) != "" {
|
||||
rb, readErr := os.ReadFile(filepath.Join(t.dir, legacyPath, "request.http"))
|
||||
if readErr != nil && !os.IsNotExist(readErr) {
|
||||
return "", nil, "", nil, readErr
|
||||
}
|
||||
pb, readErr := os.ReadFile(filepath.Join(t.dir, legacyPath, "response.http"))
|
||||
if readErr != nil && !os.IsNotExist(readErr) {
|
||||
return "", nil, "", nil, readErr
|
||||
}
|
||||
reqHead, reqBody = splitLegacyHTTP(rb)
|
||||
respHead, respBody = splitLegacyHTTP(pb)
|
||||
return reqHead, reqBody, respHead, respBody, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", nil, "", nil, err
|
||||
}
|
||||
if reqBlob.Valid && reqBlob.String != "" {
|
||||
p, err := t.blobPath(reqBlob.String)
|
||||
if err != nil {
|
||||
return "", nil, "", nil, err
|
||||
}
|
||||
reqBody, err = os.ReadFile(p)
|
||||
if err != nil {
|
||||
return "", nil, "", nil, err
|
||||
}
|
||||
}
|
||||
if respBlob.Valid && respBlob.String != "" {
|
||||
p, err := t.blobPath(respBlob.String)
|
||||
if err != nil {
|
||||
return "", nil, "", nil, err
|
||||
}
|
||||
respBody, err = os.ReadFile(p)
|
||||
if err != nil {
|
||||
return "", nil, "", nil, err
|
||||
}
|
||||
}
|
||||
return reqHead, reqBody, respHead, respBody, nil
|
||||
}
|
||||
|
||||
func splitLegacyHTTP(raw []byte) (string, []byte) {
|
||||
if idx := bytes.Index(raw, []byte("\r\n\r\n")); idx >= 0 {
|
||||
return string(raw[:idx]), raw[idx+4:]
|
||||
}
|
||||
if idx := bytes.Index(raw, []byte("\n\n")); idx >= 0 {
|
||||
return string(raw[:idx]), raw[idx+2:]
|
||||
}
|
||||
return string(raw), nil
|
||||
}
|
||||
|
||||
// encodeHARBody returns plain text for valid NUL-free UTF-8 bodies, and base64
|
||||
// otherwise so binary payloads survive export and re-import bit-for-bit.
|
||||
func encodeHARBody(body []byte) (text, encoding string) {
|
||||
if utf8.Valid(body) && bytes.IndexByte(body, 0) < 0 {
|
||||
return string(body), ""
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(body), "base64"
|
||||
}
|
||||
|
||||
func parseStoredRequestHead(raw string) (httpVersion string, headers []HARHeader) {
|
||||
headers = []HARHeader{}
|
||||
lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n")
|
||||
if len(lines) == 0 {
|
||||
return "", headers
|
||||
}
|
||||
first := strings.TrimSpace(lines[0])
|
||||
if parts := strings.SplitN(first, " ", 3); len(parts) == 3 && strings.HasPrefix(strings.ToUpper(parts[2]), "HTTP/") {
|
||||
httpVersion = strings.TrimSpace(parts[2])
|
||||
}
|
||||
headers = appendParsedHeaders(headers, lines[1:])
|
||||
return httpVersion, headers
|
||||
}
|
||||
|
||||
func parseStoredResponseHead(raw string) (httpVersion, statusText string, headers []HARHeader) {
|
||||
headers = []HARHeader{}
|
||||
lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n")
|
||||
if len(lines) == 0 {
|
||||
return "", "", headers
|
||||
}
|
||||
first := strings.TrimSpace(lines[0])
|
||||
if parts := strings.SplitN(first, " ", 3); len(parts) >= 2 && strings.HasPrefix(strings.ToUpper(parts[0]), "HTTP") {
|
||||
if strings.HasPrefix(strings.ToUpper(parts[0]), "HTTP/") {
|
||||
httpVersion = strings.TrimSpace(parts[0])
|
||||
}
|
||||
if len(parts) == 3 {
|
||||
statusText = strings.TrimSpace(parts[2])
|
||||
}
|
||||
}
|
||||
headers = appendParsedHeaders(headers, lines[1:])
|
||||
return httpVersion, statusText, headers
|
||||
}
|
||||
|
||||
func appendParsedHeaders(dst []HARHeader, lines []string) []HARHeader {
|
||||
for _, line := range lines {
|
||||
line = strings.TrimRight(line, "\r")
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
colon := strings.IndexByte(line, ':')
|
||||
if colon <= 0 {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSpace(line[:colon])
|
||||
if name == "" || !validHeaderName(name) {
|
||||
continue
|
||||
}
|
||||
val := strings.TrimSpace(line[colon+1:])
|
||||
dst = append(dst, HARHeader{Name: name, Value: val})
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
func findHeaderValue(headers []HARHeader, name string) string {
|
||||
for _, h := range headers {
|
||||
if strings.EqualFold(h.Name, name) {
|
||||
return h.Value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,817 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/traffic/rewrite"
|
||||
)
|
||||
|
||||
func TestHARRoundTripPlainBase64BinaryEmptyAndLargeBlob(t *testing.T) {
|
||||
src, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
// Install an enabled capture rewrite rule to prove HAR import NEVER runs rules
|
||||
// or writes exchange_rewrites rows.
|
||||
set := rewrite.Compile([]rewrite.Rule{{
|
||||
ID: 42, Name: "should-not-apply-to-har", Enabled: true, Phase: rewrite.PhaseRequest,
|
||||
Actions: []rewrite.Action{{Type: rewrite.ActionSetHeader, Name: "X-Injected", Value: "1"}},
|
||||
}}, rewrite.Options{})
|
||||
if len(set.LoadErrors()) != 0 {
|
||||
t.Fatalf("compile rewrite rules: %v", set.LoadErrors())
|
||||
}
|
||||
src.SetRewriteRules(set)
|
||||
|
||||
binaryResp := []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0xff, 0xfe, 0x01}
|
||||
binaryReq := []byte{0x00, 0x01, 0x02, 0x80, 0xff}
|
||||
largeTextBody := "LARGE-HAR-NEEDLE-" + strings.Repeat("abcdefghij", (maxInlineBody/10)+50) // > 256 KiB -> blob!
|
||||
|
||||
inputHAR := HAR{
|
||||
Log: &HARLog{
|
||||
Version: "1.2",
|
||||
Creator: HARCreator{Name: "TestBrowser", Version: "1.0"},
|
||||
Entries: []HAREntry{
|
||||
{
|
||||
StartedDateTime: "2026-05-01T10:00:00.789Z",
|
||||
Request: HARRequest{
|
||||
Method: "POST",
|
||||
URL: "https://app.example.com/api/v1/items/42?mode=full",
|
||||
HTTPVersion: "HTTP/1.1",
|
||||
Headers: []HARHeader{
|
||||
{Name: ":method", Value: "POST"},
|
||||
{Name: ":authority", Value: "app.example.com"},
|
||||
{Name: "Host", Value: "app.example.com"},
|
||||
{Name: "Content-Type", Value: "application/json"},
|
||||
{Name: "X-Trace", Value: "one"},
|
||||
{Name: "X-Trace", Value: "two"},
|
||||
},
|
||||
PostData: &HARPostData{
|
||||
MimeType: "application/json",
|
||||
Text: `{"action":"create","needle":"plain-req-token"}`,
|
||||
},
|
||||
},
|
||||
Response: HARResponse{
|
||||
Status: 201,
|
||||
StatusText: "Created",
|
||||
HTTPVersion: "HTTP/1.1",
|
||||
Headers: []HARHeader{
|
||||
{Name: "Content-Type", Value: "application/json; charset=utf-8"},
|
||||
{Name: "X-Resp", Value: "ok"},
|
||||
},
|
||||
Content: HARContent{
|
||||
MimeType: "application/json; charset=utf-8",
|
||||
Text: `{"id":42,"msg":"plain-resp-marker"}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
StartedDateTime: "2026-05-01T10:00:05Z",
|
||||
Request: HARRequest{
|
||||
Method: "POST",
|
||||
URL: "https://cdn.example.com/upload",
|
||||
HTTPVersion: "HTTP/2.0",
|
||||
Headers: []HARHeader{{Name: "Content-Type", Value: "application/octet-stream"}},
|
||||
PostData: &HARPostData{
|
||||
MimeType: "application/octet-stream",
|
||||
Text: base64.StdEncoding.EncodeToString(binaryReq),
|
||||
Encoding: "base64",
|
||||
},
|
||||
},
|
||||
Response: HARResponse{
|
||||
Status: 200,
|
||||
HTTPVersion: "HTTP/2.0",
|
||||
Headers: []HARHeader{{Name: "Content-Type", Value: "image/png"}},
|
||||
Content: HARContent{
|
||||
MimeType: "image/png",
|
||||
Text: base64.StdEncoding.EncodeToString(binaryResp),
|
||||
Encoding: "base64",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
StartedDateTime: "2026-05-01T10:00:10Z",
|
||||
Request: HARRequest{
|
||||
Method: "GET",
|
||||
URL: "https://app.example.com/empty",
|
||||
Headers: []HARHeader{{Name: "Accept", Value: "*/*"}},
|
||||
},
|
||||
Response: HARResponse{
|
||||
Status: 204,
|
||||
Headers: []HARHeader{},
|
||||
Content: HARContent{},
|
||||
},
|
||||
},
|
||||
{
|
||||
StartedDateTime: "2026-05-01T10:00:15Z",
|
||||
Request: HARRequest{
|
||||
Method: "GET",
|
||||
URL: "https://blob.example.com/big.txt",
|
||||
Headers: []HARHeader{},
|
||||
},
|
||||
Response: HARResponse{
|
||||
Status: 200,
|
||||
Headers: []HARHeader{{Name: "Content-Type", Value: "text/plain"}},
|
||||
Content: HARContent{
|
||||
MimeType: "text/plain",
|
||||
Text: largeTextBody,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
rawIn, err := json.Marshal(inputHAR)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
res, err := src.ImportHAR(rawIn, HARImportOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("ImportHAR: %v", err)
|
||||
}
|
||||
if res.Imported != 4 || res.Skipped != 0 || res.Failed != 0 || res.Truncated {
|
||||
t.Fatalf("ImportHAR result = %+v, want 4/0/0/false", res)
|
||||
}
|
||||
|
||||
// Re-importing the exact same HAR must be idempotent (all skipped).
|
||||
res2, err := src.ImportHAR(rawIn, HARImportOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("re-import HAR: %v", err)
|
||||
}
|
||||
if res2.Imported != 0 || res2.Skipped != 4 || res2.Failed != 0 {
|
||||
t.Fatalf("re-import result = %+v, want 0/4/0", res2)
|
||||
}
|
||||
|
||||
// Verify none of the imported exchanges are flagged rewritten or have exchange_rewrites rows.
|
||||
var rwRows int
|
||||
if err := src.DB().QueryRow(`SELECT COUNT(*) FROM exchange_rewrites`).Scan(&rwRows); err != nil || rwRows != 0 {
|
||||
t.Fatalf("exchange_rewrites count = %d err=%v, want 0", rwRows, err)
|
||||
}
|
||||
allRows, total, err := src.Page(PageQuery{RespMin: -1, RespMax: -1}, 0, 20)
|
||||
if err != nil || total != 4 || len(allRows) != 4 {
|
||||
t.Fatalf("Page after import = (%d, %d, %v), want (4, 4, nil)", len(allRows), total, err)
|
||||
}
|
||||
for _, r := range allRows {
|
||||
if !strings.HasPrefix(r.ID, "har-") {
|
||||
t.Fatalf("expected har- prefix on id %q", r.ID)
|
||||
}
|
||||
if r.Rewritten {
|
||||
t.Fatalf("imported exchange %s unexpectedly flagged rewritten", r.ID)
|
||||
}
|
||||
if rw, ids, err := src.RewriteInfo(r.ID); err != nil || rw || len(ids) != 0 {
|
||||
t.Fatalf("RewriteInfo(%s) = (%v, %v, %v), want (false, nil, nil)", r.ID, rw, ids, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify searchable via FTS (both inline body and spilled large text body).
|
||||
if rows, tot, err := src.Page(PageQuery{Body: "plain-resp-marker", RespMin: -1, RespMax: -1}, 0, 10); err != nil || tot != 1 || len(rows) != 1 {
|
||||
t.Fatalf("FTS search inline body = (%d, %d, %v), want 1", len(rows), tot, err)
|
||||
}
|
||||
if rows, tot, err := src.Page(PageQuery{Body: "LARGE-HAR-NEEDLE", RespMin: -1, RespMax: -1}, 0, 10); err != nil || tot != 1 || len(rows) != 1 {
|
||||
t.Fatalf("FTS search blob body = (%d, %d, %v), want 1", len(rows), tot, err)
|
||||
}
|
||||
|
||||
// Verify large body actually spilled to a blob and registered in blob_refs.
|
||||
var blobHash string
|
||||
var blobRefCount int
|
||||
if err := src.DB().QueryRow(`SELECT resp_blob FROM exchange_bodies b JOIN exchanges e ON e.id=b.id WHERE e.host='blob.example.com'`).Scan(&blobHash); err != nil || blobHash == "" {
|
||||
t.Fatalf("large body did not spill to blob: hash=%q err=%v", blobHash, err)
|
||||
}
|
||||
if err := src.DB().QueryRow(`SELECT COUNT(*) FROM blob_refs WHERE hash=?`, blobHash).Scan(&blobRefCount); err != nil || blobRefCount != 1 {
|
||||
t.Fatalf("blob_refs count for %s = %d err=%v, want 1", blobHash, blobRefCount, err)
|
||||
}
|
||||
|
||||
// Export HAR from src.
|
||||
exported, meta, err := src.ExportHAR(PageQuery{RespMin: -1, RespMax: -1}, HARExportOptions{CreatorVersion: "2.3.0"})
|
||||
if err != nil {
|
||||
t.Fatalf("ExportHAR: %v", err)
|
||||
}
|
||||
if meta.Exported != 4 || meta.TotalMatched != 4 || meta.Truncated || meta.TruncatedReason != "" {
|
||||
t.Fatalf("ExportHAR meta = %+v", meta)
|
||||
}
|
||||
if exported.Log.Version != "1.2" || exported.Log.Creator.Name != "ARTEX" || exported.Log.Creator.Version != "2.3.0" {
|
||||
t.Fatalf("ExportHAR header = %+v", exported.Log)
|
||||
}
|
||||
if len(exported.Log.Entries) != 4 {
|
||||
t.Fatalf("ExportHAR entries = %d, want 4", len(exported.Log.Entries))
|
||||
}
|
||||
|
||||
// Verify HAR 1.2 required keys are present with explicit unknown placeholders
|
||||
// and serverIPAddress is not fabricated.
|
||||
rawExported, err := json.Marshal(exported)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(rawExported), `"serverIPAddress"`) {
|
||||
t.Fatalf("exported HAR invented serverIPAddress: %s", string(rawExported))
|
||||
}
|
||||
var rawDoc map[string]any
|
||||
if err := json.Unmarshal(rawExported, &rawDoc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rawLog := rawDoc["log"].(map[string]any)
|
||||
rawEntries := rawLog["entries"].([]any)
|
||||
firstEntry := rawEntries[0].(map[string]any)
|
||||
if got, ok := firstEntry["time"].(float64); !ok || got != 0 {
|
||||
t.Fatalf("entry.time = %#v, want 0", firstEntry["time"])
|
||||
}
|
||||
if cacheObj, ok := firstEntry["cache"].(map[string]any); !ok || len(cacheObj) != 0 {
|
||||
t.Fatalf("entry.cache = %#v, want {}", firstEntry["cache"])
|
||||
}
|
||||
timingsObj, ok := firstEntry["timings"].(map[string]any)
|
||||
if !ok || timingsObj["send"] != float64(-1) || timingsObj["wait"] != float64(-1) || timingsObj["receive"] != float64(-1) {
|
||||
t.Fatalf("entry.timings = %#v, want {send:-1 wait:-1 receive:-1}", firstEntry["timings"])
|
||||
}
|
||||
reqObj := firstEntry["request"].(map[string]any)
|
||||
if cookies, ok := reqObj["cookies"].([]any); !ok || len(cookies) != 0 {
|
||||
t.Fatalf("request.cookies = %#v, want []", reqObj["cookies"])
|
||||
}
|
||||
if qs, ok := reqObj["queryString"].([]any); !ok || len(qs) != 0 {
|
||||
t.Fatalf("request.queryString = %#v, want []", reqObj["queryString"])
|
||||
}
|
||||
if hs, ok := reqObj["headersSize"].(float64); !ok || hs != -1 {
|
||||
t.Fatalf("request.headersSize = %#v, want -1", reqObj["headersSize"])
|
||||
}
|
||||
if _, ok := reqObj["httpVersion"].(string); !ok {
|
||||
t.Fatalf("request.httpVersion missing: %#v", reqObj)
|
||||
}
|
||||
respObj := firstEntry["response"].(map[string]any)
|
||||
if cookies, ok := respObj["cookies"].([]any); !ok || len(cookies) != 0 {
|
||||
t.Fatalf("response.cookies = %#v, want []", respObj["cookies"])
|
||||
}
|
||||
if redir, ok := respObj["redirectURL"].(string); !ok || redir != "" {
|
||||
t.Fatalf("response.redirectURL = %#v, want \"\"", respObj["redirectURL"])
|
||||
}
|
||||
if hs, ok := respObj["headersSize"].(float64); !ok || hs != -1 {
|
||||
t.Fatalf("response.headersSize = %#v, want -1", respObj["headersSize"])
|
||||
}
|
||||
if _, ok := respObj["httpVersion"].(string); !ok {
|
||||
t.Fatalf("response.httpVersion missing: %#v", respObj)
|
||||
}
|
||||
if _, ok := respObj["statusText"].(string); !ok {
|
||||
t.Fatalf("response.statusText missing: %#v", respObj)
|
||||
}
|
||||
contentObj := respObj["content"].(map[string]any)
|
||||
if _, ok := contentObj["mimeType"].(string); !ok {
|
||||
t.Fatalf("response.content.mimeType missing: %#v", contentObj)
|
||||
}
|
||||
|
||||
// Verify stable chronological ordering & exact body encodings.
|
||||
e0 := exported.Log.Entries[0]
|
||||
if e0.Request.URL != "https://app.example.com/api/v1/items/42?mode=full" || e0.Response.Content.Encoding != "" || e0.Response.Content.Text != `{"id":42,"msg":"plain-resp-marker"}` {
|
||||
t.Fatalf("entry 0 mismatch: %+v", e0)
|
||||
}
|
||||
e1 := exported.Log.Entries[1]
|
||||
if e1.Response.Content.Encoding != "base64" || e1.Request.PostData == nil || e1.Request.PostData.Encoding != "base64" {
|
||||
t.Fatalf("entry 1 binary encodings wrong: req=%+v resp=%+v", e1.Request.PostData, e1.Response.Content)
|
||||
}
|
||||
gotBinResp, err := base64.StdEncoding.DecodeString(e1.Response.Content.Text)
|
||||
if err != nil || !bytes.Equal(gotBinResp, binaryResp) {
|
||||
t.Fatalf("entry 1 binary resp mismatch: got=%x err=%v", gotBinResp, err)
|
||||
}
|
||||
gotBinReq, err := base64.StdEncoding.DecodeString(e1.Request.PostData.Text)
|
||||
if err != nil || !bytes.Equal(gotBinReq, binaryReq) {
|
||||
t.Fatalf("entry 1 binary req mismatch: got=%x err=%v", gotBinReq, err)
|
||||
}
|
||||
e2 := exported.Log.Entries[2]
|
||||
if e2.Request.PostData != nil || e2.Response.Content.Size != 0 || e2.Response.Content.Text != "" {
|
||||
t.Fatalf("entry 2 empty body mismatch: %+v", e2)
|
||||
}
|
||||
e3 := exported.Log.Entries[3]
|
||||
if e3.Response.Content.Size != len(largeTextBody) || e3.Response.Content.Text != largeTextBody || e3.Response.Content.Encoding != "" {
|
||||
t.Fatalf("entry 3 large blob body truncated or altered: size=%d len=%d", e3.Response.Content.Size, len(e3.Response.Content.Text))
|
||||
}
|
||||
|
||||
// Re-importing the exported HAR into src must be 100% idempotent (same deterministic IDs!).
|
||||
reimpSame, err := src.ImportHAR(rawExported, HARImportOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("re-import exported HAR into src: %v", err)
|
||||
}
|
||||
if reimpSame.Imported != 0 || reimpSame.Skipped != 4 || reimpSame.Failed != 0 {
|
||||
t.Fatalf("re-import exported HAR into src = %+v, want 0/4/0", reimpSame)
|
||||
}
|
||||
|
||||
// Re-importing the exported HAR into a fresh store must yield identical IDs, metadata, and bodies.
|
||||
dst, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
reimpFresh, err := dst.ImportHAR(rawExported, HARImportOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("import exported HAR into dst: %v", err)
|
||||
}
|
||||
if reimpFresh.Imported != 4 || reimpFresh.Skipped != 0 || reimpFresh.Failed != 0 {
|
||||
t.Fatalf("dst import = %+v, want 4/0/0", reimpFresh)
|
||||
}
|
||||
|
||||
exportedDst, _, err := dst.ExportHAR(PageQuery{RespMin: -1, RespMax: -1}, HARExportOptions{CreatorVersion: "2.3.0"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rawDst, err := json.Marshal(exportedDst)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(rawExported, rawDst) {
|
||||
t.Fatalf("dst round-trip HAR differs from src export")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHARImportValidationDuplicatesAndPartialErrors(t *testing.T) {
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
badDocs := []struct {
|
||||
name string
|
||||
raw string
|
||||
want string
|
||||
}{
|
||||
{"empty", "", "empty HAR"},
|
||||
{"malformed json", `{"log":`, "malformed JSON"},
|
||||
{"trailing garbage", `{"log":{"version":"1.2","entries":[]}} extra`, "trailing content"},
|
||||
{"missing log", `{}`, "missing log"},
|
||||
{"missing entries", `{"log":{"version":"1.2"}}`, "missing log.entries"},
|
||||
{"null entries", `{"log":{"version":"1.2","entries":null}}`, "missing log.entries"},
|
||||
{"unsupported version 1.1", `{"log":{"version":"1.1","entries":[]}}`, "unsupported HAR version"},
|
||||
{"unsupported version 2.0", `{"log":{"version":"2.0","entries":[]}}`, "unsupported HAR version"},
|
||||
{"empty version", `{"log":{"version":"","entries":[]}}`, "unsupported HAR version"},
|
||||
}
|
||||
for _, tc := range badDocs {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := tr.ImportHAR([]byte(tc.raw), HARImportOptions{})
|
||||
if !errors.Is(err, ErrInvalidHAR) || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("ImportHAR(%s) err = %v, want ErrInvalidHAR containing %q", tc.name, err, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Duplicate entries inside one HAR + bad entries mixed with valid ones.
|
||||
mixed := `{
|
||||
"log": {
|
||||
"version": "1.2",
|
||||
"entries": [
|
||||
{
|
||||
"startedDateTime": "2026-05-01T12:00:00Z",
|
||||
"request": {"method": "GET", "url": "https://dup.example/a", "headers": []},
|
||||
"response": {"status": 200, "headers": [], "content": {"text": "ok"}}
|
||||
},
|
||||
{
|
||||
"startedDateTime": "2026-05-01T12:00:00Z",
|
||||
"request": {"method": "GET", "url": "https://dup.example/a", "headers": []},
|
||||
"response": {"status": 200, "headers": [], "content": {"text": "ok"}}
|
||||
},
|
||||
{
|
||||
"startedDateTime": "not-a-timestamp",
|
||||
"request": {"method": "GET", "url": "https://bad.example/ts", "headers": []},
|
||||
"response": {"status": 200, "headers": [], "content": {}}
|
||||
},
|
||||
{
|
||||
"startedDateTime": "2026-05-01T12:00:01Z",
|
||||
"request": {"method": "GET", "url": "https://bad.example/hdr", "headers": [{"name": "X-Bad", "value": "a\r\nb"}]},
|
||||
"response": {"status": 200, "headers": [], "content": {}}
|
||||
},
|
||||
{
|
||||
"startedDateTime": "2026-05-01T12:00:02Z",
|
||||
"request": {"method": "GET", "url": "https://bad.example/b64", "headers": []},
|
||||
"response": {"status": 200, "headers": [], "content": {"text": "!!!not-base64!!!", "encoding": "base64"}}
|
||||
},
|
||||
{
|
||||
"startedDateTime": "2026-05-01T12:00:03Z",
|
||||
"request": {"method": "GET", "url": "https://good.example/b", "headers": []},
|
||||
"response": {"status": 200, "headers": [], "content": {"text": "second"}}
|
||||
}
|
||||
]
|
||||
}
|
||||
}`
|
||||
res, err := tr.ImportHAR([]byte(mixed), HARImportOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("mixed ImportHAR: %v", err)
|
||||
}
|
||||
if res.Imported != 2 || res.Skipped != 1 || res.Failed != 3 || len(res.Errors) != 3 {
|
||||
t.Fatalf("mixed ImportHAR result = %+v, want imported=2 skipped=1 failed=3 errors=3", res)
|
||||
}
|
||||
|
||||
// Entry cap returns ErrInvalidHAR.
|
||||
if _, err := tr.ImportHAR([]byte(mixed), HARImportOptions{MaxEntries: 2}); !errors.Is(err, ErrInvalidHAR) {
|
||||
t.Fatalf("MaxEntries exceeded err = %v, want ErrInvalidHAR", err)
|
||||
}
|
||||
|
||||
// Byte cap truncates cleanly and reports remaining skipped.
|
||||
byteCapHAR := `{
|
||||
"log": {
|
||||
"version": "1.2",
|
||||
"entries": [
|
||||
{
|
||||
"startedDateTime": "2026-05-01T12:01:00Z",
|
||||
"request": {"method": "GET", "url": "https://cap.example/1", "headers": []},
|
||||
"response": {"status": 200, "headers": [], "content": {"text": "1234567890"}}
|
||||
},
|
||||
{
|
||||
"startedDateTime": "2026-05-01T12:01:01Z",
|
||||
"request": {"method": "GET", "url": "https://cap.example/2", "headers": []},
|
||||
"response": {"status": 200, "headers": [], "content": {"text": "1234567890"}}
|
||||
}
|
||||
]
|
||||
}
|
||||
}`
|
||||
capRes, err := tr.ImportHAR([]byte(byteCapHAR), HARImportOptions{MaxTotalBytes: 15})
|
||||
if err != nil {
|
||||
t.Fatalf("byte-capped ImportHAR: %v", err)
|
||||
}
|
||||
if capRes.Imported != 1 || capRes.Skipped != 1 || !capRes.Truncated || len(capRes.Errors) != 1 {
|
||||
t.Fatalf("byte-capped ImportHAR = %+v, want imported=1 skipped=1 truncated=true", capRes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHARExportFiltersAndTruncationCaps(t *testing.T) {
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
insert := func(id string, ts int64, host string, status int, body string) {
|
||||
t.Helper()
|
||||
if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?)`, id, ts, host, "GET", "/"+id, "https://"+host+"/"+id, status, "text/plain", 0, len(body), ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tr.DB().Exec(`INSERT INTO exchange_bodies(id,req_head,req_body,resp_head,resp_body)
|
||||
VALUES(?,?,?,?,?)`, id, "GET /"+id+" HTTP/1.1\nHost: "+host, []byte{}, "HTTP/1.1 200 OK", []byte(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
insert("e-2", 20, "alpha.example", 200, "bbbbbbbbbb")
|
||||
insert("e-1", 10, "alpha.example", 404, "aaaaaaaaaa")
|
||||
insert("e-3", 30, "beta.example", 200, "cccccccccc")
|
||||
|
||||
// Filter by host + status, verify chronological order.
|
||||
doc, meta, err := tr.ExportHAR(PageQuery{Host: "alpha.example", RespMin: -1, RespMax: -1}, HARExportOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.Exported != 2 || meta.TotalMatched != 2 || meta.Truncated {
|
||||
t.Fatalf("filtered export meta = %+v", meta)
|
||||
}
|
||||
if doc.Log.Entries[0].Request.URL != "https://alpha.example/e-1" || doc.Log.Entries[1].Request.URL != "https://alpha.example/e-2" {
|
||||
t.Fatalf("export not in stable (ts,id) order: %+v", doc.Log.Entries)
|
||||
}
|
||||
|
||||
// Truncation by MaxEntries.
|
||||
docEntryCap, metaEntryCap, err := tr.ExportHAR(PageQuery{RespMin: -1, RespMax: -1}, HARExportOptions{MaxEntries: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !metaEntryCap.Truncated || metaEntryCap.TruncatedReason != "max_entries" || metaEntryCap.Exported != 2 || metaEntryCap.TotalMatched != 3 {
|
||||
t.Fatalf("entry-capped meta = %+v", metaEntryCap)
|
||||
}
|
||||
if !docEntryCap.Log.Truncated || docEntryCap.Log.TruncatedReason != "max_entries" || docEntryCap.Log.TotalMatched != 3 {
|
||||
t.Fatalf("entry-capped log fields = %+v", docEntryCap.Log)
|
||||
}
|
||||
|
||||
// Truncation by MaxTotalBytes (note: raw cap counts both headers and bodies).
|
||||
// Each row here has req_head (~46B) + resp_head (15B) + body (10B) ≈ 71B.
|
||||
docByteCap, metaByteCap, err := tr.ExportHAR(PageQuery{RespMin: -1, RespMax: -1}, HARExportOptions{MaxTotalBytes: 100})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !metaByteCap.Truncated || metaByteCap.TruncatedReason != "max_bytes" || metaByteCap.Exported != 1 || metaByteCap.TotalMatched != 3 {
|
||||
t.Fatalf("byte-capped meta = %+v", metaByteCap)
|
||||
}
|
||||
if !docByteCap.Log.Truncated || docByteCap.Log.TruncatedReason != "max_bytes" || docByteCap.Log.TotalMatched != 3 {
|
||||
t.Fatalf("byte-capped log fields = %+v", docByteCap.Log)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHARTwoIdenticalExchangesRoundTripPreservesBoth(t *testing.T) {
|
||||
src, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
// Two distinct ARTEX exchanges recorded in the same second with identical
|
||||
// method, URL, status, headers, and bodies.
|
||||
for _, id := range []string{"dup-a", "dup-b"} {
|
||||
if _, err := src.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES(?,1700000000,'same.example','POST','/ping','https://same.example/ping',200,'application/json',2,2,'')`, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := src.DB().Exec(`INSERT INTO exchange_bodies(id,req_head,req_body,resp_head,resp_body)
|
||||
VALUES(?,'POST /ping HTTP/1.1\nHost: same.example\n',?,'HTTP/1.1 200 OK\n',?)`, id, []byte("{}"), []byte("{}")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
exported, meta, err := src.ExportHAR(PageQuery{RespMin: -1, RespMax: -1}, HARExportOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if meta.Exported != 2 || len(exported.Log.Entries) != 2 {
|
||||
t.Fatalf("export = %+v, want 2 entries", meta)
|
||||
}
|
||||
if exported.Log.Entries[0].ArtexExchangeID != "dup-a" || exported.Log.Entries[1].ArtexExchangeID != "dup-b" {
|
||||
t.Fatalf("exported _artexExchangeId = [%q %q], want [dup-a dup-b]",
|
||||
exported.Log.Entries[0].ArtexExchangeID, exported.Log.Entries[1].ArtexExchangeID)
|
||||
}
|
||||
raw, err := json.Marshal(exported)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Re-importing into src is idempotent (both skipped).
|
||||
reimpSrc, err := src.ImportHAR(raw, HARImportOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reimpSrc.Imported != 0 || reimpSrc.Skipped != 2 || reimpSrc.Failed != 0 {
|
||||
t.Fatalf("re-import into src = %+v, want 0/2/0", reimpSrc)
|
||||
}
|
||||
|
||||
// Importing into a fresh store preserves BOTH dup-a and dup-b (no collapse!).
|
||||
dst, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
impDst, err := dst.ImportHAR(raw, HARImportOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if impDst.Imported != 2 || impDst.Skipped != 0 || impDst.Failed != 0 {
|
||||
t.Fatalf("import into fresh dst = %+v, want 2/0/0", impDst)
|
||||
}
|
||||
if count, err := dst.Count(); err != nil || count != 2 {
|
||||
t.Fatalf("dst.Count() = (%d, %v), want 2", count, err)
|
||||
}
|
||||
if _, _, err := dst.Get("dup-a"); err != nil {
|
||||
t.Fatalf("dup-a missing in dst: %v", err)
|
||||
}
|
||||
if _, _, err := dst.Get("dup-b"); err != nil {
|
||||
t.Fatalf("dup-b missing in dst: %v", err)
|
||||
}
|
||||
// Re-importing into dst is idempotent.
|
||||
reimpDst, err := dst.ImportHAR(raw, HARImportOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reimpDst.Imported != 0 || reimpDst.Skipped != 2 || reimpDst.Failed != 0 {
|
||||
t.Fatalf("re-import into dst = %+v, want 0/2/0", reimpDst)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHARExportBinaryNearCapFitsUnderImportLimit(t *testing.T) {
|
||||
src, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
// Binary body near the 20 MiB raw export cap (~19.5 MiB). After base64
|
||||
// expansion (4/3x ≈ 26 MiB) plus headers and JSON envelope, it must still
|
||||
// marshal strictly under the 32 MiB HTTP import limit.
|
||||
const bodySize = 19*1024*1024 + 512*1024
|
||||
binBody := make([]byte, bodySize)
|
||||
for i := range binBody {
|
||||
binBody[i] = byte((i*31 + 0x80) & 0xff)
|
||||
}
|
||||
binBody[0] = 0x00 // guarantee binary/base64 encoding
|
||||
|
||||
respB := src.spill(binBody, "application/octet-stream")
|
||||
reqHead := "GET /big.bin HTTP/1.1\nHost: cap.example\n" + strings.Repeat("X-Pad: 0123456789\n", 10)
|
||||
respHead := "HTTP/1.1 200 OK\nContent-Type: application/octet-stream\n"
|
||||
if err := src.insertExchange("bin-near-cap", 1700000100, "cap.example", "GET", "/big.bin",
|
||||
"https://cap.example/big.bin", 200, "application/octet-stream", 0, len(binBody), reqHead, storedBody{}, respHead, respB); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
doc, meta, err := src.ExportHAR(PageQuery{RespMin: -1, RespMax: -1}, HARExportOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("ExportHAR near cap: %v", err)
|
||||
}
|
||||
if meta.Truncated || meta.Exported != 1 {
|
||||
t.Fatalf("near-cap export meta = %+v, want Truncated=false Exported=1", meta)
|
||||
}
|
||||
// Verify headers were counted in TotalBytes alongside the body.
|
||||
wantRaw := int64(len(reqHead) + len(respHead) + len(binBody))
|
||||
if meta.TotalBytes != wantRaw {
|
||||
t.Fatalf("meta.TotalBytes = %d, want %d (headers + body)", meta.TotalBytes, wantRaw)
|
||||
}
|
||||
|
||||
rawJSON, err := json.Marshal(doc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const httpImportLimit = 32 << 20
|
||||
if len(rawJSON) >= httpImportLimit {
|
||||
t.Fatalf("marshaled near-cap binary HAR = %d bytes, exceeds 32 MiB import limit (%d)", len(rawJSON), httpImportLimit)
|
||||
}
|
||||
|
||||
// Re-importing into a fresh store succeeds with exact bytes.
|
||||
dst, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer dst.Close()
|
||||
imp, err := dst.ImportHAR(rawJSON, HARImportOptions{})
|
||||
if err != nil || imp.Imported != 1 || imp.Failed != 0 || imp.Truncated {
|
||||
t.Fatalf("re-import near-cap binary HAR = (%+v, %v)", imp, err)
|
||||
}
|
||||
_, _, _, gotResp, err := dst.loadFullExchange("bin-near-cap", "")
|
||||
if err != nil || !bytes.Equal(gotResp, binBody) {
|
||||
t.Fatalf("near-cap binary round-trip mismatch: len=%d err=%v", len(gotResp), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHARImportPostDataParamsAndCookies(t *testing.T) {
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
// Entry 0: form body ONLY in postData.params, cookies ONLY in cookies arrays.
|
||||
// Entry 1: Cookie and Set-Cookie already present in headers -> must NOT duplicate.
|
||||
harJSON := `{
|
||||
"log": {
|
||||
"version": "1.2",
|
||||
"entries": [
|
||||
{
|
||||
"startedDateTime": "2026-05-02T09:00:00Z",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "https://form.example/login",
|
||||
"headers": [{"name": "Host", "value": "form.example"}],
|
||||
"cookies": [
|
||||
{"name": "sid", "value": "abc123"},
|
||||
{"name": "theme", "value": "dark"}
|
||||
],
|
||||
"postData": {
|
||||
"mimeType": "application/x-www-form-urlencoded",
|
||||
"params": [
|
||||
{"name": "user", "value": "alice"},
|
||||
{"name": "msg", "value": "hello world"}
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": {
|
||||
"status": 302,
|
||||
"headers": [{"name": "Location", "value": "/home"}],
|
||||
"cookies": [
|
||||
{"name": "session", "value": "xyz", "path": "/", "httpOnly": true, "secure": true}
|
||||
],
|
||||
"content": {"size": 0, "mimeType": ""}
|
||||
}
|
||||
},
|
||||
{
|
||||
"startedDateTime": "2026-05-02T09:00:05Z",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": "https://form.example/home",
|
||||
"headers": [{"name": "Cookie", "value": "existing=1"}],
|
||||
"cookies": [{"name": "existing", "value": "1"}]
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": [{"name": "Set-Cookie", "value": "kept=2; Path=/"}],
|
||||
"cookies": [{"name": "kept", "value": "2", "path": "/"}],
|
||||
"content": {"size": 2, "mimeType": "text/plain", "text": "ok"}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}`
|
||||
|
||||
res, err := tr.ImportHAR([]byte(harJSON), HARImportOptions{})
|
||||
if err != nil || res.Imported != 2 || res.Failed != 0 {
|
||||
t.Fatalf("ImportHAR(params+cookies) = (%+v, %v)", res, err)
|
||||
}
|
||||
|
||||
rows, _, err := tr.Page(PageQuery{Host: "form.example", RespMin: -1, RespMax: -1, Sort: "ts", Order: "asc"}, 0, 10)
|
||||
if err != nil || len(rows) != 2 {
|
||||
t.Fatalf("Page = (%d, %v)", len(rows), err)
|
||||
}
|
||||
req0, resp0, err := tr.Get(rows[0].ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(req0, "Cookie: sid=abc123; theme=dark\n") {
|
||||
t.Fatalf("entry 0 missing synthesized Cookie header: %q", req0)
|
||||
}
|
||||
if !strings.Contains(req0, "user=alice&msg=hello+world") {
|
||||
t.Fatalf("entry 0 missing postData.params form body: %q", req0)
|
||||
}
|
||||
if !strings.Contains(resp0, "Set-Cookie: session=xyz; Path=/; Secure; HttpOnly\n") {
|
||||
t.Fatalf("entry 0 missing synthesized Set-Cookie header: %q", resp0)
|
||||
}
|
||||
|
||||
req1, resp1, err := tr.Get(rows[1].ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Count(req1, "Cookie:") != 1 {
|
||||
t.Fatalf("entry 1 duplicated Cookie header: %q", req1)
|
||||
}
|
||||
if strings.Count(resp1, "Set-Cookie:") != 1 {
|
||||
t.Fatalf("entry 1 duplicated Set-Cookie header: %q", resp1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHARImportStripsAndRedactsURLUserinfo(t *testing.T) {
|
||||
tr, err := Open(t.TempDir(), "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tr.Close()
|
||||
|
||||
harJSON := `{
|
||||
"log": {
|
||||
"version": "1.2",
|
||||
"entries": [
|
||||
{
|
||||
"startedDateTime": "2026-05-03T10:00:00Z",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": "https://alice:supersecret@api.example.com/v1/items?k=1",
|
||||
"headers": []
|
||||
},
|
||||
"response": {
|
||||
"status": 200,
|
||||
"headers": [],
|
||||
"content": {"mimeType": "text/plain", "text": "ok"}
|
||||
}
|
||||
},
|
||||
{
|
||||
"startedDateTime": "2026-05-03T10:00:01Z",
|
||||
"request": {
|
||||
"method": "BAD METHOD",
|
||||
"url": "https://bob:topsecret@bad.example.com/fail",
|
||||
"headers": []
|
||||
},
|
||||
"response": {"status": 200, "headers": [], "content": {}}
|
||||
},
|
||||
{
|
||||
"startedDateTime": "2026-05-03T10:00:02Z",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"url": "https://carol:hiddenpass@bad host/x",
|
||||
"headers": []
|
||||
},
|
||||
"response": {"status": 200, "headers": [], "content": {}}
|
||||
}
|
||||
]
|
||||
}
|
||||
}`
|
||||
|
||||
res, err := tr.ImportHAR([]byte(harJSON), HARImportOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Imported != 1 || res.Failed != 2 || len(res.Errors) != 2 {
|
||||
t.Fatalf("ImportHAR userinfo result = %+v, want 1 imported / 2 failed", res)
|
||||
}
|
||||
for _, e := range res.Errors {
|
||||
for _, secret := range []string{"bob", "topsecret", "carol", "hiddenpass"} {
|
||||
if strings.Contains(e.URL, secret) || strings.Contains(e.Reason, secret) {
|
||||
t.Fatalf("import error leaked userinfo %q: %+v", secret, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stored URL must have userinfo stripped.
|
||||
rows, total, err := tr.Page(PageQuery{RespMin: -1, RespMax: -1}, 0, 10)
|
||||
if err != nil || total != 1 || len(rows) != 1 {
|
||||
t.Fatalf("Page = (%d, %d, %v)", len(rows), total, err)
|
||||
}
|
||||
if rows[0].URL != "https://api.example.com/v1/items?k=1" {
|
||||
t.Fatalf("stored URL = %q, want userinfo stripped", rows[0].URL)
|
||||
}
|
||||
// FTS must NOT find the stripped password.
|
||||
if found, tot, err := tr.Page(PageQuery{Query: "supersecret", RespMin: -1, RespMax: -1}, 0, 10); err != nil || tot != 0 || len(found) != 0 {
|
||||
t.Fatalf("FTS leaked userinfo secret: %+v", found)
|
||||
}
|
||||
// Exported HAR must NOT contain the password.
|
||||
doc, _, err := tr.ExportHAR(PageQuery{RespMin: -1, RespMax: -1}, HARExportOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(doc)
|
||||
if strings.Contains(string(raw), "supersecret") || strings.Contains(string(raw), "alice") {
|
||||
t.Fatalf("exported HAR leaked userinfo: %s", string(raw))
|
||||
}
|
||||
}
|
||||
+71
-4
@@ -3,8 +3,10 @@ package traffic
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -13,6 +15,7 @@ import (
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -28,8 +31,61 @@ const (
|
||||
maxReplayResponse = 2 * 1024 * 1024
|
||||
// maxReplayHeaders bounds the number of request header lines.
|
||||
maxReplayHeaders = 200
|
||||
// replayNonceHeader is the internal header Replay attaches when ApplyRules
|
||||
// is false (the default). The capture proxy strips it unconditionally
|
||||
// before forwarding and bypasses rewrite rules only when the value matches
|
||||
// a live registered nonce.
|
||||
replayNonceHeader = "X-ARTEX-Replay-Nonce"
|
||||
)
|
||||
|
||||
// replayNonces holds live single-use nonces for in-flight Replay calls that
|
||||
// bypass capture rewrite rules (ApplyRules == false).
|
||||
var replayNonces struct {
|
||||
mu sync.Mutex
|
||||
live map[string]struct{}
|
||||
}
|
||||
|
||||
func registerReplayNonce() (string, error) {
|
||||
var raw [32]byte
|
||||
if _, err := rand.Read(raw[:]); err != nil {
|
||||
return "", fmt.Errorf("replay nonce: %w", err)
|
||||
}
|
||||
nonce := hex.EncodeToString(raw[:])
|
||||
replayNonces.mu.Lock()
|
||||
if replayNonces.live == nil {
|
||||
replayNonces.live = map[string]struct{}{}
|
||||
}
|
||||
replayNonces.live[nonce] = struct{}{}
|
||||
replayNonces.mu.Unlock()
|
||||
return nonce, nil
|
||||
}
|
||||
|
||||
func deleteReplayNonce(nonce string) {
|
||||
replayNonces.mu.Lock()
|
||||
delete(replayNonces.live, nonce)
|
||||
replayNonces.mu.Unlock()
|
||||
}
|
||||
|
||||
func consumeReplayNonce(nonce string) bool {
|
||||
if nonce == "" {
|
||||
return false
|
||||
}
|
||||
replayNonces.mu.Lock()
|
||||
_, ok := replayNonces.live[nonce]
|
||||
if ok {
|
||||
delete(replayNonces.live, nonce)
|
||||
}
|
||||
replayNonces.mu.Unlock()
|
||||
return ok
|
||||
}
|
||||
|
||||
func replayNonceCount() int {
|
||||
replayNonces.mu.Lock()
|
||||
n := len(replayNonces.live)
|
||||
replayNonces.mu.Unlock()
|
||||
return n
|
||||
}
|
||||
|
||||
// ErrReplayInvalid marks a replay request rejected before anything is sent.
|
||||
var ErrReplayInvalid = errors.New("invalid replay request")
|
||||
|
||||
@@ -45,10 +101,11 @@ type ReplayHeader struct {
|
||||
|
||||
// ReplayRequest is the edited request to send.
|
||||
type ReplayRequest struct {
|
||||
Method string
|
||||
URL string
|
||||
Headers []ReplayHeader
|
||||
Body []byte
|
||||
Method string
|
||||
URL string
|
||||
Headers []ReplayHeader
|
||||
Body []byte
|
||||
ApplyRules bool
|
||||
}
|
||||
|
||||
// ReplayResult is the response as received by the client side of the proxy.
|
||||
@@ -78,6 +135,7 @@ var replayDropped = map[string]bool{
|
||||
"Te": true,
|
||||
"Trailer": true,
|
||||
"Expect": true,
|
||||
http.CanonicalHeaderKey(replayNonceHeader): true,
|
||||
}
|
||||
|
||||
// replayRecorded reports whether the proxy records a request to scheme://host:
|
||||
@@ -217,6 +275,15 @@ func (t *Traffic) Replay(ctx context.Context, req ReplayRequest) (ReplayResult,
|
||||
httpReq.Host = host
|
||||
}
|
||||
|
||||
if !req.ApplyRules && t.replayRecorded(target.Scheme, target.Host) {
|
||||
nonce, err := registerReplayNonce()
|
||||
if err != nil {
|
||||
return ReplayResult{}, err
|
||||
}
|
||||
defer deleteReplayNonce(nonce)
|
||||
httpReq.Header.Set(replayNonceHeader, nonce)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/traffic/rewrite"
|
||||
)
|
||||
|
||||
func advHeaderValue(hs []ReplayHeader, name string) string {
|
||||
for _, h := range hs {
|
||||
if strings.EqualFold(h.Name, name) {
|
||||
return h.Value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// The internal header is stripped whatever its casing and however many
|
||||
// occurrences are sent, and it never bypasses rules without a live nonce.
|
||||
func TestAdversarialReplayNonceHeaderDuplicatesStripped(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var seen []string
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
seen = append([]string{}, r.Header.Values(replayNonceHeader)...)
|
||||
mu.Unlock()
|
||||
w.Header().Set("X-Seen-Rule", r.Header.Get("X-Rule"))
|
||||
_, _ = io.WriteString(w, "ok")
|
||||
}))
|
||||
t.Cleanup(target.Close)
|
||||
|
||||
tr, _, proxyURL := startProxy(t)
|
||||
tr.SetRewriteRules(rules(t, nil, nil,
|
||||
rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Rule", Value: "applied"},
|
||||
}},
|
||||
rewrite.Rule{ID: 2, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Rule-Resp", Value: "applied"},
|
||||
}},
|
||||
))
|
||||
|
||||
h := http.Header{}
|
||||
h[replayNonceHeader] = []string{"aaaa", "bbbb"}
|
||||
h["x-artex-replay-nonce"] = []string{"cccc"}
|
||||
resp, _ := sendThrough(t, tr, proxyURL, http.MethodGet, target.URL+"/dup", "", h)
|
||||
|
||||
mu.Lock()
|
||||
got := append([]string{}, seen...)
|
||||
mu.Unlock()
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("target saw nonce header values %v", got)
|
||||
}
|
||||
if resp.Header.Get("X-Rule-Resp") != "applied" {
|
||||
t.Fatalf("duplicate forged nonce bypassed the response rule: %v", resp.Header)
|
||||
}
|
||||
if resp.Header.Get("X-Seen-Rule") != "applied" {
|
||||
t.Fatalf("duplicate forged nonce bypassed the request rule: %v", resp.Header)
|
||||
}
|
||||
|
||||
id := waitExchangeIDBySuffix(t, tr, "/dup")
|
||||
reqText, _, err := tr.Get(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(reqText), strings.ToLower(replayNonceHeader)) {
|
||||
t.Fatalf("recorded request kept a nonce header:\n%s", reqText)
|
||||
}
|
||||
}
|
||||
|
||||
// A nonce is single use, and takeReplayNonce refuses to return a value when
|
||||
// more than one occurrence is present.
|
||||
func TestAdversarialReplayNonceRegistrySingleUse(t *testing.T) {
|
||||
n, err := registerReplayNonce()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(n) != 64 {
|
||||
t.Fatalf("nonce length = %d, want 64 hex chars", len(n))
|
||||
}
|
||||
if !consumeReplayNonce(n) {
|
||||
t.Fatal("first consume did not match")
|
||||
}
|
||||
if consumeReplayNonce(n) {
|
||||
t.Fatal("second consume matched: nonce is not single use")
|
||||
}
|
||||
if consumeReplayNonce("") {
|
||||
t.Fatal("empty nonce matched")
|
||||
}
|
||||
|
||||
h := http.Header{}
|
||||
h.Set(replayNonceHeader, n)
|
||||
if got, present := takeReplayNonce(h); !present || got != n || len(h) != 0 {
|
||||
t.Fatalf("single occurrence = (%q, %v, %d), want (%q, true, 0)", got, present, len(h), n)
|
||||
}
|
||||
|
||||
h2 := http.Header{replayNonceHeader: {"a", "b"}, "x-artex-replay-nonce": {"c"}}
|
||||
if got, present := takeReplayNonce(h2); !present || got != "" || len(h2) != 0 {
|
||||
t.Fatalf("duplicates = (%q, %v, %d), want (\"\", true, 0)", got, present, len(h2))
|
||||
}
|
||||
|
||||
h3 := http.Header{replayNonceHeader: {}}
|
||||
if got, present := takeReplayNonce(h3); !present || got != "" || len(h3) != 0 {
|
||||
t.Fatalf("empty values = (%q, %v, %d), want (\"\", true, 0)", got, present, len(h3))
|
||||
}
|
||||
}
|
||||
|
||||
// A caller-supplied nonce header is dropped in both modes and never wins over
|
||||
// the real nonce: default still bypasses, apply_rules still applies.
|
||||
func TestAdversarialReplayForgedNonceHeaderCannotSteerReplay(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var seen []string
|
||||
var rule string
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
rule = r.Header.Get("X-Rule")
|
||||
seen = append([]string{}, r.Header.Values(replayNonceHeader)...)
|
||||
mu.Unlock()
|
||||
w.Header().Set("X-Seen-Rule", r.Header.Get("X-Rule"))
|
||||
_, _ = io.WriteString(w, "ok")
|
||||
}))
|
||||
t.Cleanup(target.Close)
|
||||
|
||||
tr, _, _ := startProxy(t)
|
||||
tr.SetRewriteRules(rules(t, nil, nil,
|
||||
rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Rule", Value: "applied"},
|
||||
}},
|
||||
))
|
||||
|
||||
forged := ReplayHeader{Name: replayNonceHeader, Value: strings.Repeat("ab", 32)}
|
||||
|
||||
res, err := tr.Replay(context.Background(), ReplayRequest{
|
||||
Method: http.MethodGet,
|
||||
URL: target.URL + "/forged-bypass",
|
||||
Headers: []ReplayHeader{forged},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("default replay: %v", err)
|
||||
}
|
||||
mu.Lock()
|
||||
if rule != "" || len(seen) != 0 {
|
||||
t.Fatalf("default replay: target saw X-Rule=%q nonce=%v", rule, seen)
|
||||
}
|
||||
mu.Unlock()
|
||||
if advHeaderValue(res.Headers, "X-Seen-Rule") != "" {
|
||||
t.Fatalf("default replay with forged header did not bypass: %v", res.Headers)
|
||||
}
|
||||
|
||||
res2, err := tr.Replay(context.Background(), ReplayRequest{
|
||||
Method: http.MethodGet,
|
||||
URL: target.URL + "/forged-apply",
|
||||
Headers: []ReplayHeader{forged},
|
||||
ApplyRules: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply replay: %v", err)
|
||||
}
|
||||
mu.Lock()
|
||||
if rule != "applied" || len(seen) != 0 {
|
||||
t.Fatalf("apply replay: target saw X-Rule=%q nonce=%v", rule, seen)
|
||||
}
|
||||
mu.Unlock()
|
||||
if advHeaderValue(res2.Headers, "X-Seen-Rule") != "applied" {
|
||||
t.Fatalf("apply replay did not apply the rule: %v", res2.Headers)
|
||||
}
|
||||
if replayNonceCount() != 0 {
|
||||
t.Fatalf("nonce registry not empty: %d", replayNonceCount())
|
||||
}
|
||||
}
|
||||
|
||||
// A bypassing replay and an applying replay to the same target at the same
|
||||
// time must not share state, and a normal request after a bypass must still be
|
||||
// rewritten.
|
||||
func TestAdversarialReplayConcurrentBypassAndApply(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
seen := map[string]string{}
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
seen[r.URL.Path] = r.Header.Get("X-Rule")
|
||||
mu.Unlock()
|
||||
w.Header().Set("X-Seen-Rule", r.Header.Get("X-Rule"))
|
||||
_, _ = io.WriteString(w, "ok")
|
||||
}))
|
||||
t.Cleanup(target.Close)
|
||||
|
||||
tr, _, pxy := startProxy(t)
|
||||
tr.SetRewriteRules(rules(t, nil, nil,
|
||||
rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Rule", Value: "applied"},
|
||||
}},
|
||||
))
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var offRule, onRule string
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
res, err := tr.Replay(context.Background(), ReplayRequest{Method: http.MethodGet, URL: target.URL + "/conc-off"})
|
||||
if err != nil {
|
||||
t.Errorf("bypass replay: %v", err)
|
||||
return
|
||||
}
|
||||
offRule = advHeaderValue(res.Headers, "X-Seen-Rule")
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
res, err := tr.Replay(context.Background(), ReplayRequest{Method: http.MethodGet, URL: target.URL + "/conc-on", ApplyRules: true})
|
||||
if err != nil {
|
||||
t.Errorf("apply replay: %v", err)
|
||||
return
|
||||
}
|
||||
onRule = advHeaderValue(res.Headers, "X-Seen-Rule")
|
||||
}()
|
||||
wg.Wait()
|
||||
if offRule != "" {
|
||||
t.Fatalf("bypass replay saw X-Rule=%q", offRule)
|
||||
}
|
||||
if onRule != "applied" {
|
||||
t.Fatalf("apply replay saw X-Rule=%q", onRule)
|
||||
}
|
||||
if replayNonceCount() != 0 {
|
||||
t.Fatalf("nonce registry not empty after concurrent replays: %d", replayNonceCount())
|
||||
}
|
||||
|
||||
// A plain request after the bypass is still rewritten: the marker is per flow.
|
||||
resp, _ := sendThrough(t, tr, pxy, http.MethodGet, target.URL+"/after", "", nil)
|
||||
if resp.Header.Get("X-Seen-Rule") != "applied" {
|
||||
t.Fatalf("normal request after a bypass was not rewritten: %v", resp.Header)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if seen["/conc-off"] != "" || seen["/conc-on"] != "applied" || seen["/after"] != "applied" {
|
||||
t.Fatalf("target saw %v", seen)
|
||||
}
|
||||
}
|
||||
|
||||
// An SSE response is bypassed by default and rewritten when opted in: the
|
||||
// stream phase is the path the plain response test cannot reach.
|
||||
func TestAdversarialReplayBypassCoversSSE(t *testing.T) {
|
||||
tr, _, _ := startProxy(t)
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.WriteHeader(200)
|
||||
fl := w.(http.Flusher)
|
||||
for _, ev := range []string{"data: one\n\n", "data: two\n\n"} {
|
||||
_, _ = io.WriteString(w, ev)
|
||||
fl.Flush()
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
tr.SetRewriteRules(rules(t, nil, nil,
|
||||
rewrite.Rule{ID: 1, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Rule-Resp", Value: "applied"},
|
||||
}},
|
||||
))
|
||||
|
||||
res, err := tr.Replay(context.Background(), ReplayRequest{
|
||||
Method: http.MethodGet,
|
||||
URL: localhostURL(srv.URL) + "/sse-off",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("bypass SSE replay: %v", err)
|
||||
}
|
||||
if got := advHeaderValue(res.Headers, "X-Rule-Resp"); got != "" {
|
||||
t.Fatalf("bypass SSE replay applied header rule: %q", got)
|
||||
}
|
||||
if !strings.Contains(res.Raw, "data: two") {
|
||||
t.Fatalf("bypass SSE body changed: %s", res.Raw)
|
||||
}
|
||||
idOff := waitExchangeIDBySuffix(t, tr, "/sse-off")
|
||||
if rewritten, ids, err := tr.RewriteInfo(idOff); err != nil || rewritten || len(ids) != 0 {
|
||||
t.Fatalf("bypass SSE RewriteInfo = (%v, %v, %v)", rewritten, ids, err)
|
||||
}
|
||||
|
||||
res2, err := tr.Replay(context.Background(), ReplayRequest{
|
||||
Method: http.MethodGet,
|
||||
URL: localhostURL(srv.URL) + "/sse-on",
|
||||
ApplyRules: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("apply SSE replay: %v", err)
|
||||
}
|
||||
if got := advHeaderValue(res2.Headers, "X-Rule-Resp"); got != "applied" {
|
||||
t.Fatalf("apply SSE replay header rule = %q, want applied", got)
|
||||
}
|
||||
idOn := waitExchangeIDBySuffix(t, tr, "/sse-on")
|
||||
if rows := traceRows(t, tr, idOn); len(rows) == 0 {
|
||||
t.Fatal("apply SSE replay recorded no rewrite trace")
|
||||
}
|
||||
}
|
||||
|
||||
// A cancelled replay leaves no live nonce and no permanent bypass marker.
|
||||
func TestAdversarialReplayCancelCleansUp(t *testing.T) {
|
||||
block := make(chan struct{})
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
<-block
|
||||
_, _ = io.WriteString(w, "late")
|
||||
}))
|
||||
t.Cleanup(target.Close)
|
||||
|
||||
tr, s, _ := startProxy(t)
|
||||
tr.SetRewriteRules(rules(t, nil, nil,
|
||||
rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Rule", Value: "applied"},
|
||||
}},
|
||||
))
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
if _, err := tr.Replay(ctx, ReplayRequest{Method: http.MethodGet, URL: target.URL + "/cancel"}); err == nil {
|
||||
t.Fatal("cancelled replay returned no error")
|
||||
}
|
||||
close(block)
|
||||
if replayNonceCount() != 0 {
|
||||
t.Fatalf("nonce registry leaked after cancel: %d", replayNonceCount())
|
||||
}
|
||||
waitFor(t, "bypass marker cleanup", func() bool {
|
||||
empty := true
|
||||
s.bypass.Range(func(_, _ any) bool {
|
||||
empty = false
|
||||
return false
|
||||
})
|
||||
return empty
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
package traffic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/traffic/rewrite"
|
||||
mproxy "github.com/lqqyt2423/go-mitmproxy/proxy"
|
||||
)
|
||||
|
||||
func hasNonceHeader(h http.Header) bool {
|
||||
for k := range h {
|
||||
if strings.EqualFold(k, replayNonceHeader) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func waitExchangeIDBySuffix(t *testing.T, tr *Traffic, suffix string) string {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
var id string
|
||||
err := tr.DB().QueryRow(`SELECT id FROM exchanges WHERE url LIKE ?`, "%"+suffix).Scan(&id)
|
||||
if err == nil {
|
||||
return id
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("exchange %s not recorded: %v", suffix, err)
|
||||
}
|
||||
time.Sleep(15 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplayBypassesRewriteRulesByDefault(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var gotMethod, gotBody, gotOrig, gotRuleReq string
|
||||
var sawNonce bool
|
||||
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
mu.Lock()
|
||||
gotMethod = r.Method
|
||||
gotBody = string(b)
|
||||
gotOrig = r.Header.Get("X-Orig")
|
||||
gotRuleReq = r.Header.Get("X-Rule-Req")
|
||||
sawNonce = hasNonceHeader(r.Header)
|
||||
mu.Unlock()
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
_, _ = io.WriteString(w, "target-resp")
|
||||
}))
|
||||
t.Cleanup(target.Close)
|
||||
|
||||
tr, s, _ := startProxy(t)
|
||||
tr.SetRewriteRules(rules(t, nil, nil,
|
||||
rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Rule-Req", Value: "applied"},
|
||||
{Type: rewrite.ActionSetBody, Value: "rewritten-req"},
|
||||
}},
|
||||
rewrite.Rule{ID: 2, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Rule-Resp", Value: "applied"},
|
||||
{Type: rewrite.ActionReplaceBody, Regex: "target-resp", Value: "rewritten-resp"},
|
||||
}},
|
||||
))
|
||||
|
||||
// ApplyRules omitted (false by default) -> bypass both phases.
|
||||
res, err := tr.Replay(context.Background(), ReplayRequest{
|
||||
Method: http.MethodPost,
|
||||
URL: target.URL + "/default-bypass",
|
||||
Headers: []ReplayHeader{{Name: "X-Orig", Value: "1"}},
|
||||
Body: []byte("orig-body"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
if replayNonceCount() != 0 {
|
||||
t.Fatalf("nonce registry not empty after Replay: %d", replayNonceCount())
|
||||
}
|
||||
mu.Lock()
|
||||
if gotMethod != http.MethodPost || gotBody != "orig-body" || gotOrig != "1" || gotRuleReq != "" || sawNonce {
|
||||
t.Fatalf("target received method=%q body=%q X-Orig=%q X-Rule-Req=%q sawNonce=%v", gotMethod, gotBody, gotOrig, gotRuleReq, sawNonce)
|
||||
}
|
||||
mu.Unlock()
|
||||
if res.Status != 200 || !res.Recorded || strings.Contains(res.Raw, "X-Rule-Resp") || !strings.HasSuffix(res.Raw, "target-resp") {
|
||||
t.Fatalf("client received modified response: %+v", res)
|
||||
}
|
||||
|
||||
id := waitExchangeIDBySuffix(t, tr, "/default-bypass")
|
||||
if rewritten, ids, err := tr.RewriteInfo(id); err != nil || rewritten || len(ids) != 0 {
|
||||
t.Fatalf("RewriteInfo = (%v, %v, %v), want (false, nil, nil)", rewritten, ids, err)
|
||||
}
|
||||
var traceCount int
|
||||
if err := tr.DB().QueryRow(`SELECT COUNT(*) FROM exchange_rewrites WHERE exchange_id=?`, id).Scan(&traceCount); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if traceCount != 0 {
|
||||
t.Fatalf("exchange_rewrites rows = %d, want 0", traceCount)
|
||||
}
|
||||
reqText, _, err := tr.Get(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(reqText), strings.ToLower(replayNonceHeader)) {
|
||||
t.Fatalf("recorded request leaked internal nonce header:\n%s", reqText)
|
||||
}
|
||||
|
||||
// Response-only rules must also be bypassed and the nonce header stripped.
|
||||
tr.SetRewriteRules(rules(t, nil, nil,
|
||||
rewrite.Rule{ID: 3, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Rule-Resp", Value: "applied"},
|
||||
}},
|
||||
))
|
||||
res2, err := tr.Replay(context.Background(), ReplayRequest{
|
||||
Method: http.MethodGet,
|
||||
URL: target.URL + "/resp-only-bypass",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Replay response-only: %v", err)
|
||||
}
|
||||
mu.Lock()
|
||||
if sawNonce {
|
||||
t.Fatal("target saw nonce header when only response rules were active")
|
||||
}
|
||||
mu.Unlock()
|
||||
if strings.Contains(res2.Raw, "X-Rule-Resp") {
|
||||
t.Fatalf("response-only rule was not bypassed: %s", res2.Raw)
|
||||
}
|
||||
id2 := waitExchangeIDBySuffix(t, tr, "/resp-only-bypass")
|
||||
if rewritten, _, err := tr.RewriteInfo(id2); err != nil || rewritten {
|
||||
t.Fatalf("response-only bypass RewriteInfo = %v %v", rewritten, err)
|
||||
}
|
||||
|
||||
// RequestError clears the bypass marker.
|
||||
f := withID(&mproxy.Flow{}, 99)
|
||||
s.bypass.Store(keyOf(f), struct{}{})
|
||||
s.RequestError(f, io.EOF)
|
||||
if _, ok := s.bypass.Load(keyOf(f)); ok {
|
||||
t.Fatal("RequestError did not clear bypass marker")
|
||||
}
|
||||
|
||||
// Failed Replay (unreachable target) still leaves nonce registry empty.
|
||||
_, _ = tr.Replay(context.Background(), ReplayRequest{
|
||||
Method: http.MethodGet,
|
||||
URL: "http://127.0.0.1:1/unreachable",
|
||||
})
|
||||
if replayNonceCount() != 0 {
|
||||
t.Fatalf("nonce registry leaked after failed Replay: %d", replayNonceCount())
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplayApplyRulesTrue(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var gotBody, gotRuleReq string
|
||||
var sawNonce bool
|
||||
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
mu.Lock()
|
||||
gotBody = string(b)
|
||||
gotRuleReq = r.Header.Get("X-Rule-Req")
|
||||
sawNonce = hasNonceHeader(r.Header)
|
||||
mu.Unlock()
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
_, _ = io.WriteString(w, "target-resp")
|
||||
}))
|
||||
t.Cleanup(target.Close)
|
||||
|
||||
tr, _, _ := startProxy(t)
|
||||
tr.SetRewriteRules(rules(t, nil, nil,
|
||||
rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Rule-Req", Value: "applied"},
|
||||
{Type: rewrite.ActionSetBody, Value: "rewritten-req"},
|
||||
}},
|
||||
rewrite.Rule{ID: 2, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Rule-Resp", Value: "applied"},
|
||||
{Type: rewrite.ActionReplaceBody, Regex: "target-resp", Value: "rewritten-resp"},
|
||||
}},
|
||||
))
|
||||
|
||||
res, err := tr.Replay(context.Background(), ReplayRequest{
|
||||
Method: http.MethodPost,
|
||||
URL: target.URL + "/apply-rules",
|
||||
Body: []byte("orig-body"),
|
||||
ApplyRules: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Replay(ApplyRules=true): %v", err)
|
||||
}
|
||||
if replayNonceCount() != 0 {
|
||||
t.Fatalf("nonce registry not empty after Replay(ApplyRules=true): %d", replayNonceCount())
|
||||
}
|
||||
mu.Lock()
|
||||
if gotBody != "rewritten-req" || gotRuleReq != "applied" || sawNonce {
|
||||
t.Fatalf("target received body=%q X-Rule-Req=%q sawNonce=%v", gotBody, gotRuleReq, sawNonce)
|
||||
}
|
||||
mu.Unlock()
|
||||
if !strings.Contains(res.Raw, "X-Rule-Resp: applied") || !strings.HasSuffix(res.Raw, "rewritten-resp") {
|
||||
t.Fatalf("response rule did not apply on Replay(ApplyRules=true): %s", res.Raw)
|
||||
}
|
||||
|
||||
id := waitExchangeIDBySuffix(t, tr, "/apply-rules")
|
||||
rewritten, ids, err := tr.RewriteInfo(id)
|
||||
if err != nil || !rewritten || !slices.Equal(ids, []int64{1, 2}) {
|
||||
t.Fatalf("RewriteInfo = (%v, %v, %v), want (true, [1 2], nil)", rewritten, ids, err)
|
||||
}
|
||||
if rows := traceRows(t, tr, id); len(rows) == 0 {
|
||||
t.Fatal("expected exchange_rewrites trace rows when ApplyRules=true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgedReplayNonceHeaderStrippedAndIgnored(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var gotRuleReq string
|
||||
var sawNonce bool
|
||||
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
gotRuleReq = r.Header.Get("X-Rule-Req")
|
||||
sawNonce = hasNonceHeader(r.Header)
|
||||
mu.Unlock()
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
_, _ = io.WriteString(w, "target-resp")
|
||||
}))
|
||||
t.Cleanup(target.Close)
|
||||
|
||||
tr, _, proxyURL := startProxy(t)
|
||||
tr.SetRewriteRules(rules(t, nil, nil,
|
||||
rewrite.Rule{ID: 1, Phase: rewrite.PhaseRequest, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Rule-Req", Value: "applied"},
|
||||
}},
|
||||
rewrite.Rule{ID: 2, Phase: rewrite.PhaseResponse, Actions: []rewrite.Action{
|
||||
{Type: rewrite.ActionSetHeader, Name: "X-Rule-Resp", Value: "applied"},
|
||||
}},
|
||||
))
|
||||
|
||||
resp, _ := sendThrough(t, tr, proxyURL, http.MethodGet, target.URL+"/forged", "", http.Header{
|
||||
replayNonceHeader: {"0000000000000000000000000000000000000000000000000000000000000000"},
|
||||
})
|
||||
mu.Lock()
|
||||
if sawNonce {
|
||||
t.Fatal("forged replay nonce header reached target")
|
||||
}
|
||||
if gotRuleReq != "applied" {
|
||||
t.Fatalf("forged nonce bypassed request rule: X-Rule-Req=%q", gotRuleReq)
|
||||
}
|
||||
mu.Unlock()
|
||||
if resp.Header.Get("X-Rule-Resp") != "applied" {
|
||||
t.Fatalf("forged nonce bypassed response rule: X-Rule-Resp=%q", resp.Header.Get("X-Rule-Resp"))
|
||||
}
|
||||
|
||||
id := waitExchangeIDBySuffix(t, tr, "/forged")
|
||||
rewritten, ids, err := tr.RewriteInfo(id)
|
||||
if err != nil || !rewritten || !slices.Equal(ids, []int64{1, 2}) {
|
||||
t.Fatalf("forged nonce RewriteInfo = (%v, %v, %v), want (true, [1 2], nil)", rewritten, ids, err)
|
||||
}
|
||||
reqText, _, err := tr.Get(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(reqText), strings.ToLower(replayNonceHeader)) {
|
||||
t.Fatalf("recorded request kept forged nonce header:\n%s", reqText)
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -420,6 +421,32 @@ func engineRequest(r *mproxy.Request) rewrite.Request {
|
||||
return rewrite.Request{Method: r.Method, URL: r.URL, Header: r.Header, Body: r.Body}
|
||||
}
|
||||
|
||||
// takeReplayNonce removes every occurrence of X-ARTEX-Replay-Nonce from h
|
||||
// unconditionally and returns its value only when exactly one header value was
|
||||
// present.
|
||||
func takeReplayNonce(h http.Header) (nonce string, present bool) {
|
||||
if h == nil {
|
||||
return "", false
|
||||
}
|
||||
count := 0
|
||||
for k, vs := range h {
|
||||
if strings.EqualFold(k, replayNonceHeader) {
|
||||
present = true
|
||||
for _, v := range vs {
|
||||
count++
|
||||
if count == 1 {
|
||||
nonce = strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
delete(h, k)
|
||||
}
|
||||
}
|
||||
if count != 1 {
|
||||
return "", present
|
||||
}
|
||||
return nonce, present
|
||||
}
|
||||
|
||||
// Request runs the request phase. The result goes on the flow only once the
|
||||
// request's snapshot slot is reserved; without one the request leaves as the
|
||||
// client sent it. Rules fail open: a panic leaves the request unchanged.
|
||||
@@ -427,6 +454,12 @@ func (s *sink) Request(f *mproxy.Flow) {
|
||||
if s.t == nil || f.Request == nil || f.Request.URL == nil {
|
||||
return
|
||||
}
|
||||
// Strip the internal replay bypass header unconditionally before forwarding
|
||||
// or running rules; honour it only when its value matches a registered nonce.
|
||||
if nonce, present := takeReplayNonce(f.Request.Header); present && consumeReplayNonce(nonce) {
|
||||
s.bypass.Store(keyOf(f), struct{}{})
|
||||
return
|
||||
}
|
||||
set := s.t.rewriteRules.Load()
|
||||
if set.Len(rewrite.PhaseRequest) == 0 {
|
||||
return
|
||||
@@ -459,6 +492,10 @@ func (s *sink) Response(f *mproxy.Flow) {
|
||||
if f.Request == nil || f.Response == nil {
|
||||
return
|
||||
}
|
||||
if _, bypassed := s.bypass.LoadAndDelete(keyOf(f)); bypassed {
|
||||
s.t.record(f)
|
||||
return
|
||||
}
|
||||
snap := s.t.snaps.take(keyOf(f))
|
||||
set := s.t.rewriteRules.Load()
|
||||
if snap != nil {
|
||||
@@ -506,6 +543,13 @@ func (s *sink) applyResponse(f *mproxy.Flow, set *rewrite.Set, in rewrite.Input)
|
||||
// upgrade, with no response on the flow: nothing of the handshake can change
|
||||
// there, so every candidate rule is traced "not rewritten: stream".
|
||||
func (s *sink) rewriteStream(f *mproxy.Flow, kind string) (rq requestView, rs responseView, lines []rewrite.TraceLine, res rewrite.Result) {
|
||||
if _, bypassed := s.bypass.LoadAndDelete(keyOf(f)); bypassed {
|
||||
rq = flowRequestView(f.Request)
|
||||
if kind == "sse" {
|
||||
rs = flowResponseView(f.Response)
|
||||
}
|
||||
return rq, rs, nil, rewrite.Result{}
|
||||
}
|
||||
snap := s.t.snaps.take(keyOf(f))
|
||||
set := s.t.rewriteRules.Load()
|
||||
if snap != nil {
|
||||
@@ -568,6 +612,7 @@ func (s *sink) applyStreamResponse(f *mproxy.Flow, res rewrite.Result) {
|
||||
// and the rules, never the URL or the error text.
|
||||
func (s *sink) RequestError(f *mproxy.Flow, err error) {
|
||||
if f != nil {
|
||||
s.bypass.Delete(keyOf(f))
|
||||
if snap := s.t.snaps.take(keyOf(f)); snap != nil && snap.requestRewritten() {
|
||||
ids := snap.appliedRuleIDs()
|
||||
for _, id := range ids {
|
||||
@@ -593,6 +638,7 @@ func (s *sink) StreamResponseModifier(f *mproxy.Flow, in io.Reader) (out io.Read
|
||||
if s == nil || s.t == nil || f == nil || !f.Stream || f.SSE != nil {
|
||||
return in
|
||||
}
|
||||
s.bypass.Delete(keyOf(f))
|
||||
snap := s.t.snaps.take(keyOf(f))
|
||||
if snap == nil {
|
||||
return in
|
||||
|
||||
+64
-31
@@ -469,6 +469,7 @@ type sink struct {
|
||||
t *Traffic
|
||||
paced sync.Map // ClientConn.Id of a marked connection -> struct{}
|
||||
streams sync.Map // *mproxy.Flow of a live WebSocket/SSE stream -> *streamState (stream.go)
|
||||
bypass sync.Map // flowKey of a replay flow bypassing rewrite rules -> struct{}
|
||||
}
|
||||
|
||||
// maybePassthrough marks a host to be tunneled transparently on the next
|
||||
@@ -979,7 +980,7 @@ func clampBlobRange(offset, length, size int64) (off, n int64, empty bool) {
|
||||
return offset, length, false
|
||||
}
|
||||
|
||||
// ExchangeMeta is one row of the index (returned by Search).
|
||||
// ExchangeMeta is one row of the index (returned by Search and Page).
|
||||
type ExchangeMeta struct {
|
||||
ID string `json:"id"`
|
||||
TS int64 `json:"ts"`
|
||||
@@ -991,14 +992,17 @@ type ExchangeMeta struct {
|
||||
ContentType string `json:"content_type"`
|
||||
RespLen int `json:"resp_len"`
|
||||
Path string `json:"path"`
|
||||
Rewritten bool `json:"rewritten"`
|
||||
}
|
||||
|
||||
const rewrittenExistsSQL = `EXISTS(SELECT 1 FROM exchange_rewrites r WHERE r.exchange_id=exchanges.id AND r.outcome='applied')`
|
||||
|
||||
// Search returns paged exchange metadata (never bodies).
|
||||
func (t *Traffic) Search(host string, page, size int) ([]ExchangeMeta, error) {
|
||||
if size <= 0 || size > 500 {
|
||||
size = 100
|
||||
}
|
||||
q := `SELECT id,ts,host,method,url_template,url,status,content_type,resp_len,path FROM exchanges`
|
||||
q := `SELECT id,ts,host,method,url_template,url,status,content_type,resp_len,path,` + rewrittenExistsSQL + ` FROM exchanges`
|
||||
args := []any{}
|
||||
if host != "" {
|
||||
q += ` WHERE host=?`
|
||||
@@ -1014,7 +1018,7 @@ func (t *Traffic) Search(host string, page, size int) ([]ExchangeMeta, error) {
|
||||
var out []ExchangeMeta
|
||||
for rows.Next() {
|
||||
var m ExchangeMeta
|
||||
if err := rows.Scan(&m.ID, &m.TS, &m.Host, &m.Method, &m.URLTemplate, &m.URL, &m.Status, &m.ContentType, &m.RespLen, &m.Path); err != nil {
|
||||
if err := rows.Scan(&m.ID, &m.TS, &m.Host, &m.Method, &m.URLTemplate, &m.URL, &m.Status, &m.ContentType, &m.RespLen, &m.Path, &m.Rewritten); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, m)
|
||||
@@ -1041,21 +1045,41 @@ func ftsQuote(s string) string {
|
||||
return `"` + strings.ReplaceAll(s, `"`, `""`) + `"`
|
||||
}
|
||||
|
||||
const (
|
||||
RewriteFilterAll = "all"
|
||||
RewriteFilterRewritten = "rewritten"
|
||||
RewriteFilterOriginal = "original"
|
||||
)
|
||||
|
||||
// PageQuery bundles the optional filters and sort order for Page. Every filter
|
||||
// is combined with AND; an empty string skips that filter, and RespMin/RespMax
|
||||
// of -1 skip the response-size bounds. An empty Sort/Order defaults to
|
||||
// newest-first by timestamp.
|
||||
type PageQuery struct {
|
||||
Host string // host substring
|
||||
Method string // exact method (case-insensitive)
|
||||
Query string // broad search box: metadata columns OR captured text
|
||||
Body string // content box: captured request/response text only (full-text)
|
||||
Path string // url_template substring — the vulnerability's path
|
||||
Status string // exact code ("404") or class bucket ("4xx")
|
||||
RespMin int64 // minimum resp_len, or -1 when unset
|
||||
RespMax int64 // maximum resp_len, or -1 when unset
|
||||
Sort string // ts | status | resp_len (default ts)
|
||||
Order string // asc | desc (default desc)
|
||||
Host string // host substring
|
||||
Method string // exact method (case-insensitive)
|
||||
Query string // broad search box: metadata columns OR captured text
|
||||
Body string // content box: captured request/response text only (full-text)
|
||||
Path string // url_template substring — the vulnerability's path
|
||||
Status string // exact code ("404") or class bucket ("4xx")
|
||||
Rewritten string // all (default) | rewritten (or true) | original (or false)
|
||||
RespMin int64 // minimum resp_len, or -1 when unset
|
||||
RespMax int64 // maximum resp_len, or -1 when unset
|
||||
Sort string // ts | status | resp_len (default ts)
|
||||
Order string // asc | desc (default desc)
|
||||
}
|
||||
|
||||
// rewriteFilterSQL turns a rewritten filter token into a SQL condition over
|
||||
// exchange_rewrites. Empty, "all", or unrecognized values default to no filter.
|
||||
func rewriteFilterSQL(s string) (cond string, ok bool) {
|
||||
switch strings.ToLower(strings.TrimSpace(s)) {
|
||||
case RewriteFilterRewritten, "true", "1":
|
||||
return rewrittenExistsSQL, true
|
||||
case RewriteFilterOriginal, "false", "0":
|
||||
return "NOT " + rewrittenExistsSQL, true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// sortColumns whitelists the columns Page may order by, so the caller-supplied
|
||||
@@ -1080,22 +1104,10 @@ func statusFilter(s string) (cond string, args []any, ok bool) {
|
||||
return "", nil, false
|
||||
}
|
||||
|
||||
// Page returns one page of exchange metadata for the traffic list, filtered by
|
||||
// the criteria in f (all optional, combined with AND) and ordered by the
|
||||
// requested column. Query is the broad search box (metadata columns OR, when the
|
||||
// term is long enough for the trigram index, captured request/response text);
|
||||
// Body narrows to exchanges whose captured text matches, via that same index.
|
||||
// Also returns the total number of rows matching the filter (for the UI's
|
||||
// pagination). Bodies are never included in the rows.
|
||||
func (t *Traffic) Page(f PageQuery, page, size int) (rows []ExchangeMeta, total int, err error) {
|
||||
if size <= 0 || size > 500 {
|
||||
size = 100
|
||||
}
|
||||
if page < 0 {
|
||||
page = 0
|
||||
}
|
||||
where := ""
|
||||
var args []any
|
||||
// pageWhere builds the SQL WHERE clause and bound arguments for PageQuery.
|
||||
// Page and ExportHAR share this so list queries and HAR exports honor the exact
|
||||
// same filter semantics.
|
||||
func (t *Traffic) pageWhere(f PageQuery) (where string, args []any) {
|
||||
add := func(cond string, vs ...any) {
|
||||
if where == "" {
|
||||
where = " WHERE "
|
||||
@@ -1117,6 +1129,9 @@ func (t *Traffic) Page(f PageQuery, page, size int) (rows []ExchangeMeta, total
|
||||
if cond, sargs, ok := statusFilter(f.Status); ok {
|
||||
add(cond, sargs...)
|
||||
}
|
||||
if cond, ok := rewriteFilterSQL(f.Rewritten); ok {
|
||||
add(cond)
|
||||
}
|
||||
if f.RespMin >= 0 {
|
||||
add("resp_len>=?", f.RespMin)
|
||||
}
|
||||
@@ -1143,6 +1158,24 @@ func (t *Traffic) Page(f PageQuery, page, size int) (rows []ExchangeMeta, total
|
||||
add(cond, arg)
|
||||
}
|
||||
}
|
||||
return where, args
|
||||
}
|
||||
|
||||
// Page returns one page of exchange metadata for the traffic list, filtered by
|
||||
// the criteria in f (all optional, combined with AND) and ordered by the
|
||||
// requested column. Query is the broad search box (metadata columns OR, when the
|
||||
// term is long enough for the trigram index, captured request/response text);
|
||||
// Body narrows to exchanges whose captured text matches, via that same index.
|
||||
// Also returns the total number of rows matching the filter (for the UI's
|
||||
// pagination). Bodies are never included in the rows.
|
||||
func (t *Traffic) Page(f PageQuery, page, size int) (rows []ExchangeMeta, total int, err error) {
|
||||
if size <= 0 || size > 500 {
|
||||
size = 100
|
||||
}
|
||||
if page < 0 {
|
||||
page = 0
|
||||
}
|
||||
where, args := t.pageWhere(f)
|
||||
if err = t.db.QueryRow(`SELECT COUNT(*) FROM exchanges`+where, args...).Scan(&total); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
@@ -1156,7 +1189,7 @@ func (t *Traffic) Page(f PageQuery, page, size int) (rows []ExchangeMeta, total
|
||||
}
|
||||
// id embeds capture timestamp + sequence, so a trailing id DESC makes the order
|
||||
// total and pagination stable even when the sort column has many ties.
|
||||
sel := `SELECT id,ts,host,method,url_template,url,status,content_type,resp_len,path FROM exchanges` +
|
||||
sel := `SELECT id,ts,host,method,url_template,url,status,content_type,resp_len,path,` + rewrittenExistsSQL + ` FROM exchanges` +
|
||||
where + ` ORDER BY ` + col + ` ` + dir + `, id DESC LIMIT ? OFFSET ?`
|
||||
qargs := append(append([]any{}, args...), size, page*size)
|
||||
rs, err := t.db.Query(sel, qargs...)
|
||||
@@ -1166,7 +1199,7 @@ func (t *Traffic) Page(f PageQuery, page, size int) (rows []ExchangeMeta, total
|
||||
defer rs.Close()
|
||||
for rs.Next() {
|
||||
var m ExchangeMeta
|
||||
if err := rs.Scan(&m.ID, &m.TS, &m.Host, &m.Method, &m.URLTemplate, &m.URL, &m.Status, &m.ContentType, &m.RespLen, &m.Path); err != nil {
|
||||
if err := rs.Scan(&m.ID, &m.TS, &m.Host, &m.Method, &m.URLTemplate, &m.URL, &m.Status, &m.ContentType, &m.RespLen, &m.Path, &m.Rewritten); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
rows = append(rows, m)
|
||||
|
||||
@@ -526,3 +526,93 @@ func TestRecordConcurrent(t *testing.T) {
|
||||
t.Fatalf("after concurrent writes exchanges=%d bodies=%d, want %d each", exchanges, bodies, n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrafficPageRewrittenFlagAndFilter(t *testing.T) {
|
||||
tr, _ := openTraffic(t)
|
||||
insertEx := func(id string, ts int64, host string, status int) {
|
||||
t.Helper()
|
||||
if _, err := tr.DB().Exec(`INSERT INTO exchanges(id,ts,host,method,url_template,url,status,content_type,req_len,resp_len,path)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?)`, id, ts, host, "GET", "/p/"+id, "https://"+host+"/p/"+id, status, "text/plain", 0, 10, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
insertEx("rw-a-200", 1, "alpha.example", 200)
|
||||
insertEx("rw-b-404", 2, "beta.example", 404)
|
||||
insertEx("orig-a-200", 3, "alpha.example", 200)
|
||||
insertEx("skip-a-404", 4, "alpha.example", 404)
|
||||
insertEx("err-b-200", 5, "beta.example", 200)
|
||||
|
||||
if _, err := tr.DB().Exec(`INSERT INTO exchange_rewrites(exchange_id,seq,phase,rule_id,outcome) VALUES
|
||||
('rw-a-200',0,'request',1,'applied'),
|
||||
('rw-b-404',0,'response',2,'applied'),
|
||||
('skip-a-404',0,'request',3,'skipped'),
|
||||
('err-b-200',0,'response',4,'error')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Default ("") and "all" return all 5 with accurate Rewritten flags.
|
||||
for _, filter := range []string{"", RewriteFilterAll} {
|
||||
rows, total, err := tr.Page(PageQuery{Rewritten: filter, RespMin: -1, RespMax: -1}, 0, 20)
|
||||
if err != nil || total != 5 || len(rows) != 5 {
|
||||
t.Fatalf("filter=%q got (%d, total=%d, err=%v), want (5, 5, nil)", filter, len(rows), total, err)
|
||||
}
|
||||
got := map[string]bool{}
|
||||
for _, r := range rows {
|
||||
got[r.ID] = r.Rewritten
|
||||
}
|
||||
if !got["rw-a-200"] || !got["rw-b-404"] || got["orig-a-200"] || got["skip-a-404"] || got["err-b-200"] {
|
||||
t.Fatalf("filter=%q flags = %v", filter, got)
|
||||
}
|
||||
}
|
||||
|
||||
// Rewritten filter (canonical + true alias).
|
||||
for _, filter := range []string{RewriteFilterRewritten, "true"} {
|
||||
rows, total, err := tr.Page(PageQuery{Rewritten: filter, RespMin: -1, RespMax: -1}, 0, 20)
|
||||
if err != nil || total != 2 || len(rows) != 2 {
|
||||
t.Fatalf("filter=%q got (%d, total=%d, err=%v), want (2, 2, nil)", filter, len(rows), total, err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if !r.Rewritten {
|
||||
t.Fatalf("filter=%q returned non-rewritten row %s", filter, r.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Original filter (canonical + false alias) includes plain, skipped-only, and error-only.
|
||||
for _, filter := range []string{RewriteFilterOriginal, "false"} {
|
||||
rows, total, err := tr.Page(PageQuery{Rewritten: filter, RespMin: -1, RespMax: -1}, 0, 20)
|
||||
if err != nil || total != 3 || len(rows) != 3 {
|
||||
t.Fatalf("filter=%q got (%d, total=%d, err=%v), want (3, 3, nil)", filter, len(rows), total, err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if r.Rewritten {
|
||||
t.Fatalf("filter=%q returned rewritten row %s", filter, r.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Combined with host + status and pagination.
|
||||
rows, total, err := tr.Page(PageQuery{Host: "alpha", Status: "200", Rewritten: RewriteFilterRewritten, RespMin: -1, RespMax: -1}, 0, 10)
|
||||
if err != nil || total != 1 || len(rows) != 1 || rows[0].ID != "rw-a-200" {
|
||||
t.Fatalf("combined host+status+rewritten = (%+v, total=%d, err=%v)", rows, total, err)
|
||||
}
|
||||
p0, total0, err := tr.Page(PageQuery{Rewritten: RewriteFilterOriginal, RespMin: -1, RespMax: -1}, 0, 2)
|
||||
p1, total1, err2 := tr.Page(PageQuery{Rewritten: RewriteFilterOriginal, RespMin: -1, RespMax: -1}, 1, 2)
|
||||
if err != nil || err2 != nil || total0 != 3 || total1 != 3 || len(p0) != 2 || len(p1) != 1 {
|
||||
t.Fatalf("pagination original p0=(%d/%d,%v) p1=(%d/%d,%v)", len(p0), total0, err, len(p1), total1, err2)
|
||||
}
|
||||
|
||||
// Search also carries Rewritten.
|
||||
srows, err := tr.Search("alpha.example", 0, 10)
|
||||
if err != nil || len(srows) != 3 {
|
||||
t.Fatalf("Search = (%d, %v), want (3, nil)", len(srows), err)
|
||||
}
|
||||
found := map[string]bool{}
|
||||
for _, r := range srows {
|
||||
found[r.ID] = r.Rewritten
|
||||
}
|
||||
if !found["rw-a-200"] || found["orig-a-200"] || found["skip-a-404"] {
|
||||
t.Fatalf("Search rewritten flags = %v", found)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import { Loader2Icon, PlusIcon, SendIcon, XIcon } from "lucide-react";
|
||||
import { HttpCodeBlock, statusTone } from "@/components/http-code-block";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
@@ -23,6 +24,7 @@ type ReplayForm = {
|
||||
url: string;
|
||||
rows: HeaderRow[];
|
||||
body: string;
|
||||
applyRules: boolean;
|
||||
// Headers of the recorded request that the server would not send.
|
||||
dropped: string[];
|
||||
binary: boolean;
|
||||
@@ -40,6 +42,7 @@ function initialForm(exchange: TrafficExchange, raw: string): ReplayForm {
|
||||
url: exchange.url,
|
||||
rows: toRows(sent),
|
||||
body: parsed.binary ? "" : parsed.body,
|
||||
applyRules: false,
|
||||
dropped: notSent.filter((n) => n.toLowerCase() !== "host"),
|
||||
binary: parsed.binary,
|
||||
bodyPreviewOnly: parsed.bodyPreviewOnly,
|
||||
@@ -83,6 +86,7 @@ export function ReplayPanel({
|
||||
// Answers for an older exchange are dropped.
|
||||
const generation = React.useRef(0);
|
||||
const methodListId = React.useId();
|
||||
const applyRulesId = React.useId();
|
||||
|
||||
React.useEffect(() => {
|
||||
generation.current++;
|
||||
@@ -111,6 +115,7 @@ export function ReplayPanel({
|
||||
headers: splitReplayHeaders(form.rows).sent,
|
||||
body: form.body,
|
||||
body_base64: false,
|
||||
apply_rules: form.applyRules,
|
||||
})
|
||||
.then((r) => {
|
||||
if (gen !== generation.current) return;
|
||||
@@ -230,10 +235,20 @@ export function ReplayPanel({
|
||||
)}
|
||||
</div>
|
||||
|
||||
<Button onClick={send} disabled={!canSend}>
|
||||
{sending ? <Loader2Icon className="animate-spin" /> : <SendIcon />}
|
||||
{t("Send")}
|
||||
</Button>
|
||||
<div className="flex flex-wrap items-center gap-4">
|
||||
<Button onClick={send} disabled={!canSend}>
|
||||
{sending ? <Loader2Icon className="animate-spin" /> : <SendIcon />}
|
||||
{t("Send")}
|
||||
</Button>
|
||||
<label htmlFor={applyRulesId} className="flex cursor-pointer items-center gap-2 text-xs select-none">
|
||||
<Checkbox
|
||||
id={applyRulesId}
|
||||
checked={form.applyRules}
|
||||
onCheckedChange={(checked) => update({ applyRules: checked === true })}
|
||||
/>
|
||||
<span>{t("Apply capture rewrite rules")}</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="space-y-1 rounded-md border border-red-500/40 bg-red-500/10 p-3 text-xs">
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
ArrowUpNarrowWideIcon,
|
||||
ChevronLeftIcon,
|
||||
ChevronRightIcon,
|
||||
DownloadIcon,
|
||||
EraserIcon,
|
||||
FilterXIcon,
|
||||
ListChecksIcon,
|
||||
@@ -14,11 +15,14 @@ import {
|
||||
RadioTowerIcon,
|
||||
SearchIcon,
|
||||
Trash2Icon,
|
||||
UploadIcon,
|
||||
XIcon,
|
||||
} from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
|
||||
import { HttpCodeBlock } from "@/components/http-code-block";
|
||||
import { LinkTrafficDialog } from "@/components/link-traffic-dialog";
|
||||
import { Alert, AlertDescription } from "@/components/ui/alert";
|
||||
import {
|
||||
AlertDialog,
|
||||
AlertDialogAction,
|
||||
@@ -44,7 +48,7 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip
|
||||
import { api } from "@/lib/api";
|
||||
import { setVisibleInterval } from "@/lib/polling";
|
||||
import { useStoredSortPreference } from "@/lib/sort-preference";
|
||||
import type { TrafficDetail, TrafficExchange, TrafficHost, TrafficResp } from "@/lib/types";
|
||||
import type { TrafficDetail, TrafficExchange, TrafficHARImportResult, TrafficHost, TrafficResp } from "@/lib/types";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { intlLocale, t } from "@/i18n";
|
||||
|
||||
@@ -135,6 +139,7 @@ export default function TrafficPage() {
|
||||
const [path, setPath] = React.useState("");
|
||||
const [pathQ, setPathQ] = React.useState("");
|
||||
const [statusFilter, setStatusFilter] = React.useState("all");
|
||||
const [rewrittenFilter, setRewrittenFilter] = React.useState("all");
|
||||
const [respMin, setRespMin] = React.useState("");
|
||||
const [respMinQ, setRespMinQ] = React.useState("");
|
||||
const [respMax, setRespMax] = React.useState("");
|
||||
@@ -158,6 +163,13 @@ export default function TrafficPage() {
|
||||
const [deleting, setDeleting] = React.useState(false);
|
||||
const [reloadTick, setReloadTick] = React.useState(0); // manual refetch trigger
|
||||
|
||||
// HAR 1.2 import/export state
|
||||
const harInputRef = React.useRef<HTMLInputElement | null>(null);
|
||||
const [harImporting, setHarImporting] = React.useState(false);
|
||||
const [harExporting, setHarExporting] = React.useState(false);
|
||||
const [harSummary, setHarSummary] = React.useState<TrafficHARImportResult | null>(null);
|
||||
const [harError, setHarError] = React.useState<string | null>(null);
|
||||
|
||||
// Debounce both filters so we don't refetch on every keystroke.
|
||||
React.useEffect(() => {
|
||||
const t = setTimeout(() => setHostQ(host.trim()), 300);
|
||||
@@ -184,13 +196,15 @@ export default function TrafficPage() {
|
||||
return () => clearTimeout(t);
|
||||
}, [respMax]);
|
||||
|
||||
const hasAdvancedFilter = Boolean(bodyQ || pathQ || respMinQ || respMaxQ) || statusFilter !== "all";
|
||||
const hasAdvancedFilter =
|
||||
Boolean(bodyQ || pathQ || respMinQ || respMaxQ) || statusFilter !== "all" || rewrittenFilter !== "all";
|
||||
const resetAdvancedFilters = () => {
|
||||
setBody("");
|
||||
setBodyQ("");
|
||||
setPath("");
|
||||
setPathQ("");
|
||||
setStatusFilter("all");
|
||||
setRewrittenFilter("all");
|
||||
setRespMin("");
|
||||
setRespMinQ("");
|
||||
setRespMax("");
|
||||
@@ -201,7 +215,7 @@ export default function TrafficPage() {
|
||||
// biome-ignore lint/correctness/useExhaustiveDependencies: these values intentionally trigger a page reset.
|
||||
React.useEffect(() => {
|
||||
setPage(0);
|
||||
}, [hostQ, queryQ, method, size, bodyQ, pathQ, statusFilter, respMinQ, respMaxQ, sort]);
|
||||
}, [hostQ, queryQ, method, size, bodyQ, pathQ, statusFilter, rewrittenFilter, respMinQ, respMaxQ, sort]);
|
||||
|
||||
// Load the current page. Auto-refresh only on page 0 (newest) so paging back
|
||||
// through history isn't yanked out from under the user.
|
||||
@@ -214,6 +228,7 @@ export default function TrafficPage() {
|
||||
body: bodyQ,
|
||||
path: pathQ,
|
||||
status: statusFilter,
|
||||
rewritten: rewrittenFilter,
|
||||
respMin: respMinQ,
|
||||
respMax: respMaxQ,
|
||||
sort: sort.field,
|
||||
@@ -243,7 +258,21 @@ export default function TrafficPage() {
|
||||
alive = false;
|
||||
stopPolling();
|
||||
};
|
||||
}, [page, size, hostQ, method, queryQ, bodyQ, pathQ, statusFilter, respMinQ, respMaxQ, sort, reloadTick]);
|
||||
}, [
|
||||
page,
|
||||
size,
|
||||
hostQ,
|
||||
method,
|
||||
queryQ,
|
||||
bodyQ,
|
||||
pathQ,
|
||||
statusFilter,
|
||||
rewrittenFilter,
|
||||
respMinQ,
|
||||
respMaxQ,
|
||||
sort,
|
||||
reloadTick,
|
||||
]);
|
||||
|
||||
// Delete traffic for the current host filter (substring) or the checked
|
||||
// hosts (exact batch), then refetch.
|
||||
@@ -349,6 +378,76 @@ export default function TrafficPage() {
|
||||
: { field, direction: "desc" },
|
||||
);
|
||||
|
||||
const handleHarFileChange = async (event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const file = event.target.files?.[0];
|
||||
event.target.value = "";
|
||||
if (!file) return;
|
||||
setHarError(null);
|
||||
setHarSummary(null);
|
||||
if (file.size > 32 * 1024 * 1024) {
|
||||
const msg = t("HAR file exceeds the 32 MiB limit");
|
||||
setHarError(msg);
|
||||
toast.error(msg);
|
||||
return;
|
||||
}
|
||||
setHarImporting(true);
|
||||
try {
|
||||
const raw = await file.text();
|
||||
const res = await api.trafficImportHAR(raw);
|
||||
setHarSummary(res);
|
||||
const summary = t("HAR import complete: {imported} imported, {skipped} skipped, {failed} failed", {
|
||||
imported: res.imported,
|
||||
skipped: res.skipped,
|
||||
failed: res.failed,
|
||||
});
|
||||
if (res.failed > 0 || res.truncated) {
|
||||
toast.warning(summary);
|
||||
} else {
|
||||
toast.success(summary);
|
||||
}
|
||||
setPage(0);
|
||||
setReloadTick((n) => n + 1);
|
||||
} catch (e) {
|
||||
const msg = t("HAR import failed: {error}", { error: (e as Error).message });
|
||||
setHarError(msg);
|
||||
toast.error(msg);
|
||||
} finally {
|
||||
setHarImporting(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleHarExport = async () => {
|
||||
setHarExporting(true);
|
||||
try {
|
||||
const meta = await api.trafficExportHAR({
|
||||
host: hostQ,
|
||||
method,
|
||||
q: queryQ,
|
||||
body: bodyQ,
|
||||
path: pathQ,
|
||||
status: statusFilter,
|
||||
rewritten: rewrittenFilter,
|
||||
respMin: respMinQ,
|
||||
respMax: respMaxQ,
|
||||
});
|
||||
if (meta.truncated) {
|
||||
toast.warning(
|
||||
t("Exported {exported} of {matched} matching exchanges (truncated: {reason})", {
|
||||
exported: meta.exported,
|
||||
matched: meta.matched,
|
||||
reason: meta.truncatedReason ?? "limit",
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
toast.success(t("Exported {count} exchanges to HAR", { count: meta.exported }));
|
||||
}
|
||||
} catch (e) {
|
||||
toast.error((e as Error).message);
|
||||
} finally {
|
||||
setHarExporting(false);
|
||||
}
|
||||
};
|
||||
|
||||
const exchanges = React.useMemo(() => traffic?.exchanges ?? [], [traffic]);
|
||||
const total = traffic?.total ?? exchanges.length;
|
||||
const pageCount = Math.max(1, Math.ceil(total / size));
|
||||
@@ -378,9 +477,76 @@ export default function TrafficPage() {
|
||||
<span className="text-xs text-muted-foreground tabular-nums">
|
||||
{t("{count} records", { count: traffic?.count ?? 0 })}
|
||||
</span>
|
||||
<input
|
||||
ref={harInputRef}
|
||||
type="file"
|
||||
accept=".har,application/json"
|
||||
className="hidden"
|
||||
onChange={(e) => void handleHarFileChange(e)}
|
||||
/>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-8"
|
||||
disabled={harImporting}
|
||||
onClick={() => harInputRef.current?.click()}
|
||||
>
|
||||
{harImporting ? <Loader2Icon className="size-3.5 animate-spin" /> : <UploadIcon className="size-3.5" />}
|
||||
{harImporting ? t("Importing HAR…") : t("Import HAR")}
|
||||
</Button>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="h-8"
|
||||
disabled={harExporting || !traffic?.count}
|
||||
onClick={() => void handleHarExport()}
|
||||
>
|
||||
{harExporting ? <Loader2Icon className="size-3.5 animate-spin" /> : <DownloadIcon className="size-3.5" />}
|
||||
{harExporting ? t("Exporting HAR…") : t("Export HAR")}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{harError ? (
|
||||
<Alert variant="destructive" className="flex items-center justify-between gap-2 py-2">
|
||||
<AlertDescription className="text-xs">{harError}</AlertDescription>
|
||||
<Button variant="ghost" size="icon-xs" aria-label={t("Dismiss")} onClick={() => setHarError(null)}>
|
||||
<XIcon className="size-3.5" />
|
||||
</Button>
|
||||
</Alert>
|
||||
) : null}
|
||||
|
||||
{harSummary ? (
|
||||
<Alert className="flex items-start justify-between gap-2 py-2">
|
||||
<AlertDescription className="space-y-1 text-xs">
|
||||
<p>
|
||||
{t("HAR import complete: {imported} imported, {skipped} skipped, {failed} failed", {
|
||||
imported: harSummary.imported,
|
||||
skipped: harSummary.skipped,
|
||||
failed: harSummary.failed,
|
||||
})}
|
||||
</p>
|
||||
{harSummary.truncated ? (
|
||||
<p className="text-amber-600 dark:text-amber-400">
|
||||
{t("HAR import stopped early at the 32 MiB decoded body cap.")}
|
||||
</p>
|
||||
) : null}
|
||||
{harSummary.errors && harSummary.errors.length > 0 ? (
|
||||
<ul className="list-disc pl-4 text-muted-foreground">
|
||||
{harSummary.errors.slice(0, 5).map((item) => (
|
||||
<li key={`${item.index}-${item.reason}`} className="font-mono">
|
||||
#{item.index}: {item.reason}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : null}
|
||||
</AlertDescription>
|
||||
<Button variant="ghost" size="icon-xs" aria-label={t("Dismiss")} onClick={() => setHarSummary(null)}>
|
||||
<XIcon className="size-3.5" />
|
||||
</Button>
|
||||
</Alert>
|
||||
) : null}
|
||||
|
||||
{/* Toolbar */}
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Popover open={pickerOpen} onOpenChange={setPickerOpen}>
|
||||
@@ -598,6 +764,16 @@ export default function TrafficPage() {
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<Select value={rewrittenFilter} onValueChange={setRewrittenFilter}>
|
||||
<SelectTrigger size="sm" className="w-32">
|
||||
<SelectValue placeholder={t("Rewritten")} />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="all">{t("All traffic")}</SelectItem>
|
||||
<SelectItem value="rewritten">{t("Rewritten")}</SelectItem>
|
||||
<SelectItem value="original">{t("Original")}</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<div className="flex items-center gap-1 text-xs text-muted-foreground">
|
||||
<span>{t("Response size")}</span>
|
||||
<Input
|
||||
@@ -721,7 +897,17 @@ export default function TrafficPage() {
|
||||
<MethodBadge method={e.method} />
|
||||
</TableCell>
|
||||
<TableCell className="max-w-0">
|
||||
<span className="block truncate font-mono text-xs">{e.url}</span>
|
||||
<div className="flex items-center gap-1.5">
|
||||
<span className="min-w-0 truncate font-mono text-xs">{e.url}</span>
|
||||
{e.rewritten ? (
|
||||
<Badge
|
||||
variant="outline"
|
||||
className="shrink-0 border-amber-500/30 bg-amber-500/10 px-1.5 py-0 text-[10px] font-medium text-amber-600 dark:text-amber-400"
|
||||
>
|
||||
{t("Rewritten")}
|
||||
</Badge>
|
||||
) : null}
|
||||
</div>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<span className={cn("font-mono text-xs font-semibold tabular-nums", statusTone(e.status))}>
|
||||
@@ -762,6 +948,14 @@ export default function TrafficPage() {
|
||||
<Badge variant="secondary" className={cn("font-mono tabular-nums", statusTone(selected.status))}>
|
||||
{selected.status}
|
||||
</Badge>
|
||||
{selected.rewritten ? (
|
||||
<Badge
|
||||
variant="outline"
|
||||
className="border-amber-500/30 bg-amber-500/10 text-amber-600 dark:text-amber-400"
|
||||
>
|
||||
{t("Rewritten")}
|
||||
</Badge>
|
||||
) : null}
|
||||
<span className="ml-auto text-xs text-muted-foreground tabular-nums">{fmtTime(selected.ts)}</span>
|
||||
</div>
|
||||
<SheetTitle className="break-all font-mono">{selected.host}</SheetTitle>
|
||||
|
||||
@@ -245,6 +245,7 @@
|
||||
"All statuses": "Tüm durumlar",
|
||||
"All tasks": "Tüm görevler",
|
||||
"All tool execution records": "Tüm araç çalıştırma kayıtları",
|
||||
"All traffic": "Tüm trafik",
|
||||
"All types": "Tüm türler",
|
||||
"all used": "tümü kullanıldı",
|
||||
"All verification states": "Tüm doğrulama durumları",
|
||||
@@ -275,6 +276,7 @@
|
||||
"Appended {count} scopes": "{count} kapsam eklendi",
|
||||
"Applies only to this profile, overriding the global defaults from \"Retry & backoff\". Each field left empty = follow the global; attempts -1 = disable this layer; a filled interval replaces the exponential backoff with a fixed one. The breaker and intent rerun are process-level and can only be tuned on the global page.": "Yalnızca bu profile uygulanır ve \"Yeniden deneme ve geri çekilme\" sekmesindeki genel varsayılanları geçersiz kılar. Boş bırakılan her alan = genel ayarı izle; deneme sayısı -1 = bu katmanı devre dışı bırak; doldurulan aralık üstel geri çekilmeyi sabit bir aralıkla değiştirir. Devre kesici ve niyet yeniden çalıştırma süreç düzeyindedir ve yalnızca genel sayfada ayarlanabilir.",
|
||||
"Applies to this task only and is not written to global rules. Evaluation order: \"Block\" rules (including global ones) match first, and a hit forbids testing; with no hit and \"Allow\" rules configured for this task, an asset must match an allow rule to be permitted, otherwise testing is likewise forbidden; with no allow rules configured, no whitelist is active.": "Yalnızca bu göreve uygulanır ve genel kurallara yazılmaz. Değerlendirme sırası: önce \"Engelle\" kuralları (genel olanlar dahil) eşleştirilir ve eşleşme testi yasaklar; eşleşme yoksa ve bu görev için \"İzin ver\" kuralları yapılandırılmışsa, bir varlığa izin verilmesi için bir izin kuralıyla eşleşmesi gerekir, aksi halde test yine yasaktır; izin kuralı yapılandırılmamışsa beyaz liste etkin değildir.",
|
||||
"Apply capture rewrite rules": "Yakalama yeniden yazma kurallarını uygula",
|
||||
"Approval calls": "Onay çağrıları",
|
||||
"Approval decision": "Onay kararı",
|
||||
"Approval details #{id}": "Onay ayrıntıları #{id}",
|
||||
@@ -765,6 +767,7 @@
|
||||
"Discovery time": "Keşif zamanı",
|
||||
"Discovery time currently ascending, click to toggle sort direction": "Keşif zamanı şu anda artan sırada, sıralama yönünü değiştirmek için tıklayın",
|
||||
"Discovery time currently descending, click to toggle sort direction": "Keşif zamanı şu anda azalan sırada, sıralama yönünü değiştirmek için tıklayın",
|
||||
"Dismiss": "Kapat",
|
||||
"Dismiss update notice": "Güncelleme bildirimini kapat",
|
||||
"dismissed": "yok sayıldı",
|
||||
"Dismissed": "Yok sayıldı",
|
||||
@@ -923,9 +926,13 @@
|
||||
"Export failed: {error}": "Dışa aktarma başarısız: {error}",
|
||||
"Export findings": "Bulguları dışa aktar",
|
||||
"Export format": "Dışa aktarma biçimi",
|
||||
"Export HAR": "HAR dışa aktar",
|
||||
"Export scope": "Dışa aktarma kapsamı",
|
||||
"Export the {count} selected items": "Seçili {count} öğeyi dışa aktar",
|
||||
"Export the current filter results ({count} items)": "Mevcut süzgeç sonuçlarını dışa aktar ({count} öğe)",
|
||||
"Exported {count} exchanges to HAR": "{count} değişim HAR olarak dışa aktarıldı",
|
||||
"Exported {exported} of {matched} matching exchanges (truncated: {reason})": "Eşleşen {matched} değişimin {exported} tanesi dışa aktarıldı (kısaltıldı: {reason})",
|
||||
"Exporting HAR…": "HAR dışa aktarılıyor…",
|
||||
"Exporting…": "Dışa aktarılıyor…",
|
||||
"External MCP tool servers · visibility granted per agent": "Harici MCP araç sunucuları · görünürlük ajan bazında verilir",
|
||||
"Fact": "Olgu",
|
||||
@@ -1068,6 +1075,13 @@
|
||||
"greater than": "büyüktür",
|
||||
"greater than or equal": "büyük veya eşittir",
|
||||
"Handle automatically after the timeout instead of waiting": "Beklemek yerine zaman aşımından sonra otomatik olarak işle",
|
||||
"HAR export failed: {error}": "HAR dışa aktarma başarısız: {error}",
|
||||
"HAR export is not available in mock mode": "HAR dışa aktarma sahte modda kullanılamaz",
|
||||
"HAR file exceeds the 32 MiB limit": "HAR dosyası 32 MiB sınırını aşıyor",
|
||||
"HAR import complete: {imported} imported, {skipped} skipped, {failed} failed": "HAR içe aktarma tamamlandı: {imported} içe aktarıldı, {skipped} atlandı, {failed} başarısız",
|
||||
"HAR import failed: {error}": "HAR içe aktarma başarısız: {error}",
|
||||
"HAR import is not available in mock mode": "HAR içe aktarma sahte modda kullanılamaz",
|
||||
"HAR import stopped early at the 32 MiB decoded body cap.": "HAR içe aktarma 32 MiB çözülmüş gövde sınırında erken durdu.",
|
||||
"Hard delete": "Kalıcı silme",
|
||||
"Header": "Başlık",
|
||||
"Header name": "Başlık adı",
|
||||
@@ -1110,6 +1124,8 @@
|
||||
"Idle": "Boşta",
|
||||
"Ignored": "Yok sayıldı",
|
||||
"Implicit TLS": "Örtük TLS",
|
||||
"Import HAR": "HAR içe aktar",
|
||||
"Importing HAR…": "HAR içe aktarılıyor…",
|
||||
"In": "Girdi",
|
||||
"In {input} (incl. cache {cache}) · Out {output}": "Girdi {input} (önbellek {cache} dahil) · Çıktı {output}",
|
||||
"In flight": "Devam eden",
|
||||
@@ -1598,6 +1614,7 @@
|
||||
"OR (either one matches)": "OR (herhangi biri eşleşir)",
|
||||
"Ordered failover: workers share the current profile and switch after quota exhaustion or repeated provider failures.": "Sıralı yedeğe geçiş: çalışanlar geçerli profili paylaşır ve kota tükendiğinde veya sağlayıcı tekrar tekrar başarısız olduğunda geçiş yapar.",
|
||||
"origin": "başlangıç",
|
||||
"Original": "Orijinal",
|
||||
"Original status": "Önceki durum",
|
||||
"Other": "Diğer",
|
||||
"Out": "Çıktı",
|
||||
@@ -1904,6 +1921,7 @@
|
||||
"Revoked Skill \"{name}\" visibility": "\"{name}\" becerisinin görünürlüğü kaldırıldı",
|
||||
"Revoked visibility from \"{agent}\"": "\"{agent}\" için görünürlük kaldırıldı",
|
||||
"Revoked visibility to \"{name}\"": "\"{name}\" için görünürlük kaldırıldı",
|
||||
"Rewritten": "Yeniden yazılmış",
|
||||
"Rewritten by capture rules": "Yakalama kuralları tarafından yeniden yazıldı",
|
||||
"Rewritten by capture rules: {ids}": "Yakalama kuralları tarafından yeniden yazıldı: {ids}",
|
||||
"Risk": "Risk",
|
||||
|
||||
+82
-3
@@ -102,6 +102,8 @@ import type {
|
||||
TrafficDetail,
|
||||
TrafficEvidenceRef,
|
||||
TrafficEvidenceRole,
|
||||
TrafficHARExportMeta,
|
||||
TrafficHARImportResult,
|
||||
TrafficHost,
|
||||
TrafficReplayRequest,
|
||||
TrafficReplayResult,
|
||||
@@ -840,6 +842,7 @@ export const api = {
|
||||
body?: string;
|
||||
path?: string;
|
||||
status?: string;
|
||||
rewritten?: string;
|
||||
respMin?: string;
|
||||
respMax?: string;
|
||||
sort?: string;
|
||||
@@ -854,6 +857,7 @@ export const api = {
|
||||
(opts.body ? `&body=${encodeURIComponent(opts.body)}` : "") +
|
||||
(opts.path ? `&path=${encodeURIComponent(opts.path)}` : "") +
|
||||
(opts.status && opts.status !== "all" ? `&status=${encodeURIComponent(opts.status)}` : "") +
|
||||
(opts.rewritten && opts.rewritten !== "all" ? `&rewritten=${encodeURIComponent(opts.rewritten)}` : "") +
|
||||
(opts.respMin ? `&resp_min=${encodeURIComponent(opts.respMin)}` : "") +
|
||||
(opts.respMax ? `&resp_max=${encodeURIComponent(opts.respMax)}` : "") +
|
||||
(opts.sort ? `&sort=${encodeURIComponent(opts.sort)}` : "") +
|
||||
@@ -874,11 +878,86 @@ export const api = {
|
||||
if (!r.ok) throw new Error(t("Blob download failed: {error}", { error: r.status }));
|
||||
return new Uint8Array(await r.arrayBuffer());
|
||||
},
|
||||
// Re-sends an edited request through the proxy. Errors carry the HTTP status
|
||||
// (ApiError): 413 body too large, 502 proxy/upstream, 503 capture off, 504 timeout.
|
||||
// Re-sends an edited request through the proxy (bypasses capture rewrite rules
|
||||
// unless req.apply_rules is true). Errors carry the HTTP status (ApiError):
|
||||
// 413 body too large, 502 proxy/upstream, 503 capture off, 504 timeout.
|
||||
trafficReplay: async (req: TrafficReplayRequest) => {
|
||||
if (MOCK) throw new Error(t("Replay is not available in mock mode"));
|
||||
return post<TrafficReplayResult>(`/traffic/replay`, req);
|
||||
return post<TrafficReplayResult>(`/traffic/replay`, {
|
||||
...req,
|
||||
apply_rules: Boolean(req.apply_rules),
|
||||
});
|
||||
},
|
||||
// Imports a raw HAR 1.2 JSON document into the traffic store.
|
||||
trafficImportHAR: async (rawJSON: string): Promise<TrafficHARImportResult> => {
|
||||
if (MOCK) throw new Error(t("HAR import is not available in mock mode"));
|
||||
return http<TrafficHARImportResult>(`/traffic/import/har`, {
|
||||
method: "POST",
|
||||
body: rawJSON,
|
||||
});
|
||||
},
|
||||
// Exports filtered traffic as a HAR 1.2 file download and returns truncation metadata.
|
||||
trafficExportHAR: async (
|
||||
opts: {
|
||||
host?: string;
|
||||
method?: string;
|
||||
q?: string;
|
||||
body?: string;
|
||||
path?: string;
|
||||
status?: string;
|
||||
rewritten?: string;
|
||||
respMin?: string;
|
||||
respMax?: string;
|
||||
} = {},
|
||||
): Promise<TrafficHARExportMeta> => {
|
||||
if (MOCK) throw new Error(t("HAR export is not available in mock mode"));
|
||||
const qs = new URLSearchParams();
|
||||
if (opts.host) qs.set("host", opts.host);
|
||||
if (opts.method && opts.method !== "all") qs.set("method", opts.method);
|
||||
if (opts.q) qs.set("q", opts.q);
|
||||
if (opts.body) qs.set("body", opts.body);
|
||||
if (opts.path) qs.set("path", opts.path);
|
||||
if (opts.status && opts.status !== "all") qs.set("status", opts.status);
|
||||
if (opts.rewritten && opts.rewritten !== "all") qs.set("rewritten", opts.rewritten);
|
||||
if (opts.respMin) qs.set("resp_min", opts.respMin);
|
||||
if (opts.respMax) qs.set("resp_max", opts.respMax);
|
||||
const suffix = qs.toString();
|
||||
const token = getToken();
|
||||
const r = await fetch(`/api/traffic/export/har${suffix ? `?${suffix}` : ""}`, {
|
||||
headers: { ...(token ? { Authorization: `Bearer ${token}` } : {}) },
|
||||
});
|
||||
if (r.status === 401) {
|
||||
if (typeof window !== "undefined") {
|
||||
localStorage.removeItem("artex_token");
|
||||
document.cookie = "artex_token=; path=/; max-age=0";
|
||||
window.location.href = "/login";
|
||||
}
|
||||
throw new Error(t("Unauthorized"));
|
||||
}
|
||||
if (!r.ok) {
|
||||
let msg = String(r.status);
|
||||
try {
|
||||
const payload = (await r.json()) as { error?: unknown };
|
||||
if (typeof payload.error === "string" && payload.error.trim()) msg = payload.error.trim();
|
||||
} catch {
|
||||
// Keep status fallback.
|
||||
}
|
||||
throw new ApiError(t("HAR export failed: {error}", { error: msg }), r.status);
|
||||
}
|
||||
const blob = await r.blob();
|
||||
const exported = Number(r.headers.get("X-ARTEX-HAR-Exported") ?? "0") || 0;
|
||||
const matched = Number(r.headers.get("X-ARTEX-HAR-Matched") ?? "0") || 0;
|
||||
const truncated = r.headers.get("X-ARTEX-HAR-Truncated") === "true";
|
||||
const truncatedReason = r.headers.get("X-ARTEX-HAR-Truncated-Reason") ?? undefined;
|
||||
const objectUrl = URL.createObjectURL(blob);
|
||||
const a = document.createElement("a");
|
||||
a.href = objectUrl;
|
||||
a.download = "artex-traffic.har";
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
a.remove();
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
return { exported, matched, truncated, truncatedReason };
|
||||
},
|
||||
trafficHosts: () => get<{ hosts: TrafficHost[] }>(`/traffic/hosts`),
|
||||
trafficDeleteHost: (host: string) => del<{ deleted: number }>(`/traffic?host=${encodeURIComponent(host)}`),
|
||||
|
||||
@@ -903,6 +903,36 @@ export interface TrafficExchange {
|
||||
status: number;
|
||||
content_type: string;
|
||||
resp_len: number;
|
||||
rewritten?: boolean;
|
||||
}
|
||||
|
||||
export interface TrafficRewriteStats {
|
||||
large_responses: number;
|
||||
capacity_refusals: number;
|
||||
tombstone_drops: number;
|
||||
conversions: number;
|
||||
sweeps: number;
|
||||
}
|
||||
|
||||
export interface TrafficHARImportError {
|
||||
index: number;
|
||||
url?: string;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface TrafficHARImportResult {
|
||||
imported: number;
|
||||
skipped: number;
|
||||
failed: number;
|
||||
truncated?: boolean;
|
||||
errors?: TrafficHARImportError[];
|
||||
}
|
||||
|
||||
export interface TrafficHARExportMeta {
|
||||
exported: number;
|
||||
matched: number;
|
||||
truncated: boolean;
|
||||
truncatedReason?: string;
|
||||
}
|
||||
|
||||
export interface TrafficResp {
|
||||
@@ -913,6 +943,7 @@ export interface TrafficResp {
|
||||
page?: number;
|
||||
size?: number;
|
||||
exchanges?: TrafficExchange[];
|
||||
rewrite?: TrafficRewriteStats;
|
||||
}
|
||||
|
||||
// Full raw request/response of one exchange (lazy-loaded on row select).
|
||||
@@ -955,12 +986,14 @@ export interface TrafficReplayHeader {
|
||||
}
|
||||
|
||||
// POST /api/traffic/replay. Host is not sent: the server takes it from url.
|
||||
// apply_rules defaults to false (bypasses capture rewrite rules).
|
||||
export interface TrafficReplayRequest {
|
||||
method: string;
|
||||
url: string;
|
||||
headers: TrafficReplayHeader[];
|
||||
body: string;
|
||||
body_base64: boolean;
|
||||
apply_rules?: boolean;
|
||||
}
|
||||
|
||||
export interface TrafficReplayResult {
|
||||
|
||||
Reference in New Issue
Block a user