Compare commits

...
Author SHA1 Message Date
BaaRai 57ff63125b docs: record the extension capture fix and map the contributor email
The extension fix shipped without a changelog entry. Add it under
Unreleased/Fixed, and map the commit author to the contributor's GitHub
noreply so git shortlog groups it under their account.
2026-11-16 15:56:08 +01:00
BaaRai b9e564eaaa Merge pull request #259 from liuxiaobo0920/fix/extension-cdp-header-iteration
fix(extension): iterate object-shaped response headers from CDP
2026-11-16 15:55:39 +01:00
Xiaobo def23f6b38 fix(extension): accept object-shaped response headers from CDP
Network.responseReceived hands back response.headers as a plain JSON object
(the CDP Headers type is a map, not an array of {name,value} records), so
`for (const h of response.headers)` throws "object is not iterable" on every
single response. queueExchange() rejects, the .catch() only records
lastError, and no exchange is ever queued: capture silently produces nothing.

Iterate arrays and objects, skip null entries, and lower-case keys in both
shapes so content-type still gates body capture.
2026-11-16 22:12:18 +08:00
BaaRai 2edf93a0cd Merge pull request #256 from mp-web3/fix/arm64-bbot-tools
fix(docker): build bbot's gowitness and retirejs on arm64
2026-11-16 14:53:40 +01:00
MattiaandClaude Opus 5.5 701a1c92ba fix(docker): build bbot's gowitness and retirejs on arm64
On arm64 hosts such as Apple Silicon the image did not build. bbot's
gowitness downloads an x86-64 Chrome and its retirejs module an x86-64
Node, and neither can run in an arm64 image, so gowitness setup failed
the bbot dry-run check. retirejs failed too, but the check did not
cover it.

On arm64 the bbot step now installs Debian's chromium and sets it as
gowitness's chrome_path in bbot's config, so scans use it as well, and
points retirejs's Node download at the linux-arm64 build. The dry-run
check now covers retirejs. amd64 builds skip the new block.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 09:40:59 +01:00
BaaRai c4eb079a95 Merge pull request #255 from BaaRai/feat/traffic-stream-tab
feat(web): show WebSocket and SSE messages in the traffic detail
2026-11-16 09:33:37 +01:00
BaaRai 212e29cd15 feat(web): show WebSocket and SSE messages in the traffic detail
The traffic detail sheet gets a Stream tab for WebSocket and SSE exchanges, listing their messages page by page with direction, type, size, time and payload preview, and loading a spilled payload in full on demand. Adds trafficStream and trafficBlob to the API client.
2026-11-16 09:11:38 +01:00
BaaRai ec98975bf3 Merge pull request #254 from BaaRai/feat/traffic-stream-messages
feat(traffic): record WebSocket and SSE stream messages
2026-11-16 08:52:15 +01:00
8 changed files with 297 additions and 8 deletions
+1
View File
@@ -0,0 +1 @@
Xiaobo <269351206+liuxiaobo0920@users.noreply.github.com> <mvOfosterboydP1FD80@technologist.com>
+2
View File
@@ -32,6 +32,8 @@ All notable changes to this project are documented in this file. The format is b
### Fixed
- **Browser extension traffic capture**: CDP delivers `Network.responseReceived` response headers as an object, not an array of `{name, value}` records, so the extension threw on every response and never queued an exchange. It now accepts both shapes and lower-cases names before the body-capture gate, so captures work again and `content-type` correctly controls body capture.
- **bbot on arm64**: the image now builds on arm64 hosts such as Apple Silicon. bbot's `gowitness` downloaded an x86-64 Chrome and `retirejs` an x86-64 Node, which can't run there, so the build failed. On arm64 the image uses Debian's `chromium` for gowitness and an arm64 Node for retirejs, and the build check now covers `retirejs` too. amd64 builds are unchanged.
- **Auto-scope hygiene**: auto `task_scope` rows count for coverage and untested assets only under a declared root, and direct coverage counts anchored assets through fact anchors only; the scope is described as "coverage scope, not enforced" (#196).
- **Enrich HTTP probe**: it now dials through `intercept.PinnedDialer`, so the target gate applies and a probe without a target scope is refused (#198).
- **Agent hook data races**: the eight package-level hooks are synchronised, so a second wire-up no longer races tasks that already read them (#227).
+13 -2
View File
@@ -145,14 +145,25 @@ RUN set -eux; \
# preinstalled (--install-all-deps) so scans never need pip at runtime, which
# would fail under a program's egress allowlist.
ARG BBOT_VERSION=3.0.2
# arm64: bbot's Chrome and retirejs's Node are x86-64; use native builds.
RUN set -eux; \
python -m venv /opt/bbot; \
/opt/bbot/bin/pip install --no-cache-dir "bbot==${BBOT_VERSION}"; \
ln -s /opt/bbot/bin/bbot /usr/local/bin/bbot; \
if [ "${TARGETARCH:-amd64}" = "arm64" ]; then \
apt-get update; \
apt-get install -y --no-install-recommends chromium; \
rm -rf /var/lib/apt/lists/*; \
bbot_dir="$(/opt/bbot/bin/python -c 'import bbot, os; print(os.path.dirname(bbot.__file__))')"; \
sed -i 's/linux-x64/linux-arm64/g' "$bbot_dir/modules/retirejs.py"; \
fi; \
bbot -y --install-all-deps; \
if [ "${TARGETARCH:-amd64}" = "arm64" ]; then \
printf '\nmodules:\n gowitness:\n chrome_path: /usr/bin/chromium\n' >> /root/.config/bbot/bbot.yml; \
fi; \
bbot --version >/dev/null; \
bbot --dry-run -t example.com -m wafw00f,paramminer_getparams,lightfuzz,sslcert,gowitness >/dev/null 2>&1 || true; \
bbot --dry-run -t example.com -m wafw00f,paramminer_getparams,lightfuzz,sslcert,gowitness --no-color >/tmp/bbot-dry.log 2>&1 || true; \
bbot --dry-run -t example.com -m wafw00f,paramminer_getparams,lightfuzz,sslcert,gowitness,retirejs >/dev/null 2>&1 || true; \
bbot --dry-run -t example.com -m wafw00f,paramminer_getparams,lightfuzz,sslcert,gowitness,retirejs --no-color >/tmp/bbot-dry.log 2>&1 || true; \
! grep -qE "Installing|ERRR|CRITICAL" /tmp/bbot-dry.log || { cat /tmp/bbot-dry.log; exit 1; }
+11 -1
View File
@@ -92,7 +92,17 @@ function truncate(s) {
async function queueExchange(info, response) {
const respHeaders = {};
for (const h of response.headers || []) respHeaders[h.name.toLowerCase()] = h.value;
// CDP hands back response.headers as a plain object (dict), not an array of
// {name,value} records; iterating it with for...of throws "object is not
// iterable" and no exchange is ever queued. Accept both shapes.
const rawHeaders = response.headers || {};
if (Array.isArray(rawHeaders)) {
for (const h of rawHeaders) {
if (h && h.name != null) respHeaders[String(h.name).toLowerCase()] = h.value;
}
} else {
for (const [k, v] of Object.entries(rawHeaders)) respHeaders[String(k).toLowerCase()] = v;
}
const mime = respHeaders["content-type"] || "";
let reqBody = info.postData || "";
@@ -0,0 +1,200 @@
"use client";
import * as React from "react";
import { Loader2Icon, RefreshCwIcon } from "lucide-react";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { api } from "@/lib/api";
import type { TrafficStreamMessage } from "@/lib/types";
const PAGE_SIZE = 50;
// Binary payloads loaded in full are shown as hex of their first 64 KiB.
const HEX_PREVIEW_BYTES = 64 * 1024;
function fmtBytes(n: number) {
if (n <= 0) return "0 B";
const units = ["B", "KB", "MB", "GB"];
const i = Math.min(Math.floor(Math.log(n) / Math.log(1024)), units.length - 1);
const v = n / 1024 ** i;
return `${i === 0 ? v : v.toFixed(1)} ${units[i]}`;
}
function fmtDirection(direction: string) {
if (direction === "c2s") return "client -> server";
if (direction === "s2c") return "server -> client";
return direction;
}
// Hex dump: 16 bytes per line, offset first.
function hexDump(bytes: Uint8Array) {
const lines: string[] = [];
for (let i = 0; i < bytes.length; i += 16) {
const row = Array.from(bytes.subarray(i, i + 16), (b) => b.toString(16).padStart(2, "0")).join(" ");
lines.push(`${i.toString(16).padStart(8, "0")} ${row}`);
}
return lines.join("\n");
}
type FullPayload = { loading: boolean; text?: string; error?: string };
const payloadClass =
"max-h-64 overflow-auto rounded-md bg-muted/50 p-2 font-mono text-xs break-all whitespace-pre-wrap";
function StreamMessageRow({ message }: { message: TrafficStreamMessage }) {
const [full, setFull] = React.useState<FullPayload | null>(null);
const binary = message.type === "binary";
const loadFull = () => {
if (!message.blob) return;
setFull({ loading: true });
api
.trafficBlob(message.blob)
.then((bytes) => {
const text = binary
? `${hexDump(bytes.subarray(0, HEX_PREVIEW_BYTES))}\n(${bytes.length} bytes total)`
: new TextDecoder().decode(bytes);
setFull({ loading: false, text });
})
.catch((e: Error) => setFull({ loading: false, error: e.message }));
};
if (message.type === "meta") {
return (
<div className="mx-5 my-3 space-y-1 rounded-md border border-amber-500/40 bg-amber-500/10 p-3">
<div className="font-medium text-amber-700 text-xs dark:text-amber-400">
#{message.seq} Recording limit reached: later messages were not stored
</div>
<pre className={payloadClass}>{message.payload}</pre>
</div>
);
}
return (
<div className="space-y-1.5 border-b px-5 py-3 last:border-b-0">
<div className="flex flex-wrap items-center gap-2 text-xs">
<span className="font-mono text-muted-foreground tabular-nums">#{message.seq}</span>
<Badge variant="outline">{fmtDirection(message.direction)}</Badge>
<Badge variant="secondary">{message.type}</Badge>
{binary && message.encoding === "base64" && <Badge variant="outline">base64</Badge>}
{message.truncated && <Badge variant="destructive">truncated</Badge>}
{message.event && <span className="font-mono">event: {message.event}</span>}
{message.event_id && <span className="font-mono">id: {message.event_id}</span>}
{message.retry ? <span className="font-mono">retry: {message.retry}</span> : null}
<span className="ml-auto text-muted-foreground tabular-nums">
{fmtBytes(message.size)} · {new Date(message.ts * 1000).toLocaleTimeString()}
</span>
</div>
<pre className={payloadClass}>{full?.text ?? message.payload}</pre>
{message.blob && (
<div className="flex flex-wrap items-center gap-2 text-muted-foreground text-xs">
<span className="break-all font-mono">blob {message.blob}</span>
{!full?.text && (
<Button variant="outline" size="xs" disabled={full?.loading} onClick={loadFull}>
{full?.loading && <Loader2Icon className="animate-spin" />}
Load full payload
</Button>
)}
{full?.error && <span className="text-destructive">{full.error}</span>}
</div>
)}
</div>
);
}
// StreamMessagesPanel lists the messages of one WebSocket or SSE exchange, a
// page at a time. It does not poll: Refresh reloads from the start.
export function StreamMessagesPanel({ exchangeId, kind }: { exchangeId: string; kind: "websocket" | "sse" }) {
const [items, setItems] = React.useState<TrafficStreamMessage[]>([]);
const [total, setTotal] = React.useState(0);
const [loading, setLoading] = React.useState(true);
const [loadingMore, setLoadingMore] = React.useState(false);
const [error, setError] = React.useState<string | null>(null);
// Answers for an older exchange (or an older refresh) are dropped.
const generation = React.useRef(0);
const load = React.useCallback(
(offset: number) => {
const gen = offset === 0 ? ++generation.current : generation.current;
if (offset === 0) {
setLoading(true);
setError(null);
} else {
setLoadingMore(true);
}
api
.trafficStream(exchangeId, offset, PAGE_SIZE)
.then((page) => {
if (gen !== generation.current) return;
const next = page.items ?? [];
setItems((prev) => (offset === 0 ? next : [...prev, ...next]));
setTotal(page.total ?? 0);
})
.catch((e: Error) => {
if (gen === generation.current) setError(e.message);
})
.finally(() => {
if (gen !== generation.current) return;
setLoading(false);
setLoadingMore(false);
});
},
[exchangeId],
);
React.useEffect(() => {
setItems([]);
setTotal(0);
load(0);
}, [load]);
return (
<div className="flex flex-col">
<div className="flex items-center gap-2 px-5 py-3 text-muted-foreground text-xs">
<span>
{kind === "websocket" ? "WebSocket" : "Server-Sent Events"} · {items.length} of {total} messages
</span>
<Button variant="outline" size="xs" className="ml-auto" disabled={loading} onClick={() => load(0)}>
<RefreshCwIcon className={loading ? "animate-spin" : undefined} />
Refresh
</Button>
</div>
{renderBody()}
</div>
);
function renderBody() {
if (loading) {
return (
<div className="flex items-center gap-2 p-5 text-muted-foreground text-xs">
<Loader2Icon className="size-3.5 animate-spin" />
Loading messages…
</div>
);
}
if (error) {
return <div className="p-5 text-destructive text-xs">Failed to load messages: {error}</div>;
}
if (items.length === 0) {
return <div className="p-5 text-muted-foreground text-xs">No messages recorded for this stream.</div>;
}
return (
<>
<div className="border-t">
{items.map((m) => (
<StreamMessageRow key={`${exchangeId}:${m.seq}`} message={m} />
))}
</div>
{items.length < total && (
<div className="flex justify-center p-4">
<Button variant="outline" size="sm" disabled={loadingMore} onClick={() => load(items.length)}>
{loadingMore && <Loader2Icon className="animate-spin" />}
Load more
</Button>
</div>
)}
</>
);
}
}
+27 -5
View File
@@ -42,10 +42,12 @@ import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { api } from "@/lib/api";
import { setVisibleInterval } from "@/lib/polling";
import { useStoredSortPreference } from "@/lib/sort-preference";
import type { TrafficDetail, TrafficExchange, TrafficHost, TrafficResp } from "@/lib/types";
import { cn } from "@/lib/utils";
import { setVisibleInterval } from "@/lib/polling";
import { StreamMessagesPanel } from "./_components/stream-messages";
function fmtTime(ts: string) {
return new Date(ts).toLocaleString("zh-CN", {
@@ -141,6 +143,9 @@ export default function TrafficPage() {
const [selected, setSelected] = React.useState<TrafficExchange | null>(null);
const [detail, setDetail] = React.useState<TrafficDetail | null>(null);
const [detailLoading, setDetailLoading] = React.useState(false);
const [detailTab, setDetailTab] = React.useState("request");
const streamKind = detail?.stream_kind === "websocket" || detail?.stream_kind === "sse" ? detail.stream_kind : null;
const hasStream = streamKind !== null;
const [hosts, setHosts] = React.useState<TrafficHost[]>([]); // target picker
const [selectedHosts, setSelectedHosts] = React.useState<string[]>([]); // checked in picker
@@ -298,8 +303,10 @@ export default function TrafficPage() {
.finally(() => setDeleting(false));
};
// Lazy-load the raw request/response for the selected exchange.
// Lazy-load the raw request/response for the selected exchange. A new
// selection opens on Request; a WebSocket/SSE exchange then opens on Stream.
React.useEffect(() => {
setDetailTab("request");
if (!selected) {
setDetail(null);
return;
@@ -310,7 +317,9 @@ export default function TrafficPage() {
api
.trafficExchange(selected.id)
.then((d) => {
if (alive) setDetail(d);
if (!alive) return;
setDetail(d);
if (d.stream_kind === "websocket" || d.stream_kind === "sse") setDetailTab("stream");
})
.catch(() => {
if (alive) setDetail({ req: "(failed to load the messages)", resp: "" });
@@ -742,10 +751,18 @@ export default function TrafficPage() {
<SheetTitle className="break-all font-mono">{selected.host}</SheetTitle>
<SheetDescription className="break-all font-mono">{selected.url}</SheetDescription>
</SheetHeader>
<Tabs defaultValue="request" className="min-h-0 flex-1 gap-0">
<TabsList className="mx-5 mt-4 grid w-auto grid-cols-2">
<Tabs value={detailTab} onValueChange={setDetailTab} className="min-h-0 flex-1 gap-0">
<TabsList className={cn("mx-5 mt-4 grid w-auto", hasStream ? "grid-cols-3" : "grid-cols-2")}>
<TabsTrigger value="request">Request</TabsTrigger>
<TabsTrigger value="response">Response</TabsTrigger>
{hasStream && (
<TabsTrigger value="stream">
Stream
{(detail?.stream_count ?? 0) > 0 && (
<span className="text-muted-foreground text-xs tabular-nums">{detail?.stream_count}</span>
)}
</TabsTrigger>
)}
</TabsList>
<TabsContent value="request" className="min-h-0 overflow-auto">
{detailLoading ? (
@@ -767,6 +784,11 @@ export default function TrafficPage() {
<HttpCodeBlock raw={detail?.resp ?? ""} />
)}
</TabsContent>
{streamKind && (
<TabsContent value="stream" className="min-h-0 overflow-auto">
<StreamMessagesPanel exchangeId={selected.id} kind={streamKind} />
</TabsContent>
)}
</Tabs>
</>
)}
+15
View File
@@ -102,6 +102,7 @@ import type {
TrafficEvidenceRole,
TrafficHost,
TrafficResp,
TrafficStreamPage,
UsageStats,
WorkspaceFile,
WorkspaceListing,
@@ -816,6 +817,20 @@ export const api = {
(opts.order ? `&order=${encodeURIComponent(opts.order)}` : ""),
),
trafficExchange: (id: string) => get<TrafficDetail>(`/traffic/exchange?id=${encodeURIComponent(id)}`),
// One page of a WebSocket/SSE exchange's messages, in order.
trafficStream: (id: string, offset = 0, limit = 50) =>
get<TrafficStreamPage>(`/traffic/stream?exchange=${encodeURIComponent(id)}&offset=${offset}&limit=${limit}`),
// Raw bytes of one stored blob (a full message payload or an oversized body).
// The JSON-only `get` cannot carry binary, hence the direct fetch.
trafficBlob: async (hash: string) => {
if (MOCK) throw new Error("Blob download is not available in mock mode");
const token = getToken();
const r = await fetch(`/api/traffic/blob?hash=${encodeURIComponent(hash)}`, {
headers: { ...(token ? { Authorization: `Bearer ${token}` } : {}) },
});
if (!r.ok) throw new Error(`Blob download failed: ${r.status}`);
return new Uint8Array(await r.arrayBuffer());
},
trafficHosts: () => get<{ hosts: TrafficHost[] }>(`/traffic/hosts`),
trafficDeleteHost: (host: string) => del<{ deleted: number }>(`/traffic?host=${encodeURIComponent(host)}`),
trafficDeleteHosts: (hosts: string[]) => del<{ deleted: number }>(`/traffic/hosts`, { hosts }),
+28
View File
@@ -911,6 +911,34 @@ export interface TrafficResp {
export interface TrafficDetail {
req: string;
resp: string;
// WebSocket/SSE exchanges carry their messages separately (trafficStream);
// stream_kind is "" for an ordinary exchange.
stream_count?: number;
stream_kind?: "websocket" | "sse" | "";
}
// One recorded WebSocket message or SSE event. ts is in Unix seconds; seq
// orders. payload is a preview of at most 4 KiB (base64 when encoding is
// "base64"); blob names the full payload when it was stored separately.
export interface TrafficStreamMessage {
seq: number;
ts: number;
direction: "c2s" | "s2c" | "meta" | (string & {});
kind: number;
type: "text" | "binary" | "event" | "meta" | (string & {});
event?: string;
event_id?: string;
retry?: number;
size: number;
payload: string;
encoding?: string;
blob?: string;
truncated?: boolean;
}
export interface TrafficStreamPage {
items: TrafficStreamMessage[];
total: number;
}
// One distinct recorded host with its exchange count (target picker).