mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-13 19:50:54 +08:00
Compare commits
18
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ac003ede28 | ||
|
|
31e56a16fc | ||
|
|
bbc4f6269a | ||
|
|
e38e48e80e | ||
|
|
f6b29b835b | ||
|
|
08830365c4 | ||
|
|
1c3a5f4ffa | ||
|
|
1e616c9286 | ||
|
|
46e81f5f67 | ||
|
|
c5e12541bd | ||
|
|
743e6b0d15 | ||
|
|
ca15063db7 | ||
|
|
f07e84ac7a | ||
|
|
a77851db7d | ||
|
|
c0b741bcf6 | ||
|
|
ec3db7fdec | ||
|
|
78f92f44fc | ||
|
|
923c4a813a |
+90
-37
@@ -43,51 +43,101 @@ RUN ARTEX_SKIP_FRONTEND=1 \
|
||||
ARTEX_BUILD_VERSION="${ARTEX_VERSION}" \
|
||||
bash ./build.sh --target "${TARGETOS:-linux}/${TARGETARCH:-amd64}"
|
||||
|
||||
# ---------------------------------------------------------------- runtime ---
|
||||
FROM python:3.12-slim-bookworm
|
||||
# ------------------------------------------------------- third-party tools ---
|
||||
# Web-recon toolbox (drost-community take 1, #100): pinned release binaries, staged
|
||||
# on the build platform and copied into /usr/local/bin of the runtime image so hunt
|
||||
# workers can run them directly through their Bash tool. Versions pinned per tool;
|
||||
# upgrade deliberately. Each archive is verified against the checksums file of its
|
||||
# own GitHub release (exact asset name, sha256). That file comes from the same
|
||||
# release channel as the archive, so it catches corrupted or swapped downloads and
|
||||
# moved tags, not a compromised publisher. sqlmap has no release binaries: it is a
|
||||
# shallow clone of a pinned tag, checked against the expected commit.
|
||||
FROM --platform=$BUILDPLATFORM debian:bookworm-slim AS tools
|
||||
ARG TARGETARCH
|
||||
# Common tools: ripgrep / curl / vim, plus a set of recon staples (add/remove as needed).
|
||||
# Node 22.x comes from NodeSource: bookworm's apt nodejs is 18, Playwright requires >=20,
|
||||
# and 20 is end-of-life. 22 matches the CI and release workflows.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates ripgrep curl wget vim git jq unzip \
|
||||
dnsutils iputils-ping netcat-openbsd inetutils-telnet whois nmap \
|
||||
&& curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
|
||||
&& apt-get install -y --no-install-recommends nodejs \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
# Web-recon toolbox (drost-community take 1, #100): pinned release binaries installed
|
||||
# into /usr/local/bin so hunt workers can run them directly through their Bash tool.
|
||||
# Versions pinned per tool; upgrade deliberately. sqlmap is a python checkout (no
|
||||
# release binaries) symlinked into PATH.
|
||||
ARG NUCLEI_VERSION=v3.11.1
|
||||
ARG HTTPX_VERSION=v1.12.0
|
||||
ARG SUBFINDER_VERSION=v2.17.0
|
||||
ARG KATANA_VERSION=v1.8.0
|
||||
ARG FFUF_VERSION=v2.3.0
|
||||
ARG DALFOX_VERSION=v3.2.4
|
||||
ARG SQLMAP_VERSION=1.9.9
|
||||
ARG SQLMAP_COMMIT=cceda39208b5aad478de2c3f731df4239d9881f6
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl git unzip \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
RUN set -eux; \
|
||||
case "${TARGETARCH:-amd64}" in \
|
||||
amd64) PD_ARCH=amd64; FFUF_ARCH=amd64; DALFOX_ARCH=x86_64 ;; \
|
||||
arm64) PD_ARCH=arm64; FFUF_ARCH=arm64; DALFOX_ARCH=aarch64 ;; \
|
||||
*) echo "unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \
|
||||
esac; \
|
||||
mkdir -p /out; \
|
||||
fetch() { base="$1"; asset="$2"; sums="$3"; name="$4"; \
|
||||
rm -rf /tmp/t; mkdir -p /tmp/t/x; \
|
||||
curl -fsSL "$base/$asset" -o "/tmp/t/$asset"; \
|
||||
curl -fsSL "$base/$sums" -o /tmp/t/sums.txt; \
|
||||
want="$(awk -v a="$asset" '$2 == a || $2 == "*" a { print $1 }' /tmp/t/sums.txt)"; \
|
||||
if [ -z "$want" ]; then echo "no checksum entry for $asset" >&2; exit 1; fi; \
|
||||
got="$(sha256sum "/tmp/t/$asset" | cut -d' ' -f1)"; \
|
||||
if [ "$want" != "$got" ]; then echo "checksum mismatch for $asset: want $want got $got" >&2; exit 1; fi; \
|
||||
case "$asset" in *.zip) unzip -q "/tmp/t/$asset" -d /tmp/t/x ;; *) tar -xzf "/tmp/t/$asset" -C /tmp/t/x ;; esac; \
|
||||
find /tmp/t/x -type f -name "$name" -exec install -m 0755 {} /out/ \; ; \
|
||||
[ -x "/out/$name" ] || { echo "$name not found in $asset" >&2; exit 1; }; \
|
||||
rm -rf /tmp/t; }; \
|
||||
gh="https://github.com"; \
|
||||
fetch "$gh/projectdiscovery/nuclei/releases/download/$NUCLEI_VERSION" \
|
||||
"nuclei_${NUCLEI_VERSION#v}_linux_${PD_ARCH}.zip" "nuclei_${NUCLEI_VERSION#v}_checksums.txt" nuclei; \
|
||||
fetch "$gh/projectdiscovery/httpx/releases/download/$HTTPX_VERSION" \
|
||||
"httpx_${HTTPX_VERSION#v}_linux_${PD_ARCH}.zip" "httpx_${HTTPX_VERSION#v}_checksums.txt" httpx; \
|
||||
fetch "$gh/projectdiscovery/subfinder/releases/download/$SUBFINDER_VERSION" \
|
||||
"subfinder_${SUBFINDER_VERSION#v}_linux_${PD_ARCH}.zip" "subfinder_${SUBFINDER_VERSION#v}_checksums.txt" subfinder; \
|
||||
fetch "$gh/projectdiscovery/katana/releases/download/$KATANA_VERSION" \
|
||||
"katana_${KATANA_VERSION#v}_linux_${PD_ARCH}.zip" "katana-${KATANA_VERSION#v}-checksums.txt" katana; \
|
||||
fetch "$gh/ffuf/ffuf/releases/download/$FFUF_VERSION" \
|
||||
"ffuf_${FFUF_VERSION#v}_linux_${FFUF_ARCH}.tar.gz" "ffuf_${FFUF_VERSION#v}_checksums.txt" ffuf; \
|
||||
fetch "$gh/hahwul/dalfox/releases/download/$DALFOX_VERSION" \
|
||||
"dalfox-${DALFOX_VERSION}-linux-${DALFOX_ARCH}-musl.tar.gz" "dalfox-${DALFOX_VERSION}-linux-${DALFOX_ARCH}-musl.tar.gz.sha256" dalfox; \
|
||||
git clone --quiet --depth 1 --branch "$SQLMAP_VERSION" https://github.com/sqlmapproject/sqlmap /opt/sqlmap; \
|
||||
commit="$(git -C /opt/sqlmap rev-parse HEAD)"; \
|
||||
if [ "$commit" != "$SQLMAP_COMMIT" ]; then echo "sqlmap commit mismatch: want $SQLMAP_COMMIT got $commit" >&2; exit 1; fi; \
|
||||
rm -rf /opt/sqlmap/.git
|
||||
|
||||
# ---------------------------------------------------------------- runtime ---
|
||||
FROM python:3.12-slim-bookworm
|
||||
ARG TARGETARCH
|
||||
# Common tools: ripgrep / curl / vim, plus a set of recon staples (add/remove as needed).
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates ripgrep curl wget vim git jq unzip xz-utils \
|
||||
dnsutils iputils-ping netcat-openbsd inetutils-telnet whois nmap \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
# Node 22 from the official nodejs.org tarball, verified against its SHASUMS256.txt:
|
||||
# bookworm's apt nodejs is 18, Playwright requires >=20, and 20 is end-of-life.
|
||||
# 22 matches the CI and release workflows.
|
||||
ARG NODE_VERSION=22.23.4
|
||||
RUN set -eux; \
|
||||
case "${TARGETARCH:-amd64}" in \
|
||||
amd64) NODE_ARCH=x64 ;; \
|
||||
arm64) NODE_ARCH=arm64 ;; \
|
||||
*) echo "unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \
|
||||
esac; \
|
||||
base="https://nodejs.org/dist/v${NODE_VERSION}"; \
|
||||
asset="node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.xz"; \
|
||||
curl -fsSL "$base/$asset" -o "/tmp/$asset"; \
|
||||
curl -fsSL "$base/SHASUMS256.txt" -o /tmp/SHASUMS256.txt; \
|
||||
want="$(awk -v a="$asset" '$2 == a { print $1 }' /tmp/SHASUMS256.txt)"; \
|
||||
if [ -z "$want" ]; then echo "no checksum entry for $asset" >&2; exit 1; fi; \
|
||||
got="$(sha256sum "/tmp/$asset" | cut -d' ' -f1)"; \
|
||||
if [ "$want" != "$got" ]; then echo "checksum mismatch for $asset: want $want got $got" >&2; exit 1; fi; \
|
||||
tar -xJf "/tmp/$asset" -C /usr/local --strip-components=1 --no-same-owner; \
|
||||
rm -f "/tmp/$asset" /tmp/SHASUMS256.txt; \
|
||||
node --version; npm --version
|
||||
# Pinned, checksum-verified recon binaries and the sqlmap checkout from the tools stage.
|
||||
COPY --from=tools /out/ /usr/local/bin/
|
||||
COPY --from=tools /opt/sqlmap /opt/sqlmap
|
||||
RUN set -eux; \
|
||||
base="https://github.com"; \
|
||||
pd_fetch() { repo="$1"; ver="$2"; name="$3"; nov="${ver#v}"; \
|
||||
curl -fsSL "$base/$repo/releases/download/$ver/${name}_${nov}_linux_${TARGETARCH}.zip" -o /tmp/t.zip \
|
||||
&& unzip -o -q /tmp/t.zip -d /tmp/td \
|
||||
&& find /tmp/td -type f -name "$name" -exec install -m 0755 {} /usr/local/bin/ \; \
|
||||
&& rm -rf /tmp/t.zip /tmp/td; }; \
|
||||
case "$TARGETARCH" in amd64) FFUF_ARCH=amd64; DALFOX_ARCH=x86_64;; arm64) FFUF_ARCH=arm64; DALFOX_ARCH=aarch64;; esac; \
|
||||
pd_fetch projectdiscovery/nuclei "$NUCLEI_VERSION" nuclei; \
|
||||
pd_fetch projectdiscovery/httpx "$HTTPX_VERSION" httpx; \
|
||||
pd_fetch projectdiscovery/subfinder "$SUBFINDER_VERSION" subfinder; \
|
||||
pd_fetch projectdiscovery/katana "$KATANA_VERSION" katana; \
|
||||
tar_fetch() { url="$1"; name="$2"; \
|
||||
curl -fsSL "$url" -o /tmp/t.tgz \
|
||||
&& mkdir -p /tmp/td && tar -xzf /tmp/t.tgz -C /tmp/td \
|
||||
&& find /tmp/td -type f -name "$name" -exec install -m 0755 {} /usr/local/bin/ \; \
|
||||
&& rm -rf /tmp/t.tgz /tmp/td; }; \
|
||||
tar_fetch "$base/ffuf/ffuf/releases/download/$FFUF_VERSION/ffuf_${FFUF_VERSION#v}_linux_${FFUF_ARCH}.tar.gz" ffuf; \
|
||||
tar_fetch "$base/hahwul/dalfox/releases/download/$DALFOX_VERSION/dalfox-${DALFOX_VERSION}-linux-${DALFOX_ARCH}-musl.tar.gz" dalfox; \
|
||||
git clone --depth 1 https://github.com/sqlmapproject/sqlmap /opt/sqlmap; \
|
||||
ln -s /opt/sqlmap/sqlmap.py /usr/local/bin/sqlmap; \
|
||||
nuclei -version >/dev/null; httpx -version; subfinder -version; katana -version; \
|
||||
ffuf -V >/dev/null; dalfox --version >/dev/null 2>&1 || true; sqlmap --version >/dev/null
|
||||
ffuf -V >/dev/null; dalfox --version >/dev/null; sqlmap --version >/dev/null
|
||||
|
||||
# bbot recursive recon orchestrator (#117): blacklanternsecurity/bbot, AGPL-3.0
|
||||
# (same license as ARTEX; standalone CLI dependency, not linked code). Pinned
|
||||
@@ -111,8 +161,11 @@ RUN set -eux; \
|
||||
# @playwright/cli: provides playwright-cli, verified with --help right after install.
|
||||
# Then install playwright (browser management) and preinstall chromium plus its system
|
||||
# dependencies via --with-deps, so MCP/CLI work on first start inside the container
|
||||
# without downloading browsers over the network.
|
||||
RUN npm install -g @playwright/mcp@latest @playwright/cli@latest playwright@latest \
|
||||
# without downloading browsers over the network. All three are pinned; upgrade deliberately.
|
||||
ARG PLAYWRIGHT_MCP_VERSION=0.0.83
|
||||
ARG PLAYWRIGHT_CLI_VERSION=0.1.22
|
||||
ARG PLAYWRIGHT_VERSION=1.64.1
|
||||
RUN npm install -g "@playwright/mcp@${PLAYWRIGHT_MCP_VERSION}" "@playwright/cli@${PLAYWRIGHT_CLI_VERSION}" "playwright@${PLAYWRIGHT_VERSION}" \
|
||||
&& playwright-cli --help \
|
||||
&& playwright install --with-deps chromium \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
+1
-4
@@ -128,13 +128,10 @@ func (c *ChatAgent) Chat(ctx context.Context, agentKey, sessionID, message strin
|
||||
Provider: c.prov,
|
||||
SystemPrompt: system,
|
||||
DynamicBoundary: boundary,
|
||||
Tools: tools,
|
||||
Tools: withWebFetch(ctx, tools, c.proxyAddr, c.proxyCACert), // WebFetch: routed through the recording proxy for tracing; loads the proxy CA to verify MITM-resigned HTTPS certificates; every redirect hop is gated
|
||||
DeferredTools: def.Deferred,
|
||||
UnlockSet: def.Unlock,
|
||||
PermissionMode: permission.ModeBypass,
|
||||
EnableWebFetch: true, // routed through the recording proxy for tracing; loads the proxy CA to verify MITM-resigned HTTPS certificates
|
||||
WebFetchProxy: c.proxyAddr,
|
||||
WebFetchCACert: c.proxyCACert,
|
||||
// Web search (optional). ddgs needs no key; brave-free needs BraveKey; tavily needs TavilyKey.
|
||||
// WebSearchProxy is a dedicated egress proxy (http/https/socks5), unrelated to the traffic-recording MITM proxy; empty = direct.
|
||||
EnableWebSearch: ws.Enabled,
|
||||
|
||||
+1
-4
@@ -144,13 +144,10 @@ func (m *MainAgent) Chat(ctx context.Context, taskID int64, mainSeg int, as *db.
|
||||
Provider: m.prov,
|
||||
SystemPrompt: system,
|
||||
DynamicBoundary: boundary,
|
||||
Tools: tools,
|
||||
Tools: withWebFetch(ctx, tools, m.proxyAddr, m.proxyCACert), // WebFetch: routed through the recording proxy for tracing; loads the proxy CA to verify MITM-resigned HTTPS certificates; every redirect hop is gated
|
||||
DeferredTools: def.Deferred,
|
||||
UnlockSet: def.Unlock,
|
||||
PermissionMode: permission.ModeBypass,
|
||||
EnableWebFetch: true, // routed through the recording proxy for tracing; loads the proxy CA to verify MITM-resigned HTTPS certificates
|
||||
WebFetchProxy: m.proxyAddr,
|
||||
WebFetchCACert: m.proxyCACert,
|
||||
// Web search (optional). ddgs needs no key; brave-free needs BraveKey; tavily needs TavilyKey.
|
||||
// WebSearchProxy is a dedicated egress proxy (http/https/socks5), unrelated to the traffic-recording MITM proxy; empty = direct.
|
||||
EnableWebSearch: m.webSearch.Enabled,
|
||||
|
||||
+1
-4
@@ -426,13 +426,10 @@ func (p *Planner) Plan(ctx context.Context, taskID int64, as *db.AssetStore, ts
|
||||
Provider: p.prov,
|
||||
SystemPrompt: system,
|
||||
DynamicBoundary: boundary,
|
||||
Tools: tools,
|
||||
Tools: withWebFetch(ctx, tools, p.proxyAddr, p.proxyCACert), // WebFetch: route through the recording proxy for traceability; load the proxy CA to verify MITM-re-signed HTTPS certs; every redirect hop is gated
|
||||
DeferredTools: def.Deferred,
|
||||
UnlockSet: def.Unlock,
|
||||
PermissionMode: permission.ModeBypass,
|
||||
EnableWebFetch: true, // route through the recording proxy for traceability; load the proxy CA to verify MITM-re-signed HTTPS certs
|
||||
WebFetchProxy: p.proxyAddr,
|
||||
WebFetchCACert: p.proxyCACert,
|
||||
// Web search (optional). ddgs needs no key; brave-free needs BraveKey; tavily needs TavilyKey.
|
||||
// WebSearchProxy is a dedicated egress proxy (http/https/socks5), unrelated to the traffic-recording MITM proxy; empty = direct.
|
||||
EnableWebSearch: p.webSearch.Enabled,
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
)
|
||||
|
||||
// The domain tools read and write the platform's own records (the exploration
|
||||
// graph, assets, findings, goals). The target gate must find no target in
|
||||
// their input: an intent or a finding that mentions a URL is not a connection.
|
||||
// The gate scans every tool it does not know, so a domain tool added here has
|
||||
// to be added to the gate's list of tools without a target.
|
||||
func TestDomainToolsDeclareNoTarget(t *testing.T) {
|
||||
seeds := BuiltinToolSeeds()
|
||||
if len(seeds) < 20 {
|
||||
t.Fatalf("only %d domain tools found", len(seeds))
|
||||
}
|
||||
input := []byte(`{"summary":"probe https://app.example.com/login","items":[{"url":"https://app.example.com:8443/","ip":"203.0.113.7"}],` +
|
||||
`"poc":"curl -sk https://app.example.com/ -x http://127.0.0.1:8080","scope":"203.0.113.0/24"}`)
|
||||
if targets := intercept.ExtractTargets("a_tool_the_gate_does_not_know", input); len(targets) == 0 {
|
||||
t.Fatal("control: the input names targets, and an unknown tool is scanned")
|
||||
}
|
||||
for _, seed := range seeds {
|
||||
if targets := intercept.ExtractTargets(seed.Key, input); len(targets) != 0 {
|
||||
t.Errorf("%s: the gate reads targets %+v from a domain tool", seed.Key, targets)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
actool "github.com/Autumn-27/norma/tool"
|
||||
)
|
||||
|
||||
// withWebFetch returns tools plus the WebFetch tool of an agent session. Every
|
||||
// agent kind builds WebFetch here, one tool per run, so that its configuration
|
||||
// can depend on the run: every redirect hop goes through the target gate
|
||||
// (intercept.RedirectCheck), and every connection through the pinned dialer
|
||||
// (intercept.PinnedDialer), on the client that uses proxy and on the direct
|
||||
// retry client alike. ctx is the context of the run: when it carries a target
|
||||
// scope the dialer never connects without one, and when that scope's gate is
|
||||
// on the direct retry is turned off for the run, since it would leave without
|
||||
// the proxy. The tool is added after the session's own tools, where agentcore
|
||||
// adds it, so the tool list keeps its order. caCert is the recording proxy's
|
||||
// CA, trusted to verify the HTTPS it re-signs. tools is not modified.
|
||||
func withWebFetch(ctx context.Context, tools []actool.CoreTool, proxy, caCert string) []actool.CoreTool {
|
||||
scope, scoped := intercept.TargetScopeFromContext(ctx)
|
||||
gated := scoped && scope.Enabled != nil && scope.Enabled()
|
||||
webFetch := actool.NewWebFetch(actool.WebFetchConfig{Proxy: proxy, CACert: caCert, CheckRedirect: intercept.RedirectCheck("WebFetch"),
|
||||
DialContext: intercept.PinnedDialer("WebFetch", scoped), DisableDirectRetry: gated})
|
||||
return append(tools[:len(tools):len(tools)], webFetch)
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/guard"
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
actool "github.com/Autumn-27/norma/tool"
|
||||
)
|
||||
|
||||
// Step 2.1: the WebFetch tool of every agent kind connects through the pinned
|
||||
// dialer. In a gated scope it reaches a target only at an address a pin holds.
|
||||
|
||||
// tableResolver answers from a table and counts the lookups.
|
||||
type tableResolver struct {
|
||||
mu sync.Mutex
|
||||
answers map[string][]string
|
||||
n int
|
||||
}
|
||||
|
||||
func newTableResolver() *tableResolver { return &tableResolver{answers: map[string][]string{}} }
|
||||
|
||||
func (r *tableResolver) Name() string { return "table" }
|
||||
|
||||
func (r *tableResolver) LookupIPs(_ context.Context, host string) ([]netip.Addr, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.n++
|
||||
var out []netip.Addr
|
||||
for _, ip := range r.answers[host] {
|
||||
out = append(out, netip.MustParseAddr(ip))
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("no answer for %s", host)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *tableResolver) set(host string, ips ...string) {
|
||||
r.mu.Lock()
|
||||
r.answers[host] = ips
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
func (r *tableResolver) lookups() int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.n
|
||||
}
|
||||
|
||||
// testDialer is the base dialer of the gate in these tests: it records every
|
||||
// address, connects the routed TEST-NET addresses to local listeners and a
|
||||
// loopback address to itself, and refuses the rest.
|
||||
type testDialer struct {
|
||||
mu sync.Mutex
|
||||
routes map[string]string
|
||||
dialed []string
|
||||
}
|
||||
|
||||
func newTestDialer() *testDialer { return &testDialer{routes: map[string]string{}} }
|
||||
|
||||
func (d *testDialer) route(addr, to string) {
|
||||
d.mu.Lock()
|
||||
d.routes[addr] = to
|
||||
d.mu.Unlock()
|
||||
}
|
||||
|
||||
func (d *testDialer) dial(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
d.mu.Lock()
|
||||
d.dialed = append(d.dialed, addr)
|
||||
to, ok := d.routes[addr]
|
||||
d.mu.Unlock()
|
||||
if !ok && strings.HasPrefix(addr, "127.0.0.1:") {
|
||||
to, ok = addr, true
|
||||
}
|
||||
if !ok {
|
||||
return nil, &net.OpError{Op: "dial", Net: network, Err: errors.New("no route in the test dialer")}
|
||||
}
|
||||
return (&net.Dialer{}).DialContext(ctx, network, to)
|
||||
}
|
||||
|
||||
func (d *testDialer) calls() []string {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return slices.Clone(d.dialed)
|
||||
}
|
||||
|
||||
// dialFixture is a gated scope on task taskID whose gate resolves through a
|
||||
// table and dials through a testDialer.
|
||||
func dialFixture(t *testing.T, d *db.DB, taskID string) (intercept.TargetScope, *tableResolver, *testDialer) {
|
||||
t.Helper()
|
||||
scope := gatedScope(t, d, taskID)
|
||||
res, dialer := newTableResolver(), newTestDialer()
|
||||
scope.Interceptor.SetResolver(res)
|
||||
scope.Interceptor.SetBaseDialer(dialer.dial)
|
||||
return scope, res, dialer
|
||||
}
|
||||
|
||||
// listener is an HTTP server that counts its requests; port is its port.
|
||||
func listener(t *testing.T, body string) (addr, port string, hits *atomic.Int64) {
|
||||
t.Helper()
|
||||
hits = &atomic.Int64{}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
_, _ = w.Write([]byte(body))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
addr = srv.Listener.Addr().String()
|
||||
_, port, _ = net.SplitHostPort(addr)
|
||||
return addr, port, hits
|
||||
}
|
||||
|
||||
// runAgentKind runs one scripted WebFetch call with the agent kind on ctx.
|
||||
func runAgentKind(t *testing.T, kind string, f hookReachFixture, ctx context.Context, rawURL string) {
|
||||
t.Helper()
|
||||
prov := toolCallThenStop("WebFetch", fetchInputJSON(rawURL))
|
||||
switch kind {
|
||||
case "worker":
|
||||
payload := map[string]any{"summary": "dial probe"}
|
||||
intentID, err := f.ts.AddNode("intent", payload, 0, "running", "planner", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(payload)
|
||||
w := NewWorker(prov, "test-model", f.workDir, nil, 0, 3)
|
||||
if _, _, err := w.Execute(ctx, "work#1", f.expID, nil, f.ts, &db.Node{ID: intentID, Payload: raw}, passHooks{}, nil, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "planner":
|
||||
p := NewPlanner(prov, "test-model", f.workDir, nil, 0, 3)
|
||||
if _, _, err := p.Plan(ctx, f.expID, nil, f.ts, "probe goal", nil, passHooks{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "mainagent":
|
||||
m := NewMainAgent(prov, "test-model", f.workDir, nil, 0, 3)
|
||||
if _, err := m.Chat(ctx, f.expID, 0, nil, f.ts, "probe goal", "status?", passHooks{}, nil, nil, nil, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "chat":
|
||||
c := NewChatAgent(prov, "test-model", t.TempDir(), nil, 0)
|
||||
c.SetGuard(guard.New())
|
||||
if _, err := c.Chat(ctx, "assistant", "conv-dial-probe", "fetch it", 3, 0, false, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Each agent kind's WebFetch reaches a pinned name at its pinned address, and
|
||||
// once the name answers another address it connects nowhere and asks. A fifth
|
||||
// agent kind built without the dialer fails this table once it is added to it.
|
||||
func TestWebFetchDialerWiredOnEveryAgentKind(t *testing.T) {
|
||||
for _, kind := range []string{"worker", "planner", "mainagent", "chat"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
f := newHookReachFixture(t)
|
||||
d := testDB(t)
|
||||
t.Cleanup(func() { _ = d.Close() })
|
||||
addr, port, hits := listener(t, "pinned-reached")
|
||||
scope, res, dialer := dialFixture(t, d, f.taskID)
|
||||
dialer.route("203.0.113.7:"+port, addr)
|
||||
dialer.route("203.0.113.9:"+port, addr)
|
||||
res.set("a.example", "203.0.113.7")
|
||||
u := "http://a.example:" + port + "/"
|
||||
pinTarget(t, d, scope, u)
|
||||
ctx := intercept.WithTargetScope(t.Context(), scope)
|
||||
|
||||
runAgentKind(t, kind, f, ctx, u)
|
||||
if hits.Load() != 1 || !slices.Equal(dialer.calls(), []string{"203.0.113.7:" + port}) {
|
||||
t.Fatalf("pinned: hits=%d dialled %v, want the pinned address", hits.Load(), dialer.calls())
|
||||
}
|
||||
res.set("a.example", "203.0.113.9")
|
||||
runAgentKind(t, kind, f, ctx, u)
|
||||
if hits.Load() != 1 || len(dialer.calls()) != 1 {
|
||||
t.Fatalf("address changed: hits=%d dialled %v, want nothing more", hits.Load(), dialer.calls())
|
||||
}
|
||||
if keys := openRedirectRequests(t, d, f.taskID); len(keys) != 1 || keys[0] != "a.example:"+port {
|
||||
t.Fatalf("open requests = %v, want one for a.example:%s", keys, port)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// S2: through a proxy, the dialer is given the proxy's address, never the
|
||||
// target's. The proxy is a derived exemption: dialled as received, no lookup,
|
||||
// no row. The connection to the target is the proxy's (step 2.2 binds it).
|
||||
func TestWebFetchProxiedDialIsTheProxy(t *testing.T) {
|
||||
d := testDB(t)
|
||||
t.Cleanup(func() { _ = d.Close() })
|
||||
f := newHookReachFixture(t)
|
||||
scope, res, dialer := dialFixture(t, d, f.taskID)
|
||||
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte("via-proxy " + r.URL.Host))
|
||||
}))
|
||||
t.Cleanup(proxy.Close)
|
||||
scope.Interceptor.SetDerivedExemptions(func() []string { return []string{proxy.URL} })
|
||||
res.set("a.example", "203.0.113.7")
|
||||
ctx := intercept.WithTargetScope(t.Context(), scope)
|
||||
|
||||
out, isErr := callWebFetch(t, ctx, proxy.URL, "http://a.example:8080/")
|
||||
if isErr || !strings.Contains(out, "via-proxy a.example:8080") {
|
||||
t.Fatalf("proxied fetch: error=%v out=%q", isErr, out)
|
||||
}
|
||||
if got := dialer.calls(); !slices.Equal(got, []string{proxy.Listener.Addr().String()}) || res.lookups() != 0 {
|
||||
t.Fatalf("dialled %v after %d lookups, want the proxy only", got, res.lookups())
|
||||
}
|
||||
if keys := openRedirectRequests(t, d, f.taskID); len(keys) != 0 {
|
||||
t.Fatalf("open requests = %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// D-8: a run built with the gate off keeps its direct retry when the gate is
|
||||
// switched on during the run, and the retry goes through the pinned dialer: a
|
||||
// pinned target is dialled at its pinned address, an unpinned one nowhere.
|
||||
func TestWebFetchRetryAfterLiveSwitchIsPinned(t *testing.T) {
|
||||
d := testDB(t)
|
||||
t.Cleanup(func() { _ = d.Close() })
|
||||
f := newHookReachFixture(t)
|
||||
scope, res, dialer := dialFixture(t, d, f.taskID)
|
||||
var on atomic.Bool
|
||||
scope.Enabled = on.Load
|
||||
scope.Interceptor.SetDerivedExemptions(func() []string { return []string{"http://127.0.0.1:1"} })
|
||||
addr, port, hits := listener(t, "retry-reached")
|
||||
dialer.route("203.0.113.7:"+port, addr)
|
||||
dialer.route("203.0.113.8:"+port, addr)
|
||||
res.set("a.example", "203.0.113.7")
|
||||
res.set("b.example", "203.0.113.8")
|
||||
ctx := intercept.WithTargetScope(t.Context(), scope)
|
||||
tools := withWebFetch(ctx, nil, "http://127.0.0.1:1", "")
|
||||
|
||||
on.Store(true)
|
||||
pinTarget(t, d, scope, "http://a.example:"+port+"/")
|
||||
call := func(u string) (string, bool) {
|
||||
r, err := tools[0].Call(ctx, json.RawMessage(fetchInputJSON(u)), &actool.ToolContext{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r.Flatten(), r.IsError
|
||||
}
|
||||
if out, isErr := call("http://a.example:" + port + "/"); isErr || !strings.Contains(out, "retry-reached") || hits.Load() != 1 {
|
||||
t.Fatalf("pinned target through the retry: error=%v out=%q", isErr, out)
|
||||
}
|
||||
if out, isErr := call("http://b.example:" + port + "/"); !isErr || !strings.Contains(out, intercept.TargetApprovalPending) || hits.Load() != 1 {
|
||||
t.Fatalf("unpinned target through the retry: error=%v out=%q", isErr, out)
|
||||
}
|
||||
want := []string{"127.0.0.1:1", "203.0.113.7:" + port, "127.0.0.1:1"}
|
||||
if got := dialer.calls(); !slices.Equal(got, want) {
|
||||
t.Fatalf("dialled %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/guard"
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
actool "github.com/Autumn-27/norma/tool"
|
||||
)
|
||||
|
||||
// Step 1.9: the WebFetch tool of every agent kind checks each redirect hop
|
||||
// against the target gate. With the gate off it follows redirects as before.
|
||||
// Step 2.1: in a gated scope WebFetch has no direct retry.
|
||||
|
||||
// passHooks lets every tool call run.
|
||||
type passHooks struct{}
|
||||
|
||||
func (passHooks) PreToolUse(context.Context, string, []byte) (bool, string, []byte) {
|
||||
return false, "", nil
|
||||
}
|
||||
|
||||
func (passHooks) PostToolUse(context.Context, string, []byte, []byte, bool) {}
|
||||
|
||||
func (passHooks) Stop(context.Context, []llm.Message) (bool, []string, string) {
|
||||
return false, nil, ""
|
||||
}
|
||||
|
||||
// redirectPair is a server that redirects to a second one on another port of
|
||||
// 127.0.0.1, so the hop is another target (an IP literal: never resolved).
|
||||
type redirectPair struct {
|
||||
start, final string // URLs
|
||||
startHits, finalHits *atomic.Int64
|
||||
finalKey string // host:port of the redirect target
|
||||
}
|
||||
|
||||
func newRedirectPair(t *testing.T) redirectPair {
|
||||
t.Helper()
|
||||
p := redirectPair{startHits: &atomic.Int64{}, finalHits: &atomic.Int64{}}
|
||||
final := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
p.finalHits.Add(1)
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
_, _ = w.Write([]byte("redirect-target-reached"))
|
||||
}))
|
||||
t.Cleanup(final.Close)
|
||||
start := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
p.startHits.Add(1)
|
||||
http.Redirect(w, r, final.URL+"/final", http.StatusFound)
|
||||
}))
|
||||
t.Cleanup(start.Close)
|
||||
u, _ := url.Parse(final.URL)
|
||||
p.start, p.final, p.finalKey = start.URL+"/start", final.URL+"/final", u.Host
|
||||
return p
|
||||
}
|
||||
|
||||
// gatedScope is a gated target scope on task taskID, whose rows are removed
|
||||
// afterwards.
|
||||
func gatedScope(t *testing.T, d *db.DB, taskID string) intercept.TargetScope {
|
||||
t.Helper()
|
||||
t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM intercept_pending WHERE task_id=$1`, taskID) })
|
||||
return intercept.TargetScope{TaskID: taskID, Enabled: func() bool { return true },
|
||||
WaitBound: intercept.DefaultTargetWaitBound, Interceptor: intercept.New(d)}
|
||||
}
|
||||
|
||||
// openRedirectRequests returns the target keys of the open requests of task.
|
||||
func openRedirectRequests(t *testing.T, d *db.DB, taskID string) []string {
|
||||
t.Helper()
|
||||
rows, err := d.Query(`SELECT target_key FROM intercept_pending WHERE task_id=$1 AND status='pending' AND target_key IS NOT NULL ORDER BY id`, taskID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var keys []string
|
||||
for rows.Next() {
|
||||
var k string
|
||||
if err := rows.Scan(&k); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func fetchInputJSON(u string) string {
|
||||
b, _ := json.Marshal(map[string]string{"url": u})
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// Each agent kind runs a scripted WebFetch on a URL that redirects to a target
|
||||
// no pin covers. The hooks let the call run, so the hop is what the gate sees:
|
||||
// each kind must stop at the hop, leave the redirect target untouched and open
|
||||
// one request for it. A fifth agent kind added without the redirect policy
|
||||
// fails this table once it is added to it. The origin is pinned first: the
|
||||
// hooks never check it here, and the pinned dialer (step 2.1) connects only to
|
||||
// a pinned target.
|
||||
func TestWebFetchRedirectCheckWiredOnEveryAgentKind(t *testing.T) {
|
||||
for _, kind := range []string{"worker", "planner", "mainagent", "chat"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
f := newHookReachFixture(t)
|
||||
d := testDB(t)
|
||||
t.Cleanup(func() { _ = d.Close() })
|
||||
pair := newRedirectPair(t)
|
||||
scope := gatedScope(t, d, f.taskID)
|
||||
pinTarget(t, d, scope, pair.start)
|
||||
ctx := intercept.WithTargetScope(t.Context(), scope)
|
||||
prov := toolCallThenStop("WebFetch", fetchInputJSON(pair.start))
|
||||
|
||||
switch kind {
|
||||
case "worker":
|
||||
payload := map[string]any{"summary": "redirect probe"}
|
||||
intentID, err := f.ts.AddNode("intent", payload, 0, "running", "planner", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(payload)
|
||||
w := NewWorker(prov, "test-model", f.workDir, nil, 0, 3)
|
||||
if _, _, err := w.Execute(ctx, "work#1", f.expID, nil, f.ts, &db.Node{ID: intentID, Payload: raw}, passHooks{}, nil, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "planner":
|
||||
p := NewPlanner(prov, "test-model", f.workDir, nil, 0, 3)
|
||||
if _, _, err := p.Plan(ctx, f.expID, nil, f.ts, "probe goal", nil, passHooks{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "mainagent":
|
||||
m := NewMainAgent(prov, "test-model", f.workDir, nil, 0, 3)
|
||||
if _, err := m.Chat(ctx, f.expID, 0, nil, f.ts, "probe goal", "status?", passHooks{}, nil, nil, nil, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "chat":
|
||||
c := NewChatAgent(prov, "test-model", t.TempDir(), nil, 0)
|
||||
c.SetGuard(guard.New())
|
||||
if _, err := c.Chat(ctx, "assistant", "conv-redirect-probe", "fetch it", 3, 0, false, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if pair.startHits.Load() != 1 {
|
||||
t.Fatalf("the fetch did not run: %d hits on the origin", pair.startHits.Load())
|
||||
}
|
||||
if pair.finalHits.Load() != 0 {
|
||||
t.Fatal("the redirect was followed to a target no pin covers")
|
||||
}
|
||||
if keys := openRedirectRequests(t, d, f.taskID); len(keys) != 1 || keys[0] != pair.finalKey {
|
||||
t.Fatalf("open requests = %v, want one for %s", keys, pair.finalKey)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// In a gated scope the direct retry is off (D-8): with a proxy that refuses the
|
||||
// connection, the fetch fails with the proxy's error and never leaves without
|
||||
// it. The proxy is a derived exemption, so its dial asks for nothing.
|
||||
func TestWebFetchGatedScopeHasNoDirectRetry(t *testing.T) {
|
||||
d := testDB(t)
|
||||
t.Cleanup(func() { _ = d.Close() })
|
||||
expID, err := d.CreateExploration("no direct retry probe", "a gated scope has no direct retry")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM explorations WHERE id=$1`, expID) })
|
||||
taskID := fmt.Sprint(expID)
|
||||
pair := newRedirectPair(t)
|
||||
scope := gatedScope(t, d, taskID)
|
||||
scope.Interceptor.SetDerivedExemptions(func() []string { return []string{"http://127.0.0.1:1"} })
|
||||
ctx := intercept.WithTargetScope(t.Context(), scope)
|
||||
|
||||
out, isErr := callWebFetch(t, ctx, "http://127.0.0.1:1", pair.start)
|
||||
if !isErr || !strings.Contains(out, "proxyconnect") || strings.Contains(out, "TARGET_") {
|
||||
t.Fatalf("proxied fetch: error=%v out=%q, want the proxy's error", isErr, out)
|
||||
}
|
||||
if pair.startHits.Load() != 0 || pair.finalHits.Load() != 0 {
|
||||
t.Fatalf("hits origin=%d target=%d, want no direct retry", pair.startHits.Load(), pair.finalHits.Load())
|
||||
}
|
||||
var requests int
|
||||
if err := d.QueryRow(`SELECT count(*) FROM intercept_pending WHERE task_id=$1 AND target_key IS NOT NULL`, taskID).Scan(&requests); err != nil || requests != 0 {
|
||||
t.Fatalf("%d requests, %v; want none", requests, err)
|
||||
}
|
||||
}
|
||||
|
||||
// With the gate off, or without a scope, WebFetch follows redirects as it did
|
||||
// with agentcore's own tool: to the end of the chain, and the same error after
|
||||
// 10 hops. No database.
|
||||
func TestWebFetchRedirectGateOffUnchanged(t *testing.T) {
|
||||
loop := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, r.URL.Path+"x", http.StatusFound)
|
||||
}))
|
||||
t.Cleanup(loop.Close)
|
||||
for name, ctx := range map[string]context.Context{
|
||||
"no scope": t.Context(),
|
||||
"flag off": intercept.WithTargetScope(t.Context(), intercept.TargetScope{TaskID: "7", Enabled: func() bool { return false },
|
||||
WaitBound: time.Minute, Interceptor: intercept.New(nil)}),
|
||||
} {
|
||||
pair := newRedirectPair(t)
|
||||
for _, proxy := range []string{"", "http://127.0.0.1:1"} {
|
||||
if out, isErr := callWebFetch(t, ctx, proxy, pair.start); isErr || !strings.Contains(out, "redirect-target-reached") {
|
||||
t.Fatalf("%s, proxy %q: error=%v out=%q", name, proxy, isErr, out)
|
||||
}
|
||||
}
|
||||
if pair.finalHits.Load() != 2 {
|
||||
t.Fatalf("%s: %d hits on the redirect target, want 2", name, pair.finalHits.Load())
|
||||
}
|
||||
got, _ := callWebFetch(t, ctx, "", loop.URL+"/l")
|
||||
want := actool.NewWebFetch(actool.WebFetchConfig{}).Call
|
||||
res, err := want(ctx, json.RawMessage(fetchInputJSON(loop.URL+"/l")), &actool.ToolContext{})
|
||||
if err != nil || got != res.Flatten() || !strings.Contains(got, "stopped after 10 redirects") {
|
||||
t.Fatalf("%s: redirect loop: %q, want agentcore's %q", name, got, res.Flatten())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pinTarget opens the request for the target of rawURL in scope and approves
|
||||
// it, as a click on the approval card does.
|
||||
func pinTarget(t *testing.T, d *db.DB, scope intercept.TargetScope, rawURL string) {
|
||||
t.Helper()
|
||||
ic, input := scope.Interceptor, []byte(fetchInputJSON(rawURL))
|
||||
reqs, res := ic.CheckTargets(t.Context(), scope, "WebFetch", input)
|
||||
if res.Code != "" || len(reqs) != 1 {
|
||||
t.Fatalf("%s: requests=%+v result=%+v, want one request", rawURL, reqs, res)
|
||||
}
|
||||
scope.WaitBound = 0
|
||||
if res = ic.RequestTargets(t.Context(), scope, "WebFetch", input, reqs, nil); res.Code != intercept.TargetApprovalPending || res.PendingID == 0 {
|
||||
t.Fatalf("%s: request = %+v", rawURL, res)
|
||||
}
|
||||
if err := ic.Decide(res.PendingID, true, "test", "192.0.2.1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taskID, err := strconv.ParseInt(scope.TaskID, 10, 64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM target_pins WHERE task_id=$1`, taskID) })
|
||||
}
|
||||
|
||||
// callWebFetch runs the WebFetch tool the agents build, with proxy.
|
||||
func callWebFetch(t *testing.T, ctx context.Context, proxy, u string) (string, bool) {
|
||||
t.Helper()
|
||||
tools := withWebFetch(ctx, nil, proxy, "")
|
||||
if len(tools) != 1 || tools[0].Name() != "WebFetch" {
|
||||
t.Fatalf("withWebFetch built %d tools", len(tools))
|
||||
}
|
||||
res, err := tools[0].Call(ctx, json.RawMessage(fetchInputJSON(u)), &actool.ToolContext{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return res.Flatten(), res.IsError
|
||||
}
|
||||
+5
-7
@@ -482,16 +482,14 @@ func (w *Worker) execute(ctx context.Context, name string, taskID int64, as *db.
|
||||
Provider: w.prov,
|
||||
SystemPrompt: system,
|
||||
DynamicBoundary: boundary,
|
||||
Tools: tools,
|
||||
Tools: withWebFetch(ctx, tools, w.proxyAddr, w.proxyCACert),
|
||||
DeferredTools: def.Deferred,
|
||||
UnlockSet: def.Unlock,
|
||||
PermissionMode: permission.ModeBypass,
|
||||
// WebFetch goes through the recording proxy, so its HTTP is recorded just like
|
||||
// curl; loading the proxy CA lets MITM-re-signed HTTPS certs [verify normally]
|
||||
// (rather than disabling verification). Direct when proxy is empty.
|
||||
EnableWebFetch: true,
|
||||
WebFetchProxy: w.proxyAddr,
|
||||
WebFetchCACert: w.proxyCACert,
|
||||
// WebFetch (added to Tools above) goes through the recording proxy, so its
|
||||
// HTTP is recorded just like curl; loading the proxy CA lets MITM-re-signed
|
||||
// HTTPS certs [verify normally] (rather than disabling verification). Direct
|
||||
// when proxy is empty. Every redirect hop goes through the target gate.
|
||||
// Web search (optional). ddgs needs no key; brave-free needs BraveKey; tavily needs TavilyKey.
|
||||
// WebSearchProxy is a dedicated egress proxy (http/https/socks5), unrelated to the traffic-recording MITM proxy; empty = direct.
|
||||
EnableWebSearch: w.webSearch.Enabled,
|
||||
|
||||
@@ -45,6 +45,12 @@ func (r AssetInterceptRule) Reason() string {
|
||||
|
||||
// matchOne checks whether a single enabled rule matches the given domain/IP/URL
|
||||
// candidates, and returns the concrete matched value.
|
||||
//
|
||||
// URL rules match the URL string as given, with no normalisation. Known limit:
|
||||
// exact_url misses any other spelling of the same URL (case, percent-encoding
|
||||
// such as %2F, a trailing slash, an added query, /./), and fuzzy_url misses an
|
||||
// encoded form of its substring. At tool dispatch the string is the URL the
|
||||
// call declares (intercept, targetCheck.run).
|
||||
func matchOne(r AssetInterceptRule, domains, ips, urls []string) (string, bool) {
|
||||
p := strings.TrimSpace(r.Pattern)
|
||||
if p == "" {
|
||||
|
||||
@@ -320,6 +320,9 @@ ON CONFLICT (name) DO NOTHING`,
|
||||
if err := d.seedDefaultAssetInterceptRules(); err != nil {
|
||||
return fmt.Errorf("seed asset intercept rules: %w", err)
|
||||
}
|
||||
if err := d.seedTargetGateExemptions(); err != nil {
|
||||
return fmt.Errorf("seed target-gate exemptions: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -43,7 +43,8 @@ type InterceptAudit struct {
|
||||
Output string `json:"output,omitempty"`
|
||||
OutputTruncated bool `json:"output_truncated,omitempty"`
|
||||
ExecutionEndedAt *time.Time `json:"execution_ended_at,omitempty"`
|
||||
// Targets is set on target approval requests only (target_key IS NOT NULL).
|
||||
// Targets is set on the rows of the target gate only (decision_source = target):
|
||||
// approval requests, blocks, and the first use of an exemption or pin version.
|
||||
Targets []InterceptTarget `json:"targets,omitempty"`
|
||||
// MergedAsk marks a target request that also carries the ask of a rule or of
|
||||
// the judge: one row, one decision for both.
|
||||
@@ -68,10 +69,16 @@ type InterceptTarget struct {
|
||||
PinVersion int `json:"pin_version,omitempty"`
|
||||
PinnedIPs []string `json:"pinned_ips,omitempty"`
|
||||
Reason string `json:"reason,omitempty"` // no_pin | new_ip | revoked
|
||||
Outcome string `json:"outcome"` // requested | pinned | denied
|
||||
Outcome string `json:"outcome"` // requested | pinned | denied | pin_match | exempt | rule_blocked | resolve_failed | too_many | gate_unavailable
|
||||
// NameOnlyEffective: requested while the upstream egress proxy was set; a pin
|
||||
// created from this request covers the name only.
|
||||
NameOnlyEffective bool `json:"name_only_effective,omitempty"`
|
||||
// ExemptVoid: a stored exemption names this target, and did not apply because
|
||||
// the name resolved to a private, loopback or otherwise local address.
|
||||
ExemptVoid bool `json:"exempt_void,omitempty"`
|
||||
// RedirectFrom: the origin (host:port) of the redirect hop that led to this
|
||||
// target; empty when the call named the target itself.
|
||||
RedirectFrom string `json:"redirect_from,omitempty"`
|
||||
}
|
||||
|
||||
type InterceptDetail struct {
|
||||
|
||||
+161
-1
@@ -334,6 +334,10 @@ WHERE id=$1 AND status='pending' AND target_key IS NOT NULL FOR UPDATE`, id).Sca
|
||||
return pin, true, nil
|
||||
}
|
||||
|
||||
// ErrTargetPinEmpty refuses an approval whose request displayed no address: a
|
||||
// pin never holds zero addresses, whatever the previous pin of the key holds.
|
||||
var ErrTargetPinEmpty = errors.New("the target request lists no resolved address; an approval never pins an empty address list")
|
||||
|
||||
// writeTargetPin inserts the pin an approved request creates and fills the
|
||||
// request's target with what was pinned. The only writer of target_pins rows.
|
||||
func writeTargetPin(tx *sql.Tx, scope TargetScopeRef, interceptID int64, t *InterceptTarget, approver, remoteAddr string) (*TargetPin, error) {
|
||||
@@ -349,7 +353,12 @@ WHERE `+targetPinScope+` AND host=$3 AND port=$4`, scope.TaskID, scope.Conversat
|
||||
}
|
||||
ips := map[string]bool{}
|
||||
for _, ip := range t.ResolvedIPs {
|
||||
ips[ip] = true
|
||||
if ip != "" {
|
||||
ips[ip] = true
|
||||
}
|
||||
}
|
||||
if len(ips) == 0 {
|
||||
return nil, ErrTargetPinEmpty
|
||||
}
|
||||
prev, err := scanTargetPin(tx.QueryRow(`SELECT `+targetPinCols+` FROM target_pins
|
||||
WHERE `+targetPinScope+` AND host=$3 AND port=$4 AND status='active' FOR UPDATE`, scope.TaskID, scope.ConversationID, t.Host, t.Port))
|
||||
@@ -391,3 +400,154 @@ RETURNING `+targetPinCols,
|
||||
t.Outcome, t.PinID, t.PinVersion, t.PinnedIPs = "pinned", pin.ID, pin.Version, pin.PinnedIPs
|
||||
return pin, nil
|
||||
}
|
||||
|
||||
// TargetDecision is an outcome of the target gate that no human took: a block,
|
||||
// or the first use of an exemption or of a pin version.
|
||||
type TargetDecision struct {
|
||||
Scope TargetScopeRef
|
||||
AgentName string
|
||||
ToolName string
|
||||
ToolInput []byte
|
||||
Status string // allowed | denied
|
||||
Reason string
|
||||
Audit *InterceptAudit
|
||||
}
|
||||
|
||||
// LogTargetDecision records a target-gate outcome as an already-decided row
|
||||
// (decided_by = system, decision_source = target). It opens no request and
|
||||
// never pins.
|
||||
func (d *DB) LogTargetDecision(r TargetDecision) (int64, error) {
|
||||
if err := r.Scope.check(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw := json.RawMessage(r.ToolInput)
|
||||
if len(raw) == 0 {
|
||||
raw = json.RawMessage("{}")
|
||||
}
|
||||
taskID, convID := r.Scope.pendingScope()
|
||||
var id int64
|
||||
err := d.QueryRow(`
|
||||
INSERT INTO intercept_pending(conversation_id, task_id, agent_name, tool_name, tool_input, status, reason, decided_at, decision_source, audit, decided_by)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, NOW(), 'target', $8, 'system') RETURNING id`,
|
||||
convID, taskID, r.AgentName, r.ToolName, raw, r.Status, r.Reason, firstAudit([]*InterceptAudit{r.Audit})).Scan(&id)
|
||||
return id, err
|
||||
}
|
||||
|
||||
// SettingTargetExemptHosts is the settings key of the stored target-gate
|
||||
// exemption list: a JSON array of TargetExemption.
|
||||
const SettingTargetExemptHosts = "target_gate_exempt_hosts"
|
||||
|
||||
// TargetExemption is one entry of the target-gate exemption list: an exact
|
||||
// host and port that needs no approval and no pin. It is configuration, not an
|
||||
// approval.
|
||||
type TargetExemption struct {
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// TargetExemptions returns the stored exemption list and its raw settings
|
||||
// value ("" when the list was never stored).
|
||||
func (d *DB) TargetExemptions() (list []TargetExemption, raw string, err error) {
|
||||
raw, ok, err := d.GetSetting(SettingTargetExemptHosts)
|
||||
if err != nil || !ok {
|
||||
return nil, "", err
|
||||
}
|
||||
if err := json.Unmarshal([]byte(raw), &list); err != nil {
|
||||
return nil, raw, fmt.Errorf("stored target-gate exemption list: %w", err)
|
||||
}
|
||||
return list, raw, nil
|
||||
}
|
||||
|
||||
// ReplaceTargetExemptions stores list as the exemption list and appends the
|
||||
// change to target_gate_exemption_log, in one transaction. digest identifies
|
||||
// the new list in the log and in the audit of the calls that use it.
|
||||
func (d *DB) ReplaceTargetExemptions(list []TargetExemption, changedBy, digest string) error {
|
||||
if list == nil {
|
||||
list = []TargetExemption{}
|
||||
}
|
||||
after, err := json.Marshal(list)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tx, err := d.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var before *string
|
||||
var stored string
|
||||
switch err := tx.QueryRow(`SELECT value FROM settings WHERE key=$1 FOR UPDATE`, SettingTargetExemptHosts).Scan(&stored); {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
case err != nil:
|
||||
return err
|
||||
case json.Valid([]byte(stored)):
|
||||
before = &stored
|
||||
}
|
||||
if _, err := tx.Exec(`
|
||||
INSERT INTO settings(key, value) VALUES ($1, $2)
|
||||
ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now()`, SettingTargetExemptHosts, string(after)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO target_gate_exemption_log(changed_by, before, after, digest)
|
||||
VALUES ($1, $2::jsonb, $3::jsonb, $4)`, changedBy, before, string(after), digest); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// TargetExemptionChange is one row of target_gate_exemption_log.
|
||||
type TargetExemptionChange struct {
|
||||
ID int64 `json:"id"`
|
||||
ChangedAt time.Time `json:"changed_at"`
|
||||
ChangedBy string `json:"changed_by"`
|
||||
Before json.RawMessage `json:"before"`
|
||||
After json.RawMessage `json:"after"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// ListTargetExemptionLog returns the most recent changes of the exemption
|
||||
// list, newest first.
|
||||
func (d *DB) ListTargetExemptionLog(limit int) ([]TargetExemptionChange, error) {
|
||||
rows, err := d.Query(`SELECT id, changed_at, changed_by, before, after, digest
|
||||
FROM target_gate_exemption_log ORDER BY id DESC LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []TargetExemptionChange{}
|
||||
for rows.Next() {
|
||||
var c TargetExemptionChange
|
||||
var before, after []byte
|
||||
if err := rows.Scan(&c.ID, &c.ChangedAt, &c.ChangedBy, &before, &after, &c.Digest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.Before, c.After = before, after
|
||||
out = append(out, c)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// seedTargetGateExemptions stores the initial exemption list once: the package
|
||||
// sources of the runtime image, so that a package install does not ask for an
|
||||
// approval. Gated by a settings flag like the rule seeds: the list is owned by
|
||||
// the operator afterwards, and an entry they remove stays removed.
|
||||
func (d *DB) seedTargetGateExemptions() error {
|
||||
if v, _, _ := d.GetSetting("target_gate_exempt_seed_v1"); v == "done" {
|
||||
return nil
|
||||
}
|
||||
raw, err := json.Marshal([]TargetExemption{
|
||||
{Host: "pypi.org", Port: 443, Note: "[built-in] Python package index"},
|
||||
{Host: "files.pythonhosted.org", Port: 443, Note: "[built-in] Python package files"},
|
||||
{Host: "registry.npmjs.org", Port: 443, Note: "[built-in] npm registry"},
|
||||
{Host: "deb.debian.org", Port: 80, Note: "[built-in] Debian packages"},
|
||||
{Host: "deb.debian.org", Port: 443, Note: "[built-in] Debian packages"},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := d.InsertSettingIfAbsent(SettingTargetExemptHosts, string(raw)); err != nil {
|
||||
return err
|
||||
}
|
||||
return d.SetSetting("target_gate_exempt_seed_v1", "done")
|
||||
}
|
||||
|
||||
@@ -922,3 +922,172 @@ func TestResolveInterceptByRecordsApprover(t *testing.T) {
|
||||
t.Fatalf("a system decision recorded an approver: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// An approval never pins zero addresses. A request that displays none cannot
|
||||
// be approved, whether the key has a pin or not: the row stays pending, the
|
||||
// pin of the key stays as it is, and the request can still be denied.
|
||||
func TestTargetPinRefusesEmptyAddressList(t *testing.T) {
|
||||
d := openTargetDB(t)
|
||||
scope := TargetScopeRef{TaskID: targetTask(t, d, TaskCreateOptions{}).ID}
|
||||
const host = "no-address.example.com"
|
||||
|
||||
for name, ips := range map[string][]string{"nil": nil, "empty": {}, "blank": {"", ""}} {
|
||||
id, created := openTargetRequest(t, d, scope, host, 443, ips...)
|
||||
if !created {
|
||||
t.Fatalf("%s: the request was not created", name)
|
||||
}
|
||||
pin, ok, err := d.ResolveTargetRequest(id, true, "ARTEX", "198.51.100.9")
|
||||
if !errors.Is(err, ErrTargetPinEmpty) || ok || pin != nil {
|
||||
t.Fatalf("%s: approval of an empty address list: pin=%+v ok=%v err=%v, want ErrTargetPinEmpty", name, pin, ok, err)
|
||||
}
|
||||
row, err := d.GetInterceptPending(id)
|
||||
if err != nil || row == nil || row.Status != "pending" {
|
||||
t.Fatalf("%s: row after the refused approval = %+v, %v; want still pending", name, row, err)
|
||||
}
|
||||
if pins, err := d.ListTargetPins(scope); err != nil || len(pins) != 0 {
|
||||
t.Fatalf("%s: pins after the refused approval = %+v, %v; want none", name, pins, err)
|
||||
}
|
||||
if _, ok, err := d.ResolveTargetRequest(id, false, "ARTEX", "198.51.100.9"); err != nil || !ok {
|
||||
t.Fatalf("%s: denying the request: ok=%v err=%v", name, ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The key has a pin with addresses: an empty request does not re-issue it.
|
||||
v1 := approveTarget(t, d, scope, host, 443, "203.0.113.7")
|
||||
id, _ := openTargetRequest(t, d, scope, host, 443)
|
||||
if pin, ok, err := d.ResolveTargetRequest(id, true, "ARTEX", "198.51.100.9"); !errors.Is(err, ErrTargetPinEmpty) || ok || pin != nil {
|
||||
t.Fatalf("empty request on a pinned key: pin=%+v ok=%v err=%v, want ErrTargetPinEmpty", pin, ok, err)
|
||||
}
|
||||
pins, err := d.ListTargetPins(scope)
|
||||
if err != nil || len(pins) != 1 || pins[0].ID != v1.ID || pins[0].Status != TargetPinActive || pins[0].Version != 1 {
|
||||
t.Fatalf("pins = %+v, %v; want v1 active and untouched", pins, err)
|
||||
}
|
||||
// Blank entries are dropped from a list that has addresses.
|
||||
if _, ok, err := d.ResolveTargetRequest(id, false, "ARTEX", "198.51.100.9"); err != nil || !ok {
|
||||
t.Fatal(ok, err)
|
||||
}
|
||||
v2 := approveTarget(t, d, scope, host, 443, "", "203.0.113.8")
|
||||
if v2.Version != 2 || !slices.Equal(v2.PinnedIPs, []string{"203.0.113.7", "203.0.113.8"}) {
|
||||
t.Fatalf("v2 = %+v, want both addresses and no blank entry", v2)
|
||||
}
|
||||
}
|
||||
|
||||
// The exemption list is seeded once with the package sources of the runtime
|
||||
// image, and belongs to the operator afterwards: a start never restores an
|
||||
// entry they removed. Every change is appended to the log with its previous value.
|
||||
func TestTargetGateExemptionSeedAndLog(t *testing.T) {
|
||||
d := openTargetDB(t)
|
||||
seeded := []TargetExemption{
|
||||
{Host: "pypi.org", Port: 443, Note: "[built-in] Python package index"},
|
||||
{Host: "files.pythonhosted.org", Port: 443, Note: "[built-in] Python package files"},
|
||||
{Host: "registry.npmjs.org", Port: 443, Note: "[built-in] npm registry"},
|
||||
{Host: "deb.debian.org", Port: 80, Note: "[built-in] Debian packages"},
|
||||
{Host: "deb.debian.org", Port: 443, Note: "[built-in] Debian packages"},
|
||||
}
|
||||
// Start from the state of a new database, and put the list back afterwards.
|
||||
original, _, err := d.TargetExemptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = d.ReplaceTargetExemptions(original, "test", "restore") })
|
||||
if _, err := d.Exec(`DELETE FROM settings WHERE key IN ($1, 'target_gate_exempt_seed_v1')`, SettingTargetExemptHosts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if list, raw, err := d.TargetExemptions(); err != nil || list != nil || raw != "" {
|
||||
t.Fatalf("unset list = %+v, %q, %v", list, raw, err)
|
||||
}
|
||||
if err := d.seedTargetGateExemptions(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list, raw, err := d.TargetExemptions()
|
||||
if err != nil || !slices.Equal(list, seeded) || raw == "" {
|
||||
t.Fatalf("seeded list = %+v, %v; want the package sources", list, err)
|
||||
}
|
||||
|
||||
// The operator replaces the list; the next start does not seed again.
|
||||
changesBefore, err := d.ListTargetExemptionLog(1000)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
own := []TargetExemption{{Host: "mirror.example.org", Port: 443, Note: "internal mirror"}}
|
||||
if err := d.ReplaceTargetExemptions(own, "alice", "digest-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.seedTargetGateExemptions(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if list, _, err := d.TargetExemptions(); err != nil || !slices.Equal(list, own) {
|
||||
t.Fatalf("list after a second seed = %+v, %v; want the list of the operator", list, err)
|
||||
}
|
||||
// An emptied list stays empty, and is stored as [] rather than null.
|
||||
if err := d.ReplaceTargetExemptions(nil, "bob", "digest-2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.seedTargetGateExemptions(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if list, raw, err := d.TargetExemptions(); err != nil || len(list) != 0 || raw != "[]" {
|
||||
t.Fatalf("emptied list = %+v, %q, %v", list, raw, err)
|
||||
}
|
||||
|
||||
changes, err := d.ListTargetExemptionLog(1000)
|
||||
if err != nil || len(changes) != len(changesBefore)+2 {
|
||||
t.Fatalf("%d log rows added (err=%v), want 2", len(changes)-len(changesBefore), err)
|
||||
}
|
||||
decode := func(raw json.RawMessage) []TargetExemption {
|
||||
t.Helper()
|
||||
var out []TargetExemption
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
t.Fatalf("log value %q: %v", raw, err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
second, first := changes[0], changes[1]
|
||||
if first.ChangedBy != "alice" || first.Digest != "digest-1" || !slices.Equal(decode(first.Before), seeded) || !slices.Equal(decode(first.After), own) {
|
||||
t.Fatalf("first change = %+v", first)
|
||||
}
|
||||
if second.ChangedBy != "bob" || second.Digest != "digest-2" || !slices.Equal(decode(second.Before), own) || len(decode(second.After)) != 0 || second.ID <= first.ID {
|
||||
t.Fatalf("second change = %+v", second)
|
||||
}
|
||||
// The log is append-only: a change never rewrites an earlier row.
|
||||
if again, err := d.ListTargetExemptionLog(1000); err != nil || len(again) != len(changes) || again[1].ID != first.ID || string(again[1].After) != string(first.After) {
|
||||
t.Fatalf("log changed between two reads: %+v, %v", again, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A decision of the gate that no human took is a decided row of the scope:
|
||||
// decided by the system, source target, never a request and never a pin.
|
||||
func TestLogTargetDecision(t *testing.T) {
|
||||
d := openTargetDB(t)
|
||||
task := targetTask(t, d, TaskCreateOptions{})
|
||||
conv := targetConversation(t, d)
|
||||
for name, scope := range map[string]TargetScopeRef{"task": {TaskID: task.ID}, "conversation": {ConversationID: conv.ID}} {
|
||||
id, err := d.LogTargetDecision(TargetDecision{Scope: scope, AgentName: "planner", ToolName: "WebFetch",
|
||||
ToolInput: []byte(`{"url":"https://app.example.com/"}`), Status: "denied", Reason: "TARGET_RESOLVE_FAILED: no such host",
|
||||
Audit: &InterceptAudit{Correlation: "unavailable", EffectiveAction: "deny", ExecutionStatus: "not_executed",
|
||||
Targets: []InterceptTarget{{Host: "app.example.com", Port: 443, Kind: "domain", Outcome: "resolve_failed"}}}})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
row, err := d.GetInterceptDetail(id)
|
||||
if err != nil || row == nil {
|
||||
t.Fatalf("%s: row=%+v err=%v", name, row, err)
|
||||
}
|
||||
if row.Status != "denied" || row.DecidedBy != "system" || row.DecisionSource != "target" || row.DecidedAt == nil || row.RuleID != nil ||
|
||||
row.AgentName != "planner" || row.Audit == nil || row.Audit.Targets[0].Outcome != "resolve_failed" {
|
||||
t.Fatalf("%s: row = %+v audit=%+v", name, row.InterceptApprovalRow, row.Audit)
|
||||
}
|
||||
if (row.TaskID != nil) != (scope.TaskID != 0) || (row.ConversationID != nil) != (scope.ConversationID != 0) {
|
||||
t.Fatalf("%s: row scope = task %v conversation %v", name, row.TaskID, row.ConversationID)
|
||||
}
|
||||
if key, err := d.TargetRequestKey(id); err != nil || key != "" {
|
||||
t.Fatalf("%s: target key = %q, %v; a decided row is not a request", name, key, err)
|
||||
}
|
||||
if n := pendingTargetRows(t, d, scope); n != 0 {
|
||||
t.Fatalf("%s: %d open requests, want none", name, n)
|
||||
}
|
||||
}
|
||||
if _, err := d.LogTargetDecision(TargetDecision{Status: "denied"}); err == nil {
|
||||
t.Fatal("a decision without a scope was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,4 +61,4 @@ require (
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/Autumn-27/norma => github.com/BaaRai/Norma v0.4.6
|
||||
replace github.com/Autumn-27/norma => github.com/BaaRai/Norma v0.4.8
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
github.com/BaaRai/Norma v0.4.6 h1:sEm58Gx7gw+UU/05aHZBZ+Xqm5dMYiuSQpc1aXnbElc=
|
||||
github.com/BaaRai/Norma v0.4.6/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw=
|
||||
github.com/BaaRai/Norma v0.4.8 h1:k3at2aYSJ6aIDjEo7tC4VDSLBHXw2HobYjpKChVe6hI=
|
||||
github.com/BaaRai/Norma v0.4.8/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw=
|
||||
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
|
||||
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
|
||||
|
||||
+129
-26
@@ -10,7 +10,11 @@ package guard
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -35,6 +39,7 @@ type Guard struct {
|
||||
reg *hook.Registry
|
||||
interceptor *intercept.Interceptor // optional; nil disables user-configured rules
|
||||
targetOnly bool // planner / main agent: target gate only, no rules, no judge, no audit ring
|
||||
gate targetGate // the target gate; nil without an interceptor
|
||||
}
|
||||
|
||||
// New creates a Guard without user-configured intercept rules (used for pentest
|
||||
@@ -50,18 +55,28 @@ func NewWithInterceptor(ic *intercept.Interceptor) *Guard { return newGuard(ic)
|
||||
// ring (which keeps the worker history of the task's main Guard).
|
||||
func NewTargetOnly(ic *intercept.Interceptor) *Guard {
|
||||
g := &Guard{attrib: map[string]int{}, interceptor: ic, targetOnly: true}
|
||||
g.setGate(ic)
|
||||
g.reg = hook.NewRegistry().On(hook.PreToolUse, g.preToolUse)
|
||||
return g
|
||||
}
|
||||
|
||||
func newGuard(ic *intercept.Interceptor) *Guard {
|
||||
g := &Guard{attrib: map[string]int{}, interceptor: ic}
|
||||
g.setGate(ic)
|
||||
g.reg = hook.NewRegistry().
|
||||
On(hook.PreToolUse, g.preToolUse).
|
||||
On(hook.PostToolUse, g.postToolUse)
|
||||
return g
|
||||
}
|
||||
|
||||
// setGate wires the interceptor in as the target gate. A nil interceptor leaves
|
||||
// the gate unset rather than set to an interface that holds a nil pointer.
|
||||
func (g *Guard) setGate(ic *intercept.Interceptor) {
|
||||
if ic != nil {
|
||||
g.gate = ic
|
||||
}
|
||||
}
|
||||
|
||||
// Hooks returns the hook registry to attach to an agent session.
|
||||
func (g *Guard) Hooks() *hook.Registry { return g.reg }
|
||||
|
||||
@@ -93,45 +108,68 @@ func (g *Guard) preToolUse(ctx context.Context, ev hook.Event) hook.Result {
|
||||
return g.applyIntercept(ctx, ev)
|
||||
}
|
||||
|
||||
// targetGate is the seam of the approval gate for asset targets. The gate is not
|
||||
// wired in yet: every call is allowed, and nothing is extracted, resolved, read
|
||||
// or written.
|
||||
func (g *Guard) targetGate(_ context.Context, _ hook.Event) hook.Result {
|
||||
return hook.Result{}
|
||||
// targetGate is the approval gate for asset targets, as the guard uses it.
|
||||
// *intercept.Interceptor implements it.
|
||||
type targetGate interface {
|
||||
CheckTargets(ctx context.Context, scope intercept.TargetScope, toolName string, input []byte) ([]intercept.TargetRequest, intercept.TargetResult)
|
||||
RequestTargets(ctx context.Context, scope intercept.TargetScope, toolName string, input []byte, targets []intercept.TargetRequest, ask *intercept.Decision) intercept.TargetResult
|
||||
}
|
||||
|
||||
// applyIntercept evaluates user-configured intercept rules against the tool call.
|
||||
// Both rules and the fallback judge receive the complete tool input.
|
||||
// targetPasses bounds the request-and-recheck loop of one call: a request that
|
||||
// was decided in between, or addresses that changed again after an approval,
|
||||
// send the call back to the pin lookup at most this many times.
|
||||
const targetPasses = 3
|
||||
|
||||
// gateScope returns the target-gate scope of the session, and whether its gate
|
||||
// is on. With the gate off (the default) no call reaches the gate.
|
||||
func (g *Guard) gateScope(ctx context.Context) (intercept.TargetScope, bool) {
|
||||
scope, ok := intercept.TargetScopeFromContext(ctx)
|
||||
return scope, ok && g.gate != nil && scope.Enabled != nil && scope.Enabled()
|
||||
}
|
||||
|
||||
// applyIntercept evaluates user-configured intercept rules against the tool call,
|
||||
// then the target gate. Both rules and the fallback judge receive the complete
|
||||
// tool input.
|
||||
//
|
||||
// The rules and the judge apply to the enabled tools of a full guard. The
|
||||
// target gate applies to every tool call of a session whose gate is on: a deny
|
||||
// blocks first, before any target request exists; an allow by a rule or by the
|
||||
// judge does not satisfy the gate; an ask and an unpinned target share one
|
||||
// approval request.
|
||||
func (g *Guard) applyIntercept(ctx context.Context, ev hook.Event) hook.Result {
|
||||
if g.targetOnly {
|
||||
return g.targetGate(ctx, ev)
|
||||
}
|
||||
if g.interceptor == nil {
|
||||
return hook.Result{}
|
||||
}
|
||||
if !g.interceptor.IsToolEnabled(ev.ToolName) {
|
||||
scope, gated := g.gateScope(ctx)
|
||||
rules := !g.targetOnly && g.interceptor != nil && g.interceptor.IsToolEnabled(ev.ToolName)
|
||||
if !rules && !gated {
|
||||
return hook.Result{}
|
||||
}
|
||||
ctx = intercept.WithCall(ctx, ev.ToolName, ev.Input)
|
||||
dec, matched := g.interceptor.Match(ev.ToolName, ev.Input)
|
||||
if !matched {
|
||||
// No rule matched. Ask the LLM fallback judge (if enabled); when it is off
|
||||
// or unwired, keep current behavior and allow.
|
||||
d, judged := g.interceptor.Judge(ctx, ev.ToolName, ev.Input)
|
||||
if !judged {
|
||||
return hook.Result{}
|
||||
var dec intercept.Decision
|
||||
if rules {
|
||||
var matched bool
|
||||
dec, matched = g.interceptor.Match(ev.ToolName, ev.Input)
|
||||
if !matched {
|
||||
// No rule matched. Ask the LLM fallback judge (if enabled); when it is off
|
||||
// or unwired, keep current behavior and allow.
|
||||
dec, _ = g.interceptor.Judge(ctx, ev.ToolName, ev.Input)
|
||||
}
|
||||
dec = d
|
||||
}
|
||||
switch dec.Action {
|
||||
case "deny":
|
||||
if dec.Action == "deny" {
|
||||
// Note: a deny hit does not block for approval; it directly logs a denied entry (visible on the history / task interception pages).
|
||||
g.interceptor.Log(ctx, intercept.ConvIDFromContext(ctx), dec, ev.ToolName, ev.Input, "denied")
|
||||
return g.block(ev.ToolName, systemBlockMessage(dec.Message), "")
|
||||
}
|
||||
var unpinned []intercept.TargetRequest
|
||||
if gated {
|
||||
var res intercept.TargetResult
|
||||
if unpinned, res = g.gate.CheckTargets(ctx, scope, ev.ToolName, ev.Input); res.Code != "" {
|
||||
return g.block(ev.ToolName, TargetBlockMessage(res), "")
|
||||
}
|
||||
}
|
||||
var ask *intercept.Decision
|
||||
switch dec.Action {
|
||||
case "allow":
|
||||
// Record explicit rule and model approvals so review details remain auditable.
|
||||
g.interceptor.Log(ctx, intercept.ConvIDFromContext(ctx), dec, ev.ToolName, ev.Input, "allowed")
|
||||
return hook.Result{}
|
||||
case "ask":
|
||||
// If the worker context is already cancelled (task stopped / killed), block
|
||||
// immediately without creating a pending record — avoids orphaned DB entries
|
||||
@@ -139,15 +177,80 @@ func (g *Guard) applyIntercept(ctx context.Context, ev hook.Event) hook.Result {
|
||||
if ctx.Err() != nil {
|
||||
return g.block(ev.ToolName, systemBlockMessage("work cancelled; platform safety controls blocked execution"), "")
|
||||
}
|
||||
ask = &dec
|
||||
}
|
||||
if len(unpinned) > 0 {
|
||||
res, merged := g.requestTargets(ctx, scope, ev, unpinned, ask)
|
||||
if res.Code != "" {
|
||||
return g.block(ev.ToolName, TargetBlockMessage(res), "")
|
||||
}
|
||||
if merged {
|
||||
// The approval of the target request answered the ask as well.
|
||||
ask = nil
|
||||
}
|
||||
}
|
||||
if ask != nil {
|
||||
convID := intercept.ConvIDFromContext(ctx)
|
||||
if !g.interceptor.HandleAsk(ctx, convID, dec, ev.ToolName, ev.Input) {
|
||||
return g.block(ev.ToolName, systemBlockMessage("manual approval not granted (user rejected or approval timed out)"), "")
|
||||
}
|
||||
return hook.Result{}
|
||||
}
|
||||
return hook.Result{}
|
||||
}
|
||||
|
||||
// requestTargets asks for the approval of the targets no pin covers, waits for
|
||||
// the decisions, and checks the call again: it runs only if every target is
|
||||
// covered then. A code in the result blocks the call. merged reports that the
|
||||
// ask was carried by a request this call created, so it needs no prompt of its
|
||||
// own.
|
||||
func (g *Guard) requestTargets(ctx context.Context, scope intercept.TargetScope, ev hook.Event, unpinned []intercept.TargetRequest, ask *intercept.Decision) (res intercept.TargetResult, merged bool) {
|
||||
for pass := 0; pass < targetPasses; pass++ {
|
||||
if res = g.gate.RequestTargets(ctx, scope, ev.ToolName, ev.Input, unpinned, ask); res.Code != "" {
|
||||
return res, merged
|
||||
}
|
||||
if res.AskMerged {
|
||||
merged, ask = true, nil
|
||||
}
|
||||
if unpinned, res = g.gate.CheckTargets(ctx, scope, ev.ToolName, ev.Input); res.Code != "" || len(unpinned) == 0 {
|
||||
return res, merged
|
||||
}
|
||||
}
|
||||
t := unpinned[0]
|
||||
return intercept.TargetResult{Code: intercept.TargetApprovalPending, Host: t.Host, Port: t.Port,
|
||||
Reason: fmt.Sprintf("the target is still not covered by a pin after %d approval rounds; its addresses changed again", targetPasses)}, merged
|
||||
}
|
||||
|
||||
// targetGuidance tells the model what a target block means for its next step.
|
||||
var targetGuidance = map[string]string{
|
||||
intercept.TargetApprovalPending: "The call was not executed. An approval by the operator is needed and may still arrive: retry the same call later. " +
|
||||
"Do not rewrite the command, switch tools or substitute an IP address.",
|
||||
intercept.TargetDenied: "The call was not executed. The operator did not approve this target: do not retry it and do not reach it another way.",
|
||||
intercept.TargetResolveFailed: "The call was not executed and no approval was requested. " +
|
||||
"This is a resolver failure on the platform side, not a block by the target. " +
|
||||
"Do not rewrite the command, switch tools or substitute an IP address. Retry the same call later, or report the failure.",
|
||||
intercept.TargetRuleBlocked: "The call was not executed and no approval was requested. " +
|
||||
"The operator's rules exclude this target: do not reach it another way.",
|
||||
intercept.TargetTooMany: "The call was not executed and no approval was requested. Name fewer new targets per call.",
|
||||
intercept.TargetGateUnavailable: "The call was not executed. This is a failure on the platform side, not a block by the target. " +
|
||||
"Retry the same call later, or report the failure.",
|
||||
}
|
||||
|
||||
// TargetBlockMessage is systemBlockMessage for the target gate: the block code,
|
||||
// the reason and the target, then what to do about it. A target block is not
|
||||
// always final (an approval can still arrive, a lookup can succeed later), so
|
||||
// it does not end with "This operation is forbidden."
|
||||
func TargetBlockMessage(res intercept.TargetResult) string {
|
||||
reason := res.Reason
|
||||
if res.Host != "" && !strings.Contains(reason, res.Host) {
|
||||
target := res.Host
|
||||
if res.Port != 0 {
|
||||
target = net.JoinHostPort(res.Host, strconv.Itoa(res.Port))
|
||||
}
|
||||
reason = "target " + target + ": " + reason
|
||||
}
|
||||
return "[ARTEX platform control, not target defense] " + res.Code + ": " + reason + ". " + targetGuidance[res.Code]
|
||||
}
|
||||
|
||||
// systemBlockMessage frames an intercept block as an ARTEX platform-governance
|
||||
// decision so the agent does not mistake it for a target-side defense.
|
||||
//
|
||||
|
||||
+5
-4
@@ -37,10 +37,11 @@ func TestPreToolUsePassthrough(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The target-only guard (planner, main agent) runs the target-gate seam and
|
||||
// nothing else: no intercept rule, no fallback judge, no audit-ring entry. Its
|
||||
// interceptor has no database behind it, so any read or write through it would
|
||||
// panic — passing proves the seam allows every call without touching the store.
|
||||
// The target-only guard (planner, main agent) runs the target gate and nothing
|
||||
// else: no intercept rule, no fallback judge, no audit-ring entry. Here the gate
|
||||
// is off (no scope, or a scope without the flag). Its interceptor has no database
|
||||
// behind it, so any read or write through it would panic — passing proves that
|
||||
// with the gate off every call is allowed without touching the store.
|
||||
func TestTargetOnlyGuardSkipsRules(t *testing.T) {
|
||||
calls := []struct {
|
||||
tool string
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,14 @@
|
||||
package guard
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// Own database per package (#80), as in db, agent, server, evidence and intercept.
|
||||
db.SuffixTestDatabase("guard")
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
+15
-1
@@ -16,6 +16,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -181,6 +182,18 @@ type Interceptor struct {
|
||||
// upstreamProxy reports whether target traffic leaves through an upstream
|
||||
// egress proxy (nil = no). Injected by the server layer.
|
||||
upstreamProxy func() bool
|
||||
// derivedExempt lists the endpoints of the platform's own services (nil =
|
||||
// none). Injected by the server layer. See SetDerivedExemptions.
|
||||
derivedExempt func() []string
|
||||
// targets is what the target gate reads and writes; resolve resolves the
|
||||
// names it checks. firstUse holds the exemptions and pin versions whose first
|
||||
// use this process has logged.
|
||||
targets targetStore
|
||||
resolve *resolveCache
|
||||
firstUse sync.Map
|
||||
// dial is the base dialer of PinnedDialer: every connection it lets through
|
||||
// is opened here. See SetBaseDialer.
|
||||
dial func(ctx context.Context, network, addr string) (net.Conn, error)
|
||||
}
|
||||
|
||||
// SetUpstreamProxyCheck installs the function that reports whether an upstream
|
||||
@@ -210,7 +223,8 @@ var defaultEnabledTools = []string{
|
||||
// New creates an Interceptor backed by d. The rule cache is lazy-loaded on
|
||||
// first use.
|
||||
func New(d *db.DB) *Interceptor {
|
||||
return &Interceptor{db: d, pending: newPendingManager()}
|
||||
return &Interceptor{db: d, pending: newPendingManager(),
|
||||
targets: dbTargetStore{d}, resolve: newResolveCache(systemResolver{}), dial: baseDial}
|
||||
}
|
||||
|
||||
// Invalidate clears the in-memory rule cache and the enabled-tools cache.
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/idna"
|
||||
)
|
||||
|
||||
// Kinds of target.
|
||||
const (
|
||||
TargetDomain = "domain"
|
||||
TargetIP = "ip"
|
||||
TargetCIDR = "cidr"
|
||||
)
|
||||
|
||||
// Target is one network target a tool call declares, in normalised form. With
|
||||
// its scope, Host and Port are the key of a pin and of an approval request;
|
||||
// path, query and scheme are never part of it.
|
||||
type Target struct {
|
||||
Kind string // domain | ip | cidr
|
||||
Host string // lower-case punycode name, canonical IP literal, or masked CIDR
|
||||
Port int // 0 = port not declared
|
||||
}
|
||||
|
||||
// schemePorts are the default ports of the schemes a target is usually written
|
||||
// with. A URL of another scheme without an explicit port has port 0.
|
||||
var schemePorts = map[string]int{"http": 80, "https": 443, "ws": 80, "wss": 443, "ftp": 21}
|
||||
|
||||
// hostProfile maps a name the way a lookup does (case, width, punycode) without
|
||||
// the strict rules that would refuse an underscore label.
|
||||
var hostProfile = idna.New(idna.MapForLookup(), idna.BidiRule())
|
||||
|
||||
// NormalizeTarget parses one target written as a URL with a scheme, or as a
|
||||
// host, host:port, [IPv6]:port, IP literal or CIDR. The name is lower-cased, its
|
||||
// trailing dot, userinfo and IPv6 brackets are removed, the host of a URL is
|
||||
// percent-decoded, an IPv4-mapped IPv6 address is unmapped, the decimal, hex, octal and short forms of an IPv4
|
||||
// address become dotted, and a Unicode name becomes punycode. The port is the
|
||||
// explicit one, else the default of the scheme, else 0.
|
||||
func NormalizeTarget(raw string) (Target, bool) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
def, isURL := 0, false
|
||||
if i := strings.Index(raw, "://"); i > 0 {
|
||||
isURL = true
|
||||
def = schemePorts[strings.ToLower(raw[:i])]
|
||||
raw = raw[i+3:]
|
||||
if end := strings.IndexAny(raw, "/?#"); end >= 0 {
|
||||
raw = raw[:end]
|
||||
}
|
||||
raw = raw[strings.LastIndexByte(raw, '@')+1:]
|
||||
}
|
||||
host, port := raw, ""
|
||||
switch {
|
||||
case strings.HasPrefix(raw, "["):
|
||||
end := strings.IndexByte(raw, ']')
|
||||
if end < 0 {
|
||||
return Target{}, false
|
||||
}
|
||||
host = raw[1:end]
|
||||
if rest := raw[end+1:]; rest != "" {
|
||||
if rest[0] != ':' {
|
||||
return Target{}, false
|
||||
}
|
||||
port = rest[1:]
|
||||
}
|
||||
case strings.Count(raw, ":") == 1:
|
||||
host, port, _ = strings.Cut(raw, ":")
|
||||
}
|
||||
if isURL && !strings.HasPrefix(raw, "[") {
|
||||
// A client decodes a percent-encoded host before it resolves it.
|
||||
if decoded, err := url.PathUnescape(host); err == nil {
|
||||
host = decoded
|
||||
}
|
||||
}
|
||||
t := Target{Port: def}
|
||||
if port != "" {
|
||||
n, err := strconv.Atoi(port)
|
||||
if err != nil || n < 1 || n > 65535 {
|
||||
return Target{}, false
|
||||
}
|
||||
t.Port = n
|
||||
}
|
||||
var ok bool
|
||||
t.Kind, t.Host, ok = normalizeHost(host, true)
|
||||
return t, ok
|
||||
}
|
||||
|
||||
// normalizeHost returns the kind and the canonical form of a host. numeric
|
||||
// accepts the forms of an IPv4 address that are not dotted decimal
|
||||
// (2130706433, 0x7f000001, 0177.0.0.1, 127.1), which a URL host may use; in
|
||||
// free text they would turn every number into an address.
|
||||
func normalizeHost(raw string, numeric bool) (kind, host string, ok bool) {
|
||||
h := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(raw)), ".")
|
||||
if h == "" {
|
||||
return "", "", false
|
||||
}
|
||||
if strings.Contains(h, "/") {
|
||||
p, err := netip.ParsePrefix(h)
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
return TargetCIDR, p.Masked().String(), true
|
||||
}
|
||||
if a, err := netip.ParseAddr(h); err == nil {
|
||||
return TargetIP, a.Unmap().WithZone("").String(), true
|
||||
}
|
||||
if numeric {
|
||||
if a, ok := parseIPv4Forms(h); ok {
|
||||
return TargetIP, a.String(), true
|
||||
}
|
||||
}
|
||||
// A name the profile refuses is kept as written: it then fails to resolve and
|
||||
// the call is blocked, instead of passing as a call without a target.
|
||||
if name, err := hostProfile.ToASCII(h); err == nil && name != "" {
|
||||
h = name
|
||||
}
|
||||
return TargetDomain, h, true
|
||||
}
|
||||
|
||||
// parseIPv4Forms reads an IPv4 address the way inet_aton does: one to four
|
||||
// parts, each decimal, hex (0x) or octal (leading 0), the last one filling the
|
||||
// remaining bytes.
|
||||
func parseIPv4Forms(s string) (netip.Addr, bool) {
|
||||
parts := strings.Split(s, ".")
|
||||
if len(parts) > 4 {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
var v uint64
|
||||
for n, p := range parts {
|
||||
base := 10
|
||||
switch {
|
||||
case strings.HasPrefix(p, "0x"):
|
||||
p, base = p[2:], 16
|
||||
case len(p) > 1 && p[0] == '0':
|
||||
p, base = p[1:], 8
|
||||
}
|
||||
x, err := strconv.ParseUint(p, base, 32)
|
||||
if err != nil {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
if n < len(parts)-1 {
|
||||
if x > 255 {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
v = v<<8 | x
|
||||
continue
|
||||
}
|
||||
rest := uint(8 * (4 - n))
|
||||
if x >= 1<<rest {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
v = v<<rest | x
|
||||
}
|
||||
return netip.AddrFrom4([4]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}), true
|
||||
}
|
||||
|
||||
// validHostname reports whether h is a plain DNS name: labels of letters,
|
||||
// digits, hyphen and underscore. A wildcard or a pattern is not one.
|
||||
func validHostname(h string) bool {
|
||||
if len(h) > 253 {
|
||||
return false
|
||||
}
|
||||
for _, label := range strings.Split(h, ".") {
|
||||
if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' {
|
||||
return false
|
||||
}
|
||||
for _, c := range []byte(label) {
|
||||
if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-' || c == '_') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// The declared forms of a call decide first: the URL block rules on the URLs
|
||||
// the call writes, then the block rules on every name, IP literal and range as
|
||||
// written. Nothing is resolved before that, so a hit costs no lookup.
|
||||
|
||||
func TestExtractDeclaredURLs(t *testing.T) {
|
||||
raw := func(v any) []byte {
|
||||
b, _ := json.Marshal(v)
|
||||
return b
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
tool string
|
||||
input []byte
|
||||
want []string
|
||||
}{
|
||||
{"WebFetch", "WebFetch", raw(map[string]string{"url": " https://app.example.com/admin?x=1 "}), []string{"https://app.example.com/admin?x=1"}},
|
||||
{"WebFetch without a target", "WebFetch", raw(map[string]string{"url": "http:///path"}), nil},
|
||||
{"Bash, query kept", "Bash", raw(command("curl -s 'https://a.example.com/admin?x=1'")), []string{"https://a.example.com/admin?x=1"}},
|
||||
{"Bash, two URLs", "Bash", raw(command("curl http://a.example.com/x && curl https://b.example.com:8443/y")),
|
||||
[]string{"http://a.example.com/x", "https://b.example.com:8443/y"}},
|
||||
{"shell_send", "shell_send", raw(map[string]string{"text": "curl http://a.example.com/x\n"}), []string{"http://a.example.com/x"}},
|
||||
{"Monitor", "Monitor", raw(command("watch curl http://198.51.100.9:8080/status")), []string{"http://198.51.100.9:8080/status"}},
|
||||
{"MCP string leaves", "mcp__scan__run", []byte(`{"target":"http://a.example.com/p","opts":{"list":["https://b.example.com/q"]}}`),
|
||||
[]string{"http://a.example.com/p", "https://b.example.com/q"}},
|
||||
{"ExecuteExtraTool unwrap", executeExtraTool, raw(map[string]any{"tool_name": "WebFetch", "params": map[string]string{"url": "https://a.example.com/z"}}),
|
||||
[]string{"https://a.example.com/z"}},
|
||||
{"tool without a target", "web_search", raw(map[string]string{"query": "https://a.example.com/admin"}), nil},
|
||||
{"wrapped tool without a target", executeExtraTool, raw(map[string]any{"tool_name": "web_search", "params": map[string]string{"query": "https://a.example.com/admin"}}), nil},
|
||||
{"authority that names no host", "Bash", raw(command("curl http://$HOST/x https://${H}:8443/admin")), nil},
|
||||
} {
|
||||
d := extractDeclared(c.tool, c.input)
|
||||
if !slices.Equal(d.urls, c.want) {
|
||||
t.Errorf("%s: urls = %q, want %q", c.name, d.urls, c.want)
|
||||
}
|
||||
// The targets are those ExtractTargets returns.
|
||||
if !slices.Equal(d.targets, ExtractTargets(c.tool, c.input)) {
|
||||
t.Errorf("%s: targets differ from ExtractTargets", c.name)
|
||||
}
|
||||
if len(d.urls) > 0 && len(d.targets) == 0 {
|
||||
t.Errorf("%s: a URL without a target", c.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A name that a name rule blocks is blocked on the name as written: no lookup
|
||||
// is made for it, nor for another name of the same call that no rule blocks.
|
||||
func TestBlockRuleOnDeclaredNameNeedsNoLookup(t *testing.T) {
|
||||
for _, rule := range []db.AssetInterceptRule{
|
||||
{Enabled: true, Kind: "exact_domain", Pattern: "blocked.example.com"},
|
||||
{Enabled: true, Kind: "fuzzy_domain", Pattern: "blocked.example"},
|
||||
} {
|
||||
g := newGate(t)
|
||||
g.res.set("ok.example.com", "203.0.113.7")
|
||||
g.res.set("blocked.example.com", "203.0.113.8")
|
||||
g.store.rules = []db.AssetInterceptRule{rule}
|
||||
for name, v := range map[string]verdict{
|
||||
"WebFetch": g.fetch("https://blocked.example.com/"),
|
||||
// The unblocked name comes first in the call.
|
||||
"two names": g.check("Bash", `{"command":"curl https://ok.example.com/ https://blocked.example.com/"}`),
|
||||
} {
|
||||
res := g.blocked(rule.Kind+", "+name, TargetRuleBlocked, v)
|
||||
if res.Host != "blocked.example.com" || !strings.Contains(res.Reason, rule.Pattern) {
|
||||
t.Fatalf("%s, %s: result = %+v", rule.Kind, name, res)
|
||||
}
|
||||
}
|
||||
if n := g.res.total(); n != 0 {
|
||||
t.Fatalf("%s: %d lookups, want none", rule.Kind, n)
|
||||
}
|
||||
for _, row := range g.store.logged {
|
||||
if a := row.Audit.Targets[0]; a.Outcome != "rule_blocked" || len(a.ResolvedIPs) != 0 {
|
||||
t.Fatalf("%s: logged target = %+v, want rule_blocked with no address", rule.Kind, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// URL rules apply at dispatch, on the URLs the call declares, before anything
|
||||
// is resolved. A hit blocks with the code of every block rule, opens no
|
||||
// request, and wins over a pin and an exemption.
|
||||
func TestURLBlockRuleAtDispatch(t *testing.T) {
|
||||
g := newGate(t)
|
||||
g.res.set("app.example.com", "203.0.113.7")
|
||||
g.res.set("pypi.org", "151.101.0.223")
|
||||
g.store.exempt = []db.TargetExemption{{Host: "pypi.org", Port: 443}}
|
||||
g.store.approve(TargetRequest{Host: "app.example.com", Port: 443, ResolvedIPs: []string{"203.0.113.7"}})
|
||||
g.passes("pinned", g.fetch("https://app.example.com/admin"))
|
||||
g.passes("exempt", g.fetch("https://pypi.org/admin"))
|
||||
|
||||
rule := func(kind, pattern string, enabled bool) []db.AssetInterceptRule {
|
||||
return []db.AssetInterceptRule{{Enabled: enabled, Kind: kind, Pattern: pattern}}
|
||||
}
|
||||
// A disabled URL rule blocks nothing.
|
||||
g.store.rules = rule("exact_url", "https://app.example.com/admin", false)
|
||||
g.passes("disabled rule", g.fetch("https://app.example.com/admin"))
|
||||
|
||||
lookups, rows := g.res.total(), len(g.store.logged)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
rules []db.AssetInterceptRule
|
||||
v func() verdict
|
||||
host string
|
||||
port int
|
||||
matchedURL string
|
||||
}{
|
||||
{"exact_url, WebFetch of a pinned host", rule("exact_url", "https://app.example.com/admin", true),
|
||||
func() verdict { return g.fetch("https://app.example.com/admin") }, "app.example.com", 443, "https://app.example.com/admin"},
|
||||
{"fuzzy_url, exempt host", rule("fuzzy_url", "/admin", true),
|
||||
func() verdict { return g.fetch("https://pypi.org/admin") }, "pypi.org", 443, "https://pypi.org/admin"},
|
||||
{"fuzzy_url, Bash with a query", rule("fuzzy_url", "/admin", true),
|
||||
func() verdict { return g.check("Bash", `{"command":"curl -s 'https://app.example.com/admin?x=1'"}`) }, "app.example.com", 443, "https://app.example.com/admin?x=1"},
|
||||
{"fuzzy_url, second URL of the call", rule("fuzzy_url", "/admin", true),
|
||||
func() verdict {
|
||||
return g.check("Bash", `{"command":"curl https://app.example.com/ && curl http://198.51.100.9:8080/admin"}`)
|
||||
}, "198.51.100.9", 8080, "http://198.51.100.9:8080/admin"},
|
||||
{"fuzzy_url, MCP input", rule("fuzzy_url", "/ADMIN", true),
|
||||
func() verdict { return g.check("mcp__scan__run", `{"target":"https://app.example.com/admin/users"}`) }, "app.example.com", 443, "https://app.example.com/admin/users"},
|
||||
{"exact_url, wrapped WebFetch", rule("exact_url", "https://app.example.com/admin", true),
|
||||
func() verdict {
|
||||
return g.check(executeExtraTool, `{"tool_name":"WebFetch","params":{"url":"https://app.example.com/admin"}}`)
|
||||
}, "app.example.com", 443, "https://app.example.com/admin"},
|
||||
} {
|
||||
g.store.rules = c.rules
|
||||
g.res.set("app.example.com", "198.51.100.200") // a lookup would make a pinned host ask
|
||||
res := g.blocked(c.name, TargetRuleBlocked, c.v())
|
||||
if res.Host != c.host || res.Port != c.port || !strings.Contains(res.Reason, c.matchedURL+" is blocked by an asset rule") ||
|
||||
!strings.Contains(res.Reason, c.rules[0].Pattern) {
|
||||
t.Errorf("%s: result = %+v", c.name, res)
|
||||
}
|
||||
}
|
||||
if n := g.res.total() - lookups; n != 0 {
|
||||
t.Fatalf("%d lookups for calls a URL rule blocks, want none", n)
|
||||
}
|
||||
if got := g.store.outcomes()[rows:]; len(got) != 6 || slices.ContainsFunc(got, func(o string) bool { return o != "denied/rule_blocked" }) {
|
||||
t.Fatalf("logged rows = %v, want 6 rule_blocked rows", got)
|
||||
}
|
||||
|
||||
// The rendered URL of a custom http tool and the URLs of a rendered command
|
||||
// meet the same rules, with no lookup.
|
||||
g.store.rules = rule("exact_url", "https://app.example.com/admin", true)
|
||||
if res := GateRenderedURL(g.ctx(), "probe_http", "https://app.example.com/admin"); res.Code != TargetRuleBlocked || res.Host != "app.example.com" {
|
||||
t.Fatalf("rendered URL: %+v", res)
|
||||
}
|
||||
g.store.rules = rule("fuzzy_url", "/admin", true)
|
||||
if res := GateRenderedCommand(g.ctx(), "probe_cmd", "curl 'https://app.example.com/admin'"); res.Code != TargetRuleBlocked {
|
||||
t.Fatalf("rendered command: %+v", res)
|
||||
}
|
||||
if n := g.res.total() - lookups; n != 0 {
|
||||
t.Fatalf("%d lookups for rendered calls a URL rule blocks, want none", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Documents the known limit of URL rules: they match the URL as the call
|
||||
// writes it. Another spelling of the same URL is not blocked. For a host, a
|
||||
// domain, IP or CIDR rule is the one to write.
|
||||
func TestExactURLRuleMatchesOnlyAsWritten(t *testing.T) {
|
||||
g := newGate(t)
|
||||
g.res.set("app.example.com", "203.0.113.7")
|
||||
g.store.approve(TargetRequest{Host: "app.example.com", Port: 443, ResolvedIPs: []string{"203.0.113.7"}})
|
||||
|
||||
g.store.rules = []db.AssetInterceptRule{{Enabled: true, Kind: "exact_url", Pattern: "https://app.example.com/admin"}}
|
||||
g.blocked("as written", TargetRuleBlocked, g.fetch("https://app.example.com/admin"))
|
||||
for _, u := range []string{
|
||||
"https://APP.example.com/admin", // case of the host
|
||||
"https://app.example.com/Admin", // case of the path
|
||||
"https://app.example.com/admin/", // trailing slash
|
||||
"https://app.example.com/admin?x=1", // added query
|
||||
"https://app.example.com/%61dmin", // percent-encoding
|
||||
"https://app.example.com/./admin", // dot segment
|
||||
"https://app.example.com:443/admin", // default port written out
|
||||
} {
|
||||
g.passes("exact_url, "+u, g.fetch(u))
|
||||
}
|
||||
|
||||
g.store.rules = []db.AssetInterceptRule{{Enabled: true, Kind: "fuzzy_url", Pattern: "/admin"}}
|
||||
g.blocked("fuzzy, as written", TargetRuleBlocked, g.fetch("https://app.example.com/x/admin"))
|
||||
for _, u := range []string{"https://app.example.com/%61dmin", "https://app.example.com/adm%69n", "https://app.example.com//%2fadmin"} {
|
||||
if strings.Contains(strings.ToLower(u), "/admin") {
|
||||
t.Fatalf("%s contains the pattern; the case documents an encoded form", u)
|
||||
}
|
||||
g.passes("fuzzy_url, "+u, g.fetch(u))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The HTTP clients the platform owns (WebFetch and the custom http tool)
|
||||
// connect through PinnedDialer. The hook, the rendered-URL gate and the
|
||||
// redirect policy decide on a URL, and the client then resolves the name again
|
||||
// on its own when it connects: a name can answer differently in between. In a
|
||||
// gated scope the dialer takes a second, independent decision on the address
|
||||
// it is asked to connect to (the target of the call, of a redirect hop, or a
|
||||
// proxy). It resolves the name afresh, compares the answer with the pins, and
|
||||
// connects only to the addresses it compared, never to the name. It never
|
||||
// waits: an address no pin covers finds or opens the approval request and
|
||||
// fails the connection at once with TARGET_APPROVAL_PENDING.
|
||||
|
||||
// baseDial opens a connection with the dialer settings of http.DefaultTransport.
|
||||
var baseDial = (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext
|
||||
|
||||
// PinnedDialer returns the DialContext of a platform-owned HTTP client run by
|
||||
// the tool toolName. The scope is read from the context net/http passes to the
|
||||
// dial, which carries the values of the request's context: the scope the hook,
|
||||
// the rendered-URL gate and the redirect policy read.
|
||||
//
|
||||
// scoped is whether the client was built for a session with a target scope.
|
||||
// Such a client never dials without one: a scope missing at the dial means the
|
||||
// context value was lost on the way, and the dial fails with
|
||||
// TARGET_GATE_UNAVAILABLE. A client built without a scope (the operator's test
|
||||
// endpoint) dials as before, and so does every client while the gate is off.
|
||||
func PinnedDialer(toolName string, scoped bool) func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
scope, ok := TargetScopeFromContext(ctx)
|
||||
switch {
|
||||
case !ok && scoped:
|
||||
return nil, &dialError{code: TargetGateUnavailable, target: addr, reason: "the target scope of the session did not reach the connection"}
|
||||
case !ok:
|
||||
return baseDial(ctx, network, addr)
|
||||
case scope.Enabled == nil || !scope.Enabled():
|
||||
if scope.Interceptor != nil {
|
||||
return scope.Interceptor.baseDialer()(ctx, network, addr)
|
||||
}
|
||||
return baseDial(ctx, network, addr)
|
||||
case scope.Interceptor == nil:
|
||||
return nil, &dialError{code: TargetGateUnavailable, target: addr, reason: "the session's target scope carries no gate"}
|
||||
}
|
||||
return scope.Interceptor.dialTarget(ctx, scope, toolName, network, addr)
|
||||
}
|
||||
}
|
||||
|
||||
// GateOn reports whether ctx carries a target scope whose gate is on.
|
||||
func GateOn(ctx context.Context) bool {
|
||||
_, on := renderedScope(ctx)
|
||||
return on
|
||||
}
|
||||
|
||||
func (i *Interceptor) baseDialer() func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
i.mu.RLock()
|
||||
defer i.mu.RUnlock()
|
||||
return i.dial
|
||||
}
|
||||
|
||||
// dialTarget decides one connection of a gated scope and opens it. The asset
|
||||
// block rules, exemptions and pins are those of the hook, on the target the
|
||||
// address names, with a fresh lookup. URL rules are not matched: they ran on
|
||||
// the URL at the hook and on each hop in RedirectCheck.
|
||||
//
|
||||
// A derived exemption, and a pin that is name-only for this connection, dial
|
||||
// the address as received. A stored exemption that holds, and a pin that
|
||||
// holds every address of the lookup, dial those addresses and nothing else.
|
||||
func (i *Interceptor) dialTarget(ctx context.Context, scope TargetScope, toolName, network, addr string) (net.Conn, error) {
|
||||
input, _ := json.Marshal(map[string]string{"dial": addr})
|
||||
ctx = WithCall(ctx, toolName, input)
|
||||
t, err := addrTarget(network, addr)
|
||||
if err != nil {
|
||||
res := undeterminedTarget(ctx, scope, toolName, input, "the address of the connection could not be read ("+err.Error()+")")
|
||||
return nil, &dialError{code: res.Code, target: addr, reason: res.Reason}
|
||||
}
|
||||
blocked := func(res TargetResult) error {
|
||||
return &dialError{code: res.Code, target: t.String(), reason: res.Reason}
|
||||
}
|
||||
c := &targetCheck{i: i, ctx: ctx, scope: scope, tool: toolName, input: input, fresh: true}
|
||||
unpinned, res := c.run([]Target{t})
|
||||
if res.Code != "" {
|
||||
return nil, blocked(res)
|
||||
}
|
||||
if len(unpinned) > 0 {
|
||||
// Wait bound 0: the request is found or created, and the dial does not wait.
|
||||
scope.WaitBound = 0
|
||||
res = i.RequestTargets(ctx, scope, toolName, input, unpinned, nil)
|
||||
if res.Code == TargetApprovalPending {
|
||||
res.Reason = notCovered(unpinned[0], scope)
|
||||
}
|
||||
return nil, blocked(res)
|
||||
}
|
||||
switch ct := c.checked[0]; ct.cover {
|
||||
case coverDerived, coverNameOnly:
|
||||
return i.baseDialer()(ctx, network, addr)
|
||||
default:
|
||||
return i.dialAddrs(ctx, network, ct.ips, t.Port)
|
||||
}
|
||||
}
|
||||
|
||||
// addrTarget reads the address net/http dials, host:port with the host in
|
||||
// ASCII form and the port always written, as the target urlTarget reads from
|
||||
// the URL.
|
||||
func addrTarget(network, addr string) (Target, error) {
|
||||
switch network {
|
||||
case "tcp", "tcp4", "tcp6":
|
||||
default:
|
||||
return Target{}, fmt.Errorf("network %q is not tcp", network)
|
||||
}
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return Target{}, err
|
||||
}
|
||||
kind, h, ok := normalizeHost(host, true)
|
||||
if !ok || kind == TargetCIDR {
|
||||
return Target{}, fmt.Errorf("host %q", host)
|
||||
}
|
||||
t := Target{Kind: kind, Host: h, Port: validPort(port)}
|
||||
if t.Port == 0 {
|
||||
return Target{}, fmt.Errorf("port %q", port)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// dialAddrs connects to the addresses the decision compared, in their canonical
|
||||
// order: the first that accepts wins. An address of the wrong family for tcp4
|
||||
// or tcp6 is skipped. When every attempt fails, the last error is returned.
|
||||
func (i *Interceptor) dialAddrs(ctx context.Context, network string, ips []string, port int) (net.Conn, error) {
|
||||
dial := i.baseDialer()
|
||||
var last error = &net.AddrError{Err: "no suitable address found", Addr: strings.Join(ips, ", ")}
|
||||
for _, ip := range ips {
|
||||
a, err := netip.ParseAddr(ip)
|
||||
if err != nil || network == "tcp4" && !a.Is4() || network == "tcp6" && !a.Is6() {
|
||||
continue
|
||||
}
|
||||
conn, err := dial(ctx, network, net.JoinHostPort(ip, strconv.Itoa(port)))
|
||||
if err == nil {
|
||||
return conn, nil
|
||||
}
|
||||
last = err
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
|
||||
// notCovered says why a connection needs an approval: the addresses the dial
|
||||
// resolved, and the pin they fall outside of.
|
||||
func notCovered(r TargetRequest, scope TargetScope) string {
|
||||
if r.Pin != nil {
|
||||
return fmt.Sprintf("%s now resolves to %s, outside pin v%d (%s) of %s",
|
||||
r.Host, strings.Join(r.ResolvedIPs, ", "), r.Pin.Version, strings.Join(r.Pin.PinnedIPs, ", "), scope.scopeName())
|
||||
}
|
||||
if r.Kind != TargetDomain {
|
||||
return fmt.Sprintf("%s is not approved for %s", r.Host, scope.scopeName())
|
||||
}
|
||||
return fmt.Sprintf("%s resolves to %s and is not approved for %s", r.Host, strings.Join(r.ResolvedIPs, ", "), scope.scopeName())
|
||||
}
|
||||
|
||||
// dialError is what PinnedDialer returns for a connection it does not open.
|
||||
// Both clients hand the error to the model as the tool result.
|
||||
type dialError struct {
|
||||
code, target, reason string
|
||||
}
|
||||
|
||||
func (e *dialError) Error() string {
|
||||
msg := "[ARTEX platform control, not target defense] " + e.code + ": the connection to " + e.target + " was not opened: " + e.reason + "."
|
||||
if e.code == TargetApprovalPending {
|
||||
msg += " An approval was requested. Retry the same call after the approval."
|
||||
}
|
||||
return msg + " Do not fetch the target with another tool or by its address."
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// Step 2.1 against PostgreSQL: a connection no pin covers at the dial opens or
|
||||
// finds the durable request, connects nowhere, and fails at once.
|
||||
|
||||
// pgDialGate is a gate on d with a fake resolver and a recording base dialer.
|
||||
func pgDialGate(t *testing.T, d *db.DB) (*Interceptor, *fakeResolver, *dialRecorder) {
|
||||
t.Helper()
|
||||
ic, res := pgGate(t, d)
|
||||
rec := newDialRecorder()
|
||||
ic.SetBaseDialer(rec.dial)
|
||||
return ic, res, rec
|
||||
}
|
||||
|
||||
// openTarget returns the audit target of the single open request of scope.
|
||||
func openTarget(t *testing.T, d *db.DB, scope TargetScope) (*db.InterceptDetail, db.InterceptTarget) {
|
||||
t.Helper()
|
||||
ids := openRows(t, d, scope)
|
||||
if len(ids) != 1 {
|
||||
t.Fatalf("open requests = %v, want one", ids)
|
||||
}
|
||||
detail, err := d.GetInterceptDetail(ids[0])
|
||||
if err != nil || detail == nil || detail.Audit == nil || len(detail.Audit.Targets) != 1 {
|
||||
t.Fatalf("request detail = %+v, %v", detail, err)
|
||||
}
|
||||
return detail, detail.Audit.Targets[0]
|
||||
}
|
||||
|
||||
// The failure test: the check passes on the pinned address, the name answers
|
||||
// another one before the dial, and the call connects nowhere: not to the old
|
||||
// address, not to the new one. One new_ip request shows both; once approved,
|
||||
// the same call connects to the new address.
|
||||
func TestPinAddressChangedBetweenCheckAndDialNotReachable(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
ic, res, rec := pgDialGate(t, d)
|
||||
srv, hits, port := pinnedServer(t, "rebound")
|
||||
rec.route("203.0.113.7:"+port, srv.Listener.Addr().String())
|
||||
rec.route("203.0.113.9:"+port, srv.Listener.Addr().String())
|
||||
res.set("app.example.com", "203.0.113.7")
|
||||
scope := taskScope(t, d, ic, time.Minute)
|
||||
ctx := WithTargetScope(t.Context(), scope)
|
||||
u := "http://app.example.com:" + port + "/"
|
||||
pinURL(t, d, ic, scope, u)
|
||||
mustPass(t, "the check", ic, scope, u)
|
||||
|
||||
res.set("app.example.com", "203.0.113.9")
|
||||
began := time.Now()
|
||||
_, err := get(ctx, pinnedClient(true), u)
|
||||
want := "TARGET_APPROVAL_PENDING: the connection to app.example.com:" + port + " was not opened: app.example.com now resolves to 203.0.113.9, outside pin v1 (203.0.113.7) of task " + scope.TaskID
|
||||
if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "Retry the same call after the approval") {
|
||||
t.Fatalf("dial after the rebind: %v", err)
|
||||
}
|
||||
if elapsed := time.Since(began); elapsed > 10*time.Second {
|
||||
t.Fatalf("the dial waited %v", elapsed)
|
||||
}
|
||||
if got := rec.calls(); len(got) != 0 || hits.Load() != 0 {
|
||||
t.Fatalf("dialled %v, hits %d; want nothing", got, hits.Load())
|
||||
}
|
||||
detail, tgt := openTarget(t, d, scope)
|
||||
if tgt.Reason != "new_ip" || !slices.Equal(tgt.ResolvedIPs, []string{"203.0.113.9"}) || tgt.PinVersion != 1 ||
|
||||
!slices.Equal(tgt.PinnedIPs, []string{"203.0.113.7"}) || detail.ToolName != "WebFetch" || !strings.Contains(string(detail.ToolInput), `"dial": "app.example.com:`+port+`"`) {
|
||||
t.Fatalf("request = %+v, target %+v", detail, tgt)
|
||||
}
|
||||
|
||||
approveAll(t, d, ic, scope)
|
||||
if body, err := get(ctx, pinnedClient(true), u); err != nil || body != "rebound" {
|
||||
t.Fatalf("after the approval: %q, %v", body, err)
|
||||
}
|
||||
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.9:" + port}) {
|
||||
t.Fatalf("dialled %v, want the new address", got)
|
||||
}
|
||||
if ps := pins(t, d, scope); len(ps) != 2 || ps[0].Version != 2 || !slices.Equal(ps[0].PinnedIPs, []string{"203.0.113.7", "203.0.113.9"}) {
|
||||
t.Fatalf("pins = %+v, want v2 with both addresses", ps)
|
||||
}
|
||||
}
|
||||
|
||||
// One address outside the pin is enough: nothing is dialled, not even the
|
||||
// pinned one.
|
||||
func TestPinnedDialPartialOverlapNotReachable(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
ic, res, rec := pgDialGate(t, d)
|
||||
res.set("app.example.com", "203.0.113.7")
|
||||
scope := taskScope(t, d, ic, time.Minute)
|
||||
pinURL(t, d, ic, scope, "https://app.example.com/")
|
||||
res.set("app.example.com", "203.0.113.7", "203.0.113.8")
|
||||
_, err := PinnedDialer("WebFetch", true)(WithTargetScope(t.Context(), scope), "tcp", "app.example.com:443")
|
||||
if err == nil || !strings.Contains(err.Error(), TargetApprovalPending) || len(rec.calls()) != 0 {
|
||||
t.Fatalf("dial: %v, dialled %v", err, rec.calls())
|
||||
}
|
||||
if _, tgt := openTarget(t, d, scope); tgt.Reason != "new_ip" || !slices.Equal(tgt.ResolvedIPs, []string{"203.0.113.7", "203.0.113.8"}) {
|
||||
t.Fatalf("request target %+v", tgt)
|
||||
}
|
||||
}
|
||||
|
||||
// A pin revoked between the check and the dial connects nowhere and asks again.
|
||||
func TestPinnedDialRevokedBetweenCheckAndDial(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
ic, res, rec := pgDialGate(t, d)
|
||||
res.set("app.example.com", "203.0.113.7")
|
||||
scope := taskScope(t, d, ic, time.Minute)
|
||||
pinURL(t, d, ic, scope, "https://app.example.com/")
|
||||
mustPass(t, "the check", ic, scope, "https://app.example.com/")
|
||||
ps := pins(t, d, scope)
|
||||
if _, err := d.RevokeTargetPin(scope.dbRef(t), ps[0].ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := PinnedDialer("WebFetch", true)(WithTargetScope(t.Context(), scope), "tcp", "app.example.com:443")
|
||||
if err == nil || !strings.Contains(err.Error(), TargetApprovalPending) || !strings.Contains(err.Error(), "is not approved for task") || len(rec.calls()) != 0 {
|
||||
t.Fatalf("dial: %v, dialled %v", err, rec.calls())
|
||||
}
|
||||
if _, tgt := openTarget(t, d, scope); tgt.Reason != "no_pin" {
|
||||
t.Fatalf("request target %+v", tgt)
|
||||
}
|
||||
}
|
||||
|
||||
// A stored exemption is dialled at the addresses the dial resolved and found
|
||||
// public; the same name on a private address is void at the dial, and with no
|
||||
// pin it asks, connecting nowhere.
|
||||
func TestPinnedDialStoredExemptionDialsCheckedAddresses(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
ic, res, rec := pgDialGate(t, d)
|
||||
srv, _, port := pinnedServer(t, "package")
|
||||
rec.route("198.51.100.20:"+port, srv.Listener.Addr().String())
|
||||
rec.route("10.0.0.5:"+port, srv.Listener.Addr().String())
|
||||
storedExemptions(t, d, db.TargetExemption{Host: "pkg.example", Port: atoiPort(t, port)})
|
||||
scope := taskScope(t, d, ic, time.Minute)
|
||||
ctx := WithTargetScope(t.Context(), scope)
|
||||
u := "http://pkg.example:" + port + "/"
|
||||
|
||||
res.set("pkg.example", "198.51.100.20")
|
||||
if body, err := get(ctx, pinnedClient(true), u); err != nil || body != "package" {
|
||||
t.Fatalf("exempt fetch: %q, %v", body, err)
|
||||
}
|
||||
res.set("pkg.example", "10.0.0.5")
|
||||
if _, err := get(ctx, pinnedClient(true), u); err == nil || !strings.Contains(err.Error(), TargetApprovalPending) {
|
||||
t.Fatalf("void exemption: %v", err)
|
||||
}
|
||||
if got := rec.calls(); !slices.Equal(got, []string{"198.51.100.20:" + port}) {
|
||||
t.Fatalf("dialled %v, want the checked public address only", got)
|
||||
}
|
||||
if _, tgt := openTarget(t, d, scope); !tgt.ExemptVoid || tgt.Reason != "no_pin" {
|
||||
t.Fatalf("request target %+v, want a void exemption", tgt)
|
||||
}
|
||||
}
|
||||
|
||||
// A pinned IP literal is dialled as written, never resolved; a name that
|
||||
// resolves to the same address is another target, not covered (D22).
|
||||
func TestPinnedDialIPLiteralPin(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
ic, res, rec := pgDialGate(t, d)
|
||||
srv, _, port := pinnedServer(t, "literal")
|
||||
rec.route("198.51.100.7:"+port, srv.Listener.Addr().String())
|
||||
scope := taskScope(t, d, ic, time.Minute)
|
||||
ctx := WithTargetScope(t.Context(), scope)
|
||||
pinURL(t, d, ic, scope, "http://198.51.100.7:"+port+"/")
|
||||
if body, err := get(ctx, pinnedClient(true), "http://198.51.100.7:"+port+"/"); err != nil || body != "literal" {
|
||||
t.Fatalf("literal: %q, %v", body, err)
|
||||
}
|
||||
if res.total() != 0 {
|
||||
t.Fatalf("%d lookups for an IP literal", res.total())
|
||||
}
|
||||
res.set("lit.example", "198.51.100.7")
|
||||
if _, err := get(ctx, pinnedClient(true), "http://lit.example:"+port+"/"); err == nil || !strings.Contains(err.Error(), TargetApprovalPending) {
|
||||
t.Fatalf("a name on the pinned address: %v", err)
|
||||
}
|
||||
if got := rec.calls(); !slices.Equal(got, []string{"198.51.100.7:" + port}) {
|
||||
t.Fatalf("dialled %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Port precedence as at the check: a (host, 0) pin covers any port; a
|
||||
// (host, 443) pin does not cover port 80.
|
||||
func TestPinnedDialPortPrecedence(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
ic, res, rec := pgDialGate(t, d)
|
||||
srv, _, _ := pinnedServer(t, "any-port")
|
||||
rec.route("203.0.113.7:8443", srv.Listener.Addr().String())
|
||||
res.set("app.example.com", "203.0.113.7")
|
||||
res.set("web.example.com", "203.0.113.8")
|
||||
scope := taskScope(t, d, ic, time.Minute)
|
||||
ctx := WithTargetScope(t.Context(), scope)
|
||||
|
||||
// A URL of a scheme without a default port declares no port: its pin is (host, 0).
|
||||
pinURL(t, d, ic, scope, "ssh://app.example.com/")
|
||||
if ps := pins(t, d, scope); len(ps) != 1 || ps[0].Port != 0 {
|
||||
t.Fatalf("pins = %+v, want one on (app.example.com, 0)", ps)
|
||||
}
|
||||
pinURL(t, d, ic, scope, "https://web.example.com/")
|
||||
|
||||
conn, err := PinnedDialer("WebFetch", true)(ctx, "tcp", "app.example.com:8443")
|
||||
if err != nil {
|
||||
t.Fatalf("(host, 0) pin, port 8443: %v", err)
|
||||
}
|
||||
conn.Close()
|
||||
if _, err := PinnedDialer("WebFetch", true)(ctx, "tcp", "web.example.com:80"); err == nil || !strings.Contains(err.Error(), TargetApprovalPending) {
|
||||
t.Fatalf("(host, 443) pin, port 80: %v", err)
|
||||
}
|
||||
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.7:8443"}) {
|
||||
t.Fatalf("dialled %v", got)
|
||||
}
|
||||
if _, tgt := openTarget(t, d, scope); tgt.Host != "web.example.com" || tgt.Port != 80 || tgt.Reason != "no_pin" {
|
||||
t.Fatalf("request target %+v, want web.example.com:80", tgt)
|
||||
}
|
||||
}
|
||||
|
||||
// D-5: a redirect hop is checked by the redirect policy and then dialled
|
||||
// against the pin of the hop's (host, port), never the origin's.
|
||||
func TestRedirectHopDialledAgainstHopPinNotOrigin(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
final, finalHits, finalPort := pinnedServer(t, "hop-b")
|
||||
start, _, startPort := hopRedirect(t, "http://b.example:"+finalPort+"/final")
|
||||
aURL, bURL := "http://a.example:"+startPort+"/start", "http://b.example:"+finalPort+"/"
|
||||
setup := func(t *testing.T, pinB bool) (context.Context, TargetScope, *fakeResolver, *dialRecorder) {
|
||||
ic, res, rec := pgDialGate(t, d)
|
||||
rec.route("203.0.113.1:"+startPort, start.Listener.Addr().String())
|
||||
rec.route("203.0.113.2:"+finalPort, final.Listener.Addr().String())
|
||||
// Were the hop dialled at the origin's address, it would reach b as well.
|
||||
rec.route("203.0.113.1:"+finalPort, final.Listener.Addr().String())
|
||||
res.set("a.example", "203.0.113.1")
|
||||
res.set("b.example", "203.0.113.2")
|
||||
scope := taskScope(t, d, ic, time.Minute)
|
||||
pinURL(t, d, ic, scope, aURL)
|
||||
if pinB {
|
||||
pinURL(t, d, ic, scope, bURL)
|
||||
}
|
||||
return WithTargetScope(t.Context(), scope), scope, res, rec
|
||||
}
|
||||
|
||||
t.Run("each hop at its own pin", func(t *testing.T) {
|
||||
ctx, _, _, rec := setup(t, true)
|
||||
before := finalHits.Load()
|
||||
if body, err := get(ctx, pinnedClient(true), aURL); err != nil || body != "hop-b" || finalHits.Load() != before+1 {
|
||||
t.Fatalf("redirect: %q, %v", body, err)
|
||||
}
|
||||
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.1:" + startPort, "203.0.113.2:" + finalPort}) {
|
||||
t.Fatalf("dialled %v", got)
|
||||
}
|
||||
})
|
||||
t.Run("hop resolving to the origin's pinned address", func(t *testing.T) {
|
||||
ctx, scope, res, rec := setup(t, true)
|
||||
// The cache still holds b's pinned answer: the redirect policy passes.
|
||||
res.set("b.example", "203.0.113.1")
|
||||
before := finalHits.Load()
|
||||
_, err := get(ctx, pinnedClient(true), aURL)
|
||||
if err == nil || !strings.Contains(err.Error(), "TARGET_APPROVAL_PENDING: the connection to b.example:"+finalPort+" was not opened") {
|
||||
t.Fatalf("hop: %v", err)
|
||||
}
|
||||
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.1:" + startPort}) || finalHits.Load() != before {
|
||||
t.Fatalf("dialled %v, b hits %d; want the origin only", got, finalHits.Load()-before)
|
||||
}
|
||||
if _, tgt := openTarget(t, d, scope); tgt.Host != "b.example" || tgt.Port != atoiPort(t, finalPort) || tgt.Reason != "new_ip" {
|
||||
t.Fatalf("request target %+v, want new_ip on b.example", tgt)
|
||||
}
|
||||
})
|
||||
t.Run("hop without a pin", func(t *testing.T) {
|
||||
ctx, scope, _, rec := setup(t, false)
|
||||
before := finalHits.Load()
|
||||
_, err := get(ctx, pinnedClient(true), aURL)
|
||||
if err == nil || !strings.Contains(err.Error(), "redirects to b.example:"+finalPort) || !strings.Contains(err.Error(), "not followed") {
|
||||
t.Fatalf("hop: %v", err)
|
||||
}
|
||||
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.1:" + startPort}) || finalHits.Load() != before {
|
||||
t.Fatalf("dialled %v; want the dialer never called for b", got)
|
||||
}
|
||||
if _, tgt := openTarget(t, d, scope); tgt.Host != "b.example" || tgt.Reason != "no_pin" {
|
||||
t.Fatalf("request target %+v", tgt)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// hopRedirect is a server that redirects every request to location.
|
||||
func hopRedirect(t *testing.T, location string) (*httptest.Server, *atomic.Int64, string) {
|
||||
t.Helper()
|
||||
srv, hits, key := hopServer(t, location)
|
||||
_, port, _ := net.SplitHostPort(key)
|
||||
return srv, hits, port
|
||||
}
|
||||
|
||||
// Two concurrent dials of one host whose address changed share one request:
|
||||
// both fail at once, one approval extends the pin, both retries connect.
|
||||
func TestDialMismatchOpensOneNewIPRequest(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
ic, res, rec := pgDialGate(t, d)
|
||||
srv, _, port := pinnedServer(t, "shared")
|
||||
rec.route("203.0.113.9:"+port, srv.Listener.Addr().String())
|
||||
res.set("app.example.com", "203.0.113.7")
|
||||
scope := taskScope(t, d, ic, time.Minute)
|
||||
ctx := WithTargetScope(t.Context(), scope)
|
||||
u := "http://app.example.com:" + port + "/"
|
||||
pinURL(t, d, ic, scope, u)
|
||||
res.set("app.example.com", "203.0.113.9")
|
||||
|
||||
fetchBoth := func() []error {
|
||||
var wg sync.WaitGroup
|
||||
errs := make([]error, 2)
|
||||
for n := range errs {
|
||||
wg.Go(func() { _, errs[n] = get(ctx, pinnedClient(true), u) })
|
||||
}
|
||||
wg.Wait()
|
||||
return errs
|
||||
}
|
||||
for n, err := range fetchBoth() {
|
||||
if err == nil || !strings.Contains(err.Error(), TargetApprovalPending) {
|
||||
t.Fatalf("dial %d: %v", n, err)
|
||||
}
|
||||
}
|
||||
if ids := openRows(t, d, scope); len(ids) != 1 || allRows(t, d, scope) != 2 {
|
||||
t.Fatalf("open %v, %d request rows; want the approved one and one new_ip", ids, allRows(t, d, scope))
|
||||
}
|
||||
if len(rec.calls()) != 0 {
|
||||
t.Fatalf("dialled %v", rec.calls())
|
||||
}
|
||||
approveAll(t, d, ic, scope)
|
||||
for n, err := range fetchBoth() {
|
||||
if err != nil {
|
||||
t.Fatalf("retry %d: %v", n, err)
|
||||
}
|
||||
}
|
||||
if got := rec.calls(); len(got) != 2 || got[0] != "203.0.113.9:"+port || got[1] != got[0] {
|
||||
t.Fatalf("dialled %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,537 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// Step 2.1: the pinned dialer of the platform-owned HTTP clients. Without a
|
||||
// database: a dial that would open a request is covered by the PostgreSQL
|
||||
// tests in target_dial_pg_test.go.
|
||||
|
||||
// dialRecorder is a base dialer for tests. It records every address it is
|
||||
// asked to connect to, connects the routed ones (TEST-NET addresses) to local
|
||||
// listeners and loopback addresses to themselves, and refuses the rest.
|
||||
type dialRecorder struct {
|
||||
mu sync.Mutex
|
||||
routes map[string]string
|
||||
refuse map[string]bool
|
||||
dialed []string
|
||||
}
|
||||
|
||||
func newDialRecorder() *dialRecorder {
|
||||
return &dialRecorder{routes: map[string]string{}, refuse: map[string]bool{}}
|
||||
}
|
||||
|
||||
func (d *dialRecorder) route(addr, to string) {
|
||||
d.mu.Lock()
|
||||
d.routes[addr] = to
|
||||
d.mu.Unlock()
|
||||
}
|
||||
|
||||
func (d *dialRecorder) dial(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
d.mu.Lock()
|
||||
d.dialed = append(d.dialed, addr)
|
||||
to, ok := d.routes[addr]
|
||||
refused := d.refuse[addr]
|
||||
d.mu.Unlock()
|
||||
if !ok && strings.HasPrefix(addr, "127.0.0.1:") {
|
||||
to, ok = addr, true
|
||||
}
|
||||
if refused || !ok {
|
||||
return nil, &net.OpError{Op: "dial", Net: network, Err: errors.New("connection refused by the test dialer")}
|
||||
}
|
||||
return (&net.Dialer{}).DialContext(ctx, network, to)
|
||||
}
|
||||
|
||||
func (d *dialRecorder) calls() []string {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return slices.Clone(d.dialed)
|
||||
}
|
||||
|
||||
// pinnedServer is a listener that counts its requests; port is its port.
|
||||
func pinnedServer(t *testing.T, body string) (srv *httptest.Server, hits *atomic.Int64, port string) {
|
||||
t.Helper()
|
||||
hits = &atomic.Int64{}
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
_, _ = w.Write([]byte(body))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
_, port, _ = net.SplitHostPort(srv.Listener.Addr().String())
|
||||
return srv, hits, port
|
||||
}
|
||||
|
||||
// dialGate is a gate fixture whose base dialer is a dialRecorder.
|
||||
func dialGate(t *testing.T) (*gateFixture, *dialRecorder) {
|
||||
t.Helper()
|
||||
g := newGate(t)
|
||||
rec := newDialRecorder()
|
||||
g.ic.SetBaseDialer(rec.dial)
|
||||
return g, rec
|
||||
}
|
||||
|
||||
func (g *gateFixture) pin(host string, port int, ips ...string) *db.TargetPin {
|
||||
g.store.mu.Lock()
|
||||
defer g.store.mu.Unlock()
|
||||
p := &db.TargetPin{ID: int64(len(g.store.pins) + 1), Host: host, Port: port, Version: 1, Status: db.TargetPinActive, PinnedIPs: ips}
|
||||
g.store.pins = append(g.store.pins, p)
|
||||
return p
|
||||
}
|
||||
|
||||
// pinnedClient is a client built like the platform's: the redirect policy and
|
||||
// the pinned dialer, keep-alive off.
|
||||
func pinnedClient(scoped bool) *http.Client {
|
||||
tr := http.DefaultTransport.(*http.Transport).Clone()
|
||||
tr.Proxy = nil
|
||||
tr.DialContext = PinnedDialer("WebFetch", scoped)
|
||||
tr.DisableKeepAlives = true
|
||||
return &http.Client{Transport: tr, CheckRedirect: RedirectCheck("WebFetch")}
|
||||
}
|
||||
|
||||
func get(ctx context.Context, client *http.Client, rawURL string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, err := io.ReadAll(resp.Body)
|
||||
return string(b), err
|
||||
}
|
||||
|
||||
// With the gate off, or with no scope in a client built without one, the dial
|
||||
// is the dial of today: the address as received, no lookup, no store access.
|
||||
func TestPinnedDialGateOffIsPlainDial(t *testing.T) {
|
||||
g, rec := dialGate(t)
|
||||
srv, hits, port := pinnedServer(t, "plain")
|
||||
rec.route("app.example.com:"+port, srv.Listener.Addr().String())
|
||||
g.res.set("app.example.com", "203.0.113.7")
|
||||
|
||||
conn, err := PinnedDialer("WebFetch", false)(context.Background(), "tcp", srv.Listener.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("no scope: %v", err)
|
||||
}
|
||||
conn.Close()
|
||||
for name, scope := range map[string]TargetScope{
|
||||
"no flag": {TaskID: "7", WaitBound: time.Minute, Interceptor: g.ic},
|
||||
"flag off": {TaskID: "7", Enabled: func() bool { return false }, WaitBound: time.Minute, Interceptor: g.ic},
|
||||
} {
|
||||
if body, err := get(WithTargetScope(context.Background(), scope), pinnedClient(true), "http://app.example.com:"+port+"/"); err != nil || body != "plain" {
|
||||
t.Fatalf("%s: %q, %v", name, body, err)
|
||||
}
|
||||
}
|
||||
if got := rec.calls(); len(got) != 2 || got[0] != "app.example.com:"+port || got[1] != got[0] {
|
||||
t.Fatalf("dialled %v, want the address as received", got)
|
||||
}
|
||||
if hits.Load() != 2 || g.store.reads != 0 || len(g.store.logged) != 0 || g.res.total() != 0 {
|
||||
t.Fatalf("hits=%d, %d store reads, %d rows, %d lookups; want none but the hits", hits.Load(), g.store.reads, len(g.store.logged), g.res.total())
|
||||
}
|
||||
}
|
||||
|
||||
// A client built for a scope never dials without one: a lost context value
|
||||
// fails closed, nothing is dialled.
|
||||
func TestPinnedDialScopeLostFailsClosed(t *testing.T) {
|
||||
srv, hits, _ := pinnedServer(t, "unbound")
|
||||
_, err := PinnedDialer("WebFetch", true)(context.Background(), "tcp", srv.Listener.Addr().String())
|
||||
if err == nil || !strings.Contains(err.Error(), TargetGateUnavailable) || !strings.Contains(err.Error(), "did not reach the connection") {
|
||||
t.Fatalf("dial without the scope: %v", err)
|
||||
}
|
||||
if _, err := get(context.Background(), pinnedClient(true), srv.URL); err == nil || !strings.Contains(err.Error(), TargetGateUnavailable) {
|
||||
t.Fatalf("fetch without the scope: %v", err)
|
||||
}
|
||||
if hits.Load() != 0 {
|
||||
t.Fatalf("%d requests reached the server", hits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// Every way the gate cannot decide fails closed with TARGET_GATE_UNAVAILABLE,
|
||||
// dials nothing, and logs one decided row where there is a store and a scope.
|
||||
func TestPinnedDialScopeUnreadableFailsClosed(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
setup func(g *gateFixture)
|
||||
network, addr string
|
||||
rows int
|
||||
}{
|
||||
{"nil interceptor", func(g *gateFixture) { g.scope.Interceptor = nil }, "tcp", "app.example.com:443", 0},
|
||||
{"task id", func(g *gateFixture) { g.scope.TaskID = "seven" }, "tcp", "app.example.com:443", 0},
|
||||
{"rules", func(g *gateFixture) { g.store.fail["BlockRules"] = errors.New("down") }, "tcp", "app.example.com:443", 1},
|
||||
{"pins", func(g *gateFixture) { g.store.fail["ActivePins"] = errors.New("down") }, "tcp", "app.example.com:443", 1},
|
||||
{"exemptions", func(g *gateFixture) { g.store.fail["Exemptions"] = errors.New("down") }, "tcp", "app.example.com:443", 1},
|
||||
{"address", nil, "tcp", "app.example.com", 1},
|
||||
{"port", nil, "tcp", "app.example.com:0", 1},
|
||||
{"network", nil, "udp", "app.example.com:443", 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
g, rec := dialGate(t)
|
||||
g.res.set("app.example.com", "203.0.113.7")
|
||||
g.pin("app.example.com", 443, "203.0.113.7")
|
||||
if tc.setup != nil {
|
||||
tc.setup(g)
|
||||
}
|
||||
_, err := PinnedDialer("WebFetch", true)(g.ctx(), tc.network, tc.addr)
|
||||
if err == nil || !strings.Contains(err.Error(), TargetGateUnavailable) {
|
||||
t.Fatalf("dial: %v, want TARGET_GATE_UNAVAILABLE", err)
|
||||
}
|
||||
if got := rec.calls(); len(got) != 0 {
|
||||
t.Fatalf("dialled %v", got)
|
||||
}
|
||||
if got := g.store.outcomes(); len(got) != tc.rows || tc.rows == 1 && got[0] != "denied/gate_unavailable" {
|
||||
t.Fatalf("rows = %v, want %d gate_unavailable", got, tc.rows)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A pinned name is dialled at the address the lookup returned and the pin
|
||||
// holds, never by the name.
|
||||
func TestPinnedDialConnectsOnlyToPinnedAddresses(t *testing.T) {
|
||||
g, rec := dialGate(t)
|
||||
srv, hits, port := pinnedServer(t, "pinned")
|
||||
rec.route("203.0.113.7:"+port, srv.Listener.Addr().String())
|
||||
g.res.set("app.example.com", "203.0.113.7")
|
||||
g.pin("app.example.com", atoiPort(t, port), "203.0.113.7")
|
||||
|
||||
if body, err := get(g.ctx(), pinnedClient(true), "http://app.example.com:"+port+"/"); err != nil || body != "pinned" {
|
||||
t.Fatalf("pinned fetch: %q, %v", body, err)
|
||||
}
|
||||
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.7:" + port}) || hits.Load() != 1 {
|
||||
t.Fatalf("dialled %v, hits %d; want the pinned address only", got, hits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// A name that does not resolve at the dial is blocked without a request (the
|
||||
// fixture has no database: a request would fail the test), nothing is dialled.
|
||||
func TestPinnedDialResolveFailureBlocksWithoutRequest(t *testing.T) {
|
||||
g, rec := dialGate(t)
|
||||
g.pin("app.example.com", 443, "203.0.113.7")
|
||||
g.res.mu.Lock()
|
||||
g.res.errs["app.example.com"] = errors.New("no such host")
|
||||
g.res.mu.Unlock()
|
||||
_, err := PinnedDialer("WebFetch", true)(g.ctx(), "tcp", "app.example.com:443")
|
||||
if err == nil || !strings.Contains(err.Error(), TargetResolveFailed) || !strings.Contains(err.Error(), "app.example.com:443") {
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
if got := g.store.outcomes(); len(got) != 1 || got[0] != "denied/resolve_failed" || len(rec.calls()) != 0 {
|
||||
t.Fatalf("rows %v, dialled %v", got, rec.calls())
|
||||
}
|
||||
}
|
||||
|
||||
// An asset block rule on an address the dial resolves wins over the pin.
|
||||
func TestPinnedDialBlockRuleOnDialAddress(t *testing.T) {
|
||||
g, rec := dialGate(t)
|
||||
g.pin("app.example.com", 443, "203.0.113.7", "203.0.113.8")
|
||||
g.res.set("app.example.com", "203.0.113.8")
|
||||
g.store.rules = []db.AssetInterceptRule{{Enabled: true, Kind: "exact_ip", Pattern: "203.0.113.8"}}
|
||||
_, err := PinnedDialer("WebFetch", true)(g.ctx(), "tcp", "app.example.com:443")
|
||||
if err == nil || !strings.Contains(err.Error(), TargetRuleBlocked) {
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
if got := g.store.outcomes(); len(got) != 1 || got[0] != "denied/rule_blocked" || len(rec.calls()) != 0 {
|
||||
t.Fatalf("rows %v, dialled %v", got, rec.calls())
|
||||
}
|
||||
}
|
||||
|
||||
// A derived exemption is dialled as received and never resolved, wherever its
|
||||
// name points: an LLM gateway on an internal address, the recording proxy on
|
||||
// loopback.
|
||||
func TestPinnedDialDerivedExemptionDialsByName(t *testing.T) {
|
||||
g, rec := dialGate(t)
|
||||
srv, hits, port := pinnedServer(t, "gateway")
|
||||
rec.route("llm.example:"+port, srv.Listener.Addr().String())
|
||||
g.res.set("llm.example", "10.0.0.5")
|
||||
proxy := srv.Listener.Addr().String()
|
||||
g.ic.SetDerivedExemptions(func() []string { return []string{"https://llm.example:" + port, "http://" + proxy} })
|
||||
|
||||
for _, addr := range []string{"llm.example:" + port, proxy} {
|
||||
conn, err := PinnedDialer("WebFetch", true)(g.ctx(), "tcp", addr)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", addr, err)
|
||||
}
|
||||
conn.Close()
|
||||
}
|
||||
if got := rec.calls(); !slices.Equal(got, []string{"llm.example:" + port, proxy}) || g.res.total() != 0 || hits.Load() != 0 {
|
||||
t.Fatalf("dialled %v, %d lookups; want the addresses as received and no lookup", got, g.res.total())
|
||||
}
|
||||
}
|
||||
|
||||
// A pin that is name-only for this connection, by its mark or because the
|
||||
// upstream proxy is set now, is dialled by name: the platform does not bind it.
|
||||
func TestPinnedDialNameOnlyPinDialsByName(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
marked bool
|
||||
proxied bool
|
||||
}{{"marked pin", true, false}, {"upstream proxy now", false, true}} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
g, rec := dialGate(t)
|
||||
srv, _, port := pinnedServer(t, "name-only")
|
||||
rec.route("app.example.com:"+port, srv.Listener.Addr().String())
|
||||
g.res.set("app.example.com", "203.0.113.9")
|
||||
g.pin("app.example.com", atoiPort(t, port), "203.0.113.7").NameOnlyEffective = tc.marked
|
||||
g.ic.SetUpstreamProxyCheck(func() bool { return tc.proxied })
|
||||
if body, err := get(g.ctx(), pinnedClient(true), "http://app.example.com:"+port+"/"); err != nil || body != "name-only" {
|
||||
t.Fatalf("fetch: %q, %v", body, err)
|
||||
}
|
||||
if got := rec.calls(); !slices.Equal(got, []string{"app.example.com:" + port}) {
|
||||
t.Fatalf("dialled %v, want the name", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The dial reads its address as the check reads the URL: same key for case,
|
||||
// trailing dot, Unicode, IPv6 brackets and IPv4-mapped IPv6. net/http dials the
|
||||
// ASCII form of a Unicode name with the scheme's port written (S3).
|
||||
func TestPinnedDialNormalisesLikeTheCheck(t *testing.T) {
|
||||
for _, tc := range []struct{ rawURL, addr string }{
|
||||
{"https://App.Example.COM./x", "App.Example.COM.:443"},
|
||||
{"http://bücher.example/", "xn--bcher-kva.example:80"},
|
||||
{"https://[2001:DB8::1]:8443/", "[2001:db8::1]:8443"},
|
||||
{"http://[::ffff:192.0.2.1]/", "[::ffff:192.0.2.1]:80"},
|
||||
} {
|
||||
want, err := urlTarget(tc.rawURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := addrTarget("tcp", tc.addr)
|
||||
if err != nil || got != want {
|
||||
t.Errorf("%s: dial key %+v (%v), check key %+v", tc.addr, got, err, want)
|
||||
}
|
||||
}
|
||||
var dialled string
|
||||
tr := &http.Transport{DialContext: func(_ context.Context, _, addr string) (net.Conn, error) {
|
||||
dialled = addr
|
||||
return nil, errors.New("recorded")
|
||||
}}
|
||||
_, _ = (&http.Client{Transport: tr}).Get("http://bücher.example/")
|
||||
if dialled != "xn--bcher-kva.example:80" {
|
||||
t.Fatalf("net/http dialled %q, want the ASCII name and the scheme's port", dialled)
|
||||
}
|
||||
}
|
||||
|
||||
// Concurrent dials of one host share one lookup; the cached answer is not
|
||||
// used, and the cache holds the new answer afterwards.
|
||||
func TestPinnedDialFreshLookupCollapsed(t *testing.T) {
|
||||
g, rec := dialGate(t)
|
||||
srv, _, port := pinnedServer(t, "fresh")
|
||||
rec.route("203.0.113.7:"+port, srv.Listener.Addr().String())
|
||||
g.pin("app.example.com", atoiPort(t, port), "203.0.113.7")
|
||||
g.res.set("app.example.com", "203.0.113.9")
|
||||
if _, err := g.ic.resolve.lookup(context.Background(), "app.example.com", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The cache now holds 203.0.113.9, which no pin covers: a dial that used it
|
||||
// would open a request, which this fixture cannot.
|
||||
g.res.set("app.example.com", "203.0.113.7")
|
||||
hold := make(chan struct{})
|
||||
g.res.mu.Lock()
|
||||
g.res.hold = hold
|
||||
g.res.mu.Unlock()
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 2)
|
||||
for range 2 {
|
||||
wg.Go(func() {
|
||||
conn, err := PinnedDialer("WebFetch", true)(g.ctx(), "tcp", "app.example.com:"+port)
|
||||
if err == nil {
|
||||
conn.Close()
|
||||
}
|
||||
errs <- err
|
||||
})
|
||||
}
|
||||
eventually(t, "the shared lookup", func() bool { return g.res.lookups("app.example.com") == 2 })
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
close(hold)
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
}
|
||||
if n := g.res.lookups("app.example.com"); n != 2 {
|
||||
t.Fatalf("%d lookups, want the cached one and one shared by both dials", n)
|
||||
}
|
||||
g.ic.resolve.mu.Lock()
|
||||
cached := g.ic.resolve.entries["app.example.com"].ips
|
||||
g.ic.resolve.mu.Unlock()
|
||||
if !slices.Equal(cached, []string{"203.0.113.7"}) || !slices.Equal(rec.calls(), []string{"203.0.113.7:" + port, "203.0.113.7:" + port}) {
|
||||
t.Fatalf("cache %v, dialled %v", cached, rec.calls())
|
||||
}
|
||||
}
|
||||
|
||||
// The compared addresses are tried in canonical order, the first that accepts
|
||||
// wins; tcp4 skips the IPv6 ones.
|
||||
func TestPinnedDialTriesAddressesInOrder(t *testing.T) {
|
||||
g, rec := dialGate(t)
|
||||
srv, _, port := pinnedServer(t, "second")
|
||||
rec.refuse["203.0.113.7:"+port] = true
|
||||
rec.route("203.0.113.8:"+port, srv.Listener.Addr().String())
|
||||
g.pin("app.example.com", atoiPort(t, port), "203.0.113.7", "203.0.113.8")
|
||||
g.res.set("app.example.com", "203.0.113.8", "203.0.113.7")
|
||||
if body, err := get(g.ctx(), pinnedClient(true), "http://app.example.com:"+port+"/"); err != nil || body != "second" {
|
||||
t.Fatalf("fetch: %q, %v", body, err)
|
||||
}
|
||||
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.7:" + port, "203.0.113.8:" + port}) {
|
||||
t.Fatalf("dialled %v", got)
|
||||
}
|
||||
|
||||
g, rec = dialGate(t)
|
||||
rec.route("203.0.113.8:"+port, srv.Listener.Addr().String())
|
||||
g.pin("app.example.com", atoiPort(t, port), "2001:db8::1", "203.0.113.8")
|
||||
g.res.set("app.example.com", "2001:db8::1", "203.0.113.8")
|
||||
conn, err := PinnedDialer("WebFetch", true)(g.ctx(), "tcp4", "app.example.com:"+port)
|
||||
if err != nil {
|
||||
t.Fatalf("tcp4: %v", err)
|
||||
}
|
||||
conn.Close()
|
||||
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.8:" + port}) {
|
||||
t.Fatalf("tcp4 dialled %v, want the IPv4 address only", got)
|
||||
}
|
||||
}
|
||||
|
||||
// S1: the context net/http passes to DialContext carries the values of the
|
||||
// request's context. The client is built without a scope (scoped=false), so
|
||||
// only the request's context can send the dial down the pinned path.
|
||||
func TestPinnedDialSeesScopeInDialContext(t *testing.T) {
|
||||
g, rec := dialGate(t)
|
||||
srv, _, port := pinnedServer(t, "scoped")
|
||||
rec.route("203.0.113.7:"+port, srv.Listener.Addr().String())
|
||||
g.pin("app.example.com", atoiPort(t, port), "203.0.113.7")
|
||||
g.res.set("app.example.com", "203.0.113.7")
|
||||
if body, err := get(g.ctx(), pinnedClient(false), "http://app.example.com:"+port+"/"); err != nil || body != "scoped" {
|
||||
t.Fatalf("fetch: %q, %v", body, err)
|
||||
}
|
||||
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.7:" + port}) || g.res.lookups("app.example.com") != 1 {
|
||||
t.Fatalf("dialled %v after %d lookups, want the pinned path", got, g.res.lookups("app.example.com"))
|
||||
}
|
||||
}
|
||||
|
||||
// S4: dialling the pinned address keeps HTTPS verified against the name of
|
||||
// the request. A name the certificate does not hold fails verification.
|
||||
func TestPinnedDialHTTPSVerifiesName(t *testing.T) {
|
||||
g, rec := dialGate(t)
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("verified")) }))
|
||||
t.Cleanup(srv.Close)
|
||||
_, port, _ := net.SplitHostPort(srv.Listener.Addr().String())
|
||||
rec.route("203.0.113.7:"+port, srv.Listener.Addr().String())
|
||||
// httptest's certificate holds example.com.
|
||||
for _, host := range []string{"example.com", "other.example"} {
|
||||
g.pin(host, atoiPort(t, port), "203.0.113.7")
|
||||
g.res.set(host, "203.0.113.7")
|
||||
}
|
||||
client := pinnedClient(true)
|
||||
client.Transport.(*http.Transport).TLSClientConfig = &tls.Config{RootCAs: srv.Client().Transport.(*http.Transport).TLSClientConfig.RootCAs}
|
||||
if body, err := get(g.ctx(), client, "https://example.com:"+port+"/"); err != nil || body != "verified" {
|
||||
t.Fatalf("https by the pinned address: %q, %v", body, err)
|
||||
}
|
||||
_, err := get(g.ctx(), client, "https://other.example:"+port+"/")
|
||||
var verr *tls.CertificateVerificationError
|
||||
if !errors.As(err, &verr) {
|
||||
t.Fatalf("a name the certificate does not hold: %v, want a verification error", err)
|
||||
}
|
||||
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.7:" + port, "203.0.113.7:" + port}) {
|
||||
t.Fatalf("dialled %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// S6: with keep-alive off (as Norma sets it on a transport with a dialer, and
|
||||
// the custom tool on its own), a second fetch of one client dials again: a pin
|
||||
// changed in between applies to it.
|
||||
func TestPinnedDialNoReuseAcrossCalls(t *testing.T) {
|
||||
g, rec := dialGate(t)
|
||||
srv, _, port := pinnedServer(t, "again")
|
||||
rec.route("203.0.113.7:"+port, srv.Listener.Addr().String())
|
||||
rec.route("203.0.113.8:"+port, srv.Listener.Addr().String())
|
||||
p := atoiPort(t, port)
|
||||
g.pin("app.example.com", p, "203.0.113.7")
|
||||
g.res.set("app.example.com", "203.0.113.7")
|
||||
client := pinnedClient(true)
|
||||
u := "http://app.example.com:" + port + "/"
|
||||
if _, err := get(g.ctx(), client, u); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g.store.revoke("app.example.com", p)
|
||||
g.pin("app.example.com", p, "203.0.113.8")
|
||||
g.res.set("app.example.com", "203.0.113.8")
|
||||
if _, err := get(g.ctx(), client, u); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.7:" + port, "203.0.113.8:" + port}) {
|
||||
t.Fatalf("dialled %v, want a new dial under the new pin", got)
|
||||
}
|
||||
}
|
||||
|
||||
// S5: the redirected request keeps the original request's context, so the
|
||||
// hop's redirect check and the hop's dial read the same target scope.
|
||||
func TestRedirectContextCarriesScope(t *testing.T) {
|
||||
g, rec := dialGate(t)
|
||||
final, finalHits, finalKey := hopServer(t, "")
|
||||
start, _, startKey := hopServer(t, final.URL+"/final")
|
||||
for _, key := range []string{startKey, finalKey} {
|
||||
host, port, _ := net.SplitHostPort(key)
|
||||
g.pin(host, atoiPort(t, port), host)
|
||||
}
|
||||
var hopScope TargetScope
|
||||
var hopOK bool
|
||||
client := pinnedClient(true)
|
||||
policy := client.CheckRedirect
|
||||
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
hopScope, hopOK = TargetScopeFromContext(req.Context())
|
||||
return policy(req, via)
|
||||
}
|
||||
if body, err := get(g.ctx(), client, start.URL+"/start"); err != nil || body != "hop-reached" || finalHits.Load() != 1 {
|
||||
t.Fatalf("redirect: %q, %v", body, err)
|
||||
}
|
||||
if !hopOK || hopScope.TaskID != g.scope.TaskID || hopScope.Interceptor != g.ic {
|
||||
t.Fatalf("hop context scope = %+v (%v), want the scope of the call", hopScope, hopOK)
|
||||
}
|
||||
if got := rec.calls(); !slices.Equal(got, []string{startKey, finalKey}) {
|
||||
t.Fatalf("dialled %v, want the origin then the hop through the pinned path", got)
|
||||
}
|
||||
}
|
||||
|
||||
// D-2: a request raised while the upstream proxy is set says the platform does
|
||||
// not bind the connection.
|
||||
func TestNameOnlyReasonSaysConnectionNotBound(t *testing.T) {
|
||||
scope := TargetScope{TaskID: "7"}
|
||||
r := target("app.example.com", 443, "203.0.113.7")
|
||||
const sentence = "This approval is name-only. The platform does not bind the connection to these addresses."
|
||||
if got := targetReason(scope, r, nil, true); !strings.Contains(got, sentence) {
|
||||
t.Fatalf("name-only reason = %q", got)
|
||||
}
|
||||
if got := targetReason(scope, r, nil, false); strings.Contains(got, "does not bind") {
|
||||
t.Fatalf("reason without the upstream proxy = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// D-2: the first-use row of an unmarked pin used while the upstream proxy is
|
||||
// set records the pin as name-only.
|
||||
func TestPinUsedRecordsEffectiveNameOnly(t *testing.T) {
|
||||
g := newGate(t)
|
||||
g.res.set("app.example.com", "203.0.113.7")
|
||||
g.pin("app.example.com", 443, "203.0.113.7")
|
||||
g.ic.SetUpstreamProxyCheck(func() bool { return true })
|
||||
g.passes("pinned", g.fetch(appURL))
|
||||
if got := g.store.outcomes(); len(got) != 1 || got[0] != "allowed/pin_match" || !g.store.logged[0].Audit.Targets[0].NameOnlyEffective {
|
||||
t.Fatalf("rows %v, target %+v; want a name-only first use", got, g.store.logged[0].Audit.Targets[0])
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,13 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"strings"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// metadataAddrs are instance-metadata addresses that lie outside the link-local
|
||||
@@ -51,3 +56,125 @@ func ValidateExemptionHost(host string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ErrInvalidExemption is wrapped by the reason an exemption entry is refused.
|
||||
var ErrInvalidExemption = errors.New("invalid exemption")
|
||||
|
||||
// SetDerivedExemptions installs the function that lists the endpoints of the
|
||||
// platform's own services, as URLs or host:port: the LLM gateways, the search
|
||||
// backend in use and the platform proxies. They are exempt because the
|
||||
// configuration names them. They are computed at check time and never stored,
|
||||
// so they follow the configuration and cannot be set through the exemption
|
||||
// list. A derived exemption is never resolved and never voided: an internal
|
||||
// relay for a platform service is a normal deployment.
|
||||
func (i *Interceptor) SetDerivedExemptions(f func() []string) {
|
||||
i.mu.Lock()
|
||||
i.derivedExempt = f
|
||||
i.mu.Unlock()
|
||||
}
|
||||
|
||||
// proxyPorts are the ports net/http dials for a proxy URL without one, for the
|
||||
// schemes schemePorts has no default for.
|
||||
var proxyPorts = map[string]int{"socks5": 1080, "socks5h": 1080}
|
||||
|
||||
// DerivedExemptions returns the exemptions derived from the configuration, as
|
||||
// exact host and port. An endpoint without a port gets the one net/http dials
|
||||
// (the scheme's default, 1080 for socks5); one with neither is not listed.
|
||||
func (i *Interceptor) DerivedExemptions() []db.TargetExemption {
|
||||
i.mu.RLock()
|
||||
f := i.derivedExempt
|
||||
i.mu.RUnlock()
|
||||
out := []db.TargetExemption{}
|
||||
if f == nil {
|
||||
return out
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, raw := range f() {
|
||||
t, ok := NormalizeTarget(raw)
|
||||
if scheme, _, found := strings.Cut(strings.TrimSpace(raw), "://"); ok && found && t.Port == 0 {
|
||||
t.Port = proxyPorts[strings.ToLower(scheme)]
|
||||
}
|
||||
if !ok || t.Port == 0 || t.Kind == TargetCIDR || seen[db.TargetKey(t.Host, t.Port)] {
|
||||
continue
|
||||
}
|
||||
seen[db.TargetKey(t.Host, t.Port)] = true
|
||||
out = append(out, db.TargetExemption{Host: t.Host, Port: t.Port})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Exemptions returns the stored exemption list and its digest.
|
||||
func (i *Interceptor) Exemptions() ([]db.TargetExemption, string, error) {
|
||||
return i.targets.Exemptions()
|
||||
}
|
||||
|
||||
// SetExemptions validates list and stores it as the exemption list; the change
|
||||
// is appended to the exemption log with changedBy. An entry is an exact host
|
||||
// and port: a name or an IP literal, no wildcard, no suffix, no CIDR, and a
|
||||
// port between 1 and 65535. It returns the list as stored (hosts normalised,
|
||||
// duplicates dropped) and its digest.
|
||||
func (i *Interceptor) SetExemptions(list []db.TargetExemption, changedBy string) ([]db.TargetExemption, string, error) {
|
||||
out := []db.TargetExemption{}
|
||||
seen := map[string]bool{}
|
||||
for _, e := range list {
|
||||
kind, host, ok := normalizeHost(e.Host, true)
|
||||
switch {
|
||||
case !ok || kind == TargetCIDR || kind == TargetDomain && !validHostname(host):
|
||||
return nil, "", fmt.Errorf("%w: %q is not an exact host name or IP address", ErrInvalidExemption, e.Host)
|
||||
case e.Port < 1 || e.Port > 65535:
|
||||
return nil, "", fmt.Errorf("%w: %s needs a port between 1 and 65535, got %d", ErrInvalidExemption, host, e.Port)
|
||||
}
|
||||
if err := ValidateExemptionHost(host); err != nil {
|
||||
return nil, "", fmt.Errorf("%w: %v", ErrInvalidExemption, err)
|
||||
}
|
||||
if key := db.TargetKey(host, e.Port); !seen[key] {
|
||||
seen[key] = true
|
||||
out = append(out, db.TargetExemption{Host: host, Port: e.Port, Note: strings.TrimSpace(e.Note)})
|
||||
}
|
||||
}
|
||||
raw, err := json.Marshal(out)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
digest := digestInput(raw)
|
||||
if err := i.db.ReplaceTargetExemptions(out, changedBy, digest); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return out, digest, nil
|
||||
}
|
||||
|
||||
// exemptions is the exemption list as one gated call sees it.
|
||||
type exemptions struct {
|
||||
derived map[string]bool // db.TargetKey(host, port)
|
||||
stored map[string]bool
|
||||
digest string // of the stored list
|
||||
}
|
||||
|
||||
func (i *Interceptor) loadExemptions() (exemptions, error) {
|
||||
stored, digest, err := i.targets.Exemptions()
|
||||
if err != nil {
|
||||
return exemptions{}, err
|
||||
}
|
||||
ex := exemptions{derived: map[string]bool{}, stored: map[string]bool{}, digest: digest}
|
||||
for _, e := range i.DerivedExemptions() {
|
||||
ex.derived[db.TargetKey(e.Host, e.Port)] = true
|
||||
}
|
||||
for _, e := range stored {
|
||||
ex.stored[db.TargetKey(e.Host, e.Port)] = true
|
||||
}
|
||||
return ex, nil
|
||||
}
|
||||
|
||||
// voidsExemption reports whether one resolved address voids a stored
|
||||
// exemption on a name: a private address (10/8, 172.16/12, 192.168/16, the
|
||||
// unique local range fc00::/7), or one that is never exempt as a literal
|
||||
// either (loopback, link-local 169.254.0.0/16 and fe80::/10, unspecified,
|
||||
// multicast, instance metadata). One such address is enough.
|
||||
func voidsExemption(ip string) bool {
|
||||
a, err := netip.ParseAddr(ip)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
a = a.Unmap().WithZone("")
|
||||
return a.IsPrivate() || ValidateExemptionHost(a.String()) != nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
package intercept
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// An operator may exempt a private unicast IP literal, never a loopback,
|
||||
// link-local (cloud metadata), unspecified or multicast one.
|
||||
@@ -27,3 +32,20 @@ func TestValidateExemptionHost(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A proxy endpoint configured without a port is exempt on the port net/http
|
||||
// dials for it (OPEN-9): the scheme's default, 1080 for socks5. An endpoint
|
||||
// with no port and no known scheme is not listed.
|
||||
func TestDerivedExemptionsDefaultProxyPort(t *testing.T) {
|
||||
ic := New(nil)
|
||||
ic.SetDerivedExemptions(func() []string {
|
||||
return []string{"socks5://egress.example.net", "SOCKS5H://dns.example.net", "http://proxy.example.net", "https://gw.example.net", "bare.example.net", "socks5://set.example.net:9050"}
|
||||
})
|
||||
want := []db.TargetExemption{
|
||||
{Host: "egress.example.net", Port: 1080}, {Host: "dns.example.net", Port: 1080}, {Host: "proxy.example.net", Port: 80},
|
||||
{Host: "gw.example.net", Port: 443}, {Host: "set.example.net", Port: 9050},
|
||||
}
|
||||
if got := ic.DerivedExemptions(); !slices.Equal(got, want) {
|
||||
t.Fatalf("derived exemptions = %+v, want %+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/netip"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// executeExtraTool is the tool that runs a deferred tool (every MCP tool, and
|
||||
// the custom tools flagged deferred). The hook sees it, never the inner name.
|
||||
const executeExtraTool = "ExecuteExtraTool"
|
||||
|
||||
// noTargetTools are the tools whose input names no network target: the search
|
||||
// tool (its input is a query), the file and session tools, and the platform's
|
||||
// domain tools, which read and write the platform's own records. Every name
|
||||
// that is not listed here is scanned, so a tool added later is gated by default.
|
||||
var noTargetTools = toolSet(
|
||||
"web_search",
|
||||
"Read", "Write", "Edit", "MultiEdit", "LS", "Glob", "Grep",
|
||||
"TodoWrite", "sleep", "TaskOutput", "TaskStop", "TaskList",
|
||||
// exploration graph, assets, findings, goals (agent.ToolSet)
|
||||
"graph_overview", "list_findings", "list_facts", "node_detail", "expand_digest",
|
||||
"get_worker_output", "get_worker_trace", "search_all_worker_traces",
|
||||
"add_hint", "add_intent", "steer_work", "kill_work",
|
||||
"set_goals", "set_constraints", "list_goals", "prove_goal", "goal_met",
|
||||
"insert_assets", "add_company_scope", "add_task_scope", "list_assets",
|
||||
"list_untested_assets", "list_companies", "report_finding", "record_fact",
|
||||
"save_task_credential", "list_task_credentials",
|
||||
// cross-task orchestration, findings and retests (server)
|
||||
"list_tasks", "list_llm_profiles", "spawn_task", "pause_task", "get_task_graph",
|
||||
"list_task_findings", "add_task_hint", "get_task_worker_trace",
|
||||
"list_task_worker_traces", "search_task_worker_traces", "get_task_node_detail",
|
||||
"update_finding_report", "get_finding_traffic", "bind_finding_traffic",
|
||||
"get_finding_retest_context", "record_finding_retest_result",
|
||||
"delete_assets_by_host", "create_skill", "update_skill_file",
|
||||
// the operator runs the command, not the platform
|
||||
"ask_operator",
|
||||
// recorded traffic
|
||||
"traffic_search", "traffic_get", "traffic_blob",
|
||||
)
|
||||
|
||||
func toolSet(names ...string) map[string]bool {
|
||||
m := make(map[string]bool, len(names))
|
||||
for _, n := range names {
|
||||
m[n] = true
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// ExtractTargets returns the network targets a tool call declares in its
|
||||
// input, normalised and without duplicates, in order of appearance.
|
||||
//
|
||||
// WebFetch names its target exactly. The command tools are read with the
|
||||
// command-string rules: URLs with a scheme, IP literals, IP:port, host:port and
|
||||
// CIDR. A bare host name is not a target there, because a file name such as
|
||||
// x.py is a valid host name. Every other tool that is not known to have no
|
||||
// target has each string of its input read with the same rules. This is what
|
||||
// the model declared: a command can hide its target, and the process resolves
|
||||
// names on its own. The gate detects a declared target; it does not prevent an
|
||||
// undeclared connection.
|
||||
func ExtractTargets(toolName string, input []byte) []Target {
|
||||
return extractDeclared(toolName, input).targets
|
||||
}
|
||||
|
||||
// extractDeclared reads what a tool call declares: its targets, as
|
||||
// ExtractTargets returns them, and the URLs it writes, as written, for the URL
|
||||
// block rules. A URL is kept only when it names a target.
|
||||
func extractDeclared(toolName string, input []byte) targetList {
|
||||
if toolName == executeExtraTool {
|
||||
var in struct {
|
||||
ToolName string `json:"tool_name"`
|
||||
Params json.RawMessage `json:"params"`
|
||||
}
|
||||
if json.Unmarshal(input, &in) == nil && in.ToolName != "" && in.ToolName != executeExtraTool {
|
||||
toolName, input = in.ToolName, in.Params
|
||||
}
|
||||
}
|
||||
if noTargetTools[toolName] {
|
||||
return targetList{}
|
||||
}
|
||||
var out targetList
|
||||
switch toolName {
|
||||
case "WebFetch":
|
||||
u := declaredFields(input).URL
|
||||
if t, ok := NormalizeTarget(u); ok {
|
||||
out.add(t)
|
||||
out.urls = append(out.urls, strings.TrimSpace(u))
|
||||
}
|
||||
case "Bash", "shell_open", "Monitor":
|
||||
out.scan(declaredFields(input).Command)
|
||||
case "shell_send":
|
||||
out.scan(declaredFields(input).Text)
|
||||
default:
|
||||
out.scanStrings(input)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// declaredInput holds the fields of the built-in tools that carry a target.
|
||||
type declaredInput struct {
|
||||
URL string `json:"url"`
|
||||
Command string `json:"command"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// declaredFields decodes the input into a struct, as the tools themselves do:
|
||||
// a key matches whatever its case and the last duplicate wins. The gate
|
||||
// therefore reads the value the tool will run, not another spelling of the key.
|
||||
func declaredFields(input []byte) declaredInput {
|
||||
var in declaredInput
|
||||
_ = json.Unmarshal(input, &in) // a field of another type is skipped, the others are kept
|
||||
return in
|
||||
}
|
||||
|
||||
type targetList struct {
|
||||
targets []Target
|
||||
seen map[Target]bool
|
||||
urls []string // the URLs that name a target, as written
|
||||
}
|
||||
|
||||
func (l *targetList) add(t Target) {
|
||||
if l.seen[t] {
|
||||
return
|
||||
}
|
||||
if l.seen == nil {
|
||||
l.seen = map[Target]bool{}
|
||||
}
|
||||
l.seen[t] = true
|
||||
l.targets = append(l.targets, t)
|
||||
}
|
||||
|
||||
// scanStrings reads every string of a JSON input, in document order. It walks
|
||||
// the tokens instead of decoding into a map, so both values of a duplicated key
|
||||
// are read: the gate cannot know which one the tool behind the call keeps.
|
||||
func (l *targetList) scanStrings(input []byte) {
|
||||
dec := json.NewDecoder(bytes.NewReader(input))
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if s, ok := tok.(string); ok {
|
||||
l.scan(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// shellMeta are the characters that end a word for an unquoted shell.
|
||||
const shellMeta = `|;&(){},\`
|
||||
|
||||
// urlHosts returns the readings of the authority of a URL found in a command.
|
||||
// Whether the URL is quoted decides how its authority is read when a shell
|
||||
// character sits in it: 'http://a.example;@b.example/' names b.example to the
|
||||
// client, and without the quotes the shell ends the URL at a.example. A
|
||||
// backslash is dropped by the shell and kept inside quotes. The gate cannot
|
||||
// tell which applies, so it returns every reading and each one is a target.
|
||||
func urlHosts(authority string) []string {
|
||||
host := func(s string) string {
|
||||
s = s[strings.LastIndexByte(s, '@')+1:]
|
||||
if i := strings.IndexAny(s, shellMeta); i >= 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
return s
|
||||
}
|
||||
out := []string{host(authority)}
|
||||
if i := strings.IndexAny(authority, shellMeta); i >= 0 {
|
||||
out = append(out, host(authority[:i]), host(strings.ReplaceAll(authority, `\`, "")))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var (
|
||||
// A URL with a scheme: the scheme, the authority, then the rest of the token.
|
||||
// The rest is consumed so that what a path or a query contains is not read
|
||||
// as a target.
|
||||
reURL = regexp.MustCompile(`(?i)\b([a-z][a-z0-9+.\-]*://)([^\s/?#"'<>` + "`" + `]*)[^\s"'<>` + "`" + `]*`)
|
||||
// [IPv6] and [IPv6]:port.
|
||||
reBracketed = regexp.MustCompile(`\[([0-9A-Fa-f:.]+(?:%[\w.\-]+)?)\](?::(\d{1,5}))?`)
|
||||
// IPv4, IPv4:port and IPv4/len.
|
||||
reIPv4 = regexp.MustCompile(`\d{1,3}(?:\.\d{1,3}){3}(?:/\d{1,2}|:\d{1,5})?`)
|
||||
// IPv6 and IPv6/len: at least two colons, checked by the parser afterwards.
|
||||
reIPv6 = regexp.MustCompile(`[0-9A-Fa-f:.]*:[0-9A-Fa-f:.]*:[0-9A-Fa-f:.]*(?:%[\w.\-]+)?(?:/\d{1,3})?`)
|
||||
// name:port, for a name with a dot and an alphabetic last label, or localhost.
|
||||
// A single label is not read as a host: user:1234 and image:22 are not targets.
|
||||
reHostPort = regexp.MustCompile(`(?i)(?:localhost|[a-z0-9_](?:[a-z0-9_\-]*[a-z0-9_])?(?:\.[a-z0-9_](?:[a-z0-9_\-]*[a-z0-9_])?)*\.[a-z][a-z0-9\-]*[a-z0-9]):\d{1,5}`)
|
||||
)
|
||||
|
||||
// scan reads the targets of one string with the command-string rules. Each
|
||||
// form is blanked once it is read, so a later rule never reads part of it again.
|
||||
func (l *targetList) scan(s string) {
|
||||
if s == "" {
|
||||
return
|
||||
}
|
||||
b := []byte(s)
|
||||
blank := func(from, to int) {
|
||||
for i := from; i < to; i++ {
|
||||
b[i] = ' '
|
||||
}
|
||||
}
|
||||
each := func(re *regexp.Regexp, read func(m []int) bool) {
|
||||
for _, m := range re.FindAllSubmatchIndex(b, -1) {
|
||||
if read(m) {
|
||||
blank(m[0], m[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
each(reURL, func(m []int) bool {
|
||||
scheme, found := string(b[m[2]:m[3]]), false
|
||||
for _, host := range urlHosts(string(b[m[4]:m[5]])) {
|
||||
// A host that is not a name ($HOST, a glob) is what the shell or the
|
||||
// client will make of it, which the gate cannot read.
|
||||
if t, ok := NormalizeTarget(scheme + host); ok && (t.Kind != TargetDomain || validHostname(t.Host)) {
|
||||
l.add(t)
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if found {
|
||||
// The whole token, path and query included, from the original string.
|
||||
l.urls = append(l.urls, s[m[0]:m[1]])
|
||||
}
|
||||
// The scheme and what follows the authority are never read again. An
|
||||
// authority that named no host is: an address written in it still counts.
|
||||
blank(m[0], m[4])
|
||||
blank(m[5], m[1])
|
||||
return found
|
||||
})
|
||||
each(reBracketed, func(m []int) bool {
|
||||
a, err := netip.ParseAddr(string(b[m[2]:m[3]]))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
t := Target{Kind: TargetIP, Host: a.Unmap().WithZone("").String()}
|
||||
if m[4] >= 0 {
|
||||
t.Port = validPort(string(b[m[4]:m[5]]))
|
||||
}
|
||||
l.add(t)
|
||||
return true
|
||||
})
|
||||
each(reIPv6, func(m []int) bool {
|
||||
start, end := m[0], m[1]
|
||||
if start > 0 && isWord(b[start-1]) || end < len(b) && (isWord(b[end]) || b[end] == ':') {
|
||||
return false
|
||||
}
|
||||
kind, host, ok := normalizeHost(string(b[start:end]), false)
|
||||
// "::" alone is punctuation (a slice, a scope operator), not an address.
|
||||
if !ok || kind == TargetDomain || host == "::" {
|
||||
return false
|
||||
}
|
||||
l.add(Target{Kind: kind, Host: host})
|
||||
return true
|
||||
})
|
||||
each(reIPv4, func(m []int) bool {
|
||||
start, end := m[0], m[1]
|
||||
if start > 0 && (isWord(b[start-1]) || b[start-1] == '.') {
|
||||
return false
|
||||
}
|
||||
// The address alone, when what follows it is not a port or a prefix length.
|
||||
addrEnd := start + bytes.IndexAny(b[start:end], "/:")
|
||||
if addrEnd < start {
|
||||
addrEnd = end
|
||||
}
|
||||
if !endsToken(b, end) {
|
||||
end = addrEnd
|
||||
if !endsToken(b, end) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
a, err := netip.ParseAddr(string(b[start:addrEnd]))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
t := Target{Kind: TargetIP, Host: a.String()}
|
||||
if end > addrEnd {
|
||||
switch suffix := string(b[addrEnd+1 : end]); b[addrEnd] {
|
||||
case '/':
|
||||
if p, err := netip.ParsePrefix(string(b[start:end])); err == nil {
|
||||
t = Target{Kind: TargetCIDR, Host: p.Masked().String()}
|
||||
}
|
||||
case ':':
|
||||
t.Port = validPort(suffix)
|
||||
}
|
||||
}
|
||||
l.add(t)
|
||||
return true
|
||||
})
|
||||
each(reHostPort, func(m []int) bool {
|
||||
start, end := m[0], m[1]
|
||||
// Not the tail of a longer word, a path or an address, and not the head of
|
||||
// a version number (name:1.25).
|
||||
if start > 0 && (isWord(b[start-1]) || strings.IndexByte(`.-/\`, b[start-1]) >= 0) {
|
||||
return false
|
||||
}
|
||||
if !endsToken(b, end) {
|
||||
return false
|
||||
}
|
||||
host, port, _ := strings.Cut(string(b[start:end]), ":")
|
||||
n := validPort(port)
|
||||
_, host, ok := normalizeHost(host, false)
|
||||
if n == 0 || !ok {
|
||||
return false
|
||||
}
|
||||
l.add(Target{Kind: TargetDomain, Host: host, Port: n})
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
// endsToken reports whether the token that ends before b[i] really ends there:
|
||||
// what follows is not a word character and not a dot that continues a name or
|
||||
// a number.
|
||||
func endsToken(b []byte, i int) bool {
|
||||
if i >= len(b) {
|
||||
return true
|
||||
}
|
||||
if isWord(b[i]) {
|
||||
return false
|
||||
}
|
||||
return b[i] != '.' || i+1 >= len(b) || !isWord(b[i+1])
|
||||
}
|
||||
|
||||
func isWord(c byte) bool {
|
||||
return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_'
|
||||
}
|
||||
|
||||
// validPort returns the port s names, 0 when it is not one.
|
||||
func validPort(s string) int {
|
||||
n, err := strconv.Atoi(s)
|
||||
if err != nil || n < 1 || n > 65535 {
|
||||
return 0
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// Block codes of the gate decision. Like the codes of a target request they
|
||||
// are carried in the model-facing text of the blocked call. None of them opens
|
||||
// an approval request.
|
||||
const (
|
||||
// TargetResolveFailed: a name did not resolve, so there is no address to
|
||||
// compare with a pin or to show to a human.
|
||||
TargetResolveFailed = "TARGET_RESOLVE_FAILED"
|
||||
// TargetRuleBlocked: an asset block rule matches a target.
|
||||
TargetRuleBlocked = "TARGET_RULE_BLOCKED"
|
||||
// TargetTooMany: the call names more unpinned targets than one call may request.
|
||||
TargetTooMany = "TARGET_TOO_MANY"
|
||||
)
|
||||
|
||||
// maxUnpinnedTargets is how many distinct targets without a covering pin one
|
||||
// call may request approval for. Each approval is one click on one host.
|
||||
const maxUnpinnedTargets = 5
|
||||
|
||||
// literalResolver is the resolver recorded for an IP literal or a CIDR: the
|
||||
// address is what was written, nothing was resolved.
|
||||
const literalResolver = "literal"
|
||||
|
||||
// targetStore is what the gate reads and writes. The database implements it.
|
||||
type targetStore interface {
|
||||
// BlockRules returns the asset block rules that apply to the scope: the
|
||||
// global ones and, for a task, its own. Allow rules play no role in the gate.
|
||||
BlockRules(scope db.TargetScopeRef) ([]db.AssetInterceptRule, error)
|
||||
// ActivePins returns the active pins that may cover (host, port) in scope,
|
||||
// the pin on the exact port first.
|
||||
ActivePins(scope db.TargetScopeRef, host string, port int) ([]*db.TargetPin, error)
|
||||
// Exemptions returns the stored exemption list and its digest.
|
||||
Exemptions() ([]db.TargetExemption, string, error)
|
||||
// LogDecision records an outcome no human took.
|
||||
LogDecision(db.TargetDecision) error
|
||||
}
|
||||
|
||||
type dbTargetStore struct{ d *db.DB }
|
||||
|
||||
func (s dbTargetStore) BlockRules(scope db.TargetScopeRef) ([]db.AssetInterceptRule, error) {
|
||||
rules, err := s.d.ListAssetInterceptRules()
|
||||
if err != nil || scope.TaskID == 0 {
|
||||
return rules, err
|
||||
}
|
||||
block, _, err := s.d.Assets().TaskInterceptRulesSplit(scope.TaskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(rules, block...), nil
|
||||
}
|
||||
|
||||
func (s dbTargetStore) ActivePins(scope db.TargetScopeRef, host string, port int) ([]*db.TargetPin, error) {
|
||||
return s.d.ActiveTargetPins(scope, host, port)
|
||||
}
|
||||
|
||||
func (s dbTargetStore) Exemptions() ([]db.TargetExemption, string, error) {
|
||||
list, raw, err := s.d.TargetExemptions()
|
||||
if raw == "" {
|
||||
raw = "[]"
|
||||
}
|
||||
return list, digestInput([]byte(raw)), err
|
||||
}
|
||||
|
||||
func (s dbTargetStore) LogDecision(r db.TargetDecision) error {
|
||||
_, err := s.d.LogTargetDecision(r)
|
||||
return err
|
||||
}
|
||||
|
||||
// CheckTargets decides, for one tool call of a gated scope, whether every
|
||||
// network target the call declares may be reached: extraction, asset block
|
||||
// rules, exemptions, resolution and the pin lookup. It creates no approval
|
||||
// request and never waits.
|
||||
//
|
||||
// A block is reported in the result (Code, Reason and the target that caused
|
||||
// it) and logged as a decided row. Otherwise the returned requests are the
|
||||
// targets no pin covers, each with the addresses a human has to read; the
|
||||
// caller passes them to RequestTargets and checks again after the decisions.
|
||||
// No request and no code means the call may run.
|
||||
//
|
||||
// With the gate off the call returns at once: nothing is extracted, resolved,
|
||||
// read or written.
|
||||
func (i *Interceptor) CheckTargets(ctx context.Context, scope TargetScope, toolName string, input []byte) ([]TargetRequest, TargetResult) {
|
||||
if scope.Enabled == nil || !scope.Enabled() {
|
||||
return nil, TargetResult{}
|
||||
}
|
||||
declared := extractDeclared(toolName, input)
|
||||
if len(declared.targets) == 0 {
|
||||
return nil, TargetResult{}
|
||||
}
|
||||
c := &targetCheck{i: i, ctx: ctx, scope: scope, tool: toolName, input: input, urls: declared.urls}
|
||||
return c.run(declared.targets)
|
||||
}
|
||||
|
||||
// targetCheck is one run of the gate decision.
|
||||
type targetCheck struct {
|
||||
i *Interceptor
|
||||
ctx context.Context
|
||||
scope TargetScope
|
||||
ref db.TargetScopeRef
|
||||
tool string
|
||||
input []byte
|
||||
urls []string // the URLs the call declares, as written, for the URL block rules
|
||||
// redirectFrom is the origin (host:port) of a redirect hop; empty for a call.
|
||||
redirectFrom string
|
||||
// fresh makes every lookup skip the cache: the decision of a connection
|
||||
// (dialTarget) compares the answer it is about to dial.
|
||||
fresh bool
|
||||
// checked is what the run decided about each target, read by dialTarget.
|
||||
checked []*checkedTarget
|
||||
}
|
||||
|
||||
// What let a target pass, recorded in checkedTarget.cover.
|
||||
const (
|
||||
coverDerived = "derived" // an exemption derived from the configuration
|
||||
coverStored = "stored" // a stored exemption that holds
|
||||
coverPin = "pin" // a pin that holds every resolved address
|
||||
coverNameOnly = "name_only" // a pin that is name-only for this call
|
||||
)
|
||||
|
||||
// checkedTarget is one target of the call and what the gate learned about it.
|
||||
type checkedTarget struct {
|
||||
Target
|
||||
ips []string // resolved addresses; the literal itself for an IP or a CIDR
|
||||
resolver string
|
||||
resolvedAt time.Time
|
||||
err error // the name did not resolve
|
||||
exemptVoid bool
|
||||
cover string // what let the target pass; empty when nothing did
|
||||
}
|
||||
|
||||
func (t Target) String() string {
|
||||
return targetName(TargetRequest{Host: t.Host, Port: t.Port})
|
||||
}
|
||||
|
||||
func (t *checkedTarget) audit(outcome string) db.InterceptTarget {
|
||||
return db.InterceptTarget{Host: t.Host, Port: t.Port, Kind: t.Kind, ResolvedIPs: t.ips,
|
||||
Resolver: t.resolver, ResolvedAt: t.resolvedAt, Outcome: outcome, ExemptVoid: t.exemptVoid}
|
||||
}
|
||||
|
||||
func (c *targetCheck) run(targets []Target) ([]TargetRequest, TargetResult) {
|
||||
i := c.i
|
||||
var err error
|
||||
if c.ref, err = c.scope.ref(); err != nil {
|
||||
return nil, TargetResult{Code: TargetGateUnavailable, Host: targets[0].Host, Port: targets[0].Port, Reason: err.Error()}
|
||||
}
|
||||
first := &checkedTarget{Target: targets[0]}
|
||||
rules, err := i.targets.BlockRules(c.ref)
|
||||
if err != nil {
|
||||
return c.block(TargetGateUnavailable, "gate_unavailable", first, "the asset rules could not be read: "+err.Error())
|
||||
}
|
||||
ex, err := i.loadExemptions()
|
||||
if err != nil {
|
||||
return c.block(TargetGateUnavailable, "gate_unavailable", first, "the exemption list could not be read: "+err.Error())
|
||||
}
|
||||
|
||||
// Block rules come first, and an exempt or pinned host that matches one is
|
||||
// blocked. They are matched on what the call declares before anything is
|
||||
// resolved: a hit there costs no lookup, for any target of the call.
|
||||
//
|
||||
// URL rules (exact_url, fuzzy_url) are matched once per decision, on the
|
||||
// URLs as the call writes them. That is a known limit: exact_url compares
|
||||
// the string as written, so another case, encoding (%2F), a trailing slash,
|
||||
// an added query or /./ does not match, and fuzzy_url is evaded by encoding
|
||||
// the substring; a command can also build its URL at run time. To keep an
|
||||
// agent away from a host, a domain, IP or CIDR rule is the one to write.
|
||||
if rule, u, hit := db.MatchAssetInterceptRules(rules, nil, nil, c.urls); hit {
|
||||
ct := &checkedTarget{Target: targets[0]}
|
||||
if t, ok := NormalizeTarget(u); ok {
|
||||
ct.Target = t
|
||||
}
|
||||
return c.block(TargetRuleBlocked, "rule_blocked", ct, fmt.Sprintf("%s is blocked by an asset rule. %s", u, rule.Reason()))
|
||||
}
|
||||
// Then the declared form of every target: the name as written, the IP
|
||||
// literal, the range.
|
||||
checked := make([]*checkedTarget, len(targets))
|
||||
c.checked = checked
|
||||
for n, t := range targets {
|
||||
ct := &checkedTarget{Target: t}
|
||||
checked[n] = ct
|
||||
if t.Kind != TargetDomain {
|
||||
ct.ips, ct.resolver, ct.resolvedAt = []string{t.Host}, literalResolver, time.Now().UTC()
|
||||
}
|
||||
if rule, hit := blockRule(rules, ct); hit {
|
||||
return c.block(TargetRuleBlocked, "rule_blocked", ct, fmt.Sprintf("%s is blocked by an asset rule. %s", t, rule.Reason()))
|
||||
}
|
||||
}
|
||||
// Only then are the names resolved, and the rules matched on the addresses.
|
||||
// A name the configuration exempts is never resolved.
|
||||
for _, ct := range checked {
|
||||
if ct.Kind != TargetDomain || ex.derived[db.TargetKey(ct.Host, ct.Port)] {
|
||||
continue
|
||||
}
|
||||
c.resolve(ct, c.fresh)
|
||||
if rule, hit := blockRule(rules, ct); hit {
|
||||
return c.block(TargetRuleBlocked, "rule_blocked", ct, fmt.Sprintf("%s is blocked by an asset rule. %s", ct.Target, rule.Reason()))
|
||||
}
|
||||
}
|
||||
|
||||
i.mu.RLock()
|
||||
proxied := i.upstreamProxy
|
||||
i.mu.RUnlock()
|
||||
// Behind the upstream proxy the proxy chooses the address it connects to, so
|
||||
// every pin is name-only for this call, whatever the pin row says: the proxy
|
||||
// can be turned on after a pin was approved.
|
||||
nameOnly := proxied != nil && proxied()
|
||||
|
||||
var unpinned []TargetRequest
|
||||
var uses []func()
|
||||
for _, ct := range checked {
|
||||
key := db.TargetKey(ct.Host, ct.Port)
|
||||
if ex.derived[key] {
|
||||
ct.cover = coverDerived
|
||||
uses = append(uses, func() { c.exemptUsed(ct, ex.digest) })
|
||||
continue
|
||||
}
|
||||
if ex.stored[key] {
|
||||
if exemptionHolds(ct) {
|
||||
ct.cover = coverStored
|
||||
uses = append(uses, func() { c.exemptUsed(ct, ex.digest) })
|
||||
continue
|
||||
}
|
||||
// A void exemption is not a block: the target goes on like any other.
|
||||
ct.exemptVoid = true
|
||||
}
|
||||
if ct.err != nil {
|
||||
return c.block(TargetResolveFailed, "resolve_failed", ct, c.resolveFailure(ct))
|
||||
}
|
||||
pins, err := i.targets.ActivePins(c.ref, ct.Host, ct.Port)
|
||||
if err != nil {
|
||||
return c.block(TargetGateUnavailable, "gate_unavailable", ct, "the pins could not be read: "+err.Error())
|
||||
}
|
||||
pin := coveringPin(pins, ct.ips, nameOnly)
|
||||
if pin == nil && ct.Kind == TargetDomain && !c.fresh {
|
||||
// A request is about to be raised: the addresses a human reads and
|
||||
// pins are resolved now, never taken from the cache. A fresh run
|
||||
// already has them.
|
||||
if c.resolve(ct, true); ct.err != nil {
|
||||
return c.block(TargetResolveFailed, "resolve_failed", ct, c.resolveFailure(ct))
|
||||
}
|
||||
if rule, hit := blockRule(rules, ct); hit {
|
||||
return c.block(TargetRuleBlocked, "rule_blocked", ct, fmt.Sprintf("%s is blocked by an asset rule. %s", ct.Target, rule.Reason()))
|
||||
}
|
||||
pin = coveringPin(pins, ct.ips, nameOnly)
|
||||
}
|
||||
if pin != nil {
|
||||
ct.cover = coverPin
|
||||
if nameOnly || pin.NameOnlyEffective {
|
||||
ct.cover = coverNameOnly
|
||||
}
|
||||
uses = append(uses, func() { c.pinUsed(ct, pin, nameOnly) })
|
||||
continue
|
||||
}
|
||||
req := TargetRequest{Host: ct.Host, Port: ct.Port, Kind: ct.Kind, ResolvedIPs: ct.ips,
|
||||
Resolver: ct.resolver, ResolvedAt: ct.resolvedAt, Reason: "no_pin", ExemptVoid: ct.exemptVoid}
|
||||
if len(pins) > 0 {
|
||||
// The request is raised on the key of the pin that exists (the one on
|
||||
// the exact port when both exist), so that an approval extends that pin.
|
||||
req.Port, req.Reason, req.Pin = pins[0].Port, "new_ip", pins[0]
|
||||
}
|
||||
if !hasRequest(unpinned, req) {
|
||||
unpinned = append(unpinned, req)
|
||||
}
|
||||
}
|
||||
if len(unpinned) > maxUnpinnedTargets {
|
||||
names := make([]string, len(unpinned))
|
||||
audit := make([]db.InterceptTarget, len(unpinned))
|
||||
for n, r := range unpinned {
|
||||
names[n] = targetName(r)
|
||||
audit[n] = db.InterceptTarget{Host: r.Host, Port: r.Port, Kind: r.Kind, ResolvedIPs: r.ResolvedIPs,
|
||||
Resolver: r.Resolver, ResolvedAt: r.ResolvedAt, Reason: r.Reason, Outcome: "too_many"}
|
||||
}
|
||||
reason := fmt.Sprintf("this call names %d targets that are not approved (%s); at most %d are requested per call",
|
||||
len(unpinned), strings.Join(names, ", "), maxUnpinnedTargets)
|
||||
c.log("denied", TargetTooMany+": "+reason, "", audit...)
|
||||
return nil, TargetResult{Code: TargetTooMany, Host: unpinned[0].Host, Port: unpinned[0].Port, Reason: reason}
|
||||
}
|
||||
if len(unpinned) == 0 {
|
||||
// The call passes: record what let it pass, once per process.
|
||||
for _, use := range uses {
|
||||
use()
|
||||
}
|
||||
}
|
||||
return unpinned, TargetResult{}
|
||||
}
|
||||
|
||||
// resolve fills the addresses of a name. fresh skips the cache.
|
||||
func (c *targetCheck) resolve(ct *checkedTarget, fresh bool) {
|
||||
ct.resolver = c.i.resolve.name()
|
||||
res, err := c.i.resolve.lookup(c.ctx, ct.Host, fresh)
|
||||
ct.ips, ct.resolvedAt, ct.err = res.ips, res.at, err
|
||||
}
|
||||
|
||||
func (c *targetCheck) resolveFailure(ct *checkedTarget) string {
|
||||
reason := fmt.Sprintf("the platform could not resolve %s (%v)", ct.Host, ct.err)
|
||||
if pins, err := c.i.targets.ActivePins(c.ref, ct.Host, ct.Port); err == nil && len(pins) > 0 {
|
||||
reason += fmt.Sprintf(", so it cannot compare the address with the approved pin (v%d: %s)",
|
||||
pins[0].Version, strings.Join(pins[0].PinnedIPs, ", "))
|
||||
}
|
||||
return reason
|
||||
}
|
||||
|
||||
// block logs a blocked call as a decided row and returns the block.
|
||||
func (c *targetCheck) block(code, outcome string, ct *checkedTarget, reason string) ([]TargetRequest, TargetResult) {
|
||||
c.log("denied", code+": "+reason, "", ct.audit(outcome))
|
||||
return nil, TargetResult{Code: code, Host: ct.Host, Port: ct.Port, Reason: reason}
|
||||
}
|
||||
|
||||
// log writes one decided row of the gate. Best effort: a failed write never
|
||||
// changes the outcome of the call.
|
||||
func (c *targetCheck) log(status, reason, configDigest string, targets ...db.InterceptTarget) {
|
||||
action := "deny"
|
||||
if status == "allowed" {
|
||||
action = "allow"
|
||||
}
|
||||
audit := auditFor(c.ctx, Decision{Action: action, Message: reason, ConfigDigest: configDigest}, c.input, status)
|
||||
if status == "allowed" {
|
||||
// The row records that a pin or an exemption was used, not this one call.
|
||||
audit.ExecutionStatus = "unknown"
|
||||
}
|
||||
for n := range targets {
|
||||
targets[n].RedirectFrom = c.redirectFrom
|
||||
}
|
||||
audit.Targets = targets
|
||||
_, agentName := taskInfoFromCtx(c.ctx)
|
||||
_ = c.i.targets.LogDecision(db.TargetDecision{Scope: c.ref, AgentName: agentName, ToolName: c.tool,
|
||||
ToolInput: c.input, Status: status, Reason: reason, Audit: audit})
|
||||
}
|
||||
|
||||
// exemptUsed logs the first call of this process that passes through an
|
||||
// exemption, per scope, host, port and list digest.
|
||||
func (c *targetCheck) exemptUsed(ct *checkedTarget, digest string) {
|
||||
key := "exempt|" + c.scope.scopeName() + "|" + db.TargetKey(ct.Host, ct.Port) + "|" + digest
|
||||
if _, seen := c.i.firstUse.LoadOrStore(key, true); seen {
|
||||
return
|
||||
}
|
||||
c.log("allowed", fmt.Sprintf("Target %s is exempt from target approval for %s.", ct.Target, c.scope.scopeName()),
|
||||
digest, ct.audit("exempt"))
|
||||
}
|
||||
|
||||
// pinUsed logs the first call of this process that a pin version covers.
|
||||
// nameOnly is whether the upstream proxy made every pin name-only for the call:
|
||||
// the row records the effective state, not only the pin's mark.
|
||||
func (c *targetCheck) pinUsed(ct *checkedTarget, pin *db.TargetPin, nameOnly bool) {
|
||||
if _, seen := c.i.firstUse.LoadOrStore(fmt.Sprintf("pin|%d", pin.ID), true); seen {
|
||||
return
|
||||
}
|
||||
t := ct.audit("pin_match")
|
||||
t.PinID, t.PinVersion, t.PinnedIPs, t.NameOnlyEffective = pin.ID, pin.Version, pin.PinnedIPs, pin.NameOnlyEffective || nameOnly
|
||||
c.log("allowed", fmt.Sprintf("Target %s is covered by pin v%d of %s.", ct.Target, pin.Version, c.scope.scopeName()), "", t)
|
||||
}
|
||||
|
||||
// blockRule matches the asset block rules against a target: its name, and its
|
||||
// addresses. A range is matched as written and by its network address. URL
|
||||
// rules are not matched here: run matches them once, on the declared URLs.
|
||||
func blockRule(rules []db.AssetInterceptRule, ct *checkedTarget) (db.AssetInterceptRule, bool) {
|
||||
var domains []string
|
||||
ips := ct.ips
|
||||
switch ct.Kind {
|
||||
case TargetDomain:
|
||||
domains = []string{ct.Host}
|
||||
case TargetCIDR:
|
||||
if p, err := netip.ParsePrefix(ct.Host); err == nil {
|
||||
ips = []string{ct.Host, p.Addr().String()}
|
||||
}
|
||||
}
|
||||
rule, _, hit := db.MatchAssetInterceptRules(rules, domains, ips, nil)
|
||||
return rule, hit
|
||||
}
|
||||
|
||||
// exemptionHolds reports whether the stored exemption on a target applies. On a
|
||||
// name it is void when the name does not resolve or one of its addresses is
|
||||
// private, loopback or otherwise local. An IP literal is not resolved; it holds
|
||||
// unless it is an address that can never be exempt.
|
||||
func exemptionHolds(ct *checkedTarget) bool {
|
||||
if ct.Kind == TargetIP {
|
||||
return ValidateExemptionHost(ct.Host) == nil
|
||||
}
|
||||
if ct.Kind != TargetDomain || ct.err != nil || len(ct.ips) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, ip := range ct.ips {
|
||||
if voidsExemption(ip) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// coveringPin returns the pin that covers the resolved addresses: the first
|
||||
// candidate that is name-only, or that holds every address. The address sets
|
||||
// of two pins are never merged.
|
||||
func coveringPin(pins []*db.TargetPin, resolved []string, nameOnly bool) *db.TargetPin {
|
||||
for _, p := range pins {
|
||||
if nameOnly || p.NameOnlyEffective || subset(resolved, p.PinnedIPs) {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func hasRequest(list []TargetRequest, r TargetRequest) bool {
|
||||
for _, x := range list {
|
||||
if x.Host == r.Host && x.Port == r.Port {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,475 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// The tests below run the gate against PostgreSQL: the real store, the real
|
||||
// request and decide path, and a fake resolver.
|
||||
|
||||
// pgGate returns an interceptor on d with a fake resolver.
|
||||
func pgGate(t *testing.T, d *db.DB) (*Interceptor, *fakeResolver) {
|
||||
t.Helper()
|
||||
ic, res := New(d), newFakeResolver()
|
||||
ic.SetResolver(res)
|
||||
return ic, res
|
||||
}
|
||||
|
||||
// storedExemptions sets the exemption list for one test and puts the previous
|
||||
// one back afterwards. The list is global, so these tests do not run in parallel.
|
||||
func storedExemptions(t *testing.T, d *db.DB, list ...db.TargetExemption) {
|
||||
t.Helper()
|
||||
previous, _, err := d.TargetExemptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.ReplaceTargetExemptions(list, "test", "test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = d.ReplaceTargetExemptions(previous, "test", "test") })
|
||||
}
|
||||
|
||||
// decidedRows returns the rows of the gate that no human decided, oldest first.
|
||||
func decidedRows(t *testing.T, d *db.DB, scope TargetScope) []*db.InterceptDetail {
|
||||
t.Helper()
|
||||
rows, err := d.Query(`SELECT id FROM intercept_pending
|
||||
WHERE COALESCE(task_id,'')=$1 AND COALESCE(conversation_id,0)=$2 AND decision_source='target' AND decided_by='system' AND target_key IS NULL ORDER BY id`,
|
||||
scope.TaskID, scope.ConversationID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ids []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
rows.Close()
|
||||
out := make([]*db.InterceptDetail, len(ids))
|
||||
for n, id := range ids {
|
||||
if out[n], err = d.GetInterceptDetail(id); err != nil || out[n] == nil || out[n].Audit == nil || len(out[n].Audit.Targets) == 0 {
|
||||
t.Fatalf("row %d: detail=%+v err=%v", id, out[n], err)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fetchInput(url string) []byte { return []byte(`{"url":"` + url + `"}`) }
|
||||
|
||||
// approveAll approves every open request of scope.
|
||||
func approveAll(t *testing.T, d *db.DB, ic *Interceptor, scope TargetScope) {
|
||||
t.Helper()
|
||||
ids := openRows(t, d, scope)
|
||||
if len(ids) == 0 {
|
||||
t.Fatal("no open request to approve")
|
||||
}
|
||||
for _, id := range ids {
|
||||
if err := ic.Decide(id, true, "ARTEX", "192.0.2.1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pin opens the request for an unpinned WebFetch target and approves it.
|
||||
func pinURL(t *testing.T, d *db.DB, ic *Interceptor, scope TargetScope, url string) {
|
||||
t.Helper()
|
||||
reqs, res := ic.CheckTargets(context.Background(), scope, "WebFetch", fetchInput(url))
|
||||
if res.Code != "" || len(reqs) != 1 {
|
||||
t.Fatalf("%s: requests=%+v result=%+v, want one request", url, reqs, res)
|
||||
}
|
||||
done := request(context.Background(), ic, scope, nil, reqs...)
|
||||
eventually(t, "the request row", func() bool { return len(openRows(t, d, scope)) == 1 })
|
||||
approveAll(t, d, ic, scope)
|
||||
if out := result(t, done); out.Code != "" {
|
||||
t.Fatalf("%s: approved request ended with %+v", url, out)
|
||||
}
|
||||
}
|
||||
|
||||
func mustPass(t *testing.T, what string, ic *Interceptor, scope TargetScope, url string) {
|
||||
t.Helper()
|
||||
if reqs, res := ic.CheckTargets(context.Background(), scope, "WebFetch", fetchInput(url)); res.Code != "" || len(reqs) != 0 {
|
||||
t.Fatalf("%s: requests=%+v result=%+v, want the call to pass", what, reqs, res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExemptionChangeIsLogged(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
ic := New(d)
|
||||
original, _, err := ic.Exemptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = d.ReplaceTargetExemptions(original, "test", "test") })
|
||||
logged := func() []db.TargetExemptionChange {
|
||||
t.Helper()
|
||||
changes, err := d.ListTargetExemptionLog(100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return changes
|
||||
}
|
||||
before := logged()
|
||||
|
||||
first := []db.TargetExemption{{Host: "PyPI.org.", Port: 443, Note: " index "}, {Host: "10.0.0.5", Port: 8080}, {Host: "pypi.org", Port: 443, Note: "duplicate"}}
|
||||
stored, digest, err := ic.SetExemptions(first, "alice")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []db.TargetExemption{{Host: "pypi.org", Port: 443, Note: "index"}, {Host: "10.0.0.5", Port: 8080}}
|
||||
if !slices.Equal(stored, want) {
|
||||
t.Fatalf("stored = %+v, want %+v (normalised, without the duplicate)", stored, want)
|
||||
}
|
||||
if list, got, err := ic.Exemptions(); err != nil || !slices.Equal(list, want) || got != digest || digest == "" {
|
||||
t.Fatalf("read back: list=%+v digest=%q (want %q) err=%v", list, got, digest, err)
|
||||
}
|
||||
second, digest2, err := ic.SetExemptions(nil, "bob")
|
||||
if err != nil || len(second) != 0 || digest2 == digest {
|
||||
t.Fatalf("empty list: stored=%+v digest=%q err=%v", second, digest2, err)
|
||||
}
|
||||
|
||||
// A refused list changes nothing and logs nothing.
|
||||
if _, _, err := ic.SetExemptions([]db.TargetExemption{{Host: "169.254.169.254", Port: 80}}, "mallory"); !errors.Is(err, ErrInvalidExemption) {
|
||||
t.Fatalf("metadata address: err=%v, want it refused", err)
|
||||
}
|
||||
if list, got, err := ic.Exemptions(); err != nil || len(list) != 0 || got != digest2 {
|
||||
t.Fatalf("after the refused change: list=%+v digest=%q err=%v", list, got, err)
|
||||
}
|
||||
|
||||
changes := logged()
|
||||
if len(changes) != len(before)+2 {
|
||||
t.Fatalf("%d log rows added, want 2", len(changes)-len(before))
|
||||
}
|
||||
newest, older := changes[0], changes[1]
|
||||
var olderBefore, olderAfter, newestBefore, newestAfter []db.TargetExemption
|
||||
for raw, into := range map[*json.RawMessage]*[]db.TargetExemption{&older.Before: &olderBefore, &older.After: &olderAfter, &newest.Before: &newestBefore, &newest.After: &newestAfter} {
|
||||
if len(*raw) > 0 {
|
||||
if err := json.Unmarshal(*raw, into); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if older.ChangedBy != "alice" || older.Digest != digest || !slices.Equal(olderBefore, original) || !slices.Equal(olderAfter, want) || older.ChangedAt.IsZero() {
|
||||
t.Fatalf("first change = %+v", older)
|
||||
}
|
||||
if newest.ChangedBy != "bob" || newest.Digest != digest2 || !slices.Equal(newestBefore, want) || len(newestAfter) != 0 || string(newest.After) != "[]" {
|
||||
t.Fatalf("second change = %+v", newest)
|
||||
}
|
||||
}
|
||||
|
||||
// The first call that passes through an exemption is logged once per scope,
|
||||
// host, port and list digest, and per process.
|
||||
func TestExemptionFirstUseLoggedOnce(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
storedExemptions(t, d, db.TargetExemption{Host: "pypi.org", Port: 443})
|
||||
ic, res := pgGate(t, d)
|
||||
ic.SetDerivedExemptions(func() []string { return []string{"https://llm.example.net/v1"} })
|
||||
res.set("pypi.org", "151.101.0.223")
|
||||
scope := taskScope(t, d, ic, time.Minute)
|
||||
_, digest, err := ic.Exemptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for n := 0; n < 3; n++ {
|
||||
mustPass(t, "stored exemption", ic, scope, "https://pypi.org/simple/")
|
||||
mustPass(t, "derived exemption", ic, scope, "https://llm.example.net/v1/models")
|
||||
}
|
||||
rows := decidedRows(t, d, scope)
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("%d first-use rows, want one per exemption", len(rows))
|
||||
}
|
||||
for n, host := range []string{"pypi.org", "llm.example.net"} {
|
||||
row := rows[n]
|
||||
target := row.Audit.Targets[0]
|
||||
if row.Status != "allowed" || row.DecidedBy != "system" || row.DecisionSource != "target" || row.ToolName != "WebFetch" ||
|
||||
target.Host != host || target.Port != 443 || target.Outcome != "exempt" || row.Audit.ConfigDigest != digest || row.DecidedAt == nil {
|
||||
t.Fatalf("first-use row %d = %+v audit=%+v", n, row.InterceptApprovalRow, row.Audit)
|
||||
}
|
||||
}
|
||||
if n := len(pins(t, d, scope)) + len(openRows(t, d, scope)); n != 0 {
|
||||
t.Fatalf("an exemption created %d pins or requests", n)
|
||||
}
|
||||
|
||||
// Another scope logs its own first use.
|
||||
other := taskScope(t, d, ic, time.Minute)
|
||||
mustPass(t, "other task", ic, other, "https://pypi.org/simple/")
|
||||
if n := len(decidedRows(t, d, other)); n != 1 {
|
||||
t.Fatalf("other task: %d first-use rows, want 1", n)
|
||||
}
|
||||
// A changed list has another digest: the use is logged again.
|
||||
storedExemptions(t, d, db.TargetExemption{Host: "pypi.org", Port: 443}, db.TargetExemption{Host: "files.pythonhosted.org", Port: 443})
|
||||
mustPass(t, "changed list", ic, scope, "https://pypi.org/simple/")
|
||||
mustPass(t, "changed list, again", ic, scope, "https://pypi.org/simple/")
|
||||
if n := len(decidedRows(t, d, scope)); n != 3 {
|
||||
t.Fatalf("after the list change: %d rows, want 3", n)
|
||||
}
|
||||
// Another process (a second interceptor on the same database) logs again.
|
||||
second, res2 := pgGate(t, d)
|
||||
res2.set("pypi.org", "151.101.0.223")
|
||||
scope.Interceptor = second
|
||||
mustPass(t, "second process", second, scope, "https://pypi.org/simple/")
|
||||
if n := len(decidedRows(t, d, scope)); n != 4 {
|
||||
t.Fatalf("second process: %d rows, want 4", n)
|
||||
}
|
||||
}
|
||||
|
||||
// The seeded package sources are stored entries like the ones an operator
|
||||
// adds: the void rule applies to them.
|
||||
func TestSeededExemptionIsVoidOnPrivateAddress(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
stored, _, err := d.TargetExemptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.ContainsFunc(stored, func(e db.TargetExemption) bool { return e.Host == "pypi.org" && e.Port == 443 }) {
|
||||
t.Skipf("the stored list no longer holds the seeded pypi.org:443 entry: %+v", stored)
|
||||
}
|
||||
ic, res := pgGate(t, d)
|
||||
scope := taskScope(t, d, ic, time.Minute)
|
||||
res.set("pypi.org", "151.101.0.223")
|
||||
mustPass(t, "seeded entry, public address", ic, scope, "https://pypi.org/simple/")
|
||||
|
||||
second, res2 := pgGate(t, d)
|
||||
res2.set("pypi.org", "151.101.0.223", "10.0.0.5")
|
||||
reqs, out := second.CheckTargets(context.Background(), scope, "WebFetch", fetchInput("https://pypi.org/simple/"))
|
||||
if out.Code != "" || len(reqs) != 1 || !reqs[0].ExemptVoid {
|
||||
t.Fatalf("seeded entry, one private address: requests=%+v result=%+v, want a request with a void exemption", reqs, out)
|
||||
}
|
||||
// The request row records that the exemption was void.
|
||||
done := request(context.Background(), second, scope, nil, reqs...)
|
||||
id := oneOpenRow(t, d, scope)
|
||||
detail, err := d.GetInterceptDetail(id)
|
||||
if err != nil || detail == nil || !detail.Audit.Targets[0].ExemptVoid {
|
||||
t.Fatalf("request row audit = %+v err=%v, want exempt_void", detail, err)
|
||||
}
|
||||
if err := second.Decide(id, false, "ARTEX", "192.0.2.1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out := result(t, done); out.Code != TargetDenied {
|
||||
t.Fatalf("denied request ended with %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
// Task allow rules play no role in the gate: they neither satisfy it nor
|
||||
// restrict it. Block rules, global and of the task, always win.
|
||||
func TestAllowRuleDoesNotSatisfyGate(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
ic, res := pgGate(t, d)
|
||||
scope := taskScope(t, d, ic, time.Minute)
|
||||
ref := scope.dbRef(t)
|
||||
res.set("app.example.com", "203.0.113.7")
|
||||
res.set("other.example.com", "203.0.113.9")
|
||||
rule := func(action, kind, pattern string) {
|
||||
t.Helper()
|
||||
if _, err := d.Assets().CreateTaskInterceptRule(ref.TaskID, action, kind, pattern, "target gate test", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// An allow rule on the host does not replace the approval.
|
||||
rule("allow", "exact_domain", "app.example.com")
|
||||
reqs, out := ic.CheckTargets(context.Background(), scope, "WebFetch", fetchInput("https://app.example.com/"))
|
||||
if out.Code != "" || len(reqs) != 1 || reqs[0].Reason != "no_pin" {
|
||||
t.Fatalf("allow rule on the host: requests=%+v result=%+v, want a request", reqs, out)
|
||||
}
|
||||
// A pinned host outside the allow list still passes: the whitelist is not
|
||||
// applied at dispatch.
|
||||
pinURL(t, d, ic, scope, "https://other.example.com/")
|
||||
mustPass(t, "pinned host outside the allow list", ic, scope, "https://other.example.com/")
|
||||
|
||||
// A block rule of the task beats the pin.
|
||||
rule("block", "exact_ip", "203.0.113.9")
|
||||
if reqs, out := ic.CheckTargets(context.Background(), scope, "WebFetch", fetchInput("https://other.example.com/")); out.Code != TargetRuleBlocked || len(reqs) != 0 {
|
||||
t.Fatalf("task block rule: requests=%+v result=%+v", reqs, out)
|
||||
}
|
||||
// It is the task's rule: another task is not blocked by it.
|
||||
other := taskScope(t, d, ic, time.Minute)
|
||||
if reqs, out := ic.CheckTargets(context.Background(), other, "WebFetch", fetchInput("https://other.example.com/")); out.Code != "" || len(reqs) != 1 {
|
||||
t.Fatalf("other task: requests=%+v result=%+v, want a request", reqs, out)
|
||||
}
|
||||
// A global block rule applies to every scope.
|
||||
global, err := d.CreateAssetInterceptRule("exact_domain", "app.example.com", "target gate test", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = d.DeleteAssetInterceptRule(global.ID) })
|
||||
for name, s := range map[string]TargetScope{"task": scope, "other task": other} {
|
||||
if reqs, out := ic.CheckTargets(context.Background(), s, "WebFetch", fetchInput("https://app.example.com/")); out.Code != TargetRuleBlocked || len(reqs) != 0 {
|
||||
t.Fatalf("global block rule, %s: requests=%+v result=%+v", name, reqs, out)
|
||||
}
|
||||
}
|
||||
// Each block is a decided row; none opened a request.
|
||||
rows := decidedRows(t, d, scope)
|
||||
blocks := 0
|
||||
for _, row := range rows {
|
||||
if row.Audit.Targets[0].Outcome == "rule_blocked" {
|
||||
blocks++
|
||||
if row.Status != "denied" || !strings.HasPrefix(row.Reason, TargetRuleBlocked+": ") || row.Audit.ExecutionStatus != "not_executed" {
|
||||
t.Fatalf("block row = %+v audit=%+v", row.InterceptApprovalRow, row.Audit)
|
||||
}
|
||||
}
|
||||
}
|
||||
if blocks != 2 || len(openRows(t, d, scope)) != 0 {
|
||||
t.Fatalf("%d block rows and %d open requests, want 2 and 0", blocks, len(openRows(t, d, scope)))
|
||||
}
|
||||
}
|
||||
|
||||
// The proxy can be turned on between the approval and a later call. The pin
|
||||
// row then still says name_only_effective = false, and the gate treats it as
|
||||
// name-only at call time: a changed address neither blocks nor asks.
|
||||
func TestProxyTurnedOnAfterApprovalMakesPinNameOnly(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
ic, res := pgGate(t, d)
|
||||
proxied := false
|
||||
ic.SetUpstreamProxyCheck(func() bool { return proxied })
|
||||
scope := taskScope(t, d, ic, time.Minute)
|
||||
res.set("app.example.com", "203.0.113.7")
|
||||
pinURL(t, d, ic, scope, "https://app.example.com/")
|
||||
approved := pins(t, d, scope)
|
||||
if len(approved) != 1 || approved[0].NameOnlyEffective || !slices.Equal(approved[0].PinnedIPs, []string{"203.0.113.7"}) {
|
||||
t.Fatalf("pins = %+v, want one pin that is not name-only", approved)
|
||||
}
|
||||
requestRows := allRows(t, d, scope)
|
||||
|
||||
// global_proxy is set, and the name answers another address. A new
|
||||
// interceptor stands for a later call that has nothing cached.
|
||||
proxied = true
|
||||
res.set("app.example.com", "198.51.100.99")
|
||||
ic.SetResolver(res)
|
||||
mustPass(t, "changed address behind the proxy", ic, scope, "https://app.example.com/")
|
||||
|
||||
if n := len(openRows(t, d, scope)); n != 0 {
|
||||
t.Fatalf("%d open requests, want none", n)
|
||||
}
|
||||
if n := allRows(t, d, scope); n != requestRows {
|
||||
t.Fatalf("%d request rows, want %d: the call opened a request", n, requestRows)
|
||||
}
|
||||
after := pins(t, d, scope)
|
||||
if len(after) != 1 || after[0].ID != approved[0].ID || after[0].NameOnlyEffective || after[0].Status != db.TargetPinActive ||
|
||||
!slices.Equal(after[0].PinnedIPs, []string{"203.0.113.7"}) {
|
||||
t.Fatalf("pins after the call = %+v, want the same row, unchanged", after)
|
||||
}
|
||||
// The use of the pin is on record, with the address the platform saw.
|
||||
rows := decidedRows(t, d, scope)
|
||||
last := rows[len(rows)-1]
|
||||
if target := last.Audit.Targets[0]; last.Status != "allowed" || target.Outcome != "pin_match" || target.PinID != approved[0].ID ||
|
||||
!slices.Equal(target.ResolvedIPs, []string{"198.51.100.99"}) {
|
||||
t.Fatalf("last row = %+v audit=%+v", last.InterceptApprovalRow, last.Audit)
|
||||
}
|
||||
|
||||
// Control: with the proxy off the same call asks.
|
||||
proxied = false
|
||||
if reqs, out := ic.CheckTargets(context.Background(), scope, "WebFetch", fetchInput("https://app.example.com/")); out.Code != "" || len(reqs) != 1 || reqs[0].Reason != "new_ip" {
|
||||
t.Fatalf("proxy off: requests=%+v result=%+v, want a new_ip request", reqs, out)
|
||||
}
|
||||
}
|
||||
|
||||
// An approval never pins zero addresses: not on a first approval, and not when
|
||||
// the key already has a pin with addresses.
|
||||
func TestApprovalNeverPinsEmptyAddressList(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
ic, res := pgGate(t, d)
|
||||
scope := taskScope(t, d, ic, time.Minute)
|
||||
ref := scope.dbRef(t)
|
||||
|
||||
// RequestTargets refuses to open such a request.
|
||||
for _, ips := range [][]string{nil, {}} {
|
||||
out := ic.RequestTargets(context.Background(), scope, "WebFetch", webFetchInput,
|
||||
[]TargetRequest{{Host: "app.example.com", Port: 443, Kind: TargetDomain, ResolvedIPs: ips, Reason: "no_pin"}}, nil)
|
||||
if out.Code != TargetResolveFailed || out.PendingID != 0 {
|
||||
t.Fatalf("request with %v: %+v, want %s", ips, out, TargetResolveFailed)
|
||||
}
|
||||
}
|
||||
// One empty target refuses the whole call, before any row is written.
|
||||
out := ic.RequestTargets(context.Background(), scope, "Bash", []byte(`{"command":"x"}`),
|
||||
[]TargetRequest{target("a.example.com", 443, "203.0.113.7"), target("b.example.com", 443)}, nil)
|
||||
if out.Code != TargetResolveFailed || out.Host != "b.example.com" || allRows(t, d, scope) != 0 {
|
||||
t.Fatalf("mixed call: %+v with %d rows, want %s and no row", out, allRows(t, d, scope), TargetResolveFailed)
|
||||
}
|
||||
|
||||
// A row that reached the table without an address cannot be approved.
|
||||
open := func(host string, ips []string) int64 {
|
||||
t.Helper()
|
||||
id, created, err := d.FindOrCreateTargetRequest(db.TargetRequest{Scope: ref, Host: host, Port: 443, ToolName: "WebFetch", Reason: "test",
|
||||
Audit: &db.InterceptAudit{Targets: []db.InterceptTarget{{Host: host, Port: 443, ResolvedIPs: ips, Outcome: "requested"}}}})
|
||||
if err != nil || !created {
|
||||
t.Fatalf("open request: id=%d created=%v err=%v", id, created, err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
for _, ips := range [][]string{nil, {}, {""}} {
|
||||
id := open("empty.example.com", ips)
|
||||
if err := ic.Decide(id, true, "ARTEX", "192.0.2.1"); !errors.Is(err, db.ErrTargetPinEmpty) {
|
||||
t.Fatalf("approve %v: err=%v, want ErrTargetPinEmpty", ips, err)
|
||||
}
|
||||
if row, err := d.GetInterceptPending(id); err != nil || row == nil || row.Status != "pending" {
|
||||
t.Fatalf("refused approval: row=%+v err=%v, want it still pending", row, err)
|
||||
}
|
||||
if n := len(pins(t, d, scope)); n != 0 {
|
||||
t.Fatalf("%d pins after a refused approval, want none", n)
|
||||
}
|
||||
// The request can still be denied.
|
||||
if err := ic.Decide(id, false, "ARTEX", "192.0.2.1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// With a pin on the key: the approval is refused all the same, and the
|
||||
// pin is neither superseded nor re-issued with its old addresses.
|
||||
res.set("app.example.com", "203.0.113.7")
|
||||
pinURL(t, d, ic, scope, "https://app.example.com/")
|
||||
id := open("app.example.com", nil)
|
||||
if err := ic.Decide(id, true, "ARTEX", "192.0.2.1"); !errors.Is(err, db.ErrTargetPinEmpty) {
|
||||
t.Fatalf("approve an empty new_ip request: err=%v, want ErrTargetPinEmpty", err)
|
||||
}
|
||||
if after := pins(t, d, scope); len(after) != 1 || after[0].Version != 1 || after[0].Status != db.TargetPinActive || !slices.Equal(after[0].PinnedIPs, []string{"203.0.113.7"}) {
|
||||
t.Fatalf("pins = %+v, want version 1 untouched", after)
|
||||
}
|
||||
|
||||
// A name that stops resolving opens no request, so there is nothing to approve.
|
||||
res.errs["app.example.com"] = errors.New("no such host")
|
||||
ic.SetResolver(res)
|
||||
if reqs, out := ic.CheckTargets(context.Background(), scope, "WebFetch", fetchInput("https://app.example.com:8443/")); out.Code != TargetResolveFailed || len(reqs) != 0 {
|
||||
t.Fatalf("unresolvable name: requests=%+v result=%+v", reqs, out)
|
||||
}
|
||||
}
|
||||
|
||||
// A block is logged for a conversation as it is for a task, on the
|
||||
// conversation's own rows.
|
||||
func TestTargetBlockIsLoggedForConversation(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
ic, res := pgGate(t, d)
|
||||
conv, err := d.CreateConversation("auto", "target gate block", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = d.DeleteConversation(conv.ID)
|
||||
_, _ = d.Exec(`DELETE FROM target_pins WHERE conversation_id=$1`, conv.ID)
|
||||
})
|
||||
scope := TargetScope{ConversationID: conv.ID, Enabled: func() bool { return true }, WaitBound: time.Minute, Interceptor: ic}
|
||||
res.errs["app.example.com"] = errors.New("no such host")
|
||||
ctx := WithCall(WithConvID(context.Background(), conv.ID), "WebFetch", fetchInput("https://app.example.com/"))
|
||||
if reqs, out := ic.CheckTargets(ctx, scope, "WebFetch", fetchInput("https://app.example.com/")); out.Code != TargetResolveFailed || len(reqs) != 0 {
|
||||
t.Fatalf("requests=%+v result=%+v", reqs, out)
|
||||
}
|
||||
rows := decidedRows(t, d, scope)
|
||||
if len(rows) != 1 {
|
||||
t.Fatalf("%d rows, want 1", len(rows))
|
||||
}
|
||||
row := rows[0]
|
||||
if row.Status != "denied" || row.ConversationID == nil || *row.ConversationID != conv.ID || row.TaskID != nil ||
|
||||
row.Audit.Targets[0].Outcome != "resolve_failed" || row.Audit.InputDigest == "" {
|
||||
t.Fatalf("row = %+v audit=%+v", row.InterceptApprovalRow, row.Audit)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,107 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// The HTTP clients the platform owns (WebFetch and the custom http tool)
|
||||
// follow redirects. The hook and the rendered-URL gate decide on the URL a call
|
||||
// names; a redirect sends the client to a URL nobody checked. RedirectCheck is
|
||||
// the redirect policy of those clients: the client asks it before it follows
|
||||
// each hop, and the hop goes through the same target decision as the call, on
|
||||
// the URL the client is about to request: the asset block rules on that URL and
|
||||
// on its target, before any lookup, then exemptions, resolution and pins.
|
||||
|
||||
// maxRedirects is the limit net/http applies to a client without a redirect
|
||||
// policy. Setting a policy replaces it, so RedirectCheck restates it, with the
|
||||
// same error text.
|
||||
const maxRedirects = 10
|
||||
|
||||
// RedirectCheck returns the redirect policy (http.Client.CheckRedirect) of a
|
||||
// platform-owned client run by the tool toolName. Everything it needs comes
|
||||
// from the context of the request, which net/http keeps on every hop: the
|
||||
// target scope of the session and the Interceptor that owns its gate.
|
||||
//
|
||||
// With the gate off, or with no scope in the context (the operator's test
|
||||
// endpoint), it only applies the limit of 10 hops: nothing is parsed, resolved,
|
||||
// read or written, and the client behaves as with the default policy.
|
||||
//
|
||||
// A hop no pin or exemption covers is not followed, and the policy never waits:
|
||||
// the fetch has its own deadline, which a human decision does not fit in. The
|
||||
// request row for the hop's target is found or created (it is durable, so an
|
||||
// approval given later lets the retried call follow), and the call fails at
|
||||
// once with TARGET_APPROVAL_PENDING. A block rule, a failed lookup or a hop
|
||||
// whose target cannot be read fails the call with its block code, and opens no
|
||||
// request.
|
||||
func RedirectCheck(toolName string) func(req *http.Request, via []*http.Request) error {
|
||||
return func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= maxRedirects {
|
||||
return errors.New("stopped after 10 redirects")
|
||||
}
|
||||
ctx := req.Context()
|
||||
scope, on := renderedScope(ctx)
|
||||
if !on {
|
||||
return nil
|
||||
}
|
||||
return checkRedirectHop(ctx, scope, toolName, req.URL.String(), via[len(via)-1].URL.String())
|
||||
}
|
||||
}
|
||||
|
||||
// checkRedirectHop runs the gate decision on one hop, from the URL that
|
||||
// redirected (from) to the URL the client will request (hop).
|
||||
func checkRedirectHop(ctx context.Context, scope TargetScope, toolName, hop, from string) error {
|
||||
input, _ := json.Marshal(map[string]string{"url": hop, "redirect_from": from})
|
||||
blocked := func(res TargetResult) error {
|
||||
return &redirectError{res: res, from: from, hop: hop, scope: scope.scopeName()}
|
||||
}
|
||||
t, err := urlTarget(hop)
|
||||
if err != nil {
|
||||
return blocked(undeterminedTarget(ctx, scope, toolName, input, "the target of the redirect could not be read ("+err.Error()+")"))
|
||||
}
|
||||
i := scope.Interceptor
|
||||
if i == nil {
|
||||
return blocked(TargetResult{Code: TargetGateUnavailable, Host: t.Host, Port: t.Port, Reason: "the session's target scope carries no gate"})
|
||||
}
|
||||
var origin string
|
||||
if o, err := urlTarget(from); err == nil {
|
||||
origin = o.String()
|
||||
}
|
||||
ctx = WithCall(ctx, toolName, input)
|
||||
c := &targetCheck{i: i, ctx: ctx, scope: scope, tool: toolName, input: input, urls: []string{hop}, redirectFrom: origin}
|
||||
unpinned, res := c.run([]Target{t})
|
||||
if res.Code == "" && len(unpinned) > 0 {
|
||||
// Wait bound 0: the request is found or created, and the call does not wait.
|
||||
scope.WaitBound = 0
|
||||
for n := range unpinned {
|
||||
unpinned[n].RedirectFrom = origin
|
||||
}
|
||||
res = i.RequestTargets(ctx, scope, toolName, input, unpinned, nil)
|
||||
}
|
||||
if res.Code == "" {
|
||||
return nil
|
||||
}
|
||||
return blocked(res)
|
||||
}
|
||||
|
||||
// redirectError is what the client returns when a hop is not followed. Both
|
||||
// tools hand the client error to the model as the tool result.
|
||||
type redirectError struct {
|
||||
res TargetResult
|
||||
from, hop, scope string
|
||||
}
|
||||
|
||||
func (e *redirectError) Error() string {
|
||||
target := e.hop
|
||||
if e.res.Host != "" {
|
||||
target = targetName(TargetRequest{Host: e.res.Host, Port: e.res.Port})
|
||||
}
|
||||
msg := "[ARTEX platform control, not target defense] " + e.res.Code + ": " + e.from + " redirects to " + target
|
||||
if e.res.Code == TargetApprovalPending {
|
||||
return msg + ", which is not approved for " + e.scope + ". The redirect was not followed and an approval was requested. " +
|
||||
"Retry the same call after the approval. Do not fetch the redirect target directly or with another tool."
|
||||
}
|
||||
return msg + ": " + e.res.Reason + ". The redirect was not followed. Do not fetch the redirect target directly or with another tool."
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Step 1.9 against PostgreSQL: a hop no pin covers opens the durable request
|
||||
// and fails the call at once; an approval lets the same call follow.
|
||||
|
||||
// A hop to an unpinned target is not followed: the redirect target receives
|
||||
// nothing, exactly one request row is opened for it, and the call fails with
|
||||
// TARGET_APPROVAL_PENDING without waiting, although the scope would wait five
|
||||
// minutes at the hook. The same call again attaches to that row. After the
|
||||
// approval it follows.
|
||||
func TestRedirectToUnpinnedHostNotFollowed(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
ic, res := pgGate(t, d)
|
||||
scope := taskScope(t, d, ic, 5*time.Minute)
|
||||
ctx := WithTargetScope(t.Context(), scope)
|
||||
final, finalHits, finalKey := hopServer(t, "")
|
||||
start, startHits, startKey := hopServer(t, final.URL+"/final")
|
||||
|
||||
began := time.Now()
|
||||
_, err := gatedFetch(ctx, start.URL+"/start")
|
||||
if err == nil || !strings.Contains(err.Error(), TargetApprovalPending) ||
|
||||
!strings.Contains(err.Error(), start.URL+"/start redirects to "+finalKey) || !strings.Contains(err.Error(), "not followed") {
|
||||
t.Fatalf("unpinned hop: %v", err)
|
||||
}
|
||||
if elapsed := time.Since(began); elapsed > 10*time.Second {
|
||||
t.Fatalf("the hop waited %v", elapsed)
|
||||
}
|
||||
if startHits.Load() != 1 || finalHits.Load() != 0 {
|
||||
t.Fatalf("hits start=%d final=%d, want the redirect target untouched", startHits.Load(), finalHits.Load())
|
||||
}
|
||||
ids := openRows(t, d, scope)
|
||||
if len(ids) != 1 || allRows(t, d, scope) != 1 {
|
||||
t.Fatalf("open requests = %v (%d rows), want one", ids, allRows(t, d, scope))
|
||||
}
|
||||
detail, err := d.GetInterceptDetail(ids[0])
|
||||
if err != nil || detail == nil || detail.Audit == nil || len(detail.Audit.Targets) != 1 {
|
||||
t.Fatalf("request detail = %+v, %v", detail, err)
|
||||
}
|
||||
if tgt := detail.Audit.Targets[0]; tgt.Host+":"+strconv.Itoa(tgt.Port) != finalKey || tgt.RedirectFrom != startKey || tgt.Outcome != "requested" {
|
||||
t.Fatalf("requested target = %+v, want %s from %s", tgt, finalKey, startKey)
|
||||
}
|
||||
if !strings.Contains(detail.Reason, "A redirect from "+startKey) || !strings.Contains(string(detail.ToolInput), start.URL+"/start") {
|
||||
t.Fatalf("request reason %q / input %s, want the origin of the redirect", detail.Reason, detail.ToolInput)
|
||||
}
|
||||
|
||||
// The retried call stops at the same hop and attaches to the same row.
|
||||
if _, err := gatedFetch(ctx, start.URL+"/start"); err == nil || !strings.Contains(err.Error(), TargetApprovalPending) || finalHits.Load() != 0 {
|
||||
t.Fatalf("retry before the approval: %v, final hits %d", err, finalHits.Load())
|
||||
}
|
||||
if again := openRows(t, d, scope); len(again) != 1 || again[0] != ids[0] || allRows(t, d, scope) != 1 {
|
||||
t.Fatalf("after the retry: open %v, %d rows; want the same single request", again, allRows(t, d, scope))
|
||||
}
|
||||
|
||||
approveAll(t, d, ic, scope)
|
||||
if resp, err := gatedFetch(ctx, start.URL+"/start"); err != nil || resp.StatusCode != 200 || finalHits.Load() != 1 {
|
||||
t.Fatalf("after the approval: %v, final hits %d", err, finalHits.Load())
|
||||
}
|
||||
if res.total() != 0 {
|
||||
t.Fatalf("%d lookups for IP literals", res.total())
|
||||
}
|
||||
}
|
||||
|
||||
// http -> https on the same host is another key (port 443, not 80): a pin on
|
||||
// (host, 80) does not cover it, and the hop asks again.
|
||||
func TestRedirectHTTPToHTTPSAsksAgain(t *testing.T) {
|
||||
d := targetTestDB(t)
|
||||
ic, res := pgGate(t, d)
|
||||
res.set("app.example.com", "203.0.113.7")
|
||||
scope := taskScope(t, d, ic, time.Minute)
|
||||
pinURL(t, d, ic, scope, "http://app.example.com/")
|
||||
ctx := WithTargetScope(t.Context(), scope)
|
||||
|
||||
if err := hop(ctx, "http://app.example.com/next", "http://app.example.com/"); err != nil {
|
||||
t.Fatalf("hop on the pinned key: %v", err)
|
||||
}
|
||||
err := hop(ctx, "https://app.example.com/", "http://app.example.com/")
|
||||
if err == nil || !strings.Contains(err.Error(), TargetApprovalPending) || !strings.Contains(err.Error(), "redirects to app.example.com:443") {
|
||||
t.Fatalf("http -> https: %v", err)
|
||||
}
|
||||
ids := openRows(t, d, scope)
|
||||
if len(ids) != 1 {
|
||||
t.Fatalf("open requests = %v, want the one for port 443", ids)
|
||||
}
|
||||
detail, err := d.GetInterceptDetail(ids[0])
|
||||
if err != nil || detail == nil || detail.Audit == nil || detail.Audit.Targets[0].Port != 443 || detail.Audit.Targets[0].RedirectFrom != "app.example.com:80" {
|
||||
t.Fatalf("request = %+v, %v", detail, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// Step 1.9: every redirect hop of a platform-owned client goes through the gate.
|
||||
// Without a database: a hop that would open a request is covered by the
|
||||
// PostgreSQL tests.
|
||||
|
||||
// hopServer counts its requests and answers a fixed page, or a redirect to
|
||||
// location when it is set.
|
||||
func hopServer(t *testing.T, location string) (srv *httptest.Server, hits *atomic.Int64, key string) {
|
||||
t.Helper()
|
||||
hits = &atomic.Int64{}
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
if location != "" {
|
||||
http.Redirect(w, r, location, http.StatusFound)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte("hop-reached"))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
u, err := url.Parse(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return srv, hits, u.Host
|
||||
}
|
||||
|
||||
// hop calls the redirect policy of tool on one hop, as the client does.
|
||||
func hop(ctx context.Context, to string, from ...string) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, to, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
via := make([]*http.Request, len(from))
|
||||
for n, f := range from {
|
||||
if via[n], err = http.NewRequestWithContext(ctx, http.MethodGet, f, nil); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
return RedirectCheck("WebFetch")(req, via)
|
||||
}
|
||||
|
||||
func gatedFetch(ctx context.Context, rawURL string) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := (&http.Client{CheckRedirect: RedirectCheck("WebFetch")}).Do(req)
|
||||
if resp != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// With the gate off, or without a scope, the policy is net/http's default:
|
||||
// every hop is followed, up to 10, with the same error after that. Nothing is
|
||||
// parsed, resolved, read or logged, not even a hop whose target cannot be read.
|
||||
func TestRedirectGateOffKeepsDefaultPolicy(t *testing.T) {
|
||||
g := newGate(t)
|
||||
g.res.set("app.example.com", "203.0.113.7")
|
||||
chain := make([]string, maxRedirects)
|
||||
for n := range chain {
|
||||
chain[n] = "http://app.example.com/"
|
||||
}
|
||||
for name, ctx := range map[string]context.Context{
|
||||
"no scope": context.Background(),
|
||||
"no flag": WithTargetScope(context.Background(), TargetScope{TaskID: "7", WaitBound: time.Minute, Interceptor: g.ic}),
|
||||
"flag off": WithTargetScope(context.Background(), TargetScope{TaskID: "7", Enabled: func() bool { return false }, Interceptor: g.ic}),
|
||||
} {
|
||||
if err := hop(ctx, "https://other.example.com/", chain[:1]...); err != nil {
|
||||
t.Errorf("%s: %v, want the hop followed", name, err)
|
||||
}
|
||||
if err := hop(ctx, "http:///no-host", chain[:1]...); err != nil {
|
||||
t.Errorf("%s, no host: %v, want the hop followed", name, err)
|
||||
}
|
||||
if err := hop(ctx, "https://other.example.com/", chain[:maxRedirects-1]...); err != nil {
|
||||
t.Errorf("%s, hop 9: %v", name, err)
|
||||
}
|
||||
if err := hop(ctx, "https://other.example.com/", chain...); err == nil || err.Error() != "stopped after 10 redirects" {
|
||||
t.Errorf("%s, hop 10: %v, want net/http's limit", name, err)
|
||||
}
|
||||
|
||||
// Through a real client: a hop to another port, which the gate would ask
|
||||
// for, is followed.
|
||||
final, finalHits, _ := hopServer(t, "")
|
||||
start, _, _ := hopServer(t, final.URL+"/final")
|
||||
if resp, err := gatedFetch(ctx, start.URL+"/start"); err != nil || resp.StatusCode != http.StatusOK || finalHits.Load() != 1 {
|
||||
t.Errorf("%s: chain not followed: %v, hits=%d", name, err, finalHits.Load())
|
||||
}
|
||||
loop := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, r.URL.Path+"x", http.StatusFound)
|
||||
}))
|
||||
_, gotErr := gatedFetch(ctx, loop.URL+"/l")
|
||||
_, wantErr := (&http.Client{}).Get(loop.URL + "/l")
|
||||
loop.Close()
|
||||
if gotErr == nil || wantErr == nil || gotErr.Error() != wantErr.Error() {
|
||||
t.Errorf("%s: redirect loop: %v, want the default policy's %v", name, gotErr, wantErr)
|
||||
}
|
||||
}
|
||||
if g.store.reads != 0 || len(g.store.logged) != 0 || g.res.total() != 0 {
|
||||
t.Errorf("gate off: %d store reads, %d rows, %d lookups, want none", g.store.reads, len(g.store.logged), g.res.total())
|
||||
}
|
||||
// Gate on, the limit holds as well, before anything is read.
|
||||
if err := hop(g.ctx(), "https://app.example.com/", chain...); err == nil || err.Error() != "stopped after 10 redirects" || g.store.reads != 0 {
|
||||
t.Errorf("gate on, hop 10: %v, %d store reads", err, g.store.reads)
|
||||
}
|
||||
}
|
||||
|
||||
// A hop to a pinned target is followed, through a real client: the redirected
|
||||
// request keeps the context of the call, so the policy finds the scope. The
|
||||
// first use of the pin is logged with the origin of the redirect.
|
||||
func TestRedirectToPinnedHostFollowed(t *testing.T) {
|
||||
g := newGate(t)
|
||||
final, finalHits, finalKey := hopServer(t, "")
|
||||
start, startHits, startKey := hopServer(t, final.URL+"/final")
|
||||
host, port, _ := net.SplitHostPort(finalKey)
|
||||
pin := &db.TargetPin{ID: 1, Host: host, Port: atoiPort(t, port), Version: 1, Status: db.TargetPinActive, PinnedIPs: []string{host}}
|
||||
g.store.pins = []*db.TargetPin{pin}
|
||||
|
||||
resp, err := gatedFetch(g.ctx(), start.URL+"/start")
|
||||
if err != nil || resp.StatusCode != http.StatusOK || startHits.Load() != 1 || finalHits.Load() != 1 {
|
||||
t.Fatalf("pinned hop: %v, hits start=%d final=%d", err, startHits.Load(), finalHits.Load())
|
||||
}
|
||||
if got := g.store.outcomes(); len(got) != 1 || got[0] != "allowed/pin_match" {
|
||||
t.Fatalf("rows = %v, want the first use of the pin", got)
|
||||
}
|
||||
row := g.store.logged[0]
|
||||
if tgt := row.Audit.Targets[0]; tgt.Host != host || tgt.RedirectFrom != startKey || tgt.PinID != pin.ID {
|
||||
t.Fatalf("logged target = %+v, want %s from %s", tgt, finalKey, startKey)
|
||||
}
|
||||
if !strings.Contains(string(row.ToolInput), final.URL+"/final") || !strings.Contains(string(row.ToolInput), start.URL+"/start") {
|
||||
t.Fatalf("logged input = %s, want the hop and its origin", row.ToolInput)
|
||||
}
|
||||
if g.res.total() != 0 {
|
||||
t.Fatalf("%d lookups for an IP literal", g.res.total())
|
||||
}
|
||||
}
|
||||
|
||||
// A (host, 0) pin covers every port: http -> https on that host is followed.
|
||||
func TestRedirectToPortZeroPinnedHostFollowed(t *testing.T) {
|
||||
g := newGate(t)
|
||||
g.res.set("app.example.com", "203.0.113.7")
|
||||
g.store.pins = []*db.TargetPin{{ID: 1, Host: "app.example.com", Port: 0, Version: 1, Status: db.TargetPinActive, PinnedIPs: []string{"203.0.113.7"}}}
|
||||
if err := hop(g.ctx(), "https://app.example.com/login", "http://app.example.com/login"); err != nil {
|
||||
t.Fatalf("http -> https under a (host, 0) pin: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A block rule stops a hop before anything is resolved, whether it names the
|
||||
// hop's host or its URL, and even when the hop stays on a pinned host. No
|
||||
// request is opened.
|
||||
func TestRedirectToBlockedNameNotResolved(t *testing.T) {
|
||||
g := newGate(t)
|
||||
g.res.set("app.example.com", "203.0.113.7")
|
||||
g.res.set("evil.example", "198.51.100.9")
|
||||
g.store.pins = []*db.TargetPin{{ID: 1, Host: "app.example.com", Port: 443, Version: 1, Status: db.TargetPinActive, PinnedIPs: []string{"203.0.113.7"}}}
|
||||
g.store.rules = []db.AssetInterceptRule{
|
||||
{Enabled: true, Kind: "exact_domain", Pattern: "evil.example"},
|
||||
{Enabled: true, Kind: "exact_url", Pattern: "https://app.example.com/admin"},
|
||||
}
|
||||
|
||||
err := hop(g.ctx(), "http://evil.example/x", "https://app.example.com/")
|
||||
if err == nil || !strings.Contains(err.Error(), TargetRuleBlocked) || !strings.Contains(err.Error(), "https://app.example.com/ redirects to evil.example:80") {
|
||||
t.Fatalf("hop to a blocked name: %v", err)
|
||||
}
|
||||
if n := g.res.lookups("evil.example"); n != 0 {
|
||||
t.Fatalf("the blocked name was resolved %d times", n)
|
||||
}
|
||||
err = hop(g.ctx(), "https://app.example.com/admin", "https://app.example.com/")
|
||||
if err == nil || !strings.Contains(err.Error(), TargetRuleBlocked) {
|
||||
t.Fatalf("hop to a blocked URL on a pinned host: %v", err)
|
||||
}
|
||||
if got := g.store.outcomes(); len(got) != 2 || got[0] != "denied/rule_blocked" || got[1] != "denied/rule_blocked" {
|
||||
t.Fatalf("rows = %v, want two blocks", got)
|
||||
}
|
||||
if from := g.store.logged[0].Audit.Targets[0].RedirectFrom; from != "app.example.com:443" {
|
||||
t.Fatalf("redirect_from = %q", from)
|
||||
}
|
||||
if g.res.total() != 0 {
|
||||
t.Fatalf("%d lookups, want none", g.res.total())
|
||||
}
|
||||
}
|
||||
|
||||
// A hop whose target cannot be read is blocked with TARGET_GATE_UNAVAILABLE, as
|
||||
// a rendered URL is, and opens no request.
|
||||
func TestRedirectUnreadableHopBlocked(t *testing.T) {
|
||||
g := newGate(t)
|
||||
for _, to := range []*url.URL{
|
||||
{Scheme: "http", Path: "/no-host"},
|
||||
{Scheme: "http", Host: "app.example.com:99999", Path: "/"},
|
||||
} {
|
||||
req, _ := http.NewRequestWithContext(g.ctx(), http.MethodGet, "http://placeholder.example/", nil)
|
||||
req.URL = to
|
||||
via, _ := http.NewRequestWithContext(g.ctx(), http.MethodGet, "https://app.example.com/", nil)
|
||||
err := RedirectCheck("WebFetch")(req, []*http.Request{via})
|
||||
if err == nil || !strings.Contains(err.Error(), TargetGateUnavailable) {
|
||||
t.Errorf("%s: %v, want TARGET_GATE_UNAVAILABLE", to, err)
|
||||
}
|
||||
}
|
||||
if got := g.store.outcomes(); len(got) != 2 || got[0] != "denied/gate_unavailable" {
|
||||
t.Fatalf("rows = %v", got)
|
||||
}
|
||||
if g.res.total() != 0 {
|
||||
t.Fatalf("%d lookups, want none", g.res.total())
|
||||
}
|
||||
}
|
||||
|
||||
func atoiPort(t *testing.T, s string) int {
|
||||
t.Helper()
|
||||
p := validPort(s)
|
||||
if p == 0 {
|
||||
t.Fatalf("port %q", s)
|
||||
}
|
||||
return p
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The custom tools render their target from a template and the parameters of
|
||||
// the call, after the hook has seen the call: the hook reads the parameters, not
|
||||
// the URL or the command they produce, and not a target written in the
|
||||
// template itself. These functions run the gate inside the tool, on what the
|
||||
// tool is about to run, before anything is sent or started.
|
||||
|
||||
// renderedPasses bounds the request-and-recheck loop of a rendered target, as
|
||||
// the hook bounds its own: a request that was decided in between, or addresses
|
||||
// that changed again after an approval, send the call back to the pin lookup at
|
||||
// most this many times.
|
||||
const renderedPasses = 3
|
||||
|
||||
// GateRenderedURL is the target gate for the URL a custom http tool renders. It
|
||||
// reads the target as the HTTP client will: the host of the parsed URL and its
|
||||
// port, else the default port of the scheme. A URL whose target cannot be read
|
||||
// is blocked with TARGET_GATE_UNAVAILABLE: the call is not executed and no
|
||||
// approval is requested.
|
||||
//
|
||||
// The result is empty when the tool may run. With the gate off, or with no
|
||||
// scope in ctx (the operator's test endpoint), it returns at once: nothing is
|
||||
// parsed, resolved, read or written.
|
||||
func GateRenderedURL(ctx context.Context, toolName, rawURL string) TargetResult {
|
||||
scope, on := renderedScope(ctx)
|
||||
if !on {
|
||||
return TargetResult{}
|
||||
}
|
||||
input := renderedInput("url", rawURL)
|
||||
t, err := urlTarget(rawURL)
|
||||
if err != nil {
|
||||
return undeterminedTarget(ctx, scope, toolName, input, "the target of the rendered URL could not be read ("+err.Error()+")")
|
||||
}
|
||||
return gateRendered(ctx, scope, toolName, input, []Target{t}, []string{rawURL})
|
||||
}
|
||||
|
||||
// GateRenderedCommand is the target gate for the command a custom command tool
|
||||
// renders, read with the command-string rules of Bash. readings are other
|
||||
// renderings of the same command that are read as well: the shell quoting of a
|
||||
// parameter placed inside a host name hides that name from the rules, and the
|
||||
// shell removes the quotes before the command runs. A command that names no
|
||||
// target passes, as it does in Bash: what a command hides is advisory (D9).
|
||||
//
|
||||
// The result is empty when the tool may run. With the gate off, or with no
|
||||
// scope in ctx, it returns at once.
|
||||
func GateRenderedCommand(ctx context.Context, toolName, command string, readings ...string) TargetResult {
|
||||
scope, on := renderedScope(ctx)
|
||||
if !on {
|
||||
return TargetResult{}
|
||||
}
|
||||
var l targetList
|
||||
l.scan(command)
|
||||
for _, r := range readings {
|
||||
l.scan(r)
|
||||
}
|
||||
return gateRendered(ctx, scope, toolName, renderedInput("command", command), l.targets, l.urls)
|
||||
}
|
||||
|
||||
func renderedScope(ctx context.Context) (TargetScope, bool) {
|
||||
scope, ok := TargetScopeFromContext(ctx)
|
||||
return scope, ok && scope.Enabled != nil && scope.Enabled()
|
||||
}
|
||||
|
||||
// renderedInput is the tool input recorded with a rendered target: what the
|
||||
// tool runs, which is what the human reads on the approval.
|
||||
func renderedInput(field, value string) []byte {
|
||||
b, _ := json.Marshal(map[string]string{field: value})
|
||||
return b
|
||||
}
|
||||
|
||||
// gateRendered runs the gate decision on the targets of a rendered call, asks
|
||||
// for the approval of those no pin covers, and checks again after the
|
||||
// decisions. urls are the URLs the call renders, for the URL block rules. It
|
||||
// never merges an ask: the rules and the judge decided at the hook, before the
|
||||
// tool ran.
|
||||
func gateRendered(ctx context.Context, scope TargetScope, toolName string, input []byte, targets []Target, urls []string) TargetResult {
|
||||
if len(targets) == 0 {
|
||||
return TargetResult{}
|
||||
}
|
||||
i := scope.Interceptor
|
||||
if i == nil {
|
||||
return TargetResult{Code: TargetGateUnavailable, Host: targets[0].Host, Port: targets[0].Port,
|
||||
Reason: "the session's target scope carries no gate"}
|
||||
}
|
||||
ctx = WithCall(ctx, toolName, input)
|
||||
check := func() ([]TargetRequest, TargetResult) {
|
||||
if !scope.Enabled() {
|
||||
return nil, TargetResult{}
|
||||
}
|
||||
c := &targetCheck{i: i, ctx: ctx, scope: scope, tool: toolName, input: input, urls: urls}
|
||||
return c.run(targets)
|
||||
}
|
||||
unpinned, res := check()
|
||||
for pass := 0; res.Code == "" && len(unpinned) > 0; pass++ {
|
||||
if pass == renderedPasses {
|
||||
t := unpinned[0]
|
||||
return TargetResult{Code: TargetApprovalPending, Host: t.Host, Port: t.Port,
|
||||
Reason: fmt.Sprintf("the target is still not covered by a pin after %d approval rounds; its addresses changed again", renderedPasses)}
|
||||
}
|
||||
if res = i.RequestTargets(ctx, scope, toolName, input, unpinned, nil); res.Code != "" {
|
||||
return res
|
||||
}
|
||||
unpinned, res = check()
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// undeterminedTarget blocks a call whose target cannot be read, and logs the
|
||||
// block as a decided row.
|
||||
func undeterminedTarget(ctx context.Context, scope TargetScope, toolName string, input []byte, reason string) TargetResult {
|
||||
res := TargetResult{Code: TargetGateUnavailable, Reason: reason}
|
||||
if scope.Interceptor == nil {
|
||||
return res
|
||||
}
|
||||
ref, err := scope.ref()
|
||||
if err != nil {
|
||||
return res
|
||||
}
|
||||
c := &targetCheck{i: scope.Interceptor, ctx: WithCall(ctx, toolName, input), scope: scope, ref: ref, tool: toolName, input: input}
|
||||
_, res = c.block(TargetGateUnavailable, "gate_unavailable", &checkedTarget{}, reason)
|
||||
return res
|
||||
}
|
||||
|
||||
// urlTarget reads the target of a URL the way net/http dials it: the host of
|
||||
// the parsed URL, and its port, else the default port of the scheme.
|
||||
func urlTarget(raw string) (Target, error) {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return Target{}, err
|
||||
}
|
||||
name := u.Hostname()
|
||||
if name == "" {
|
||||
return Target{}, errors.New("no host")
|
||||
}
|
||||
kind, host, ok := normalizeHost(name, true)
|
||||
if !ok {
|
||||
return Target{}, fmt.Errorf("host %q", name)
|
||||
}
|
||||
t := Target{Kind: kind, Host: host, Port: schemePorts[strings.ToLower(u.Scheme)]}
|
||||
if p := u.Port(); p != "" {
|
||||
if t.Port = validPort(p); t.Port == 0 {
|
||||
return Target{}, fmt.Errorf("port %q", p)
|
||||
}
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// The gate inside the custom tools: GateRenderedURL and GateRenderedCommand
|
||||
// read what the tool is about to run. Without a database: the cases that reach
|
||||
// a request are covered by the server tests.
|
||||
|
||||
func (g *gateFixture) ctx() context.Context {
|
||||
return WithTargetScope(context.Background(), g.scope)
|
||||
}
|
||||
|
||||
// With the gate off, or without a scope (the operator's test endpoint), the
|
||||
// rendered target is not even parsed: nothing is read, resolved or logged, and
|
||||
// a URL the gate could not read passes too, as it did before the gate.
|
||||
func TestRenderedGateOffTouchesNothing(t *testing.T) {
|
||||
g := newGate(t)
|
||||
g.res.set("app.example.com", "203.0.113.7")
|
||||
for name, ctx := range map[string]context.Context{
|
||||
"no scope": context.Background(),
|
||||
"no flag": WithTargetScope(context.Background(), TargetScope{TaskID: "7", WaitBound: time.Minute, Interceptor: g.ic}),
|
||||
"flag off": WithTargetScope(context.Background(), TargetScope{TaskID: "7", Enabled: func() bool { return false }, Interceptor: g.ic}),
|
||||
} {
|
||||
for _, res := range []TargetResult{
|
||||
GateRenderedURL(ctx, "probe", appURL),
|
||||
GateRenderedURL(ctx, "probe", "http://a b/"),
|
||||
GateRenderedCommand(ctx, "probe", "curl "+appURL),
|
||||
} {
|
||||
if res != (TargetResult{}) {
|
||||
t.Errorf("%s: %+v, want nothing", name, res)
|
||||
}
|
||||
}
|
||||
}
|
||||
if g.store.reads != 0 || len(g.store.logged) != 0 || g.res.total() != 0 {
|
||||
t.Errorf("gate off: %d store reads, %d rows, %d lookups, want none", g.store.reads, len(g.store.logged), g.res.total())
|
||||
}
|
||||
}
|
||||
|
||||
// The target of a rendered URL is read as net/http dials it: the host of the
|
||||
// parsed URL, the explicit port, else the default of the scheme.
|
||||
func TestRenderedURLTarget(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
raw string
|
||||
want Target
|
||||
}{
|
||||
{"https://App.Example.COM./x?next=http://other.example/", Target{TargetDomain, "app.example.com", 443}},
|
||||
{"http://user:pw@203.0.113.7:8080/a", Target{TargetIP, "203.0.113.7", 8080}},
|
||||
{"http://[::ffff:203.0.113.7]/", Target{TargetIP, "203.0.113.7", 80}},
|
||||
{"http://[2001:db8::1]:8443/", Target{TargetIP, "2001:db8::1", 8443}},
|
||||
{"http://2130706433/", Target{TargetIP, "127.0.0.1", 80}},
|
||||
{"HTTPS://bücher.example/", Target{TargetDomain, "xn--bcher-kva.example", 443}},
|
||||
// The client's reading of the authority, not the shell's: a.example; is userinfo.
|
||||
{"http://a.example;@b.example/", Target{TargetDomain, "b.example", 80}},
|
||||
} {
|
||||
got, err := urlTarget(c.raw)
|
||||
if err != nil || got != c.want {
|
||||
t.Errorf("%q: %+v, %v; want %+v", c.raw, got, err, c.want)
|
||||
}
|
||||
}
|
||||
for _, raw := range []string{
|
||||
"http://a b/", // not a URL the client accepts
|
||||
`http://a.example\@b.example/`, // idem
|
||||
"http://[::1/", // idem
|
||||
"http:///path", // no host
|
||||
"app.example.com/path", // no scheme: no host
|
||||
"http://app.example.com:99999/x", // no valid port
|
||||
} {
|
||||
if got, err := urlTarget(raw); err == nil {
|
||||
t.Errorf("%q: %+v, want no target", raw, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A rendered URL whose target cannot be read is blocked: no lookup, no
|
||||
// request, one decided row that records the URL.
|
||||
func TestRenderedURLWithoutTargetFailsClosed(t *testing.T) {
|
||||
g := newGate(t)
|
||||
for n, raw := range []string{"http://a b/", "http:///path", "app.example.com/path", "http://app.example.com:99999/"} {
|
||||
res := GateRenderedURL(g.ctx(), "probe_http", raw)
|
||||
if res.Code != TargetGateUnavailable || !strings.Contains(res.Reason, "could not be read") {
|
||||
t.Fatalf("%q: %+v, want %s", raw, res, TargetGateUnavailable)
|
||||
}
|
||||
if len(g.store.logged) != n+1 {
|
||||
t.Fatalf("%q: %d rows, want %d", raw, len(g.store.logged), n+1)
|
||||
}
|
||||
row := g.store.logged[n]
|
||||
var input map[string]string
|
||||
_ = json.Unmarshal(row.ToolInput, &input)
|
||||
if row.Status != "denied" || row.ToolName != "probe_http" || input["url"] != raw || row.Audit.Targets[0].Outcome != "gate_unavailable" {
|
||||
t.Fatalf("%q: row = %+v (input %s)", raw, row, row.ToolInput)
|
||||
}
|
||||
}
|
||||
if g.res.total() != 0 {
|
||||
t.Fatalf("%d lookups, want none", g.res.total())
|
||||
}
|
||||
}
|
||||
|
||||
// A gated scope without the interceptor that owns the gate cannot decide: the
|
||||
// call is blocked, never let through.
|
||||
func TestRenderedGateWithoutInterceptorFailsClosed(t *testing.T) {
|
||||
ctx := WithTargetScope(context.Background(), TargetScope{TaskID: "7", Enabled: func() bool { return true }})
|
||||
for name, res := range map[string]TargetResult{
|
||||
"url": GateRenderedURL(ctx, "probe", appURL),
|
||||
"unreadable url": GateRenderedURL(ctx, "probe", "http://a b/"),
|
||||
"command": GateRenderedCommand(ctx, "probe", "curl "+appURL),
|
||||
} {
|
||||
if res.Code != TargetGateUnavailable {
|
||||
t.Errorf("%s: %+v, want %s", name, res, TargetGateUnavailable)
|
||||
}
|
||||
}
|
||||
// A command that names no target needs no gate.
|
||||
if res := GateRenderedCommand(ctx, "probe", "ls -la"); res != (TargetResult{}) {
|
||||
t.Errorf("no target: %+v, want nothing", res)
|
||||
}
|
||||
}
|
||||
|
||||
// The rendered target goes through the same decision as a target at the hook:
|
||||
// block rules, resolution, pins. A pinned target passes, a blocked one is
|
||||
// blocked without a request.
|
||||
func TestRenderedTargetUsesTheGateDecision(t *testing.T) {
|
||||
g := newGate(t)
|
||||
g.res.set("app.example.com", "203.0.113.7")
|
||||
g.store.approve(TargetRequest{Host: "app.example.com", Port: 443, ResolvedIPs: []string{"203.0.113.7"}})
|
||||
g.store.approve(TargetRequest{Host: "198.51.100.9", Port: 8080, ResolvedIPs: []string{"198.51.100.9"}})
|
||||
|
||||
for name, res := range map[string]TargetResult{
|
||||
"url": GateRenderedURL(g.ctx(), "probe", appURL),
|
||||
"url, ip and port": GateRenderedURL(g.ctx(), "probe", "http://198.51.100.9:8080/x"),
|
||||
"command": GateRenderedCommand(g.ctx(), "probe", "curl -s '"+appURL+"'"),
|
||||
"command, no target": GateRenderedCommand(g.ctx(), "probe", "nmap -sV app.example.com"),
|
||||
"command, ip literal": GateRenderedCommand(g.ctx(), "probe", "nc 198.51.100.9:8080"),
|
||||
} {
|
||||
if res != (TargetResult{}) {
|
||||
t.Errorf("%s: %+v, want the call to pass", name, res)
|
||||
}
|
||||
}
|
||||
|
||||
g.res.errs["down.example"] = context.DeadlineExceeded
|
||||
if res := GateRenderedURL(g.ctx(), "probe", "https://down.example/"); res.Code != TargetResolveFailed || res.Host != "down.example" {
|
||||
t.Fatalf("unresolvable: %+v", res)
|
||||
}
|
||||
g.store.rules = []db.AssetInterceptRule{{Enabled: true, Kind: "exact_domain", Pattern: "app.example.com"}}
|
||||
for name, res := range map[string]TargetResult{
|
||||
"url": GateRenderedURL(g.ctx(), "probe", appURL),
|
||||
"command": GateRenderedCommand(g.ctx(), "probe", "curl "+appURL),
|
||||
} {
|
||||
if res.Code != TargetRuleBlocked || res.Host != "app.example.com" || res.Port != 443 {
|
||||
t.Errorf("%s: %+v, want %s on the pinned host", name, res, TargetRuleBlocked)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(strings.Join(g.store.outcomes(), " "), "denied/rule_blocked") {
|
||||
t.Fatalf("rows = %v, want the block logged", g.store.outcomes())
|
||||
}
|
||||
}
|
||||
|
||||
// The shell quoting of a parameter inside a host name hides the name from the
|
||||
// command-string rules; the reading without the quoting is read as well.
|
||||
func TestRenderedCommandReadsTheUnquotedRendering(t *testing.T) {
|
||||
g := newGate(t)
|
||||
g.store.rules = []db.AssetInterceptRule{{Enabled: true, Kind: "exact_domain", Pattern: "api.example.com"}}
|
||||
quoted := "curl http://'api'.example.com:8080/"
|
||||
if res := GateRenderedCommand(g.ctx(), "probe", quoted); res != (TargetResult{}) {
|
||||
t.Fatalf("quoted only: %+v; the rules were expected to miss the name", res)
|
||||
}
|
||||
res := GateRenderedCommand(g.ctx(), "probe", quoted, "curl http://api.example.com:8080/")
|
||||
if res.Code != TargetRuleBlocked || res.Host != "api.example.com" || res.Port != 8080 {
|
||||
t.Fatalf("with the unquoted reading: %+v, want %s on api.example.com:8080", res, TargetRuleBlocked)
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,10 @@ type TargetRequest struct {
|
||||
ResolvedAt time.Time
|
||||
Reason string // no_pin | new_ip | revoked
|
||||
Pin *db.TargetPin // the active pin an approval would extend (new_ip); nil otherwise
|
||||
ExemptVoid bool // a stored exemption names the target and was void for this call
|
||||
// RedirectFrom is the origin (host:port) of the redirect that leads to the
|
||||
// target; empty when the call names the target itself.
|
||||
RedirectFrom string
|
||||
}
|
||||
|
||||
// TargetResult is the outcome of RequestTargets. Code is empty when every
|
||||
@@ -105,6 +109,14 @@ func (i *Interceptor) RequestTargets(ctx context.Context, scope TargetScope, too
|
||||
if err != nil {
|
||||
return TargetResult{Code: TargetGateUnavailable, Reason: err.Error()}
|
||||
}
|
||||
// An approval pins the addresses the request displays, so a request without
|
||||
// any is never opened: approving it would pin nothing.
|
||||
for _, t := range targets {
|
||||
if len(t.ResolvedIPs) == 0 {
|
||||
return TargetResult{Code: TargetResolveFailed, Host: t.Host, Port: t.Port,
|
||||
Reason: "no address was resolved for " + targetName(t) + ", and an approval never pins an empty address list"}
|
||||
}
|
||||
}
|
||||
if scope.WaitBound <= 0 {
|
||||
// A call that never waits cannot own a call-bound row.
|
||||
ask = nil
|
||||
@@ -238,7 +250,8 @@ func (i *Interceptor) openTargetRequest(ctx context.Context, ref db.TargetScopeR
|
||||
target := db.InterceptTarget{
|
||||
Host: t.Host, Port: t.Port, Kind: t.Kind,
|
||||
ResolvedIPs: t.ResolvedIPs, Resolver: t.Resolver, ResolvedAt: t.ResolvedAt,
|
||||
Reason: t.Reason, Outcome: "requested", NameOnlyEffective: nameOnly,
|
||||
Reason: t.Reason, Outcome: "requested", NameOnlyEffective: nameOnly, ExemptVoid: t.ExemptVoid,
|
||||
RedirectFrom: t.RedirectFrom,
|
||||
}
|
||||
if t.Pin != nil {
|
||||
target.PinID, target.PinVersion, target.PinnedIPs = t.Pin.ID, t.Pin.Version, t.Pin.PinnedIPs
|
||||
@@ -354,6 +367,9 @@ func targetReason(scope TargetScope, t TargetRequest, ask *Decision, nameOnly bo
|
||||
b.WriteString(ask.Message)
|
||||
b.WriteString("\nThis call also names a target that needs approval. ")
|
||||
}
|
||||
if t.RedirectFrom != "" {
|
||||
fmt.Fprintf(&b, "A redirect from %s leads to this target; the redirect was not followed. ", t.RedirectFrom)
|
||||
}
|
||||
name, bound := targetName(t), scope.scopeName()
|
||||
if t.Port == 0 {
|
||||
name += " (port not declared: every port)"
|
||||
@@ -368,7 +384,8 @@ func targetReason(scope TargetScope, t TargetRequest, ask *Decision, nameOnly bo
|
||||
fmt.Fprintf(&b, " Approving pins these addresses for %s until revoked.", bound)
|
||||
if nameOnly {
|
||||
b.WriteString(" Egress goes through the upstream proxy, which chooses the address it connects to." +
|
||||
" The addresses listed are what the platform resolved. This approval is name-only.")
|
||||
" The addresses listed are what the platform resolved. This approval is name-only." +
|
||||
" The platform does not bind the connection to these addresses.")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Resolver turns a host name into its addresses. The gate compares what it
|
||||
// returns with the addresses a human pinned.
|
||||
type Resolver interface {
|
||||
// Name identifies the resolver in pins and in audit rows.
|
||||
Name() string
|
||||
// LookupIPs returns the A and AAAA answers for host.
|
||||
LookupIPs(ctx context.Context, host string) ([]netip.Addr, error)
|
||||
}
|
||||
|
||||
// SetResolver replaces the resolver of the target gate, which is the system
|
||||
// resolver unless one is set.
|
||||
func (i *Interceptor) SetResolver(r Resolver) { i.resolve.use(r) }
|
||||
|
||||
// SetBaseDialer replaces the dialer that opens the connections PinnedDialer
|
||||
// lets through, which is a net.Dialer with the settings of
|
||||
// http.DefaultTransport unless one is set. Tests use it to map addresses to
|
||||
// local listeners and to see what was dialled.
|
||||
func (i *Interceptor) SetBaseDialer(f func(ctx context.Context, network, addr string) (net.Conn, error)) {
|
||||
i.mu.Lock()
|
||||
i.dial = f
|
||||
i.mu.Unlock()
|
||||
}
|
||||
|
||||
// systemResolver is the resolver of the ARTEX process. It reads the same
|
||||
// resolver configuration as the tools that run in the same container.
|
||||
type systemResolver struct{}
|
||||
|
||||
func (systemResolver) Name() string { return "system" }
|
||||
|
||||
func (systemResolver) LookupIPs(ctx context.Context, host string) ([]netip.Addr, error) {
|
||||
return net.DefaultResolver.LookupNetIP(ctx, "ip", host)
|
||||
}
|
||||
|
||||
const (
|
||||
// resolveTimeout bounds one lookup.
|
||||
resolveTimeout = 3 * time.Second
|
||||
// resolveCacheTTL is how long a positive answer is reused. It bounds latency
|
||||
// and resolver load when several calls name one host, and adds at most this
|
||||
// much to the time a changed address goes unnoticed.
|
||||
resolveCacheTTL = 10 * time.Second
|
||||
)
|
||||
|
||||
// resolution is one positive answer: canonical addresses, sorted, never empty.
|
||||
type resolution struct {
|
||||
ips []string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
type resolveFlight struct {
|
||||
done chan struct{}
|
||||
res resolution
|
||||
err error
|
||||
}
|
||||
|
||||
// resolveCache resolves names through one Resolver. Positive answers are kept
|
||||
// per host for resolveCacheTTL, concurrent lookups for one host collapse into
|
||||
// one, and a failure or an empty answer is never kept: the next call tries
|
||||
// again.
|
||||
type resolveCache struct {
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
resolver Resolver
|
||||
entries map[string]resolution
|
||||
flights map[string]*resolveFlight
|
||||
}
|
||||
|
||||
func newResolveCache(r Resolver) *resolveCache {
|
||||
return &resolveCache{resolver: r, now: time.Now, entries: map[string]resolution{}, flights: map[string]*resolveFlight{}}
|
||||
}
|
||||
|
||||
// use replaces the resolver and drops the answers of the previous one.
|
||||
func (c *resolveCache) use(r Resolver) {
|
||||
c.mu.Lock()
|
||||
c.resolver, c.entries = r, map[string]resolution{}
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
func (c *resolveCache) name() string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.resolver.Name()
|
||||
}
|
||||
|
||||
// lookup resolves host. fresh skips the cached answer: it is what a new
|
||||
// approval request uses, because a human reads and pins those addresses.
|
||||
func (c *resolveCache) lookup(ctx context.Context, host string, fresh bool) (resolution, error) {
|
||||
c.mu.Lock()
|
||||
if e, ok := c.entries[host]; ok && !fresh && c.now().Sub(e.at) < resolveCacheTTL {
|
||||
c.mu.Unlock()
|
||||
return e, nil
|
||||
}
|
||||
f := c.flights[host]
|
||||
if f == nil {
|
||||
f = &resolveFlight{done: make(chan struct{})}
|
||||
c.flights[host] = f
|
||||
go c.fly(c.resolver, host, f)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
select {
|
||||
case <-f.done:
|
||||
return f.res, f.err
|
||||
case <-ctx.Done():
|
||||
return resolution{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// fly runs the one lookup the waiters of host share. It is not bound to the
|
||||
// context of the call that started it: that call leaving must not fail the others.
|
||||
func (c *resolveCache) fly(r Resolver, host string, f *resolveFlight) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), resolveTimeout)
|
||||
defer cancel()
|
||||
addrs, err := r.LookupIPs(ctx, host)
|
||||
switch {
|
||||
case ctx.Err() != nil:
|
||||
f.err = fmt.Errorf("lookup timed out after %s", resolveTimeout)
|
||||
case err != nil:
|
||||
f.err = lookupError(err)
|
||||
case len(addrs) == 0:
|
||||
f.err = errors.New("the lookup returned no address")
|
||||
default:
|
||||
f.res = resolution{ips: canonicalIPs(addrs), at: c.now()}
|
||||
}
|
||||
c.mu.Lock()
|
||||
delete(c.flights, host)
|
||||
if f.err == nil {
|
||||
c.entries[host] = f.res
|
||||
}
|
||||
c.mu.Unlock()
|
||||
close(f.done)
|
||||
}
|
||||
|
||||
// lookupError keeps the part of a resolver error that says what went wrong.
|
||||
func lookupError(err error) error {
|
||||
var dns *net.DNSError
|
||||
if errors.As(err, &dns) {
|
||||
if dns.IsTimeout {
|
||||
return fmt.Errorf("lookup timed out after %s", resolveTimeout)
|
||||
}
|
||||
if dns.IsNotFound {
|
||||
return errors.New("no such host")
|
||||
}
|
||||
return errors.New(dns.Err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// canonicalIPs returns the addresses as sorted, unique strings, IPv4-mapped
|
||||
// addresses unmapped and zones dropped.
|
||||
func canonicalIPs(addrs []netip.Addr) []string {
|
||||
out := make([]string, 0, len(addrs))
|
||||
for _, a := range addrs {
|
||||
out = append(out, a.Unmap().WithZone("").String())
|
||||
}
|
||||
slices.Sort(out)
|
||||
return slices.Compact(out)
|
||||
}
|
||||
|
||||
// subset reports whether every resolved address is one of the pinned ones. An
|
||||
// empty answer is never a subset: a lookup that returned nothing matches no pin.
|
||||
func subset(resolved, pinned []string) bool {
|
||||
if len(resolved) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, ip := range resolved {
|
||||
if !slices.Contains(pinned, ip) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
package intercept
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeTarget(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
raw string
|
||||
want Target
|
||||
}{
|
||||
{"case", "HTTPS://App.Example.COM/Admin", Target{TargetDomain, "app.example.com", 443}},
|
||||
{"trailing dot", "https://app.example.com./", Target{TargetDomain, "app.example.com", 443}},
|
||||
{"userinfo", "https://user:p@ss@app.example.com/x", Target{TargetDomain, "app.example.com", 443}},
|
||||
{"userinfo that looks like a host", "http://pinned.example@evil.example/", Target{TargetDomain, "evil.example", 80}},
|
||||
{"ipv6 brackets", "http://[2001:DB8::1]/", Target{TargetIP, "2001:db8::1", 80}},
|
||||
{"ipv6 brackets and port", "http://[2001:db8::1]:8443/x", Target{TargetIP, "2001:db8::1", 8443}},
|
||||
{"ipv4-mapped ipv6", "http://[::ffff:203.0.113.7]/", Target{TargetIP, "203.0.113.7", 80}},
|
||||
{"decimal ipv4", "http://2130706433/", Target{TargetIP, "127.0.0.1", 80}},
|
||||
{"hex ipv4", "http://0x7f000001/", Target{TargetIP, "127.0.0.1", 80}},
|
||||
{"octal ipv4", "http://0177.0.0.1/", Target{TargetIP, "127.0.0.1", 80}},
|
||||
{"short ipv4", "http://127.1/", Target{TargetIP, "127.0.0.1", 80}},
|
||||
{"mixed ipv4 forms", "http://0xa9.254.0xa9fe/", Target{TargetIP, "169.254.169.254", 80}},
|
||||
{"punycode", "https://bücher.example/", Target{TargetDomain, "xn--bcher-kva.example", 443}},
|
||||
{"punycode kept", "https://xn--bcher-kva.example/", Target{TargetDomain, "xn--bcher-kva.example", 443}},
|
||||
{"default port http", "http://app.example.com", Target{TargetDomain, "app.example.com", 80}},
|
||||
{"default port https", "https://app.example.com?x=1", Target{TargetDomain, "app.example.com", 443}},
|
||||
{"default port ws", "ws://app.example.com/socket", Target{TargetDomain, "app.example.com", 80}},
|
||||
{"default port wss", "wss://app.example.com/socket", Target{TargetDomain, "app.example.com", 443}},
|
||||
{"default port ftp", "ftp://files.example.com/pub", Target{TargetDomain, "files.example.com", 21}},
|
||||
{"explicit port wins", "https://app.example.com:8443/", Target{TargetDomain, "app.example.com", 8443}},
|
||||
{"empty port is the default", "https://app.example.com:/", Target{TargetDomain, "app.example.com", 443}},
|
||||
{"port 0: scheme without a default", "ssh://git.example.com/repo", Target{TargetDomain, "git.example.com", 0}},
|
||||
{"port 0: bare host", "app.example.com", Target{TargetDomain, "app.example.com", 0}},
|
||||
{"host and port", "App.Example.com:8080", Target{TargetDomain, "app.example.com", 8080}},
|
||||
{"ip and port", "203.0.113.7:22", Target{TargetIP, "203.0.113.7", 22}},
|
||||
{"bare ipv6", "2001:db8::1", Target{TargetIP, "2001:db8::1", 0}},
|
||||
{"ipv6 zone dropped", "fe80::1%eth0", Target{TargetIP, "fe80::1", 0}},
|
||||
{"cidr is masked", "10.1.2.3/8", Target{TargetCIDR, "10.0.0.0/8", 0}},
|
||||
{"ipv6 cidr", "FD00::/8", Target{TargetCIDR, "fd00::/8", 0}},
|
||||
{"underscore label", "http://my_host.internal/", Target{TargetDomain, "my_host.internal", 80}},
|
||||
{"percent-encoded host", "http://%65vil.example/", Target{TargetDomain, "evil.example", 80}},
|
||||
{"percent-encoded address", "http://127.0.0.1%2e/x", Target{TargetIP, "127.0.0.1", 80}},
|
||||
{"a host no profile accepts is kept", "https://-odd-.example/", Target{TargetDomain, "-odd-.example", 443}},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, ok := NormalizeTarget(c.raw)
|
||||
if !ok || got != c.want {
|
||||
t.Errorf("%s: NormalizeTarget(%q) = %+v, %v; want %+v", c.name, c.raw, got, ok, c.want)
|
||||
}
|
||||
}
|
||||
for _, raw := range []string{"", "file:///etc/passwd", "http:///path", "http://app.example.com:99999/", "http://app.example.com:http/", "http://[::1", "10.0.0.0/99"} {
|
||||
if got, ok := NormalizeTarget(raw); ok {
|
||||
t.Errorf("NormalizeTarget(%q) = %+v, want no target", raw, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func targetsOf(tool string, input any) []Target {
|
||||
raw, _ := json.Marshal(input)
|
||||
return ExtractTargets(tool, raw)
|
||||
}
|
||||
|
||||
func command(cmd string) map[string]string { return map[string]string{"command": cmd} }
|
||||
|
||||
func wantTargets(t *testing.T, what string, got []Target, want ...Target) {
|
||||
t.Helper()
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("%s: targets = %+v, want %+v", what, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTargets(t *testing.T) {
|
||||
app := Target{TargetDomain, "app.example.com", 443}
|
||||
|
||||
// WebFetch names its target exactly.
|
||||
wantTargets(t, "WebFetch", targetsOf("WebFetch", map[string]any{"url": "https://App.Example.com/login?next=http://other.example/", "extract": true}), app)
|
||||
wantTargets(t, "WebFetch numeric host", targetsOf("WebFetch", map[string]string{"url": "http://2130706433/"}), Target{TargetIP, "127.0.0.1", 80})
|
||||
wantTargets(t, "WebFetch without a url", targetsOf("WebFetch", map[string]string{}))
|
||||
|
||||
// The command tools: Bash, shell_open, Monitor (command) and shell_send (text).
|
||||
commands := []struct {
|
||||
cmd string
|
||||
want []Target
|
||||
}{
|
||||
{"curl -s https://app.example.com/api | jq .", []Target{app}},
|
||||
{"curl -sk 'https://app.example.com:8443/a?b=1&c=http://inner.example/' -o out", []Target{{TargetDomain, "app.example.com", 8443}}},
|
||||
{"curl http://a.example/;curl http://b.example|sh", []Target{{TargetDomain, "a.example", 80}, {TargetDomain, "b.example", 80}}},
|
||||
{"curl -x http://127.0.0.1:8080 https://app.example.com/", []Target{{TargetIP, "127.0.0.1", 8080}, app}},
|
||||
{"nmap -sV 203.0.113.7", []Target{{TargetIP, "203.0.113.7", 0}}},
|
||||
{"nmap -p 443 203.0.113.0/24 198.51.100.9", []Target{{TargetCIDR, "203.0.113.0/24", 0}, {TargetIP, "198.51.100.9", 0}}},
|
||||
{"nc -v 203.0.113.7:4444", []Target{{TargetIP, "203.0.113.7", 4444}}},
|
||||
{"ssh root@203.0.113.7 -p 22", []Target{{TargetIP, "203.0.113.7", 0}}},
|
||||
{"mysql --host=203.0.113.7 -uroot", []Target{{TargetIP, "203.0.113.7", 0}}},
|
||||
{"openssl s_client -connect app.example.com:443 -servername x", []Target{app}},
|
||||
{"nc localhost:8080", []Target{{TargetDomain, "localhost", 8080}}},
|
||||
{"ping6 2001:db8::1", []Target{{TargetIP, "2001:db8::1", 0}}},
|
||||
{"curl 'http://[2001:db8::1]:8080/'; nc [2001:db8::2]:22", []Target{{TargetIP, "2001:db8::1", 8080}, {TargetIP, "2001:db8::2", 22}}},
|
||||
{"nmap fd00::/8 ::ffff:203.0.113.7", []Target{{TargetCIDR, "fd00::/8", 0}, {TargetIP, "203.0.113.7", 0}}},
|
||||
{"curl https://app.example.com/a https://APP.example.com/b", []Target{app}},
|
||||
{"git clone git+ssh://git@code.example.com/x.git", []Target{{TargetDomain, "code.example.com", 0}}},
|
||||
{"scp f 203.0.113.7:/tmp/f", []Target{{TargetIP, "203.0.113.7", 0}}},
|
||||
{"docker run -p 203.0.113.7:8080:80 img", []Target{{TargetIP, "203.0.113.7", 8080}}},
|
||||
{"curl 203.0.113.7/2024/report", []Target{{TargetIP, "203.0.113.7", 0}}},
|
||||
{"dig @198.51.100.53 x", []Target{{TargetIP, "198.51.100.53", 0}}},
|
||||
// The authority of a URL reads differently quoted and unquoted: both readings are targets.
|
||||
{`curl 'http://pinned.example;@evil.example/'`, []Target{{TargetDomain, "evil.example", 80}, {TargetDomain, "pinned.example", 80}}},
|
||||
{`curl http://pinned.example\@evil.example/`, []Target{{TargetDomain, "evil.example", 80}, {TargetDomain, "pinned.example", 80}}},
|
||||
{`curl "http://pinned.example,x@evil.example:8443/"`, []Target{{TargetDomain, "evil.example", 8443}, {TargetDomain, "pinned.example", 80}}},
|
||||
{"curl http://a.example;curl http://b.example&&id", []Target{{TargetDomain, "a.example", 80}, {TargetDomain, "b.example", 80}}},
|
||||
{"curl http://user:pw@app.example.com:8443/ http://%65vil.example/", []Target{{TargetDomain, "app.example.com", 8443}, {TargetDomain, "evil.example", 80}}},
|
||||
// A port the gate cannot read leaves the address a target.
|
||||
{"for p in 80 443; do curl http://203.0.113.7:$p/; done", []Target{{TargetIP, "203.0.113.7", 0}}},
|
||||
}
|
||||
for _, c := range commands {
|
||||
wantTargets(t, "Bash "+c.cmd, targetsOf("Bash", command(c.cmd)), c.want...)
|
||||
wantTargets(t, "shell_open "+c.cmd, targetsOf("shell_open", command(c.cmd)), c.want...)
|
||||
wantTargets(t, "Monitor "+c.cmd, targetsOf("Monitor", command(c.cmd)), c.want...)
|
||||
wantTargets(t, "shell_send "+c.cmd, targetsOf("shell_send", map[string]string{"text": c.cmd + "\n"}), c.want...)
|
||||
}
|
||||
// Only the command field is read.
|
||||
wantTargets(t, "Bash description", targetsOf("Bash", map[string]string{"command": "ls", "description": "see https://app.example.com/"}))
|
||||
wantTargets(t, "shell_send session", targetsOf("shell_send", map[string]string{"text": "id\n", "session_id": "203.0.113.7"}))
|
||||
|
||||
// The field is read the way the tool decodes it: a key matches whatever its
|
||||
// case, and the last duplicate wins. The gate reads the value that runs.
|
||||
for _, c := range []struct{ tool, input string }{
|
||||
{"Bash", `{"command":"ls","Command":"curl https://app.example.com/"}`},
|
||||
{"Bash", `{"command":"ls","command":"curl https://app.example.com/"}`},
|
||||
{"Bash", `{"COMMAND":"curl https://app.example.com/"}`},
|
||||
{"Bash", `{"command":"curl https://app.example.com/","timeout":"not a number","text":5}`},
|
||||
{"Monitor", `{"command":"ls","cOmMaNd":"curl https://app.example.com/"}`},
|
||||
{"shell_send", `{"text":"id","Text":"curl https://app.example.com/\n"}`},
|
||||
{"WebFetch", `{"url":"https://pinned.example/","URL":"https://app.example.com/"}`},
|
||||
{"WebFetch", `{"url":"https://pinned.example/","url":"https://app.example.com/"}`},
|
||||
{"ExecuteExtraTool", `{"tool_name":"list_assets","Tool_Name":"mcp__x__y","params":{"u":"https://pinned.example/"},"PARAMS":{"u":"https://app.example.com/"}}`},
|
||||
} {
|
||||
wantTargets(t, c.tool+" "+c.input, ExtractTargets(c.tool, []byte(c.input)), app)
|
||||
}
|
||||
wantTargets(t, "first duplicate does not run", ExtractTargets("Bash", []byte(`{"command":"curl https://app.example.com/","Command":"ls"}`)))
|
||||
// A tool the gate does not know may keep either value of a duplicated key: both are read.
|
||||
wantTargets(t, "duplicate key, unknown tool", ExtractTargets("mcp__scan__run", []byte(`{"url":"https://app.example.com/","url":"https://other.example.com/"}`)),
|
||||
app, Target{TargetDomain, "other.example.com", 443})
|
||||
|
||||
// Things that look close to a target and are not one.
|
||||
for _, cmd := range []string{
|
||||
"sleep 300; echo 2130706433",
|
||||
"pip install requests==2.31.0 && python3 --version",
|
||||
"echo v1.2.3.4 > version; tar xf app-1.2.3.4.tgz",
|
||||
"date +%H:%M:%S; echo 12:30:45",
|
||||
"curl -u admin:1234 -H 'X-Id: 7' file:///etc/hosts",
|
||||
"docker run redis:7 && docker pull python:3.12",
|
||||
"chown user:1000 f; chmod 600 key.pem",
|
||||
"ip link show; echo 00:11:22:33:44:55",
|
||||
"echo 999.1.1.1 1.2.3 1.2.3.4.5",
|
||||
"python3 -c 'print(a[::-1])'; g++ -std=c++17 x.cc && echo std::cout",
|
||||
"cat ./src/main.go:42",
|
||||
} {
|
||||
wantTargets(t, "Bash "+cmd, targetsOf("Bash", command(cmd)))
|
||||
}
|
||||
}
|
||||
|
||||
// Every MCP tool and every deferred custom tool is called through
|
||||
// ExecuteExtraTool: the gate reads the inner tool name and its parameters.
|
||||
func TestExtractUnwrapsExecuteExtraTool(t *testing.T) {
|
||||
app := Target{TargetDomain, "app.example.com", 443}
|
||||
call := func(tool string, params any) []Target {
|
||||
return targetsOf("ExecuteExtraTool", map[string]any{"tool_name": tool, "params": params})
|
||||
}
|
||||
wantTargets(t, "mcp tool", call("mcp__browser__navigate", map[string]string{"url": "https://app.example.com/"}), app)
|
||||
wantTargets(t, "inner WebFetch", call("WebFetch", map[string]string{"url": "https://app.example.com/x"}), app)
|
||||
wantTargets(t, "inner Bash", call("Bash", map[string]string{"command": "nmap 203.0.113.7", "description": "https://app.example.com/"}),
|
||||
Target{TargetIP, "203.0.113.7", 0})
|
||||
wantTargets(t, "inner tool without a target", call("list_assets", map[string]string{"q": "https://app.example.com/"}))
|
||||
wantTargets(t, "no params", targetsOf("ExecuteExtraTool", map[string]any{"tool_name": "mcp__scan__run"}))
|
||||
// A call that does not name its inner tool is read as a whole.
|
||||
wantTargets(t, "no tool name", targetsOf("ExecuteExtraTool", map[string]any{"params": map[string]string{"url": "https://app.example.com/"}}), app)
|
||||
// The wrapper is unwrapped once: a nested wrapper is read as an unknown tool.
|
||||
wantTargets(t, "nested", call("ExecuteExtraTool", map[string]any{"tool_name": "list_assets", "params": map[string]string{"q": "https://app.example.com/"}}), app)
|
||||
}
|
||||
|
||||
func TestExtractSkipsNoTargetTools(t *testing.T) {
|
||||
input := map[string]any{
|
||||
"query": "site:app.example.com 203.0.113.7", "file_path": "/tmp/203.0.113.7.txt", "content": "curl https://app.example.com/",
|
||||
"url": "https://app.example.com/", "command": "nmap 203.0.113.7", "items": []any{map[string]any{"url": "https://app.example.com:8443/"}},
|
||||
}
|
||||
for _, tool := range []string{
|
||||
"web_search", "Read", "Write", "Edit", "MultiEdit", "LS", "Glob", "Grep", "TodoWrite", "sleep",
|
||||
"TaskOutput", "TaskStop", "TaskList",
|
||||
"add_intent", "insert_assets", "report_finding", "record_fact", "add_task_scope", "graph_overview",
|
||||
"spawn_task", "ask_operator", "traffic_search", "traffic_get", "traffic_blob",
|
||||
} {
|
||||
wantTargets(t, tool, targetsOf(tool, input))
|
||||
}
|
||||
}
|
||||
|
||||
// A tool the gate does not know is scanned, not skipped: a tool added later is
|
||||
// gated by default.
|
||||
func TestExtractUnknownToolScansStringLeaves(t *testing.T) {
|
||||
input := map[string]any{
|
||||
"target": "203.0.113.7",
|
||||
"options": map[string]any{"proxy": "http://127.0.0.1:8080", "ports": []any{"443", 8443}, "deep": true},
|
||||
"urls": []any{"https://app.example.com/a", map[string]any{"u": "https://app.example.com/b"}, "ftp://files.example.com"},
|
||||
"note": "x.py and README.md are files",
|
||||
}
|
||||
// In the order of the sorted keys: options, target, urls.
|
||||
want := []Target{
|
||||
{TargetIP, "127.0.0.1", 8080}, {TargetIP, "203.0.113.7", 0},
|
||||
{TargetDomain, "app.example.com", 443}, {TargetDomain, "files.example.com", 21},
|
||||
}
|
||||
for _, tool := range []string{"mcp__scan__run", "my_custom_http_tool", "SomeToolAddedLater", "create_mcp", "create_custom_tool"} {
|
||||
wantTargets(t, tool, targetsOf(tool, input), want...)
|
||||
}
|
||||
wantTargets(t, "input that is not JSON", ExtractTargets("mcp__scan__run", []byte("https://app.example.com/")))
|
||||
}
|
||||
|
||||
// A bare name is not a target in a command: a file name is a valid host name,
|
||||
// so reading bare names would ask on every script run.
|
||||
func TestExtractDoesNotTreatFileNamesAsHosts(t *testing.T) {
|
||||
for _, cmd := range []string{
|
||||
"python3 x.py", "node server.js --watch", "cat notes.txt README.md", "bash ./run.sh && go test ./...",
|
||||
"tar czf out.tar.gz data.json", "ls app.example.com.har", "vim config.yaml", "./exploit.py --out result.io",
|
||||
} {
|
||||
wantTargets(t, cmd, targetsOf("Bash", command(cmd)))
|
||||
}
|
||||
}
|
||||
|
||||
// The tests below document what the gate does NOT see. A subprocess path is
|
||||
// advisory: the gate detects a declared target and cannot prevent an undeclared
|
||||
// connection. Each asserts that no target is extracted, which is the gap. They
|
||||
// change only if that decision changes.
|
||||
|
||||
func TestBypass_BareHostnameNotExtracted(t *testing.T) {
|
||||
for _, cmd := range []string{"nmap app.example.com", "ping -c1 app.example.com", "curl app.example.com/login", "ssh root@app.example.com", "nc db 5432"} {
|
||||
wantTargets(t, cmd, targetsOf("Bash", command(cmd)))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBypass_EncodedCommandHidesTarget(t *testing.T) {
|
||||
for _, cmd := range []string{
|
||||
"echo Y3VybCBodHRwczovL2FwcC5leGFtcGxlLmNvbS8= | base64 -d | sh",
|
||||
`python3 -c "import urllib.request as u; u.urlopen('https:/'+'/app.example'+'.com/')"`,
|
||||
"H=app.example; curl $H.com",
|
||||
"curl http://$TARGET/login https://${HOST}:8443/",
|
||||
"curl 'http://{app.example.com}/'",
|
||||
} {
|
||||
wantTargets(t, cmd, targetsOf("Bash", command(cmd)))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBypass_ScriptFileHidesTarget(t *testing.T) {
|
||||
// The target is in the file, or in a list the tool reads.
|
||||
for _, cmd := range []string{"python3 exploit.py", "bash scan.sh", "nmap -iL targets.txt", "nuclei -l hosts.txt"} {
|
||||
wantTargets(t, cmd, targetsOf("Bash", command(cmd)))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBypass_SplitKeystrokesHideTarget(t *testing.T) {
|
||||
// One address typed into an interactive shell in pieces: no call names it.
|
||||
for _, text := range []string{"nc 203.0.", "113.7 44", "44\n"} {
|
||||
wantTargets(t, text, targetsOf("shell_send", map[string]string{"text": text}))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBypass_CustomScriptCodeNotScanned(t *testing.T) {
|
||||
// A custom script tool is called with its parameters only; the target is in
|
||||
// the stored code, which the hook never sees.
|
||||
wantTargets(t, "custom script tool", targetsOf("my_script_tool", map[string]any{"path": "/login", "retries": 3}))
|
||||
}
|
||||
+29
-12
@@ -620,9 +620,14 @@ func (s *Server) StartTriggeredRun(agentKey, title, message string, taskID int64
|
||||
return
|
||||
}
|
||||
cfg := s.readTriggerBehavior(agentKey) // DB read BEFORE the lock (never under queueMu)
|
||||
// A conversation triggered by a gated task is gated (the flag only, never pins).
|
||||
gated := false
|
||||
if t, ok := s.m.Task(strconv.FormatInt(taskID, 10)); ok && taskID > 0 {
|
||||
gated = t.targetGate.Load()
|
||||
}
|
||||
s.queueMu.Lock()
|
||||
s.triggerCfg[agentKey] = cfg
|
||||
s.triggerQ[agentKey] = append(s.triggerQ[agentKey], triggeredRun{agentKey: agentKey, title: title, message: message, taskID: taskID, taskDesc: taskDesc, taskGoal: taskGoal, mergeable: mergeable})
|
||||
s.triggerQ[agentKey] = append(s.triggerQ[agentKey], triggeredRun{agentKey: agentKey, title: title, message: message, taskID: taskID, taskDesc: taskDesc, taskGoal: taskGoal, mergeable: mergeable, targetGate: gated})
|
||||
s.pumpLocked(agentKey)
|
||||
s.queueMu.Unlock()
|
||||
}
|
||||
@@ -749,15 +754,26 @@ func mergeTriggeredRuns(items []triggeredRun) triggeredRun {
|
||||
fmt.Fprintf(&b, "\n── trigger %d ──\n%s\n", i+1, it.message)
|
||||
}
|
||||
return triggeredRun{
|
||||
agentKey: first.agentKey,
|
||||
title: fmt.Sprintf("Merged triggers · task#%d · %d items", first.taskID, len(items)),
|
||||
message: b.String(),
|
||||
taskID: first.taskID,
|
||||
mergeable: true,
|
||||
agentKey: first.agentKey,
|
||||
title: fmt.Sprintf("Merged triggers · task#%d · %d items", first.taskID, len(items)),
|
||||
message: b.String(),
|
||||
taskID: first.taskID,
|
||||
mergeable: true,
|
||||
targetGate: anyTriggerGated(items),
|
||||
// taskDesc/taskGoal left empty: header already embedded above.
|
||||
}
|
||||
}
|
||||
|
||||
// anyTriggerGated reports whether any fire of a merged run came from a gated task.
|
||||
func anyTriggerGated(items []triggeredRun) bool {
|
||||
for _, it := range items {
|
||||
if it.targetGate {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// mergeAllRuns folds the ENTIRE queued burst into one run (merge_mode='all'),
|
||||
// regardless of type. Events are grouped by task (tasks in first-appearance order,
|
||||
// events in their original order within a task), so each task's context header — and
|
||||
@@ -791,11 +807,12 @@ func mergeAllRuns(items []triggeredRun) triggeredRun {
|
||||
}
|
||||
}
|
||||
return triggeredRun{
|
||||
agentKey: first.agentKey,
|
||||
title: fmt.Sprintf("Merged triggers · all · %d items", len(items)),
|
||||
message: b.String(),
|
||||
taskID: first.taskID,
|
||||
mergeable: true,
|
||||
agentKey: first.agentKey,
|
||||
title: fmt.Sprintf("Merged triggers · all · %d items", len(items)),
|
||||
message: b.String(),
|
||||
taskID: first.taskID,
|
||||
mergeable: true,
|
||||
targetGate: anyTriggerGated(items),
|
||||
// taskDesc/taskGoal left empty: per-task headers already embedded above.
|
||||
}
|
||||
}
|
||||
@@ -810,7 +827,7 @@ func (s *Server) runTriggeredRun(item triggeredRun) {
|
||||
}
|
||||
}()
|
||||
pg := s.m.pg
|
||||
c, err := pg.CreateConversation(item.agentKey, firstLine(item.title, 60), nil)
|
||||
c, err := pg.CreateConversationGated(item.agentKey, firstLine(item.title, 60), nil, item.targetGate)
|
||||
if err != nil {
|
||||
log.Printf("[trigger] create conversation for %s failed: %v", item.agentKey, err)
|
||||
return
|
||||
|
||||
+60
-10
@@ -18,6 +18,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/guard"
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
"github.com/Autumn-27/norma/permission"
|
||||
actool "github.com/Autumn-27/norma/tool"
|
||||
)
|
||||
@@ -163,11 +165,11 @@ func (s *Server) pgTestCustomTool(w http.ResponseWriter, r *http.Request) {
|
||||
var res actool.Result
|
||||
switch req.Kind {
|
||||
case "command":
|
||||
res, _ = s.runCommandTool(ctx, req.Exec, params, tc)
|
||||
res, _ = s.runCommandTool(ctx, "test", req.Exec, params, tc)
|
||||
case "script":
|
||||
res, _ = s.runScriptTool(ctx, "test", req.Exec, params, tc)
|
||||
case "http":
|
||||
res, _ = s.runHTTPTool(ctx, req.Exec, params, tc)
|
||||
res, _ = s.runHTTPTool(ctx, "test", req.Exec, params, tc)
|
||||
case "shell":
|
||||
writeErr(w, 400, "shell-kind tools are bash environment declarations with no executable content")
|
||||
return
|
||||
@@ -275,11 +277,11 @@ func (s *Server) buildCustomTool(t *db.Tool) actool.CoreTool {
|
||||
}
|
||||
switch kind {
|
||||
case "command":
|
||||
return s.runCommandTool(ctx, execRaw, params, tc)
|
||||
return s.runCommandTool(ctx, key, execRaw, params, tc)
|
||||
case "script":
|
||||
return s.runScriptTool(ctx, key, execRaw, params, tc)
|
||||
case "http":
|
||||
return s.runHTTPTool(ctx, execRaw, params, tc)
|
||||
return s.runHTTPTool(ctx, key, execRaw, params, tc)
|
||||
default:
|
||||
return actool.Errorf("unknown custom tool kind: " + kind), nil
|
||||
}
|
||||
@@ -328,13 +330,18 @@ func ensureSchema(raw json.RawMessage) map[string]any {
|
||||
|
||||
// ---------- command: render the command → reuse the Bash low-level run ----------
|
||||
|
||||
func (s *Server) runCommandTool(ctx context.Context, execRaw json.RawMessage, params map[string]any, tc *actool.ToolContext) (actool.Result, error) {
|
||||
func (s *Server) runCommandTool(ctx context.Context, key string, execRaw json.RawMessage, params map[string]any, tc *actool.ToolContext) (actool.Result, error) {
|
||||
var spec commandExec
|
||||
_ = json.Unmarshal(execRaw, &spec)
|
||||
if strings.TrimSpace(spec.Command) == "" {
|
||||
return actool.Errorf("command is empty"), nil
|
||||
}
|
||||
cmd := renderTemplate(spec.Command, params, shellQuote)
|
||||
// The hook saw the parameters, not the command they render: the target gate
|
||||
// reads the rendered command here, also without the quoting of the parameters.
|
||||
if res := intercept.GateRenderedCommand(ctx, key, cmd, renderTemplate(spec.Command, params, identity)); res.Code != "" {
|
||||
return actool.Errorf(guard.TargetBlockMessage(res)), nil
|
||||
}
|
||||
// Reuse the low-level run Bash also uses (via the Bash CoreTool.Call): it
|
||||
// automatically inherits the safety floor/timeout/proxy env/output overflow. The tool is a peer of Bash sharing that low level, not routed through the Bash tool for the model to call.
|
||||
bashIn, _ := json.Marshal(map[string]any{"command": cmd})
|
||||
@@ -417,7 +424,7 @@ func execPython(ctx context.Context, interp, key, code string, params map[string
|
||||
|
||||
// ---------- http: native request + proxy ----------
|
||||
|
||||
func (s *Server) runHTTPTool(ctx context.Context, execRaw json.RawMessage, params map[string]any, tc *actool.ToolContext) (actool.Result, error) {
|
||||
func (s *Server) runHTTPTool(ctx context.Context, key string, execRaw json.RawMessage, params map[string]any, tc *actool.ToolContext) (actool.Result, error) {
|
||||
var spec httpExec
|
||||
_ = json.Unmarshal(execRaw, &spec)
|
||||
method := strings.ToUpper(strings.TrimSpace(spec.Method))
|
||||
@@ -428,6 +435,12 @@ func (s *Server) runHTTPTool(ctx context.Context, execRaw json.RawMessage, param
|
||||
if strings.TrimSpace(rawURL) == "" {
|
||||
return actool.Errorf("http url is empty"), nil
|
||||
}
|
||||
// The hook saw the parameters, not the URL they render (nor a URL written in
|
||||
// the template): the target gate reads the rendered URL here, before the
|
||||
// tool's own timeout starts, so a wait for an approval does not consume it.
|
||||
if res := intercept.GateRenderedURL(ctx, key, rawURL); res.Code != "" {
|
||||
return actool.Errorf(guard.TargetBlockMessage(res)), nil
|
||||
}
|
||||
var bodyReader io.Reader
|
||||
if spec.Body != "" {
|
||||
bodyReader = strings.NewReader(renderTemplate(spec.Body, params, identity))
|
||||
@@ -442,7 +455,9 @@ func (s *Server) runHTTPTool(ctx context.Context, execRaw json.RawMessage, param
|
||||
req.Header.Set(k, renderTemplate(v, params, identity))
|
||||
}
|
||||
client := &http.Client{Timeout: timeoutOr(spec.TimeoutMs, 30000)}
|
||||
if tr := s.httpProxyTransport(spec); tr != nil {
|
||||
// Every redirect hop goes through the target gate, as the rendered URL did.
|
||||
client.CheckRedirect = intercept.RedirectCheck(key)
|
||||
if tr := s.httpToolTransport(ctx, key, spec); tr != nil {
|
||||
client.Transport = tr
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
@@ -456,14 +471,38 @@ func (s *Server) runHTTPTool(ctx context.Context, execRaw json.RawMessage, param
|
||||
return actool.Text(actool.Capture(tc, string(b))), nil
|
||||
}
|
||||
|
||||
// httpToolTransport is the transport of one http tool call, or nil
|
||||
// (http.DefaultTransport). A call through a proxy keeps httpProxyTransport: it
|
||||
// dials the proxy, which makes the connection to the target. A direct call in
|
||||
// a gated scope connects through the pinned dialer, to addresses a pin covers.
|
||||
func (s *Server) httpToolTransport(ctx context.Context, key string, spec httpExec) *http.Transport {
|
||||
if tr := s.httpProxyTransport(spec); tr != nil {
|
||||
return tr
|
||||
}
|
||||
if !intercept.GateOn(ctx) {
|
||||
return nil
|
||||
}
|
||||
base, ok := http.DefaultTransport.(*http.Transport)
|
||||
if !ok {
|
||||
base = &http.Transport{}
|
||||
}
|
||||
tr := base.Clone()
|
||||
// An environment proxy would be dialled instead of the target.
|
||||
tr.Proxy = nil
|
||||
tr.DialContext = intercept.PinnedDialer(key, true)
|
||||
// One request per call: no idle connection outlives the call.
|
||||
tr.DisableKeepAlives = true
|
||||
return tr
|
||||
}
|
||||
|
||||
// httpProxyTransport builds a Transport for the http tool's proxy config, or nil
|
||||
// (direct). use_recording_proxy routes through the recording proxy + trusts its CA.
|
||||
func (s *Server) httpProxyTransport(spec httpExec) *http.Transport {
|
||||
proxyStr := strings.TrimSpace(spec.Proxy)
|
||||
proxyStr := proxyURLString(spec.Proxy)
|
||||
var caFile string
|
||||
if spec.UseRecordingProxy {
|
||||
if addr := s.m.ProxyAddr(); addr != "" {
|
||||
proxyStr = "http://" + addr
|
||||
if addr := proxyURLString(s.m.ProxyAddr()); addr != "" {
|
||||
proxyStr = addr
|
||||
caFile = s.m.ProxyCACert()
|
||||
}
|
||||
}
|
||||
@@ -486,6 +525,17 @@ func (s *Server) httpProxyTransport(spec httpExec) *http.Transport {
|
||||
return tr
|
||||
}
|
||||
|
||||
// proxyURLString returns a proxy address as a URL: a bare host:port gets
|
||||
// "http://", an address that already has a scheme (as ProxyAddr returns it) is
|
||||
// kept as is.
|
||||
func proxyURLString(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" || strings.Contains(s, "://") {
|
||||
return s
|
||||
}
|
||||
return "http://" + s
|
||||
}
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
func identity(s string) string { return s }
|
||||
|
||||
@@ -0,0 +1,358 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
)
|
||||
|
||||
// Step 2.1: the custom http tool connects directly through the pinned dialer
|
||||
// in a gated scope, and a gated worker's WebFetch end to end.
|
||||
|
||||
// routeDialer is the base dialer of the gate in these tests: it records every
|
||||
// address, connects the routed TEST-NET addresses to local listeners, and
|
||||
// refuses the rest.
|
||||
type routeDialer struct {
|
||||
mu sync.Mutex
|
||||
routes map[string]string
|
||||
dialed []string
|
||||
}
|
||||
|
||||
func newRouteDialer(g *gateServer) *routeDialer {
|
||||
d := &routeDialer{routes: map[string]string{}}
|
||||
g.m.interceptor.SetBaseDialer(d.dial)
|
||||
return d
|
||||
}
|
||||
|
||||
func (d *routeDialer) route(addr, to string) {
|
||||
d.mu.Lock()
|
||||
d.routes[addr] = to
|
||||
d.mu.Unlock()
|
||||
}
|
||||
|
||||
func (d *routeDialer) dial(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
d.mu.Lock()
|
||||
d.dialed = append(d.dialed, addr)
|
||||
to, ok := d.routes[addr]
|
||||
d.mu.Unlock()
|
||||
if !ok {
|
||||
return nil, &net.OpError{Op: "dial", Net: network, Err: errors.New("no route in the test dialer")}
|
||||
}
|
||||
return (&net.Dialer{}).DialContext(ctx, network, to)
|
||||
}
|
||||
|
||||
func (d *routeDialer) calls() []string {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
return slices.Clone(d.dialed)
|
||||
}
|
||||
|
||||
func (r *countingResolver) set(host string, ips ...string) {
|
||||
r.mu.Lock()
|
||||
r.answers[host] = ips
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// approveTarget opens the request of a gated http tool call on key and approves it.
|
||||
func approveTarget(t *testing.T, g *gateServer, task *Task, tool interface {
|
||||
Name() string
|
||||
}, call func() (string, bool), key string) {
|
||||
t.Helper()
|
||||
if out, isErr := call(); !isErr || !strings.Contains(out, "TARGET_APPROVAL_PENDING") {
|
||||
t.Fatalf("%s before the approval: error=%v out=%q", tool.Name(), isErr, out)
|
||||
}
|
||||
id, _, _, ok := openTargetRequest(t, g.m.pg, task.ID, key)
|
||||
if !ok {
|
||||
t.Fatalf("no request on %s", key)
|
||||
}
|
||||
decide(t, g, id, "allowed")
|
||||
}
|
||||
|
||||
// A gated direct call reaches its target at the pinned address; once the name
|
||||
// answers another address, the rendered-URL gate passes on the cached answer
|
||||
// and the dial connects nowhere.
|
||||
func TestCustomHTTPDirectDialPinned(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
task := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
|
||||
srv, hits, _, port := countingServer(t)
|
||||
res := &countingResolver{answers: map[string][]string{"api.example.com": {"203.0.113.7"}}}
|
||||
g.m.interceptor.SetResolver(res)
|
||||
dialer := newRouteDialer(g)
|
||||
dialer.route("203.0.113.7:"+port, srv.Listener.Addr().String())
|
||||
dialer.route("203.0.113.9:"+port, srv.Listener.Addr().String())
|
||||
key := "api.example.com:" + port
|
||||
tool := g.s.buildCustomTool(&db.Tool{Key: "probe_dial", Kind: "http",
|
||||
Exec: execJSON(t, map[string]any{"url": "http://" + key + "/dial"})})
|
||||
call := func() (string, bool) { return callTool(t, tool, gatedCtx(task, 0), `{}`, nil) }
|
||||
approveTarget(t, g, task, tool, call, key)
|
||||
|
||||
if out, isErr := call(); isErr || !strings.Contains(out, "probe-reached") || hits.Load() != 1 {
|
||||
t.Fatalf("pinned call: error=%v out=%q hits=%d", isErr, out, hits.Load())
|
||||
}
|
||||
res.set("api.example.com", "203.0.113.9")
|
||||
out, isErr := call()
|
||||
if !isErr || !strings.Contains(out, "TARGET_APPROVAL_PENDING: the connection to "+key+" was not opened") || hits.Load() != 1 {
|
||||
t.Fatalf("rebound name: error=%v out=%q hits=%d", isErr, out, hits.Load())
|
||||
}
|
||||
if got := dialer.calls(); !slices.Equal(got, []string{"203.0.113.7:" + port}) {
|
||||
t.Fatalf("dialled %v, want the pinned address once", got)
|
||||
}
|
||||
if _, toolName, input, ok := openTargetRequest(t, g.m.pg, task.ID, key); !ok || toolName != "probe_dial" || !strings.Contains(input, "dial") {
|
||||
t.Fatalf("request at the dial: ok=%v tool=%q input=%s", ok, toolName, input)
|
||||
}
|
||||
}
|
||||
|
||||
// With the gate off (and without a scope) the call keeps http.DefaultTransport:
|
||||
// no transport is built. A spec with a proxy keeps its proxy transport, without
|
||||
// a dialer. Gate on and direct: a transport with the pinned dialer, no
|
||||
// environment proxy, keep-alive off.
|
||||
func TestCustomHTTPGateOffTransportUnchanged(t *testing.T) {
|
||||
s := &Server{}
|
||||
direct := httpExec{URL: "http://api.example.com/"}
|
||||
scope := func(on bool) context.Context {
|
||||
return intercept.WithTargetScope(context.Background(), intercept.TargetScope{TaskID: "7", Enabled: func() bool { return on }})
|
||||
}
|
||||
for name, ctx := range map[string]context.Context{"no scope": context.Background(), "flag off": scope(false)} {
|
||||
if tr := s.httpToolTransport(ctx, "probe", direct); tr != nil {
|
||||
t.Fatalf("%s: transport %+v, want nil (http.DefaultTransport)", name, tr)
|
||||
}
|
||||
}
|
||||
if tr := s.httpToolTransport(scope(true), "probe", httpExec{URL: direct.URL, Proxy: "http://127.0.0.1:1"}); tr == nil || tr.Proxy == nil || tr.DialContext != nil {
|
||||
t.Fatalf("operator proxy: transport %+v, want the proxy transport without a dialer", tr)
|
||||
}
|
||||
if tr := s.httpToolTransport(scope(true), "probe", direct); tr == nil || tr.Proxy != nil || tr.DialContext == nil || !tr.DisableKeepAlives {
|
||||
t.Fatalf("gated direct call: transport %+v", tr)
|
||||
}
|
||||
}
|
||||
|
||||
// The proxy URL of the http tool's transport: an address with a scheme is kept
|
||||
// as is, a bare host:port gets "http://"; never a doubled scheme.
|
||||
func TestHTTPProxyTransportProxyScheme(t *testing.T) {
|
||||
req := httptest.NewRequest("GET", "http://api.example.com/", nil)
|
||||
for _, c := range []struct {
|
||||
name, globalProxy string
|
||||
spec httpExec
|
||||
want string
|
||||
}{
|
||||
{"recording proxy, global proxy with a scheme", "http://127.0.0.1:3128", httpExec{UseRecordingProxy: true}, "http://127.0.0.1:3128"},
|
||||
{"operator proxy with a scheme", "", httpExec{Proxy: "http://10.0.0.5:8080"}, "http://10.0.0.5:8080"},
|
||||
{"operator proxy as bare host:port", "", httpExec{Proxy: " 10.0.0.5:8080 "}, "http://10.0.0.5:8080"},
|
||||
} {
|
||||
s := &Server{m: &Manager{globalProxy: c.globalProxy}}
|
||||
tr := s.httpProxyTransport(c.spec)
|
||||
if tr == nil || tr.Proxy == nil {
|
||||
t.Fatalf("%s: transport %+v, want a proxy transport", c.name, tr)
|
||||
}
|
||||
u, err := tr.Proxy(req)
|
||||
if err != nil || u == nil || u.String() != c.want {
|
||||
t.Fatalf("%s: Proxy(req) = %v, %v; want %s", c.name, u, err, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// passRoundTripper stands in for an http.DefaultTransport that is not an
|
||||
// *http.Transport; it hands every request to the original transport.
|
||||
type passRoundTripper struct{ next http.RoundTripper }
|
||||
|
||||
func (p passRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
return p.next.RoundTrip(r)
|
||||
}
|
||||
|
||||
// With http.DefaultTransport replaced by another RoundTripper, a gated direct
|
||||
// call still gets a transport with the pinned dialer instead of a panic.
|
||||
func TestHTTPToolTransportDefaultTransportNotStd(t *testing.T) {
|
||||
orig := http.DefaultTransport
|
||||
http.DefaultTransport = passRoundTripper{next: orig}
|
||||
defer func() { http.DefaultTransport = orig }()
|
||||
ctx := intercept.WithTargetScope(context.Background(), intercept.TargetScope{TaskID: "7", Enabled: func() bool { return true }})
|
||||
tr := (&Server{}).httpToolTransport(ctx, "probe", httpExec{URL: "http://api.example.com/"})
|
||||
if tr == nil || tr.DialContext == nil {
|
||||
t.Fatalf("transport %+v, want one with the pinned dialer", tr)
|
||||
}
|
||||
}
|
||||
|
||||
// A redirect of a pinned origin to a pinned hop: the hop is dialled at the
|
||||
// hop's pinned address.
|
||||
func TestCustomHTTPRedirectHopDialPinned(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
task := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
|
||||
final, finalHits, _, finalPort := countingServer(t)
|
||||
hopKey := "hop.example.com:" + finalPort
|
||||
start, _, startAddr := redirectingServer(t, "http://"+hopKey+"/final")
|
||||
_, startPort, _ := net.SplitHostPort(startAddr)
|
||||
originKey := "api.example.com:" + startPort
|
||||
res := &countingResolver{answers: map[string][]string{"api.example.com": {"203.0.113.7"}, "hop.example.com": {"203.0.113.8"}}}
|
||||
g.m.interceptor.SetResolver(res)
|
||||
dialer := newRouteDialer(g)
|
||||
dialer.route("203.0.113.7:"+startPort, start.Listener.Addr().String())
|
||||
dialer.route("203.0.113.8:"+finalPort, final.Listener.Addr().String())
|
||||
// Were the hop dialled at the origin's address, it would still reach it.
|
||||
dialer.route("203.0.113.7:"+finalPort, final.Listener.Addr().String())
|
||||
tool := g.s.buildCustomTool(&db.Tool{Key: "probe_hop", Kind: "http",
|
||||
Exec: execJSON(t, map[string]any{"url": "http://" + originKey + "/start"})})
|
||||
call := func() (string, bool) { return callTool(t, tool, gatedCtx(task, 0), `{}`, nil) }
|
||||
approveTarget(t, g, task, tool, call, originKey)
|
||||
approveTarget(t, g, task, tool, call, hopKey)
|
||||
|
||||
before := len(dialer.calls())
|
||||
if out, isErr := call(); isErr || !strings.Contains(out, "probe-reached") || finalHits.Load() != 1 {
|
||||
t.Fatalf("pinned hop: error=%v out=%q", isErr, out)
|
||||
}
|
||||
if got := dialer.calls()[before:]; !slices.Equal(got, []string{"203.0.113.7:" + startPort, "203.0.113.8:" + finalPort}) {
|
||||
t.Fatalf("dialled %v, want the origin's then the hop's pinned address", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The custom-tool test endpoint has no scope (D14): its connection never goes
|
||||
// through the gate's dialer, and nothing is written.
|
||||
func TestCustomToolTestEndpointDialUngated(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
|
||||
dialer := newRouteDialer(g)
|
||||
_, hits, host, port := countingServer(t)
|
||||
before := 0
|
||||
if err := g.m.pg.QueryRow(`SELECT count(*) FROM intercept_pending WHERE decision_source='target' OR target_key IS NOT NULL`).Scan(&before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, out := g.do(t, "POST", "/api/tools/custom/test", fmt.Sprintf(`{"kind":"http","exec":{"url":"http://%s:%s/test"}}`, host, port))
|
||||
if code != 200 || out["is_error"] != false || hits.Load() != 1 || len(dialer.calls()) != 0 {
|
||||
t.Fatalf("test endpoint: %d %v hits=%d dialled %v", code, out, hits.Load(), dialer.calls())
|
||||
}
|
||||
after := 0
|
||||
if err := g.m.pg.QueryRow(`SELECT count(*) FROM intercept_pending WHERE decision_source='target' OR target_key IS NOT NULL`).Scan(&after); err != nil || after != before {
|
||||
t.Fatalf("the test endpoint wrote %d target rows (%v)", after-before, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Documents a known limit: a custom http tool that goes through a proxy is not
|
||||
// bound to pins. The pinned dialer covers direct connections only; through a
|
||||
// proxy the connection to the target is the proxy's, and the proxy here
|
||||
// connects to a peer nobody pinned. Step 2.2 binds the recording proxy (and
|
||||
// flips its half of this test); an operator-written proxy stays unbound
|
||||
// (OPEN-6).
|
||||
func TestBypass_CustomHTTPProxyPeerNotPinned(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
task := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
|
||||
_, hits, _, port := countingServer(t)
|
||||
peer := "127.0.0.1:" + port
|
||||
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Whatever the request names, the proxy connects to peer.
|
||||
resp, err := http.Get("http://" + peer + r.URL.Path)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
}))
|
||||
t.Cleanup(proxy.Close)
|
||||
res := &countingResolver{answers: map[string][]string{"api.example.com": {"203.0.113.7"}}}
|
||||
g.m.interceptor.SetResolver(res)
|
||||
dialer := newRouteDialer(g)
|
||||
key := "api.example.com:" + port
|
||||
url := "http://" + key + "/bypass"
|
||||
direct := g.s.buildCustomTool(&db.Tool{Key: "probe_bypass", Kind: "http", Exec: execJSON(t, map[string]any{"url": url})})
|
||||
approveTarget(t, g, task, direct, func() (string, bool) { return callTool(t, direct, gatedCtx(task, 0), `{}`, nil) }, key)
|
||||
|
||||
// The global proxy carries a scheme, as in production (ProxyAddr returns it
|
||||
// as is with capture off); httpProxyTransport must not write a second one.
|
||||
g.m.mu.Lock()
|
||||
g.m.globalProxy = proxy.URL
|
||||
g.m.mu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
g.m.mu.Lock()
|
||||
g.m.globalProxy = ""
|
||||
g.m.mu.Unlock()
|
||||
})
|
||||
for name, exec := range map[string]map[string]any{
|
||||
"operator proxy": {"url": url, "proxy": proxy.URL},
|
||||
"use_recording_proxy": {"url": url, "use_recording_proxy": true},
|
||||
} {
|
||||
before := hits.Load()
|
||||
tool := g.s.buildCustomTool(&db.Tool{Key: "probe_bypass", Kind: "http", Exec: execJSON(t, exec)})
|
||||
if out, isErr := callTool(t, tool, gatedCtx(task, 0), `{}`, nil); isErr || !strings.Contains(out, "probe-reached") || hits.Load() != before+1 {
|
||||
t.Fatalf("%s: error=%v out=%q", name, isErr, out)
|
||||
}
|
||||
}
|
||||
if got := dialer.calls(); len(got) != 0 {
|
||||
t.Fatalf("dialled %v; the pinned address 203.0.113.7 is never dialled through a proxy", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A gated worker's WebFetch end to end: approved, the next call passes; the
|
||||
// name then answers another address from the next lookup on, so the following
|
||||
// call passes the hook on the cached answer and fails at the dial with
|
||||
// TARGET_APPROVAL_PENDING. The new_ip request is approved through the API and
|
||||
// the call after passes.
|
||||
func TestTargetGateEndToEndDialRebindBlocked(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
task := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
|
||||
pg := g.m.pg
|
||||
srv, hits, _, port := countingServer(t)
|
||||
res := &countingResolver{answers: map[string][]string{"api.example.com": {"203.0.113.7"}}}
|
||||
g.m.interceptor.SetResolver(res)
|
||||
dialer := newRouteDialer(g)
|
||||
dialer.route("203.0.113.7:"+port, srv.Listener.Addr().String())
|
||||
dialer.route("203.0.113.9:"+port, srv.Listener.Addr().String())
|
||||
key := "api.example.com:" + port
|
||||
target := "http://" + key + "/e2e"
|
||||
p := &toolScript{calls: fetches(target, target, target), results: make(chan string, 16),
|
||||
hold: map[int64]chan struct{}{2: make(chan struct{}), 3: make(chan struct{})}}
|
||||
done := runScriptedWorker(t, g, task, p)
|
||||
|
||||
var id int64
|
||||
waitUntil(t, "the first request", func() bool {
|
||||
var ok bool
|
||||
id, _, _, ok = openTargetRequest(t, pg, task.ID, key)
|
||||
return ok
|
||||
})
|
||||
decide(t, g, id, "allowed")
|
||||
if r := p.next(t); !strings.Contains(r, "probe-reached") || hits.Load() != 1 {
|
||||
t.Fatalf("first call after the approval: %q hits=%d", r, hits.Load())
|
||||
}
|
||||
|
||||
res.set("api.example.com", "203.0.113.9")
|
||||
close(p.hold[2])
|
||||
if r := p.next(t); !strings.Contains(r, "TARGET_APPROVAL_PENDING: the connection to "+key+" was not opened") || hits.Load() != 1 {
|
||||
t.Fatalf("second call, rebound at the dial: %q hits=%d", r, hits.Load())
|
||||
}
|
||||
newIP, toolName, _, ok := openTargetRequest(t, pg, task.ID, key)
|
||||
if !ok || newIP == id || toolName != "WebFetch" {
|
||||
t.Fatalf("new_ip request: ok=%v id=%d tool=%q", ok, newIP, toolName)
|
||||
}
|
||||
detail, err := pg.GetInterceptDetail(newIP)
|
||||
if err != nil || detail == nil || detail.Audit == nil || detail.Audit.Targets[0].Reason != "new_ip" ||
|
||||
!slices.Equal(detail.Audit.Targets[0].ResolvedIPs, []string{"203.0.113.9"}) {
|
||||
t.Fatalf("new_ip request detail = %+v, %v", detail, err)
|
||||
}
|
||||
decide(t, g, newIP, "allowed")
|
||||
close(p.hold[3])
|
||||
if r := p.next(t); !strings.Contains(r, "probe-reached") || hits.Load() != 2 {
|
||||
t.Fatalf("third call after the new_ip approval: %q hits=%d", r, hits.Load())
|
||||
}
|
||||
if got := dialer.calls(); !slices.Equal(got, []string{"203.0.113.7:" + port, "203.0.113.9:" + port}) {
|
||||
t.Fatalf("dialled %v", got)
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(60 * time.Second):
|
||||
t.Fatal("the worker run did not end")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,662 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"iter"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
actool "github.com/Autumn-27/norma/tool"
|
||||
)
|
||||
|
||||
// Step 1.8: the targets the custom tools render. The hook reads the parameters
|
||||
// of a custom tool, not the URL or the command they produce: the tool runs the
|
||||
// gate on what it is about to run. Plus the end-to-end run of a gated worker
|
||||
// and the run of a task with the gate off.
|
||||
|
||||
// countingServer is an HTTP server on 127.0.0.1 that counts the requests it
|
||||
// receives. Its host:port is an IP literal: the gate never resolves it.
|
||||
func countingServer(t *testing.T) (srv *httptest.Server, hits *atomic.Int64, host, port string) {
|
||||
t.Helper()
|
||||
hits = &atomic.Int64{}
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
_, _ = w.Write([]byte("probe-reached"))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
u, err := url.Parse(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
host, port, err = net.SplitHostPort(u.Host)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return srv, hits, host, port
|
||||
}
|
||||
|
||||
// gatedCtx is the context of a worker tool call of task: its approval scope
|
||||
// with the given wait bound.
|
||||
func gatedCtx(task *Task, wait time.Duration) context.Context {
|
||||
ctx := intercept.WithTaskContext(context.Background(), task.ID, "worker", nil)
|
||||
return intercept.WithTargetScope(ctx, task.targetScope(wait))
|
||||
}
|
||||
|
||||
func execJSON(t *testing.T, v any) json.RawMessage {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// callTool runs a custom tool the way the harness does and returns its text.
|
||||
func callTool(t *testing.T, tool actool.CoreTool, ctx context.Context, input string, tc *actool.ToolContext) (string, bool) {
|
||||
t.Helper()
|
||||
res, err := tool.Call(ctx, json.RawMessage(input), tc)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", tool.Name(), err)
|
||||
}
|
||||
return res.Flatten(), res.IsError
|
||||
}
|
||||
|
||||
// openTargetRequest returns the open target request of the task on key.
|
||||
func openTargetRequest(t *testing.T, pg *db.DB, taskID, key string) (id int64, toolName, toolInput string, ok bool) {
|
||||
t.Helper()
|
||||
err := pg.QueryRow(`SELECT id, tool_name, tool_input::text FROM intercept_pending
|
||||
WHERE task_id=$1 AND status='pending' AND target_key=$2`, taskID, key).Scan(&id, &toolName, &toolInput)
|
||||
if err != nil {
|
||||
return 0, "", "", false
|
||||
}
|
||||
return id, toolName, toolInput, true
|
||||
}
|
||||
|
||||
func targetRowCount(t *testing.T, pg *db.DB, taskID string) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := pg.QueryRow(`SELECT count(*) FROM intercept_pending WHERE task_id=$1 AND (decision_source='target' OR target_key IS NOT NULL)`, taskID).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// A custom http tool is gated on the URL it renders, whether the target comes
|
||||
// from a parameter (here split over two parameters, so the hook reads no
|
||||
// target in them) or is written in the template. Nothing is sent before an
|
||||
// approval; an approval lets both through; a revoked pin asks again.
|
||||
func TestCustomHTTPRenderedTargetGated(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
task := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
|
||||
pg := g.m.pg
|
||||
srv, hits, host, port := countingServer(t)
|
||||
key := net.JoinHostPort(host, port)
|
||||
dot := strings.LastIndexByte(host, '.')
|
||||
|
||||
rendered := g.s.buildCustomTool(&db.Tool{Key: "probe_http", Kind: "http",
|
||||
Schema: json.RawMessage(`{"type":"object","properties":{"last":{"type":"string"},"port":{"type":"string"}}}`),
|
||||
Exec: execJSON(t, map[string]any{"method": "GET", "url": "http://" + host[:dot+1] + "{last}:{port}/rendered"})})
|
||||
params := fmt.Sprintf(`{"last":%q,"port":%q}`, host[dot+1:], port)
|
||||
fixed := g.s.buildCustomTool(&db.Tool{Key: "probe_fixed", Kind: "http",
|
||||
Exec: execJSON(t, map[string]any{"method": "GET", "url": srv.URL + "/fixed"})})
|
||||
// The hook cannot see either target: neither input names one.
|
||||
for _, in := range []struct{ tool, input string }{{"probe_http", params}, {"probe_fixed", `{}`}} {
|
||||
if got := intercept.ExtractTargets(in.tool, []byte(in.input)); len(got) != 0 {
|
||||
t.Fatalf("%s: the hook reads %+v; the test needs a target only the rendering shows", in.tool, got)
|
||||
}
|
||||
}
|
||||
|
||||
// The template's own URL, wait bound 0: blocked at once, nothing sent, the
|
||||
// request open with the rendered URL as its input.
|
||||
out, isErr := callTool(t, fixed, gatedCtx(task, 0), `{}`, nil)
|
||||
if !isErr || !strings.Contains(out, "TARGET_APPROVAL_PENDING") || !strings.Contains(out, key) || hits.Load() != 0 {
|
||||
t.Fatalf("hard-coded URL: error=%v out=%q hits=%d", isErr, out, hits.Load())
|
||||
}
|
||||
id, toolName, toolInput, ok := openTargetRequest(t, pg, task.ID, key)
|
||||
if !ok || toolName != "probe_fixed" || !strings.Contains(toolInput, srv.URL+"/fixed") {
|
||||
t.Fatalf("request row: ok=%v tool=%q input=%s", ok, toolName, toolInput)
|
||||
}
|
||||
|
||||
// The rendered URL names the same host and port: the call attaches to the
|
||||
// open request and waits. One click lets it through.
|
||||
type result struct {
|
||||
out string
|
||||
isErr bool
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
res, err := rendered.Call(gatedCtx(task, 30*time.Second), json.RawMessage(params), nil)
|
||||
if err != nil {
|
||||
done <- result{err.Error(), true}
|
||||
return
|
||||
}
|
||||
done <- result{res.Flatten(), res.IsError}
|
||||
}()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if hits.Load() != 0 {
|
||||
t.Fatal("the rendered URL was fetched before the approval")
|
||||
}
|
||||
if n := targetRowCount(t, pg, task.ID); n != 1 {
|
||||
t.Fatalf("%d target rows, want the one request", n)
|
||||
}
|
||||
if code, body := g.do(t, "POST", fmt.Sprintf("/api/intercept/pending/%d/decide", id), `{"decision":"allowed"}`); code != 200 {
|
||||
t.Fatalf("decide: %d %v", code, body)
|
||||
}
|
||||
select {
|
||||
case r := <-done:
|
||||
if r.isErr || !strings.Contains(r.out, "probe-reached") || hits.Load() != 1 {
|
||||
t.Fatalf("after the approval: error=%v out=%q hits=%d", r.isErr, r.out, hits.Load())
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the waiting call did not end after the approval")
|
||||
}
|
||||
if out, isErr := callTool(t, fixed, gatedCtx(task, 0), `{}`, nil); isErr || hits.Load() != 2 {
|
||||
t.Fatalf("pinned, hard-coded URL: error=%v out=%q hits=%d", isErr, out, hits.Load())
|
||||
}
|
||||
|
||||
// Revoked: the next call asks again and sends nothing.
|
||||
pins, err := pg.ListTargetPins(taskRef(t, task))
|
||||
if err != nil || len(pins) != 1 || pins[0].Host != host || fmt.Sprint(pins[0].Port) != port {
|
||||
t.Fatalf("pins = %+v, %v", pins, err)
|
||||
}
|
||||
if code, _ := g.do(t, "POST", fmt.Sprintf("/api/tasks/%s/target-pins/%d/revoke", task.ID, pins[0].ID), ""); code != 200 {
|
||||
t.Fatalf("revoke: %d", code)
|
||||
}
|
||||
out, isErr = callTool(t, rendered, gatedCtx(task, 0), params, nil)
|
||||
if !isErr || !strings.Contains(out, "TARGET_APPROVAL_PENDING") || hits.Load() != 2 {
|
||||
t.Fatalf("revoked: error=%v out=%q hits=%d", isErr, out, hits.Load())
|
||||
}
|
||||
if again, toolName, _, ok := openTargetRequest(t, pg, task.ID, key); !ok || again == id || toolName != "probe_http" {
|
||||
t.Fatalf("after the revoke: open request %d (ok=%v, tool %q), want a new one", again, ok, toolName)
|
||||
}
|
||||
|
||||
// A URL whose target cannot be read is blocked, never sent.
|
||||
broken := g.s.buildCustomTool(&db.Tool{Key: "probe_broken", Kind: "http",
|
||||
Schema: json.RawMessage(`{"type":"object","properties":{"host":{"type":"string"}}}`),
|
||||
Exec: execJSON(t, map[string]any{"url": "http://{host}/x"})})
|
||||
out, isErr = callTool(t, broken, gatedCtx(task, 0), `{"host":"a b"}`, nil)
|
||||
if !isErr || !strings.Contains(out, "TARGET_GATE_UNAVAILABLE") {
|
||||
t.Fatalf("unreadable URL: error=%v out=%q", isErr, out)
|
||||
}
|
||||
}
|
||||
|
||||
// A custom command tool is gated on the command it renders, with a target
|
||||
// written in the template or formed by a parameter inside a host name. The
|
||||
// command does not run before an approval.
|
||||
func TestCustomCommandRenderedTargetGated(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
task := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
|
||||
pg := g.m.pg
|
||||
res := &countingResolver{answers: map[string][]string{"api.example.com": {"198.51.100.7"}}}
|
||||
g.m.interceptor.SetResolver(res)
|
||||
dir := t.TempDir()
|
||||
marker := filepath.Join(dir, "marker")
|
||||
ran := func() bool {
|
||||
_, err := os.Stat(marker)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
fixed := g.s.buildCustomTool(&db.Tool{Key: "probe_cmd_fixed", Kind: "command",
|
||||
Exec: execJSON(t, map[string]any{"command": "echo reached > '" + marker + "' && : http://198.51.100.7:8080/"})})
|
||||
glued := g.s.buildCustomTool(&db.Tool{Key: "probe_cmd", Kind: "command",
|
||||
Schema: json.RawMessage(`{"type":"object","properties":{"sub":{"type":"string"}}}`),
|
||||
Exec: execJSON(t, map[string]any{"command": "echo reached > '" + marker + "' && : http://{sub}.example.com:8080/"})})
|
||||
if got := intercept.ExtractTargets("probe_cmd", []byte(`{"sub":"api"}`)); len(got) != 0 {
|
||||
t.Fatalf("the hook reads %+v; the test needs a target only the rendering shows", got)
|
||||
}
|
||||
tc := &actool.ToolContext{WorkingDir: dir}
|
||||
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
tool actool.CoreTool
|
||||
input string
|
||||
key string
|
||||
}{
|
||||
{"template", fixed, `{}`, "198.51.100.7:8080"},
|
||||
{"parameter inside the host", glued, `{"sub":"api"}`, "api.example.com:8080"},
|
||||
} {
|
||||
out, isErr := callTool(t, c.tool, gatedCtx(task, 0), c.input, tc)
|
||||
if !isErr || !strings.Contains(out, "TARGET_APPROVAL_PENDING") || ran() {
|
||||
t.Fatalf("%s: error=%v out=%q ran=%v, want a block before the command runs", c.name, isErr, out, ran())
|
||||
}
|
||||
id, toolName, _, ok := openTargetRequest(t, pg, task.ID, c.key)
|
||||
if !ok || toolName != c.tool.Name() {
|
||||
t.Fatalf("%s: no open request on %s for %s", c.name, c.key, c.tool.Name())
|
||||
}
|
||||
if code, body := g.do(t, "POST", fmt.Sprintf("/api/intercept/pending/%d/decide", id), `{"decision":"allowed"}`); code != 200 {
|
||||
t.Fatalf("%s: decide: %d %v", c.name, code, body)
|
||||
}
|
||||
if out, isErr := callTool(t, c.tool, gatedCtx(task, 0), c.input, tc); isErr || !ran() {
|
||||
t.Fatalf("%s, approved: error=%v out=%q ran=%v", c.name, isErr, out, ran())
|
||||
}
|
||||
_ = os.Remove(marker)
|
||||
}
|
||||
if res.lookups() == 0 {
|
||||
t.Fatal("the name formed by the parameter was never resolved")
|
||||
}
|
||||
}
|
||||
|
||||
// The custom-tool test endpoint is the operator's own action (D14): it has no
|
||||
// approval scope, so the gate is not on there, whatever the tasks say.
|
||||
func TestCustomToolTestEndpointUngated(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
|
||||
_, hits, host, port := countingServer(t)
|
||||
before := 0
|
||||
if err := g.m.pg.QueryRow(`SELECT count(*) FROM intercept_pending WHERE decision_source='target' OR target_key IS NOT NULL`).Scan(&before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := fmt.Sprintf(`{"kind":"http","exec":{"url":"http://%s:{port}/test"},"params":{"port":%q}}`, host, port)
|
||||
code, out := g.do(t, "POST", "/api/tools/custom/test", body)
|
||||
if code != 200 || out["is_error"] != false || !strings.Contains(fmt.Sprint(out["output"]), "probe-reached") || hits.Load() != 1 {
|
||||
t.Fatalf("http: %d %v hits=%d", code, out, hits.Load())
|
||||
}
|
||||
code, out = g.do(t, "POST", "/api/tools/custom/test", `{"kind":"command","exec":{"command":": http://198.51.100.7:8080/ && echo ran-ungated"}}`)
|
||||
if code != 200 || !strings.Contains(fmt.Sprint(out["output"]), "ran-ungated") {
|
||||
t.Fatalf("command: %d %v", code, out)
|
||||
}
|
||||
after := 0
|
||||
if err := g.m.pg.QueryRow(`SELECT count(*) FROM intercept_pending WHERE decision_source='target' OR target_key IS NOT NULL`).Scan(&after); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after != before {
|
||||
t.Fatalf("the test endpoint wrote %d target rows", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// countingResolver answers from a table and counts every lookup.
|
||||
type countingResolver struct {
|
||||
mu sync.Mutex
|
||||
answers map[string][]string
|
||||
n int
|
||||
}
|
||||
|
||||
func (r *countingResolver) Name() string { return "counting" }
|
||||
|
||||
func (r *countingResolver) LookupIPs(_ context.Context, host string) ([]netip.Addr, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.n++
|
||||
var out []netip.Addr
|
||||
for _, ip := range r.answers[host] {
|
||||
out = append(out, netip.MustParseAddr(ip))
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("no answer for %s", host)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *countingResolver) lookups() int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.n
|
||||
}
|
||||
|
||||
// scriptedCall is one tool call of a toolScript.
|
||||
type scriptedCall struct{ tool, input string }
|
||||
|
||||
func fetches(urls ...string) []scriptedCall {
|
||||
calls := make([]scriptedCall, len(urls))
|
||||
for n, u := range urls {
|
||||
calls[n] = scriptedCall{"WebFetch", fmt.Sprintf(`{"url":%q}`, u)}
|
||||
}
|
||||
return calls
|
||||
}
|
||||
|
||||
// toolScript is a model that makes each of its calls in turn, one per turn,
|
||||
// then ends the run. It hands the result of every call to the test, and waits
|
||||
// before turn n while hold[n] is open.
|
||||
type toolScript struct {
|
||||
calls []scriptedCall
|
||||
hold map[int64]chan struct{}
|
||||
results chan string
|
||||
turn atomic.Int64
|
||||
}
|
||||
|
||||
func (p *toolScript) Stream(ctx context.Context, req llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
||||
n := p.turn.Add(1)
|
||||
if n > 1 {
|
||||
select {
|
||||
case p.results <- lastToolResult(req.Messages):
|
||||
default:
|
||||
}
|
||||
}
|
||||
if h := p.hold[n]; h != nil {
|
||||
select {
|
||||
case <-h:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}
|
||||
events := []llm.StreamEvent{{Type: llm.SETextDelta, Text: "done"}, {Type: llm.SEMessageDelta, StopReason: "end_turn"}}
|
||||
if int(n) <= len(p.calls) {
|
||||
call := p.calls[n-1]
|
||||
events = []llm.StreamEvent{
|
||||
{Type: llm.SEToolUseStart, ToolID: fmt.Sprintf("call-%d", n), ToolName: call.tool},
|
||||
{Type: llm.SEToolInputJSON, Text: call.input},
|
||||
{Type: llm.SEMessageDelta, StopReason: "tool_use"},
|
||||
}
|
||||
}
|
||||
return func(yield func(llm.StreamEvent, error) bool) {
|
||||
for _, event := range events {
|
||||
if !yield(event, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *toolScript) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
|
||||
return accumulateStreamForTest(ctx, p.Stream, req)
|
||||
}
|
||||
|
||||
func (p *toolScript) next(t *testing.T) string {
|
||||
t.Helper()
|
||||
select {
|
||||
case r := <-p.results:
|
||||
return r
|
||||
case <-time.After(60 * time.Second):
|
||||
t.Fatal("the worker did not hand back a tool result")
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// lastToolResult is the text of the last tool result the model is shown.
|
||||
func lastToolResult(msgs []llm.Message) string {
|
||||
for i := len(msgs) - 1; i >= 0; i-- {
|
||||
for _, b := range msgs[i].Content {
|
||||
if b.Type != llm.BlockToolResult {
|
||||
continue
|
||||
}
|
||||
var s strings.Builder
|
||||
if b.IsError {
|
||||
s.WriteString("[error] ")
|
||||
}
|
||||
for _, c := range b.Content {
|
||||
s.WriteString(c.Text)
|
||||
}
|
||||
return s.String()
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// runScriptedWorker claims one intent of task and runs it with a worker driven
|
||||
// by p, in the background. extra are tools the worker has besides its own.
|
||||
func runScriptedWorker(t *testing.T, g *gateServer, task *Task, p llm.Provider, extra ...actool.CoreTool) <-chan struct{} {
|
||||
t.Helper()
|
||||
if _, err := task.Store.AddIntent(map[string]any{"summary": "fetch the probe"}, 1, nil, "planner"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
intent, token := g.s.engine.claimNext(task, "work#1")
|
||||
if intent == nil {
|
||||
t.Fatal("no intent claimed")
|
||||
}
|
||||
worker := agent.NewWorker(p, "test-model", g.m.dir, nil, 0, 8, extra...)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
g.s.engine.runIntent(context.Background(), task, "work#1", worker, intent, token, "", "")
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(60 * time.Second):
|
||||
t.Error("the worker run did not end")
|
||||
}
|
||||
})
|
||||
return done
|
||||
}
|
||||
|
||||
// A gated task end to end: a worker's WebFetch waits on a request, a click
|
||||
// through the API pins the address and the call is made, the next call passes
|
||||
// on the pin, the pin is listed and revoked, and the call after asks again.
|
||||
func TestTargetGateEndToEnd(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
task := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
|
||||
pg := g.m.pg
|
||||
srv, hits, host, port := countingServer(t)
|
||||
key := net.JoinHostPort(host, port)
|
||||
target := srv.URL + "/e2e"
|
||||
p := &toolScript{calls: fetches(target, target, target), results: make(chan string, 16),
|
||||
hold: map[int64]chan struct{}{3: make(chan struct{})}}
|
||||
done := runScriptedWorker(t, g, task, p)
|
||||
|
||||
var id int64
|
||||
waitUntil(t, "the first request", func() bool {
|
||||
var ok bool
|
||||
id, _, _, ok = openTargetRequest(t, pg, task.ID, key)
|
||||
return ok
|
||||
})
|
||||
if hits.Load() != 0 {
|
||||
t.Fatal("fetched before the approval")
|
||||
}
|
||||
if code, body := g.do(t, "POST", fmt.Sprintf("/api/intercept/pending/%d/decide", id), `{"decision":"allowed"}`); code != 200 {
|
||||
t.Fatalf("decide: %d %v", code, body)
|
||||
}
|
||||
if r := p.next(t); !strings.Contains(r, "probe-reached") || hits.Load() != 1 {
|
||||
t.Fatalf("first call after the approval: %q hits=%d", r, hits.Load())
|
||||
}
|
||||
if r := p.next(t); !strings.Contains(r, "probe-reached") || hits.Load() != 2 {
|
||||
t.Fatalf("second call, pinned: %q hits=%d", r, hits.Load())
|
||||
}
|
||||
var requests int
|
||||
if err := pg.QueryRow(`SELECT count(*) FROM intercept_pending WHERE task_id=$1 AND target_key IS NOT NULL`, task.ID).Scan(&requests); err != nil || requests != 1 {
|
||||
t.Fatalf("%d requests, %v; want the one that was approved", requests, err)
|
||||
}
|
||||
|
||||
code, body := g.do(t, "GET", "/api/tasks/"+task.ID+"/target-pins", "")
|
||||
pins, _ := body["pins"].([]any)
|
||||
if code != 200 || len(pins) != 1 {
|
||||
t.Fatalf("pins: %d %v", code, body)
|
||||
}
|
||||
pin := pins[0].(map[string]any)
|
||||
if pin["host"] != host || fmt.Sprint(pin["port"]) != port || pin["status"] != "active" || fmt.Sprint(pin["pinned_ips"]) != "["+host+"]" {
|
||||
t.Fatalf("listed pin = %v", pin)
|
||||
}
|
||||
pinID, _ := pin["id"].(float64)
|
||||
if code, _ := g.do(t, "POST", fmt.Sprintf("/api/tasks/%s/target-pins/%d/revoke", task.ID, int64(pinID)), ""); code != 200 {
|
||||
t.Fatalf("revoke: %d", code)
|
||||
}
|
||||
close(p.hold[3])
|
||||
|
||||
var again int64
|
||||
waitUntil(t, "the request after the revoke", func() bool {
|
||||
var ok bool
|
||||
again, _, _, ok = openTargetRequest(t, pg, task.ID, key)
|
||||
return ok && again != id
|
||||
})
|
||||
if code, body := g.do(t, "POST", fmt.Sprintf("/api/intercept/pending/%d/decide", again), `{"decision":"denied"}`); code != 200 {
|
||||
t.Fatalf("deny: %d %v", code, body)
|
||||
}
|
||||
if r := p.next(t); !strings.Contains(r, "TARGET_DENIED") || hits.Load() != 2 {
|
||||
t.Fatalf("third call, revoked then denied: %q hits=%d", r, hits.Load())
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(60 * time.Second):
|
||||
t.Fatal("the worker run did not end")
|
||||
}
|
||||
}
|
||||
|
||||
// A custom tool in a gated worker session. A deny rule on the call decides
|
||||
// first, at the hook: the tool never runs, so its rendered target is never
|
||||
// read, resolved or requested. The next call, which no rule refuses, is gated
|
||||
// inside the tool on the URL it renders.
|
||||
func TestDenyRuleWinsBeforeRenderedTarget(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
task := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
|
||||
pg, ic := g.m.pg, g.m.interceptor
|
||||
_, hits, _, port := countingServer(t)
|
||||
res := &countingResolver{answers: map[string][]string{"api.example.com": {"127.0.0.1"}}}
|
||||
ic.SetResolver(res)
|
||||
|
||||
prior, err := ic.GetEnabledTools()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ic.SetEnabledTools(prior) })
|
||||
if err := ic.SetEnabledTools(append(slices.Clone(prior), "probe_http")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const sentinel = "ARTEX_RENDERED_DENY_PROBE"
|
||||
rule, err := pg.CreateInterceptRule("rendered deny probe", "tool_input", "string", sentinel, "deny", "", 1000000, true, false, 0, "deny")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = pg.DeleteInterceptRule(rule.ID)
|
||||
ic.Invalidate()
|
||||
})
|
||||
ic.Invalidate()
|
||||
|
||||
tool := g.s.buildCustomTool(&db.Tool{Key: "probe_http", Kind: "http",
|
||||
Schema: json.RawMessage(`{"type":"object","properties":{"sub":{"type":"string"},"note":{"type":"string"}}}`),
|
||||
Exec: execJSON(t, map[string]any{"url": "http://{sub}.example.com:" + port + "/deny"})})
|
||||
p := &toolScript{calls: []scriptedCall{
|
||||
{"probe_http", `{"sub":"api","note":"` + sentinel + `"}`},
|
||||
{"probe_http", `{"sub":"api"}`},
|
||||
}, results: make(chan string, 16), hold: map[int64]chan struct{}{2: make(chan struct{})}}
|
||||
done := runScriptedWorker(t, g, task, p, tool)
|
||||
|
||||
if r := p.next(t); !strings.Contains(r, "intercepted by the platform") || strings.Contains(r, "TARGET_") {
|
||||
t.Fatalf("call refused by the rule: %q", r)
|
||||
}
|
||||
if n, rows := res.lookups(), targetRowCount(t, pg, task.ID); n != 0 || rows != 0 || hits.Load() != 0 {
|
||||
t.Fatalf("after the deny: %d lookups, %d target rows, %d requests sent; want none", n, rows, hits.Load())
|
||||
}
|
||||
close(p.hold[2])
|
||||
|
||||
key := "api.example.com:" + port
|
||||
var id int64
|
||||
waitUntil(t, "the request for the rendered target", func() bool {
|
||||
var ok bool
|
||||
var tool string
|
||||
id, tool, _, ok = openTargetRequest(t, pg, task.ID, key)
|
||||
return ok && tool == "probe_http"
|
||||
})
|
||||
if code, body := g.do(t, "POST", fmt.Sprintf("/api/intercept/pending/%d/decide", id), `{"decision":"denied"}`); code != 200 {
|
||||
t.Fatalf("deny: %d %v", code, body)
|
||||
}
|
||||
if r := p.next(t); !strings.Contains(r, "TARGET_DENIED") || hits.Load() != 0 || res.lookups() == 0 {
|
||||
t.Fatalf("rendered target: %q, %d requests sent, %d lookups", r, hits.Load(), res.lookups())
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(60 * time.Second):
|
||||
t.Fatal("the worker run did not end")
|
||||
}
|
||||
}
|
||||
|
||||
// Documents a known limit: script tools stay advisory (D9). A custom script
|
||||
// tool's target is in its stored code, which neither the hook nor the tool
|
||||
// reads, so in a gated task the call is allowed and the script reaches a target
|
||||
// nobody approved: no lookup, no request, no row. This test changes only if
|
||||
// that decision changes.
|
||||
func TestBypass_CustomScriptToolTargetNotGated(t *testing.T) {
|
||||
if detectPython() == "" {
|
||||
t.Skip("python3 unavailable")
|
||||
}
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
task := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
|
||||
res := &countingResolver{answers: map[string][]string{"localhost": {"127.0.0.1"}}}
|
||||
g.m.interceptor.SetResolver(res)
|
||||
_, hits, _, port := countingServer(t)
|
||||
|
||||
code := "import urllib.request\n" +
|
||||
"opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))\n" +
|
||||
"print(opener.open('http://localhost:" + port + "/script', timeout=20).read().decode())\n"
|
||||
tool := g.s.buildCustomTool(&db.Tool{Key: "probe_script", Kind: "script", Exec: execJSON(t, map[string]any{"code": code})})
|
||||
// The gate sees no target in the call: only the parameters reach the hook.
|
||||
if got := intercept.ExtractTargets("probe_script", []byte(`{}`)); len(got) != 0 {
|
||||
t.Fatalf("the hook reads %+v; the test needs a target only the stored code names", got)
|
||||
}
|
||||
|
||||
p := &toolScript{calls: []scriptedCall{{"probe_script", `{}`}}, results: make(chan string, 16)}
|
||||
done := runScriptedWorker(t, g, task, p, tool)
|
||||
if r := p.next(t); !strings.Contains(r, "probe-reached") || strings.Contains(r, "TARGET_") {
|
||||
t.Fatalf("script tool in a gated task: %q, want it allowed and the target reached", r)
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(60 * time.Second):
|
||||
t.Fatal("the worker run did not end")
|
||||
}
|
||||
if hits.Load() != 1 {
|
||||
t.Fatalf("%d requests reached the unapproved target, want 1 (the gap)", hits.Load())
|
||||
}
|
||||
if n, rows := res.lookups(), targetRowCount(t, g.m.pg, task.ID); n != 0 || rows != 0 {
|
||||
t.Fatalf("%d lookups, %d target rows; the gate did see the script's target", n, rows)
|
||||
}
|
||||
}
|
||||
|
||||
// A task with the gate off (the default) runs as before: a worker's WebFetch
|
||||
// and the custom tools reach their targets, and the gate writes no row and
|
||||
// resolves no name.
|
||||
func TestGateOffTaskIsUnchanged(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
task := g.newTask(t, db.TaskCreateOptions{})
|
||||
res := &countingResolver{answers: map[string][]string{"localhost": {"127.0.0.1"}}}
|
||||
g.m.interceptor.SetResolver(res)
|
||||
_, hits, _, port := countingServer(t)
|
||||
byName := "http://localhost:" + port
|
||||
|
||||
p := &toolScript{calls: fetches(byName+"/a", byName+"/b"), results: make(chan string, 16)}
|
||||
done := runScriptedWorker(t, g, task, p)
|
||||
for n := 1; n <= 2; n++ {
|
||||
if r := p.next(t); !strings.Contains(r, "probe-reached") {
|
||||
t.Fatalf("WebFetch %d: %q", n, r)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(60 * time.Second):
|
||||
t.Fatal("the worker run did not end")
|
||||
}
|
||||
|
||||
httpTool := g.s.buildCustomTool(&db.Tool{Key: "probe_http", Kind: "http",
|
||||
Schema: json.RawMessage(`{"type":"object","properties":{"port":{"type":"string"}}}`),
|
||||
Exec: execJSON(t, map[string]any{"url": "http://localhost:{port}/c"})})
|
||||
if out, isErr := callTool(t, httpTool, gatedCtx(task, intercept.DefaultTargetWaitBound), fmt.Sprintf(`{"port":%q}`, port), nil); isErr || !strings.Contains(out, "probe-reached") {
|
||||
t.Fatalf("custom http tool: error=%v out=%q", isErr, out)
|
||||
}
|
||||
cmdTool := g.s.buildCustomTool(&db.Tool{Key: "probe_cmd", Kind: "command",
|
||||
Exec: execJSON(t, map[string]any{"command": ": http://localhost:" + port + "/ && echo ran-gate-off"})})
|
||||
if out, isErr := callTool(t, cmdTool, gatedCtx(task, intercept.DefaultTargetWaitBound), `{}`, &actool.ToolContext{WorkingDir: t.TempDir()}); isErr || !strings.Contains(out, "ran-gate-off") {
|
||||
t.Fatalf("custom command tool: error=%v out=%q", isErr, out)
|
||||
}
|
||||
|
||||
if hits.Load() != 3 {
|
||||
t.Fatalf("%d requests reached the server, want 3", hits.Load())
|
||||
}
|
||||
if n := targetRowCount(t, g.m.pg, task.ID); n != 0 {
|
||||
t.Fatalf("%d target rows, want none", n)
|
||||
}
|
||||
if n := res.lookups(); n != 0 {
|
||||
t.Fatalf("%d lookups, want none", n)
|
||||
}
|
||||
}
|
||||
@@ -128,7 +128,7 @@ func TestRunHTTPTool(t *testing.T) {
|
||||
"headers": map[string]string{"Authorization": "Bearer {token}"},
|
||||
"body": `{"flag":"{flag}"}`,
|
||||
})
|
||||
res, err := (&Server{}).runHTTPTool(context.Background(), execRaw,
|
||||
res, err := (&Server{}).runHTTPTool(context.Background(), "probe", execRaw,
|
||||
map[string]any{"flag": "CTF{x}", "token": "sekret"}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("runHTTPTool: %v", err)
|
||||
@@ -176,7 +176,7 @@ func TestHTTPToolTruncatesLargeBody(t *testing.T) {
|
||||
defer srv.Close()
|
||||
|
||||
execRaw, _ := json.Marshal(map[string]any{"method": "GET", "url": srv.URL})
|
||||
res, err := (&Server{}).runHTTPTool(context.Background(), execRaw, map[string]any{}, nil)
|
||||
res, err := (&Server{}).runHTTPTool(context.Background(), "probe", execRaw, map[string]any{}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("runHTTPTool: %v", err)
|
||||
}
|
||||
|
||||
@@ -379,6 +379,11 @@ func (s *Server) interceptDecide(w http.ResponseWriter, r *http.Request) {
|
||||
writeErr(w, 409, err.Error())
|
||||
return
|
||||
}
|
||||
if errors.Is(err, db.ErrTargetPinEmpty) {
|
||||
// Refused, not failed: the request stays open and nothing was pinned.
|
||||
writeErr(w, 422, err.Error())
|
||||
return
|
||||
}
|
||||
writeErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
@@ -463,6 +468,61 @@ func (s *Server) revokeTargetPin(w http.ResponseWriter, r *http.Request, scope d
|
||||
writeJSON(w, 200, pin)
|
||||
}
|
||||
|
||||
// --- target-gate exemptions (hosts that need no target approval, global) ---
|
||||
|
||||
// targetGateGetExemptions returns the stored exemption list with its digest,
|
||||
// and the exemptions derived from the configuration (LLM gateways, search
|
||||
// backend, platform proxies), which are computed and cannot be edited.
|
||||
func (s *Server) targetGateGetExemptions(w http.ResponseWriter, r *http.Request) {
|
||||
list, digest, err := s.m.interceptor.Exemptions()
|
||||
if err != nil {
|
||||
writeErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
if list == nil {
|
||||
list = []db.TargetExemption{}
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"exemptions": list, "digest": digest, "derived": s.m.interceptor.DerivedExemptions()})
|
||||
}
|
||||
|
||||
// targetGateSetExemptions replaces the stored exemption list. Entries are an
|
||||
// exact host and port. The change is appended to the exemption log with the
|
||||
// approver of the session.
|
||||
func (s *Server) targetGateSetExemptions(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Exemptions []db.TargetExemption `json:"exemptions"`
|
||||
}
|
||||
if err := decode(r, &req); err != nil {
|
||||
writeErr(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
list, digest, err := s.m.interceptor.SetExemptions(req.Exemptions, s.approver(r))
|
||||
if err != nil {
|
||||
if errors.Is(err, intercept.ErrInvalidExemption) {
|
||||
writeErr(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
writeErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"exemptions": list, "digest": digest})
|
||||
}
|
||||
|
||||
// targetGateExemptionHistory returns the most recent changes of the exemption
|
||||
// list, newest first.
|
||||
func (s *Server) targetGateExemptionHistory(w http.ResponseWriter, r *http.Request) {
|
||||
pg := s.pg(w)
|
||||
if pg == nil {
|
||||
return
|
||||
}
|
||||
changes, err := pg.ListTargetExemptionLog(100)
|
||||
if err != nil {
|
||||
writeErr(w, 500, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, map[string]any{"changes": changes})
|
||||
}
|
||||
|
||||
// --- tool-config (global tool interception scope) ---
|
||||
|
||||
// interceptGetToolConfig returns the list of tool names that are currently
|
||||
|
||||
@@ -404,6 +404,8 @@ func NewManager(dir, proxyAddr string) (*Manager, error) {
|
||||
m := &Manager{dir: dir, pg: pg, assets: pg.Assets(), tasks: map[string]*Task{}, interceptor: intercept.New(pg)}
|
||||
// Behind the global egress proxy a target approval is name-only.
|
||||
m.interceptor.SetUpstreamProxyCheck(func() bool { return m.GlobalProxy() != "" })
|
||||
// The platform's own endpoints need no target approval.
|
||||
m.interceptor.SetDerivedExemptions(m.targetGateEndpoints)
|
||||
if proxyAddr != "" {
|
||||
tr, err := traffic.Open(filepath.Join(dir, "traffic"), proxyAddr)
|
||||
if err != nil {
|
||||
@@ -892,6 +894,32 @@ func (t *Task) targetScope(wait time.Duration) intercept.TargetScope {
|
||||
return intercept.TargetScope{TaskID: t.ID, Enabled: t.targetGate.Load, WaitBound: wait, Interceptor: t.interceptor}
|
||||
}
|
||||
|
||||
// searchBackendEndpoints are the endpoints of the search backends. The
|
||||
// "deepseek" backend has none of its own: it uses the active LLM profile.
|
||||
var searchBackendEndpoints = map[string]string{
|
||||
"ddgs": "https://html.duckduckgo.com",
|
||||
"brave-free": "https://api.search.brave.com",
|
||||
"tavily": "https://api.tavily.com",
|
||||
}
|
||||
|
||||
// targetGateEndpoints lists the endpoints of the platform's own services, which
|
||||
// the target gate exempts from approval: every LLM gateway, the search backend
|
||||
// in use, and the proxies an agent may name on a command line (the recording
|
||||
// proxy, the global egress proxy, the search proxy and each profile's proxy).
|
||||
// It is read on every gated call, so the exemptions follow the configuration.
|
||||
func (m *Manager) targetGateEndpoints() []string {
|
||||
var out []string
|
||||
profiles, _ := m.pg.ListProfiles()
|
||||
for _, p := range profiles {
|
||||
out = append(out, p.BaseURL, p.Proxy)
|
||||
}
|
||||
on, backend, _, _, searchProxy := m.WebSearch()
|
||||
if on {
|
||||
out = append(out, searchBackendEndpoints[backend])
|
||||
}
|
||||
return append(out, searchProxy, m.ProxyAddr(), m.GlobalProxy())
|
||||
}
|
||||
|
||||
// UpdateTaskMetadata changes list-only task metadata without interrupting any
|
||||
// planner, main-agent, or worker call.
|
||||
func (m *Manager) UpdateTaskMetadata(taskID string, patch pgdb.TaskPatch) (*Task, error) {
|
||||
|
||||
+23
-1
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
"github.com/Autumn-27/norma/permission"
|
||||
actool "github.com/Autumn-27/norma/tool"
|
||||
)
|
||||
@@ -253,7 +254,7 @@ func (s *Server) toolSpawnTask() actool.CoreTool {
|
||||
"plan_heartbeat_seconds": map[string]any{"type": "integer", "description": "optional: planner heartbeat interval (seconds). When this much time has passed since the last planning round ended/the task started with no trigger in between → run one planning round (deadlock backstop + wakes the planner to supervise in-flight workers). Empty or 0 = default 600 (10min);"},
|
||||
"seed_first_intent": map[string]any{"type": "boolean", "description": "optional: enable for simple tasks to issue one seed intent at creation (content = description + goal) so a worker can start testing right away without waiting for the first planner round; default false (the standard plan-then-execute flow)."},
|
||||
}, "description", "goal"),
|
||||
func(_ context.Context, in json.RawMessage) (actool.Result, error) {
|
||||
func(ctx context.Context, in json.RawMessage) (actool.Result, error) {
|
||||
var a struct {
|
||||
Description string `json:"description"`
|
||||
Goal string `json:"goal"`
|
||||
@@ -307,11 +308,32 @@ func (s *Server) toolSpawnTask() actool.CoreTool {
|
||||
if pin != nil {
|
||||
llmIDs = []int64{*pin}
|
||||
}
|
||||
// Target gate: a child spawned from a gated scope is gated too, and keeps
|
||||
// the block rules of the calling task. Read from the caller's scope, never
|
||||
// from parent_ref (optional and written by the model). Pins and allow rules
|
||||
// are never copied. Gate-off caller or no scope: created as before.
|
||||
var gate *bool
|
||||
var blockRules []db.TaskInterceptRuleInput
|
||||
if scope, ok := intercept.TargetScopeFromContext(ctx); ok && scope.Enabled != nil && scope.Enabled() {
|
||||
on := true
|
||||
gate = &on
|
||||
if callerID, err := strconv.ParseInt(scope.TaskID, 10, 64); err == nil && callerID > 0 {
|
||||
block, _, err := s.m.Assets().TaskInterceptRulesSplit(callerID)
|
||||
if err != nil {
|
||||
return actool.Errorf("read the block rules of the calling task: " + err.Error()), nil
|
||||
}
|
||||
for _, r := range block {
|
||||
blockRules = append(blockRules, db.TaskInterceptRuleInput{Enabled: r.Enabled, Action: r.Action, Kind: r.Kind, Pattern: r.Pattern, Note: r.Note})
|
||||
}
|
||||
}
|
||||
}
|
||||
t, err := s.m.CreateTaskWithOptions(a.Description, a.Goal, db.TaskCreateOptions{
|
||||
SourceTaskIDs: sourceIDs,
|
||||
LLMProfileIDs: llmIDs,
|
||||
TimeoutSeconds: a.TimeoutSeconds,
|
||||
PlanHeartbeatSeconds: a.PlanHeartbeatSeconds,
|
||||
InterceptRules: blockRules,
|
||||
TargetGate: gate,
|
||||
})
|
||||
if err != nil {
|
||||
return actool.Errorf(err.Error()), nil
|
||||
|
||||
@@ -147,6 +147,9 @@ type triggeredRun struct {
|
||||
taskDesc string // task description (task-level, same across one task); rendered once when merged
|
||||
taskGoal string // task goal (task-level, same across one task); rendered once when merged
|
||||
mergeable bool // true for finding/goal event triggers (merge by taskID)
|
||||
// targetGate: the source task had the target gate on when the event fired; a
|
||||
// merged run is gated if any of its fires is. Never carries pins.
|
||||
targetGate bool
|
||||
}
|
||||
|
||||
func New(ctx context.Context, m *Manager, skillDir string, dataDir string, keyDir string, opts ...Option) *Server {
|
||||
@@ -961,6 +964,9 @@ func (s *Server) Handler() http.Handler {
|
||||
mux.HandleFunc("GET /api/intercept/task/{taskID}", s.interceptListTaskItems)
|
||||
mux.HandleFunc("GET /api/intercept/tool-config", s.interceptGetToolConfig)
|
||||
mux.HandleFunc("PUT /api/intercept/tool-config", s.interceptSetToolConfig)
|
||||
mux.HandleFunc("GET /api/target-gate/exemptions", s.targetGateGetExemptions)
|
||||
mux.HandleFunc("PUT /api/target-gate/exemptions", s.targetGateSetExemptions)
|
||||
mux.HandleFunc("GET /api/target-gate/exemptions/history", s.targetGateExemptionHistory)
|
||||
mux.HandleFunc("GET /api/intercept/judge", s.interceptGetJudgeConfig)
|
||||
mux.HandleFunc("PUT /api/intercept/judge", s.interceptSetJudgeConfig)
|
||||
mux.HandleFunc("GET /api/intercept/judge/usage", s.interceptJudgeUsage) // cumulative token usage of fallback approval
|
||||
|
||||
+246
-21
@@ -5,16 +5,19 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"iter"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
"github.com/Autumn-27/norma/llm"
|
||||
)
|
||||
|
||||
// Target-gate opt-in flag, pins and decisions over the real mux and the real
|
||||
@@ -180,39 +183,261 @@ func TestCreateTaskAcceptsTargetGate(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// No inheritance: a task spawned by the orchestration agent starts with the
|
||||
// gate off, even from a gated parent.
|
||||
func TestSpawnedTaskStartsWithGateOff(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
parent := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
|
||||
if !parent.targetScope(0).Enabled() {
|
||||
t.Fatal("control: the parent was expected to have the gate on")
|
||||
}
|
||||
in, _ := json.Marshal(map[string]any{"description": "spawned child", "goal": gateTestGoal, "parent_ref": parent.ID, "source_task_ids": []string{parent.ID}})
|
||||
res, err := g.s.toolSpawnTask().Call(context.Background(), in, nil)
|
||||
// ruleLines renders a task's intercept rules for comparison.
|
||||
func ruleLines(t *testing.T, as *db.AssetStore, taskID string) []string {
|
||||
t.Helper()
|
||||
n, _ := strconv.ParseInt(taskID, 10, 64)
|
||||
rules, err := as.ListTaskInterceptRules(n)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
childID, found := strings.CutPrefix(res.Flatten(), "task created: ")
|
||||
if !found {
|
||||
t.Fatalf("spawn_task result: %q", res.Flatten())
|
||||
var out []string
|
||||
for _, r := range rules {
|
||||
out = append(out, fmt.Sprintf("%s %s %s enabled=%v %q", r.Action, r.Kind, r.Pattern, r.Enabled, r.Note))
|
||||
}
|
||||
g.cleanupTask(t, childID)
|
||||
return out
|
||||
}
|
||||
|
||||
// checkSpawnedChild checks the target-gate state of a spawned task: the flag,
|
||||
// live and stored, no pin, and its own intercept rules.
|
||||
func (g *gateServer) checkSpawnedChild(t *testing.T, childID string, gate bool, rules []string) {
|
||||
t.Helper()
|
||||
child, ok := g.m.Task(childID)
|
||||
if !ok || child.ParentRef != parent.ID {
|
||||
t.Fatalf("spawned task = %+v (found=%v), want a child of %s", child, ok, parent.ID)
|
||||
if !ok {
|
||||
t.Fatalf("spawned task %s not found", childID)
|
||||
}
|
||||
if child.targetScope(0).Enabled() {
|
||||
t.Fatal("the spawned task inherited the gate")
|
||||
if child.targetScope(0).Enabled() != gate {
|
||||
t.Fatalf("spawned task gate = %v, want %v", !gate, gate)
|
||||
}
|
||||
n, _ := strconv.ParseInt(childID, 10, 64)
|
||||
if stored, err := g.m.pg.GetTask(n); err != nil || stored == nil || stored.TargetGate {
|
||||
t.Fatalf("stored child = %+v, %v; want the gate off", stored, err)
|
||||
if stored, err := g.m.pg.GetTask(n); err != nil || stored == nil || stored.TargetGate != gate {
|
||||
t.Fatalf("stored child = %+v, %v; want the gate %v", stored, err, gate)
|
||||
}
|
||||
if pins, err := g.m.pg.ListTargetPins(db.TargetScopeRef{TaskID: n}); err != nil || len(pins) != 0 {
|
||||
t.Fatalf("the spawned task has pins: %+v, %v", pins, err)
|
||||
}
|
||||
if got := ruleLines(t, g.m.Assets(), childID); !slices.Equal(got, rules) {
|
||||
t.Fatalf("spawned task rules = %q, want %q", got, rules)
|
||||
}
|
||||
}
|
||||
|
||||
// A task spawned through spawn_task takes the target gate of the calling scope
|
||||
// (D23) and, from a gated task, that task's block rules (D24). It never takes
|
||||
// pins or allow rules, and parent_ref plays no part. A gate-off caller and a
|
||||
// call with no scope create the child as before: gate off, no rules.
|
||||
func TestSpawnedTaskInheritsGateFromCaller(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
parentRules := []db.TaskInterceptRuleInput{
|
||||
{Enabled: true, Action: "block", Kind: "exact_domain", Pattern: "blocked.example.com", Note: "out of scope"},
|
||||
{Enabled: false, Action: "block", Kind: "cidr", Pattern: "198.51.100.0/24"},
|
||||
{Enabled: true, Action: "allow", Kind: "exact_domain", Pattern: "allowed.example.com"},
|
||||
}
|
||||
gated := g.newTask(t, db.TaskCreateOptions{TargetGate: &on, InterceptRules: parentRules})
|
||||
open := g.newTask(t, db.TaskCreateOptions{InterceptRules: parentRules})
|
||||
if !gated.targetScope(0).Enabled() || open.targetScope(0).Enabled() {
|
||||
t.Fatal("control: one parent gated, the other not")
|
||||
}
|
||||
// The gated parent holds one pin, which no child may get.
|
||||
if p, ok, err := g.m.pg.ResolveTargetRequest(openTargetRow(t, g.m.pg, taskRef(t, gated), "app.example.com", "203.0.113.7"), true, "ARTEX", "192.0.2.1"); err != nil || !ok || p == nil {
|
||||
t.Fatalf("pin the parent: %+v ok=%v err=%v", p, ok, err)
|
||||
}
|
||||
conv, err := g.m.pg.CreateConversationGated("auto", "spawn from a gated chat", nil, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = g.m.pg.DeleteConversation(conv.ID) })
|
||||
gatedChat := intercept.TargetScope{ConversationID: conv.ID, Enabled: func() bool { return true },
|
||||
WaitBound: intercept.DefaultTargetWaitBound, Interceptor: g.m.interceptor}
|
||||
scoped := func(s intercept.TargetScope) context.Context {
|
||||
return intercept.WithTargetScope(context.Background(), s)
|
||||
}
|
||||
blockOnly := []string{
|
||||
`block exact_domain blocked.example.com enabled=true "out of scope"`,
|
||||
`block cidr 198.51.100.0/24 enabled=false ""`,
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
ctx context.Context
|
||||
parentRef string
|
||||
gate bool
|
||||
rules []string
|
||||
}{
|
||||
{"gated task", scoped(gated.targetScope(intercept.DefaultTargetWaitBound)), "", true, blockOnly},
|
||||
{"gated task, parent_ref names a gate-off task", scoped(gated.targetScope(0)), open.ID, true, blockOnly},
|
||||
{"gated conversation", scoped(gatedChat), "", true, nil},
|
||||
{"gate-off task", scoped(open.targetScope(0)), "", false, nil},
|
||||
{"gate-off task, parent_ref names a gated task", scoped(open.targetScope(0)), gated.ID, false, nil},
|
||||
{"no scope, parent_ref names a gated task", context.Background(), gated.ID, false, nil},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
args := map[string]any{"description": "spawned child", "goal": gateTestGoal}
|
||||
if tc.parentRef != "" {
|
||||
args["parent_ref"] = tc.parentRef
|
||||
}
|
||||
in, _ := json.Marshal(args)
|
||||
res, err := g.s.toolSpawnTask().Call(tc.ctx, in, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
childID, found := strings.CutPrefix(res.Flatten(), "task created: ")
|
||||
if !found {
|
||||
t.Fatalf("spawn_task result: %q", res.Flatten())
|
||||
}
|
||||
g.cleanupTask(t, childID)
|
||||
if child, _ := g.m.Task(childID); child == nil || child.ParentRef != tc.parentRef {
|
||||
t.Fatalf("spawned task = %+v, want parent_ref %q", child, tc.parentRef)
|
||||
}
|
||||
g.checkSpawnedChild(t, childID, tc.gate, tc.rules)
|
||||
})
|
||||
}
|
||||
// The callers are unchanged.
|
||||
if pins, err := g.m.pg.ListTargetPins(taskRef(t, gated)); err != nil || len(pins) != 1 {
|
||||
t.Fatalf("parent pins = %+v, %v; want still one", pins, err)
|
||||
}
|
||||
if got := ruleLines(t, g.m.Assets(), gated.ID); len(got) != 3 {
|
||||
t.Fatalf("parent rules = %q, want the three it was created with", got)
|
||||
}
|
||||
}
|
||||
|
||||
// toolCallProvider is a model that makes one tool call on its first turn and
|
||||
// ends the run on the next.
|
||||
type toolCallProvider struct {
|
||||
tool, input string
|
||||
turn atomic.Int64
|
||||
}
|
||||
|
||||
func (p *toolCallProvider) Stream(context.Context, llm.CompletionRequest) iter.Seq2[llm.StreamEvent, error] {
|
||||
events := []llm.StreamEvent{{Type: llm.SETextDelta, Text: "done"}, {Type: llm.SEMessageDelta, StopReason: "end_turn"}}
|
||||
if p.turn.Add(1) == 1 {
|
||||
events = []llm.StreamEvent{
|
||||
{Type: llm.SEToolUseStart, ToolID: "call-1", ToolName: p.tool},
|
||||
{Type: llm.SEToolInputJSON, Text: p.input},
|
||||
{Type: llm.SEMessageDelta, StopReason: "tool_use"},
|
||||
}
|
||||
}
|
||||
return func(yield func(llm.StreamEvent, error) bool) {
|
||||
for _, event := range events {
|
||||
if !yield(event, nil) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *toolCallProvider) Complete(ctx context.Context, req llm.CompletionRequest) (llm.Message, string, llm.Usage, error) {
|
||||
return accumulateStreamForTest(ctx, p.Stream, req)
|
||||
}
|
||||
|
||||
// spawn_task called by the model of an auto conversation, through the real
|
||||
// harness, tool binding and tool context: the child takes the conversation's
|
||||
// flag. The unit test above builds the caller's context by hand; this one shows
|
||||
// the conversation's scope reaches the tool.
|
||||
func TestSpawnTaskSeesCallerScopeThroughHarness(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
for _, gate := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("conversation gate %v", gate), func(t *testing.T) {
|
||||
desc := fmt.Sprintf("harness child %v %d", gate, time.Now().UnixNano())
|
||||
in, _ := json.Marshal(map[string]any{"description": desc, "goal": gateTestGoal})
|
||||
conv, err := g.m.pg.CreateConversationGated("auto", "spawn through the harness", nil, gate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = g.m.pg.DeleteConversation(conv.ID) })
|
||||
g.s.cfgMu.Lock()
|
||||
g.s.chatAgent = agent.NewChatAgent(&toolCallProvider{tool: "spawn_task", input: string(in)}, "test-model", g.m.dir, nil, 10000)
|
||||
g.s.cfgMu.Unlock()
|
||||
// The server context of this fixture is cancelled; the turn gets its own.
|
||||
ctx, cancel := context.WithCancelCause(intercept.WithConvID(context.Background(), conv.ID))
|
||||
g.s.runConversationTurn(ctx, cancel, conv, "spawn the child task", g.s.convBusyKey(conv.ID))
|
||||
var childID string
|
||||
for _, task := range g.m.List() {
|
||||
if task.Description == desc {
|
||||
childID = task.ID
|
||||
}
|
||||
}
|
||||
if childID == "" {
|
||||
t.Fatal("the model's spawn_task call created no task")
|
||||
}
|
||||
g.cleanupTask(t, childID)
|
||||
g.checkSpawnedChild(t, childID, gate, nil)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A conversation started by a trigger of a gated task is gated (D25): a single
|
||||
// fire, a by-task merge, and a merge across tasks of which one is gated. A fire
|
||||
// of a gate-off task, or with no task, starts it gate off as before. The flag is
|
||||
// copied, never pins.
|
||||
func TestTriggeredConversationInheritsGate(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
g.s.cfgMu.Lock()
|
||||
g.s.chatAgent = agent.NewChatAgent(newScopeProbeProvider(), "test-model", g.m.dir, nil, 10000)
|
||||
g.s.cfgMu.Unlock()
|
||||
on := true
|
||||
gated, open := g.newTask(t, db.TaskCreateOptions{TargetGate: &on}), g.newTask(t, db.TaskCreateOptions{})
|
||||
if p, ok, err := g.m.pg.ResolveTargetRequest(openTargetRow(t, g.m.pg, taskRef(t, gated), "app.example.com", "203.0.113.7"), true, "ARTEX", "192.0.2.1"); err != nil || !ok || p == nil {
|
||||
t.Fatalf("pin the gated task: %+v ok=%v err=%v", p, ok, err)
|
||||
}
|
||||
gatedID, openID := taskRef(t, gated).TaskID, taskRef(t, open).TaskID
|
||||
|
||||
// The server context is cancelled, so StartTriggeredRun only queues. The test
|
||||
// takes the next run with the merge mode under test and runs it as the pump would.
|
||||
trigger := func(t *testing.T, mode string, taskIDs ...int64) *db.Conversation {
|
||||
t.Helper()
|
||||
key := fmt.Sprintf("gate-trigger-%d", time.Now().UnixNano())
|
||||
for _, id := range taskIDs {
|
||||
g.s.StartTriggeredRun(key, "trigger", "event body", id, id > 0, "desc", "goal")
|
||||
}
|
||||
g.s.queueMu.Lock()
|
||||
item := g.s.nextTriggerRun(key, triggerBehavior{runMode: "serial", mergeMode: mode})
|
||||
left := len(g.s.triggerQ[key])
|
||||
delete(g.s.triggerQ, key)
|
||||
delete(g.s.triggerCfg, key)
|
||||
g.s.queueMu.Unlock()
|
||||
if left != 0 {
|
||||
t.Fatalf("%d fires left in the queue, want one run", left)
|
||||
}
|
||||
item.title = key
|
||||
g.s.runTriggeredRun(item)
|
||||
convs, err := g.m.pg.ListConversations()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range convs {
|
||||
if c.Title == key {
|
||||
t.Cleanup(func() { _ = g.m.pg.DeleteConversation(c.ID) })
|
||||
return c
|
||||
}
|
||||
}
|
||||
t.Fatal("the trigger created no conversation")
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
mode string
|
||||
tasks []int64
|
||||
gate bool
|
||||
}{
|
||||
{"gated task", "none", []int64{gatedID}, true},
|
||||
{"gated task, merged by task", "by_task", []int64{gatedID, gatedID}, true},
|
||||
{"merge across tasks, gated second", "all", []int64{openID, gatedID}, true},
|
||||
{"gate-off task", "none", []int64{openID}, false},
|
||||
{"gate-off task, merged by task", "by_task", []int64{openID, openID}, false},
|
||||
{"no task", "none", []int64{0}, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
c := trigger(t, tc.mode, tc.tasks...)
|
||||
if c.TargetGate != tc.gate {
|
||||
t.Fatalf("triggered conversation gate = %v, want %v", c.TargetGate, tc.gate)
|
||||
}
|
||||
if pins, err := g.m.pg.ListTargetPins(db.TargetScopeRef{ConversationID: c.ID}); err != nil || len(pins) != 0 {
|
||||
t.Fatalf("the triggered conversation has pins: %+v, %v", pins, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The conversation flag is set on create and on patch, and the turn that starts
|
||||
|
||||
@@ -382,7 +382,7 @@ func TestFinalPlannerRoundHasZeroWait(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The gate seam is a no-op: through the task's target-only hooks a call is
|
||||
// With the gate off (the task default) the target-only hooks are a no-op: a call is
|
||||
// allowed and nothing is written — no intercept row, no audit-ring entry. The
|
||||
// control runs the same call through the task's full Guard, where a deny rule
|
||||
// blocks it and logs a row, so the rule and the row count are known to be live.
|
||||
|
||||
@@ -0,0 +1,368 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
"github.com/Autumn-27/artex/traffic"
|
||||
actool "github.com/Autumn-27/norma/tool"
|
||||
)
|
||||
|
||||
// The gate wired into the server: the exemption API, the exemptions derived
|
||||
// from the configuration, and the hooks of a task's guards.
|
||||
|
||||
// exemptionsOf decodes the "exemptions" (or "derived") list of a response.
|
||||
func exemptionsOf(t *testing.T, body map[string]any, field string) []db.TargetExemption {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(body[field])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []db.TargetExemption
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
t.Fatalf("%s = %s: %v", field, raw, err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestTargetGateExemptionsAPI(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
pg := g.m.pg
|
||||
original, _, err := pg.TargetExemptions()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = pg.ReplaceTargetExemptions(original, "test", "restore") })
|
||||
|
||||
// The routes are behind the session, like the rest of the API.
|
||||
for _, c := range []struct{ method, path string }{
|
||||
{"GET", "/api/target-gate/exemptions"}, {"PUT", "/api/target-gate/exemptions"}, {"GET", "/api/target-gate/exemptions/history"},
|
||||
} {
|
||||
rec := httptest.NewRecorder()
|
||||
g.s.Handler().ServeHTTP(rec, httptest.NewRequest(c.method, c.path, strings.NewReader(`{"exemptions":[]}`)))
|
||||
if rec.Code != 401 {
|
||||
t.Fatalf("%s %s without a token: %d, want 401", c.method, c.path, rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
code, body := g.do(t, "GET", "/api/target-gate/exemptions", "")
|
||||
if code != 200 || !slices.Equal(exemptionsOf(t, body, "exemptions"), original) || body["digest"] == "" || body["derived"] == nil {
|
||||
t.Fatalf("GET: %d %v, want the stored list %+v", code, body, original)
|
||||
}
|
||||
|
||||
// PUT replaces the list. Hosts are normalised; host and port are exact.
|
||||
code, body = g.do(t, "PUT", "/api/target-gate/exemptions",
|
||||
`{"exemptions":[{"host":"Mirror.Example.ORG.","port":443,"note":"internal mirror"},{"host":"10.0.0.5","port":8080},{"host":"FD12:3456::5","port":8080}]}`)
|
||||
want := []db.TargetExemption{{Host: "mirror.example.org", Port: 443, Note: "internal mirror"}, {Host: "10.0.0.5", Port: 8080}, {Host: "fd12:3456::5", Port: 8080}}
|
||||
if code != 200 || !slices.Equal(exemptionsOf(t, body, "exemptions"), want) {
|
||||
t.Fatalf("PUT: %d %v, want %+v", code, body, want)
|
||||
}
|
||||
digest, _ := body["digest"].(string)
|
||||
code, body = g.do(t, "GET", "/api/target-gate/exemptions", "")
|
||||
if code != 200 || !slices.Equal(exemptionsOf(t, body, "exemptions"), want) || body["digest"] != digest || digest == "" {
|
||||
t.Fatalf("GET after PUT: %d %v", code, body)
|
||||
}
|
||||
|
||||
// Refused entries: 400, and the stored list does not change.
|
||||
for name, entry := range map[string]string{
|
||||
"metadata address": `{"host":"169.254.169.254","port":80}`,
|
||||
"metadata address, v6": `{"host":"fd00:ec2::254","port":80}`,
|
||||
"metadata address, cloud": `{"host":"100.100.100.200","port":80}`,
|
||||
"link-local v6": `{"host":"fe80::1","port":80}`,
|
||||
"loopback": `{"host":"127.0.0.1","port":8080}`,
|
||||
"loopback as a number": `{"host":"2130706433","port":8080}`,
|
||||
"loopback v6": `{"host":"::1","port":8080}`,
|
||||
"unspecified": `{"host":"0.0.0.0","port":80}`,
|
||||
"multicast": `{"host":"224.0.0.1","port":80}`,
|
||||
"wildcard": `{"host":"*.example.org","port":443}`,
|
||||
"suffix": `{"host":".example.org","port":443}`,
|
||||
"range": `{"host":"10.0.0.0/8","port":443}`,
|
||||
"url": `{"host":"https://mirror.example.org/","port":443}`,
|
||||
"host with a port": `{"host":"mirror.example.org:443","port":443}`,
|
||||
"no port": `{"host":"mirror.example.org"}`,
|
||||
"port out of range": `{"host":"mirror.example.org","port":70000}`,
|
||||
"no host": `{"port":443}`,
|
||||
"one bad entry among good": `{"host":"pypi.org","port":443},{"host":"169.254.169.254","port":80}`,
|
||||
} {
|
||||
code, body := g.do(t, "PUT", "/api/target-gate/exemptions", `{"exemptions":[`+entry+`]}`)
|
||||
if code != 400 || body["error"] == nil {
|
||||
t.Errorf("%s: %d %v, want 400", name, code, body)
|
||||
}
|
||||
}
|
||||
if code, _ := g.do(t, "PUT", "/api/target-gate/exemptions", `not json`); code != 400 {
|
||||
t.Fatalf("malformed body: %d, want 400", code)
|
||||
}
|
||||
if list, _, err := pg.TargetExemptions(); err != nil || !slices.Equal(list, want) {
|
||||
t.Fatalf("stored list after the refused changes = %+v, %v; want it unchanged", list, err)
|
||||
}
|
||||
|
||||
// The history holds the one change that was made, with who made it.
|
||||
code, body = g.do(t, "GET", "/api/target-gate/exemptions/history", "")
|
||||
changes, _ := body["changes"].([]any)
|
||||
if code != 200 || len(changes) == 0 {
|
||||
t.Fatalf("history: %d %v", code, body)
|
||||
}
|
||||
last, _ := changes[0].(map[string]any)
|
||||
if last["changed_by"] != "ARTEX" || last["digest"] != digest || len(exemptionsOf(t, last, "after")) != 3 || !slices.Equal(exemptionsOf(t, last, "before"), original) {
|
||||
t.Fatalf("last change = %v", last)
|
||||
}
|
||||
|
||||
// An empty list is a valid list.
|
||||
if code, body := g.do(t, "PUT", "/api/target-gate/exemptions", `{}`); code != 200 || len(exemptionsOf(t, body, "exemptions")) != 0 {
|
||||
t.Fatalf("empty list: %d %v", code, body)
|
||||
}
|
||||
if code, body := g.do(t, "GET", "/api/target-gate/exemptions", ""); code != 200 || body["exemptions"] == nil || len(exemptionsOf(t, body, "exemptions")) != 0 {
|
||||
t.Fatalf("GET of the empty list: %d %v", code, body)
|
||||
}
|
||||
}
|
||||
|
||||
// The exemptions of the platform's own endpoints are derived from the
|
||||
// configuration on every call: LLM gateways, profile proxies, the search
|
||||
// backend in use, the search proxy and the global egress proxy.
|
||||
func TestTargetGateEndpointsFollowConfig(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
derived := func() []string {
|
||||
var out []string
|
||||
for _, e := range g.m.interceptor.DerivedExemptions() {
|
||||
out = append(out, db.TargetKey(e.Host, e.Port))
|
||||
}
|
||||
return out
|
||||
}
|
||||
has := func(key string) bool { return slices.Contains(derived(), key) }
|
||||
|
||||
name := fmt.Sprintf("target-gate-endpoints-%d", time.Now().UnixNano())
|
||||
id, err := g.m.pg.SaveProfile(&db.LLMProfile{Name: name, Format: "openai", BaseURL: "https://LLM-gateway.example.net:8443/v1",
|
||||
Proxy: "http://user:secret@llm-proxy.example.net:3128", Model: "m", APIKey: "k"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = g.m.pg.DeleteProfile(id) })
|
||||
for _, key := range []string{"llm-gateway.example.net:8443", "llm-proxy.example.net:3128"} {
|
||||
if !has(key) {
|
||||
t.Fatalf("derived = %v, want the profile endpoint %s", derived(), key)
|
||||
}
|
||||
}
|
||||
for _, e := range derived() {
|
||||
if strings.Contains(e, "secret") || strings.Contains(e, "user") {
|
||||
t.Fatalf("a derived exemption carries credentials: %s", e)
|
||||
}
|
||||
}
|
||||
|
||||
on, backend, _, _, proxy := g.m.WebSearch()
|
||||
globalProxy := g.m.GlobalProxy()
|
||||
t.Cleanup(func() {
|
||||
_ = g.m.SetWebSearch(on, backend, nil, nil, &proxy)
|
||||
_ = g.m.SetGlobalProxy(globalProxy)
|
||||
})
|
||||
searchProxy := "http://search-proxy.example.net:8118"
|
||||
if err := g.m.SetWebSearch(true, "tavily", nil, nil, &searchProxy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := g.m.SetGlobalProxy("socks5://egress.example.net:1080"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, key := range []string{"api.tavily.com:443", "search-proxy.example.net:8118", "egress.example.net:1080"} {
|
||||
if !has(key) {
|
||||
t.Fatalf("derived = %v, want %s", derived(), key)
|
||||
}
|
||||
}
|
||||
// The configuration changes: so do the exemptions.
|
||||
none := ""
|
||||
if err := g.m.SetWebSearch(true, "ddgs", nil, nil, &none); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := g.m.SetGlobalProxy(""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := g.m.pg.DeleteProfile(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !has("html.duckduckgo.com:443") {
|
||||
t.Fatalf("derived = %v, want the keyless search backend", derived())
|
||||
}
|
||||
for _, key := range []string{"api.tavily.com:443", "search-proxy.example.net:8118", "egress.example.net:1080", "llm-gateway.example.net:8443", "llm-proxy.example.net:3128"} {
|
||||
if has(key) {
|
||||
t.Fatalf("derived = %v, still holds %s after the configuration changed", derived(), key)
|
||||
}
|
||||
}
|
||||
// Search off: its backend is not exempt.
|
||||
if err := g.m.SetWebSearch(false, "ddgs", nil, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if has("html.duckduckgo.com:443") {
|
||||
t.Fatalf("derived = %v, want no search backend while search is off", derived())
|
||||
}
|
||||
// The API shows them, read-only.
|
||||
if code, body := g.do(t, "GET", "/api/target-gate/exemptions", ""); code != 200 || body["derived"] == nil {
|
||||
t.Fatalf("GET: %d %v", code, body)
|
||||
}
|
||||
}
|
||||
|
||||
// An approval of a request that displays no address is refused with 422. The
|
||||
// request stays open and nothing is pinned.
|
||||
func TestInterceptDecideRefusesEmptyPinOverHTTP(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
task := g.newTask(t, db.TaskCreateOptions{})
|
||||
scope := taskRef(t, task)
|
||||
id := openTargetRow(t, g.m.pg, scope, "no-address.example.com")
|
||||
code, body := g.do(t, "POST", fmt.Sprintf("/api/intercept/pending/%d/decide", id), `{"decision":"allowed"}`)
|
||||
if msg, _ := body["error"].(string); code != 422 || !strings.Contains(msg, "never pins an empty address list") {
|
||||
t.Fatalf("approve: %d %v, want 422", code, body)
|
||||
}
|
||||
if row, err := g.m.pg.GetInterceptPending(id); err != nil || row == nil || row.Status != "pending" {
|
||||
t.Fatalf("row = %+v, %v; want still pending", row, err)
|
||||
}
|
||||
if pins, err := g.m.pg.ListTargetPins(scope); err != nil || len(pins) != 0 {
|
||||
t.Fatalf("pins = %+v, %v; want none", pins, err)
|
||||
}
|
||||
if code, _ := g.do(t, "POST", fmt.Sprintf("/api/intercept/pending/%d/decide", id), `{"decision":"denied"}`); code != 200 {
|
||||
t.Fatalf("deny: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// The hooks of a task run the gate with the task's own scope: the worker
|
||||
// guard and the target-only guard of the planner and the main agent. The flag
|
||||
// is the task's, read on every call.
|
||||
func TestTaskGuardsRunTargetGate(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
task := g.newTask(t, db.TaskCreateOptions{})
|
||||
scope := taskRef(t, task)
|
||||
pg := g.m.pg
|
||||
const fetch = `{"url":"http://203.0.113.7:8080/admin"}`
|
||||
ctxOf := func(agentName string, wait time.Duration) context.Context {
|
||||
return intercept.WithTargetScope(intercept.WithTaskContext(context.Background(), task.ID, agentName, nil), task.targetScope(wait))
|
||||
}
|
||||
targetRows := func() int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := pg.QueryRow(`SELECT count(*) FROM intercept_pending WHERE task_id=$1 AND decision_source='target'`, task.ID).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
hook := func(hooks interface {
|
||||
PreToolUse(context.Context, string, []byte) (bool, string, []byte)
|
||||
}, ctx context.Context) (bool, string) {
|
||||
block, msg, _ := hooks.PreToolUse(ctx, "WebFetch", []byte(fetch))
|
||||
return block, msg
|
||||
}
|
||||
|
||||
// Gate off, the default: both guards pass the call and write nothing.
|
||||
for name, hooks := range map[string]interface {
|
||||
PreToolUse(context.Context, string, []byte) (bool, string, []byte)
|
||||
}{"worker": task.Guard.Hooks(), "planner": task.TargetGuard.Hooks()} {
|
||||
if block, msg := hook(hooks, ctxOf(name, 0)); block {
|
||||
t.Fatalf("gate off, %s: blocked: %s", name, msg)
|
||||
}
|
||||
}
|
||||
if n := targetRows(); n != 0 {
|
||||
t.Fatalf("gate off: %d target rows, want none", n)
|
||||
}
|
||||
|
||||
// Gate on through the API: the next call of each guard is stopped. With
|
||||
// wait bound 0 it does not wait, and the one request stays open.
|
||||
if code, _ := g.do(t, "PATCH", "/api/tasks/"+task.ID, `{"target_gate":true}`); code != 200 {
|
||||
t.Fatalf("switch the gate on: %d", code)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
hooks interface {
|
||||
PreToolUse(context.Context, string, []byte) (bool, string, []byte)
|
||||
}
|
||||
}{{"planner", task.TargetGuard.Hooks()}, {"worker", task.Guard.Hooks()}} {
|
||||
block, msg := hook(c.hooks, ctxOf(c.name, 0))
|
||||
if !block || !strings.Contains(msg, "TARGET_APPROVAL_PENDING: target 203.0.113.7:8080: ") {
|
||||
t.Fatalf("gate on, %s: block=%v msg=%q", c.name, block, msg)
|
||||
}
|
||||
}
|
||||
var id int64
|
||||
var agentName string
|
||||
var open int
|
||||
if err := pg.QueryRow(`SELECT count(*), min(id), min(agent_name) FROM intercept_pending
|
||||
WHERE task_id=$1 AND status='pending' AND target_key='203.0.113.7:8080'`, task.ID).Scan(&open, &id, &agentName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if open != 1 || agentName != "planner" {
|
||||
t.Fatalf("%d open requests by %q, want one, opened by the planner's call", open, agentName)
|
||||
}
|
||||
// The planner's block is in the ring of its own guard, not in the task's.
|
||||
if audit := task.TargetGuard.Audit(); len(audit) != 1 || audit[0].Action != "block" {
|
||||
t.Fatalf("target-only ring = %+v, want the one block", audit)
|
||||
}
|
||||
for _, e := range task.Guard.Audit() {
|
||||
if e.Action == "block" && !strings.Contains(e.Reason, "TARGET_APPROVAL_PENDING") {
|
||||
t.Fatalf("unexpected entry in the task ring: %+v", e)
|
||||
}
|
||||
}
|
||||
|
||||
// One click through the API pins the address for the task; both guards pass.
|
||||
if code, _ := g.do(t, "POST", fmt.Sprintf("/api/intercept/pending/%d/decide", id), `{"decision":"allowed"}`); code != 200 {
|
||||
t.Fatalf("decide: %d", code)
|
||||
}
|
||||
if pins, err := pg.ListTargetPins(scope); err != nil || len(pins) != 1 || pins[0].Host != "203.0.113.7" || pins[0].Port != 8080 ||
|
||||
!slices.Equal(pins[0].PinnedIPs, []string{"203.0.113.7"}) || pins[0].DecidedBy != "ARTEX" {
|
||||
t.Fatalf("pins = %+v, %v", pins, err)
|
||||
}
|
||||
ring := len(task.TargetGuard.Audit())
|
||||
for name, hooks := range map[string]interface {
|
||||
PreToolUse(context.Context, string, []byte) (bool, string, []byte)
|
||||
}{"worker": task.Guard.Hooks(), "planner": task.TargetGuard.Hooks()} {
|
||||
if block, msg := hook(hooks, ctxOf(name, 0)); block {
|
||||
t.Fatalf("pinned, %s: blocked: %s", name, msg)
|
||||
}
|
||||
}
|
||||
if n := len(task.TargetGuard.Audit()); n != ring {
|
||||
t.Fatalf("a passing planner call added %d entries to the ring", n-ring)
|
||||
}
|
||||
// Gate off again: the gate is out of the way at once.
|
||||
if code, _ := g.do(t, "PATCH", "/api/tasks/"+task.ID, `{"target_gate":false}`); code != 200 {
|
||||
t.Fatal("switch the gate off")
|
||||
}
|
||||
rows := targetRows()
|
||||
if block, _, _ := task.Guard.Hooks().PreToolUse(ctxOf("worker", 0), "WebFetch", []byte(`{"url":"http://198.51.100.200/"}`)); block || targetRows() != rows {
|
||||
t.Fatalf("gate off again: block=%v, %d new target rows", block, targetRows()-rows)
|
||||
}
|
||||
}
|
||||
|
||||
// The platform's own tools read and write the platform's records: the gate
|
||||
// must find no target in their input, or every finding and every intent that
|
||||
// mentions a URL would ask for an approval. A tool added to these sets has to
|
||||
// be added to the gate's list of tools without a target.
|
||||
func TestHostToolsDeclareNoTarget(t *testing.T) {
|
||||
s := &Server{}
|
||||
var tools []actool.CoreTool
|
||||
tools = append(tools, s.orchestrationTools()...)
|
||||
tools = append(tools, s.findingRetestTools()...)
|
||||
tools = append(tools, s.platformTools()...)
|
||||
tools = append(tools, traffic.SeedToolMetas()...)
|
||||
if len(tools) < 25 {
|
||||
t.Fatalf("only %d host tools found", len(tools))
|
||||
}
|
||||
// The tools that set up a custom tool or an MCP server take an endpoint of
|
||||
// the model's choosing. They are scanned like any unknown tool: that
|
||||
// endpoint is a target.
|
||||
scanned := map[string]bool{"create_custom_tool": true, "update_custom_tool": true, "create_mcp": true, "update_mcp": true, "mcp_status": true}
|
||||
input := []byte(`{"task_id":1,"description":"test https://app.example.com/ and 203.0.113.7:8443","command":"curl https://app.example.com/","q":"host:app.example.com:443"}`)
|
||||
for _, tool := range tools {
|
||||
targets := intercept.ExtractTargets(tool.Name(), input)
|
||||
if scanned[tool.Name()] {
|
||||
if len(targets) == 0 {
|
||||
t.Errorf("%s: no target read", tool.Name())
|
||||
}
|
||||
delete(scanned, tool.Name())
|
||||
} else if len(targets) != 0 {
|
||||
t.Errorf("%s: the gate reads targets %+v from a platform tool", tool.Name(), targets)
|
||||
}
|
||||
}
|
||||
if len(scanned) != 0 {
|
||||
t.Errorf("tools that no longer exist: %v", scanned)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// Step 1.9: the redirect hops of the custom http tool and of a worker's
|
||||
// WebFetch go through the target gate.
|
||||
|
||||
// redirectingServer answers every request with a redirect to location and
|
||||
// counts the requests. Its host:port is an IP literal: the gate never resolves it.
|
||||
func redirectingServer(t *testing.T, location string) (srv *httptest.Server, hits *atomic.Int64, key string) {
|
||||
t.Helper()
|
||||
hits = &atomic.Int64{}
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
http.Redirect(w, r, location, http.StatusFound)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv, hits, srv.Listener.Addr().String()
|
||||
}
|
||||
|
||||
func decide(t *testing.T, g *gateServer, id int64, decision string) {
|
||||
t.Helper()
|
||||
if code, body := g.do(t, "POST", fmt.Sprintf("/api/intercept/pending/%d/decide", id), `{"decision":"`+decision+`"}`); code != 200 {
|
||||
t.Fatalf("decide %d: %d %v", id, code, body)
|
||||
}
|
||||
}
|
||||
|
||||
// A custom http tool whose rendered URL is pinned, and which redirects to a
|
||||
// target no pin covers: the hop is not followed, one request is opened for its
|
||||
// target, and the call fails at once. After the approval the same call follows.
|
||||
func TestCustomHTTPRedirectTargetGated(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
task := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
|
||||
pg := g.m.pg
|
||||
final, finalHits, fhost, fport := countingServer(t)
|
||||
finalKey := net.JoinHostPort(fhost, fport)
|
||||
start, startHits, startKey := redirectingServer(t, final.URL+"/final")
|
||||
|
||||
tool := g.s.buildCustomTool(&db.Tool{Key: "probe_redirect", Kind: "http",
|
||||
Exec: execJSON(t, map[string]any{"method": "GET", "url": start.URL + "/start"})})
|
||||
// The rendered URL itself first: its request, approved.
|
||||
if out, isErr := callTool(t, tool, gatedCtx(task, 0), `{}`, nil); !isErr || !strings.Contains(out, "TARGET_APPROVAL_PENDING") || startHits.Load() != 0 {
|
||||
t.Fatalf("rendered URL: error=%v out=%q", isErr, out)
|
||||
}
|
||||
id, _, _, ok := openTargetRequest(t, pg, task.ID, startKey)
|
||||
if !ok {
|
||||
t.Fatal("no request for the rendered URL")
|
||||
}
|
||||
decide(t, g, id, "allowed")
|
||||
|
||||
// The wait bound is long: the hop must not use it.
|
||||
began := time.Now()
|
||||
out, isErr := callTool(t, tool, gatedCtx(task, time.Minute), `{}`, nil)
|
||||
if !isErr || !strings.Contains(out, "TARGET_APPROVAL_PENDING") || !strings.Contains(out, start.URL+"/start redirects to "+finalKey) {
|
||||
t.Fatalf("redirect to an unpinned target: error=%v out=%q", isErr, out)
|
||||
}
|
||||
if elapsed := time.Since(began); elapsed > 10*time.Second {
|
||||
t.Fatalf("the hop waited %v", elapsed)
|
||||
}
|
||||
if startHits.Load() != 1 || finalHits.Load() != 0 {
|
||||
t.Fatalf("hits origin=%d target=%d, want the target untouched", startHits.Load(), finalHits.Load())
|
||||
}
|
||||
hopID, toolName, toolInput, ok := openTargetRequest(t, pg, task.ID, finalKey)
|
||||
if !ok || toolName != "probe_redirect" || !strings.Contains(toolInput, final.URL+"/final") || !strings.Contains(toolInput, start.URL+"/start") {
|
||||
t.Fatalf("request for the hop: ok=%v tool=%q input=%s", ok, toolName, toolInput)
|
||||
}
|
||||
var requests int
|
||||
if err := pg.QueryRow(`SELECT count(*) FROM intercept_pending WHERE task_id=$1 AND target_key IS NOT NULL`, task.ID).Scan(&requests); err != nil || requests != 2 {
|
||||
t.Fatalf("%d requests, %v; want the origin's and the hop's", requests, err)
|
||||
}
|
||||
|
||||
decide(t, g, hopID, "allowed")
|
||||
if out, isErr := callTool(t, tool, gatedCtx(task, 0), `{}`, nil); isErr || !strings.Contains(out, "probe-reached") || finalHits.Load() != 1 {
|
||||
t.Fatalf("after the approval: error=%v out=%q target hits=%d", isErr, out, finalHits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// The custom-tool test endpoint has no approval scope (D14): its redirects are
|
||||
// followed as before, and nothing is written.
|
||||
func TestCustomToolTestEndpointRedirectUngated(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
|
||||
final, finalHits, _, _ := countingServer(t)
|
||||
start, _, _ := redirectingServer(t, final.URL+"/final")
|
||||
count := func() int {
|
||||
var n int
|
||||
if err := g.m.pg.QueryRow(`SELECT count(*) FROM intercept_pending WHERE decision_source='target' OR target_key IS NOT NULL`).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
before := count()
|
||||
code, out := g.do(t, "POST", "/api/tools/custom/test", fmt.Sprintf(`{"kind":"http","exec":{"url":%q}}`, start.URL+"/start"))
|
||||
if code != 200 || out["is_error"] != false || !strings.Contains(fmt.Sprint(out["output"]), "probe-reached") || finalHits.Load() != 1 {
|
||||
t.Fatalf("test endpoint: %d %v target hits=%d", code, out, finalHits.Load())
|
||||
}
|
||||
if after := count(); after != before {
|
||||
t.Fatalf("the test endpoint wrote %d target rows", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
// A gated worker's WebFetch end to end: the origin is approved at the hook, the
|
||||
// redirect stops at the hop with TARGET_APPROVAL_PENDING and leaves one request
|
||||
// open; once it is approved through the API, the same call follows.
|
||||
func TestRedirectRequestSurvivesAndRetryPasses(t *testing.T) {
|
||||
g := newGateServer(t)
|
||||
on := true
|
||||
task := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
|
||||
pg := g.m.pg
|
||||
final, finalHits, fhost, fport := countingServer(t)
|
||||
finalKey := net.JoinHostPort(fhost, fport)
|
||||
start, startHits, startKey := redirectingServer(t, final.URL+"/final")
|
||||
p := &toolScript{calls: fetches(start.URL+"/start", start.URL+"/start"), results: make(chan string, 16),
|
||||
hold: map[int64]chan struct{}{2: make(chan struct{})}}
|
||||
done := runScriptedWorker(t, g, task, p)
|
||||
|
||||
var id int64
|
||||
waitUntil(t, "the request for the origin", func() bool {
|
||||
var ok bool
|
||||
id, _, _, ok = openTargetRequest(t, pg, task.ID, startKey)
|
||||
return ok
|
||||
})
|
||||
decide(t, g, id, "allowed")
|
||||
if r := p.next(t); !strings.Contains(r, "TARGET_APPROVAL_PENDING") || !strings.Contains(r, "redirects to "+finalKey) {
|
||||
t.Fatalf("first call: %q", r)
|
||||
}
|
||||
if startHits.Load() != 1 || finalHits.Load() != 0 {
|
||||
t.Fatalf("hits origin=%d target=%d, want the target untouched", startHits.Load(), finalHits.Load())
|
||||
}
|
||||
hopID, toolName, _, ok := openTargetRequest(t, pg, task.ID, finalKey)
|
||||
if !ok || toolName != "WebFetch" {
|
||||
t.Fatalf("request for the hop: ok=%v tool=%q", ok, toolName)
|
||||
}
|
||||
detail, err := pg.GetInterceptDetail(hopID)
|
||||
if err != nil || detail == nil || detail.Audit == nil || len(detail.Audit.Targets) != 1 || detail.Audit.Targets[0].RedirectFrom != startKey {
|
||||
t.Fatalf("hop request detail = %+v, %v", detail, err)
|
||||
}
|
||||
|
||||
decide(t, g, hopID, "allowed")
|
||||
close(p.hold[2])
|
||||
if r := p.next(t); !strings.Contains(r, "probe-reached") || finalHits.Load() != 1 {
|
||||
t.Fatalf("the same call after the approval: %q target hits=%d", r, finalHits.Load())
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(60 * time.Second):
|
||||
t.Fatal("the worker run did not end")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user