Compare commits

...
Author SHA1 Message Date
BaaRai bbc4f6269a Fix doubled scheme in the custom http tool's recording proxy URL
ProxyAddr returns a URL that already carries a scheme (the recording
proxy and the validated global proxy), but httpProxyTransport wrote
"http://" before it, so use_recording_proxy produced http://http://...
proxyURLString now adds http:// only to a bare host:port, for both the
operator proxy and ProxyAddr. httpToolTransport falls back to a fresh
http.Transport when http.DefaultTransport is not an *http.Transport.
2026-11-11 20:20:16 +01:00
BaaRai e38e48e80e Merge pull request #186 from BaaRai/feat/target-gate-pinned-dialer
feat(intercept): pinned dialer for WebFetch and the custom http tool (S3b 2.1)
2026-11-11 19:54:37 +01:00
BaaRai f6b29b835b feat(intercept): pinned dialer for WebFetch and the custom http tool (S3b 2.1)
In a gated scope, the two HTTP clients the platform owns connect only to
addresses that a pin or a stored exemption covers at the moment of the
connection, on every hop.

- intercept.PinnedDialer(tool, scoped): the DialContext of both clients.
  It reads the target scope from the context net/http passes to the dial.
  Gate off, or no scope in a client built without one: plain dial, as
  today. A client built for a scope whose scope is missing at the dial
  fails closed with TARGET_GATE_UNAVAILABLE.
- dialTarget runs the gate decision on the dialled host:port with a fresh
  lookup (block rules first, exemptions, pins, port precedence) and
  connects only to the addresses it compared, never to the name. Derived
  exemptions and name-only pins dial the address as received. An address
  no pin covers finds or opens the request (new_ip / no_pin, wait 0) and
  fails at once with TARGET_APPROVAL_PENDING. Redirect hops are dialled
  against the hop's own (host, port).
- WebFetch (withWebFetch, now given the run context): DialContext set on
  every agent kind; DisableDirectRetry when the run's scope is gated.
- Custom http tool: a direct call in a gated scope gets a DefaultTransport
  clone with the pinned dialer, no environment proxy and keep-alive off.
  Proxied calls are unchanged (the recording proxy is step 2.2).
- Name-only: request reason says the platform does not bind the
  connection; the first-use pin_match row records the effective state.
- Derived exemptions: a proxy URL without a port is listed with the port
  net/http dials (1080 for socks5) (OPEN-9).
- go.mod: Norma v0.4.8 (DialContext, DisableDirectRetry).
2026-11-11 19:19:55 +01:00
BaaRai 08830365c4 Merge pull request #185 from BaaRai/feat/target-gate-redirect-hops
feat(intercept): check every redirect hop against the target gate (S3b 1.9)
2026-11-11 17:15:06 +01:00
20 changed files with 1861 additions and 46 deletions
+1 -1
View File
@@ -128,7 +128,7 @@ func (c *ChatAgent) Chat(ctx context.Context, agentKey, sessionID, message strin
Provider: c.prov,
SystemPrompt: system,
DynamicBoundary: boundary,
Tools: withWebFetch(tools, c.proxyAddr, c.proxyCACert), // WebFetch: routed through the recording proxy for tracing; loads the proxy CA to verify MITM-resigned HTTPS certificates; every redirect hop is gated
Tools: withWebFetch(ctx, tools, c.proxyAddr, c.proxyCACert), // WebFetch: routed through the recording proxy for tracing; loads the proxy CA to verify MITM-resigned HTTPS certificates; every redirect hop is gated
DeferredTools: def.Deferred,
UnlockSet: def.Unlock,
PermissionMode: permission.ModeBypass,
+1 -1
View File
@@ -144,7 +144,7 @@ func (m *MainAgent) Chat(ctx context.Context, taskID int64, mainSeg int, as *db.
Provider: m.prov,
SystemPrompt: system,
DynamicBoundary: boundary,
Tools: withWebFetch(tools, m.proxyAddr, m.proxyCACert), // WebFetch: routed through the recording proxy for tracing; loads the proxy CA to verify MITM-resigned HTTPS certificates; every redirect hop is gated
Tools: withWebFetch(ctx, tools, m.proxyAddr, m.proxyCACert), // WebFetch: routed through the recording proxy for tracing; loads the proxy CA to verify MITM-resigned HTTPS certificates; every redirect hop is gated
DeferredTools: def.Deferred,
UnlockSet: def.Unlock,
PermissionMode: permission.ModeBypass,
+1 -1
View File
@@ -426,7 +426,7 @@ func (p *Planner) Plan(ctx context.Context, taskID int64, as *db.AssetStore, ts
Provider: p.prov,
SystemPrompt: system,
DynamicBoundary: boundary,
Tools: withWebFetch(tools, p.proxyAddr, p.proxyCACert), // WebFetch: route through the recording proxy for traceability; load the proxy CA to verify MITM-re-signed HTTPS certs; every redirect hop is gated
Tools: withWebFetch(ctx, tools, p.proxyAddr, p.proxyCACert), // WebFetch: route through the recording proxy for traceability; load the proxy CA to verify MITM-re-signed HTTPS certs; every redirect hop is gated
DeferredTools: def.Deferred,
UnlockSet: def.Unlock,
PermissionMode: permission.ModeBypass,
+17 -9
View File
@@ -1,19 +1,27 @@
package agent
import (
"context"
"github.com/Autumn-27/artex/intercept"
actool "github.com/Autumn-27/norma/tool"
)
// withWebFetch returns tools plus the WebFetch tool of an agent session. Every
// agent kind builds WebFetch here instead of through agentcore's EnableWebFetch,
// which has no option for a redirect policy: every redirect hop goes through the
// target gate (intercept.RedirectCheck), on the client that uses proxy and on
// the direct retry client alike. The tool is added after the session's own
// tools, where agentcore adds it, so the tool list keeps its order. caCert is the
// recording proxy's CA, trusted to verify the HTTPS it re-signs. tools is not
// modified.
func withWebFetch(tools []actool.CoreTool, proxy, caCert string) []actool.CoreTool {
webFetch := actool.NewWebFetch(actool.WebFetchConfig{Proxy: proxy, CACert: caCert, CheckRedirect: intercept.RedirectCheck("WebFetch")})
// agent kind builds WebFetch here, one tool per run, so that its configuration
// can depend on the run: every redirect hop goes through the target gate
// (intercept.RedirectCheck), and every connection through the pinned dialer
// (intercept.PinnedDialer), on the client that uses proxy and on the direct
// retry client alike. ctx is the context of the run: when it carries a target
// scope the dialer never connects without one, and when that scope's gate is
// on the direct retry is turned off for the run, since it would leave without
// the proxy. The tool is added after the session's own tools, where agentcore
// adds it, so the tool list keeps its order. caCert is the recording proxy's
// CA, trusted to verify the HTTPS it re-signs. tools is not modified.
func withWebFetch(ctx context.Context, tools []actool.CoreTool, proxy, caCert string) []actool.CoreTool {
scope, scoped := intercept.TargetScopeFromContext(ctx)
gated := scoped && scope.Enabled != nil && scope.Enabled()
webFetch := actool.NewWebFetch(actool.WebFetchConfig{Proxy: proxy, CACert: caCert, CheckRedirect: intercept.RedirectCheck("WebFetch"),
DialContext: intercept.PinnedDialer("WebFetch", scoped), DisableDirectRetry: gated})
return append(tools[:len(tools):len(tools)], webFetch)
}
+262
View File
@@ -0,0 +1,262 @@
package agent
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"net/http/httptest"
"net/netip"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/guard"
"github.com/Autumn-27/artex/intercept"
actool "github.com/Autumn-27/norma/tool"
)
// Step 2.1: the WebFetch tool of every agent kind connects through the pinned
// dialer. In a gated scope it reaches a target only at an address a pin holds.
// tableResolver answers from a table and counts the lookups.
type tableResolver struct {
mu sync.Mutex
answers map[string][]string
n int
}
func newTableResolver() *tableResolver { return &tableResolver{answers: map[string][]string{}} }
func (r *tableResolver) Name() string { return "table" }
func (r *tableResolver) LookupIPs(_ context.Context, host string) ([]netip.Addr, error) {
r.mu.Lock()
defer r.mu.Unlock()
r.n++
var out []netip.Addr
for _, ip := range r.answers[host] {
out = append(out, netip.MustParseAddr(ip))
}
if len(out) == 0 {
return nil, fmt.Errorf("no answer for %s", host)
}
return out, nil
}
func (r *tableResolver) set(host string, ips ...string) {
r.mu.Lock()
r.answers[host] = ips
r.mu.Unlock()
}
func (r *tableResolver) lookups() int {
r.mu.Lock()
defer r.mu.Unlock()
return r.n
}
// testDialer is the base dialer of the gate in these tests: it records every
// address, connects the routed TEST-NET addresses to local listeners and a
// loopback address to itself, and refuses the rest.
type testDialer struct {
mu sync.Mutex
routes map[string]string
dialed []string
}
func newTestDialer() *testDialer { return &testDialer{routes: map[string]string{}} }
func (d *testDialer) route(addr, to string) {
d.mu.Lock()
d.routes[addr] = to
d.mu.Unlock()
}
func (d *testDialer) dial(ctx context.Context, network, addr string) (net.Conn, error) {
d.mu.Lock()
d.dialed = append(d.dialed, addr)
to, ok := d.routes[addr]
d.mu.Unlock()
if !ok && strings.HasPrefix(addr, "127.0.0.1:") {
to, ok = addr, true
}
if !ok {
return nil, &net.OpError{Op: "dial", Net: network, Err: errors.New("no route in the test dialer")}
}
return (&net.Dialer{}).DialContext(ctx, network, to)
}
func (d *testDialer) calls() []string {
d.mu.Lock()
defer d.mu.Unlock()
return slices.Clone(d.dialed)
}
// dialFixture is a gated scope on task taskID whose gate resolves through a
// table and dials through a testDialer.
func dialFixture(t *testing.T, d *db.DB, taskID string) (intercept.TargetScope, *tableResolver, *testDialer) {
t.Helper()
scope := gatedScope(t, d, taskID)
res, dialer := newTableResolver(), newTestDialer()
scope.Interceptor.SetResolver(res)
scope.Interceptor.SetBaseDialer(dialer.dial)
return scope, res, dialer
}
// listener is an HTTP server that counts its requests; port is its port.
func listener(t *testing.T, body string) (addr, port string, hits *atomic.Int64) {
t.Helper()
hits = &atomic.Int64{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.Header().Set("Content-Type", "text/plain")
_, _ = w.Write([]byte(body))
}))
t.Cleanup(srv.Close)
addr = srv.Listener.Addr().String()
_, port, _ = net.SplitHostPort(addr)
return addr, port, hits
}
// runAgentKind runs one scripted WebFetch call with the agent kind on ctx.
func runAgentKind(t *testing.T, kind string, f hookReachFixture, ctx context.Context, rawURL string) {
t.Helper()
prov := toolCallThenStop("WebFetch", fetchInputJSON(rawURL))
switch kind {
case "worker":
payload := map[string]any{"summary": "dial probe"}
intentID, err := f.ts.AddNode("intent", payload, 0, "running", "planner", nil)
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(payload)
w := NewWorker(prov, "test-model", f.workDir, nil, 0, 3)
if _, _, err := w.Execute(ctx, "work#1", f.expID, nil, f.ts, &db.Node{ID: intentID, Payload: raw}, passHooks{}, nil, nil, nil); err != nil {
t.Fatal(err)
}
case "planner":
p := NewPlanner(prov, "test-model", f.workDir, nil, 0, 3)
if _, _, err := p.Plan(ctx, f.expID, nil, f.ts, "probe goal", nil, passHooks{}, nil); err != nil {
t.Fatal(err)
}
case "mainagent":
m := NewMainAgent(prov, "test-model", f.workDir, nil, 0, 3)
if _, err := m.Chat(ctx, f.expID, 0, nil, f.ts, "probe goal", "status?", passHooks{}, nil, nil, nil, nil, nil); err != nil {
t.Fatal(err)
}
case "chat":
c := NewChatAgent(prov, "test-model", t.TempDir(), nil, 0)
c.SetGuard(guard.New())
if _, err := c.Chat(ctx, "assistant", "conv-dial-probe", "fetch it", 3, 0, false, nil); err != nil {
t.Fatal(err)
}
}
}
// Each agent kind's WebFetch reaches a pinned name at its pinned address, and
// once the name answers another address it connects nowhere and asks. A fifth
// agent kind built without the dialer fails this table once it is added to it.
func TestWebFetchDialerWiredOnEveryAgentKind(t *testing.T) {
for _, kind := range []string{"worker", "planner", "mainagent", "chat"} {
t.Run(kind, func(t *testing.T) {
f := newHookReachFixture(t)
d := testDB(t)
t.Cleanup(func() { _ = d.Close() })
addr, port, hits := listener(t, "pinned-reached")
scope, res, dialer := dialFixture(t, d, f.taskID)
dialer.route("203.0.113.7:"+port, addr)
dialer.route("203.0.113.9:"+port, addr)
res.set("a.example", "203.0.113.7")
u := "http://a.example:" + port + "/"
pinTarget(t, d, scope, u)
ctx := intercept.WithTargetScope(t.Context(), scope)
runAgentKind(t, kind, f, ctx, u)
if hits.Load() != 1 || !slices.Equal(dialer.calls(), []string{"203.0.113.7:" + port}) {
t.Fatalf("pinned: hits=%d dialled %v, want the pinned address", hits.Load(), dialer.calls())
}
res.set("a.example", "203.0.113.9")
runAgentKind(t, kind, f, ctx, u)
if hits.Load() != 1 || len(dialer.calls()) != 1 {
t.Fatalf("address changed: hits=%d dialled %v, want nothing more", hits.Load(), dialer.calls())
}
if keys := openRedirectRequests(t, d, f.taskID); len(keys) != 1 || keys[0] != "a.example:"+port {
t.Fatalf("open requests = %v, want one for a.example:%s", keys, port)
}
})
}
}
// S2: through a proxy, the dialer is given the proxy's address, never the
// target's. The proxy is a derived exemption: dialled as received, no lookup,
// no row. The connection to the target is the proxy's (step 2.2 binds it).
func TestWebFetchProxiedDialIsTheProxy(t *testing.T) {
d := testDB(t)
t.Cleanup(func() { _ = d.Close() })
f := newHookReachFixture(t)
scope, res, dialer := dialFixture(t, d, f.taskID)
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("via-proxy " + r.URL.Host))
}))
t.Cleanup(proxy.Close)
scope.Interceptor.SetDerivedExemptions(func() []string { return []string{proxy.URL} })
res.set("a.example", "203.0.113.7")
ctx := intercept.WithTargetScope(t.Context(), scope)
out, isErr := callWebFetch(t, ctx, proxy.URL, "http://a.example:8080/")
if isErr || !strings.Contains(out, "via-proxy a.example:8080") {
t.Fatalf("proxied fetch: error=%v out=%q", isErr, out)
}
if got := dialer.calls(); !slices.Equal(got, []string{proxy.Listener.Addr().String()}) || res.lookups() != 0 {
t.Fatalf("dialled %v after %d lookups, want the proxy only", got, res.lookups())
}
if keys := openRedirectRequests(t, d, f.taskID); len(keys) != 0 {
t.Fatalf("open requests = %v", keys)
}
}
// D-8: a run built with the gate off keeps its direct retry when the gate is
// switched on during the run, and the retry goes through the pinned dialer: a
// pinned target is dialled at its pinned address, an unpinned one nowhere.
func TestWebFetchRetryAfterLiveSwitchIsPinned(t *testing.T) {
d := testDB(t)
t.Cleanup(func() { _ = d.Close() })
f := newHookReachFixture(t)
scope, res, dialer := dialFixture(t, d, f.taskID)
var on atomic.Bool
scope.Enabled = on.Load
scope.Interceptor.SetDerivedExemptions(func() []string { return []string{"http://127.0.0.1:1"} })
addr, port, hits := listener(t, "retry-reached")
dialer.route("203.0.113.7:"+port, addr)
dialer.route("203.0.113.8:"+port, addr)
res.set("a.example", "203.0.113.7")
res.set("b.example", "203.0.113.8")
ctx := intercept.WithTargetScope(t.Context(), scope)
tools := withWebFetch(ctx, nil, "http://127.0.0.1:1", "")
on.Store(true)
pinTarget(t, d, scope, "http://a.example:"+port+"/")
call := func(u string) (string, bool) {
r, err := tools[0].Call(ctx, json.RawMessage(fetchInputJSON(u)), &actool.ToolContext{})
if err != nil {
t.Fatal(err)
}
return r.Flatten(), r.IsError
}
if out, isErr := call("http://a.example:" + port + "/"); isErr || !strings.Contains(out, "retry-reached") || hits.Load() != 1 {
t.Fatalf("pinned target through the retry: error=%v out=%q", isErr, out)
}
if out, isErr := call("http://b.example:" + port + "/"); !isErr || !strings.Contains(out, intercept.TargetApprovalPending) || hits.Load() != 1 {
t.Fatalf("unpinned target through the retry: error=%v out=%q", isErr, out)
}
want := []string{"127.0.0.1:1", "203.0.113.7:" + port, "127.0.0.1:1"}
if got := dialer.calls(); !slices.Equal(got, want) {
t.Fatalf("dialled %v, want %v", got, want)
}
}
+46 -16
View File
@@ -7,6 +7,7 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"strings"
"sync/atomic"
"testing"
@@ -20,8 +21,8 @@ import (
)
// Step 1.9: the WebFetch tool of every agent kind checks each redirect hop
// against the target gate, on the primary client and on the direct retry
// client. With the gate off it follows redirects as before.
// against the target gate. With the gate off it follows redirects as before.
// Step 2.1: in a gated scope WebFetch has no direct retry.
// passHooks lets every tool call run.
type passHooks struct{}
@@ -100,7 +101,9 @@ func fetchInputJSON(u string) string {
// no pin covers. The hooks let the call run, so the hop is what the gate sees:
// each kind must stop at the hop, leave the redirect target untouched and open
// one request for it. A fifth agent kind added without the redirect policy
// fails this table once it is added to it.
// fails this table once it is added to it. The origin is pinned first: the
// hooks never check it here, and the pinned dialer (step 2.1) connects only to
// a pinned target.
func TestWebFetchRedirectCheckWiredOnEveryAgentKind(t *testing.T) {
for _, kind := range []string{"worker", "planner", "mainagent", "chat"} {
t.Run(kind, func(t *testing.T) {
@@ -109,6 +112,7 @@ func TestWebFetchRedirectCheckWiredOnEveryAgentKind(t *testing.T) {
t.Cleanup(func() { _ = d.Close() })
pair := newRedirectPair(t)
scope := gatedScope(t, d, f.taskID)
pinTarget(t, d, scope, pair.start)
ctx := intercept.WithTargetScope(t.Context(), scope)
prov := toolCallThenStop("WebFetch", fetchInputJSON(pair.start))
@@ -155,30 +159,33 @@ func TestWebFetchRedirectCheckWiredOnEveryAgentKind(t *testing.T) {
}
}
// The direct retry client carries the policy too: with a proxy that refuses
// the connection, WebFetch retries without it, reaches the origin, and the
// hop is still stopped.
func TestWebFetchRedirectCheckOnDirectRetry(t *testing.T) {
// In a gated scope the direct retry is off (D-8): with a proxy that refuses the
// connection, the fetch fails with the proxy's error and never leaves without
// it. The proxy is a derived exemption, so its dial asks for nothing.
func TestWebFetchGatedScopeHasNoDirectRetry(t *testing.T) {
d := testDB(t)
t.Cleanup(func() { _ = d.Close() })
expID, err := d.CreateExploration("redirect retry probe", "the direct retry client checks redirects")
expID, err := d.CreateExploration("no direct retry probe", "a gated scope has no direct retry")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM explorations WHERE id=$1`, expID) })
taskID := fmt.Sprint(expID)
pair := newRedirectPair(t)
ctx := intercept.WithTargetScope(t.Context(), gatedScope(t, d, taskID))
scope := gatedScope(t, d, taskID)
scope.Interceptor.SetDerivedExemptions(func() []string { return []string{"http://127.0.0.1:1"} })
ctx := intercept.WithTargetScope(t.Context(), scope)
out, isErr := callWebFetch(t, ctx, "http://127.0.0.1:1", pair.start)
if !isErr || !strings.Contains(out, intercept.TargetApprovalPending) || !strings.Contains(out, "redirects to "+pair.finalKey) {
t.Fatalf("retried fetch: error=%v out=%q", isErr, out)
if !isErr || !strings.Contains(out, "proxyconnect") || strings.Contains(out, "TARGET_") {
t.Fatalf("proxied fetch: error=%v out=%q, want the proxy's error", isErr, out)
}
if pair.startHits.Load() != 1 || pair.finalHits.Load() != 0 {
t.Fatalf("hits origin=%d target=%d, want the origin reached by the retry and the target untouched", pair.startHits.Load(), pair.finalHits.Load())
if pair.startHits.Load() != 0 || pair.finalHits.Load() != 0 {
t.Fatalf("hits origin=%d target=%d, want no direct retry", pair.startHits.Load(), pair.finalHits.Load())
}
if keys := openRedirectRequests(t, d, taskID); len(keys) != 1 || keys[0] != pair.finalKey {
t.Fatalf("open requests = %v, want one for %s", keys, pair.finalKey)
var requests int
if err := d.QueryRow(`SELECT count(*) FROM intercept_pending WHERE task_id=$1 AND target_key IS NOT NULL`, taskID).Scan(&requests); err != nil || requests != 0 {
t.Fatalf("%d requests, %v; want none", requests, err)
}
}
@@ -213,10 +220,33 @@ func TestWebFetchRedirectGateOffUnchanged(t *testing.T) {
}
}
// pinTarget opens the request for the target of rawURL in scope and approves
// it, as a click on the approval card does.
func pinTarget(t *testing.T, d *db.DB, scope intercept.TargetScope, rawURL string) {
t.Helper()
ic, input := scope.Interceptor, []byte(fetchInputJSON(rawURL))
reqs, res := ic.CheckTargets(t.Context(), scope, "WebFetch", input)
if res.Code != "" || len(reqs) != 1 {
t.Fatalf("%s: requests=%+v result=%+v, want one request", rawURL, reqs, res)
}
scope.WaitBound = 0
if res = ic.RequestTargets(t.Context(), scope, "WebFetch", input, reqs, nil); res.Code != intercept.TargetApprovalPending || res.PendingID == 0 {
t.Fatalf("%s: request = %+v", rawURL, res)
}
if err := ic.Decide(res.PendingID, true, "test", "192.0.2.1"); err != nil {
t.Fatal(err)
}
taskID, err := strconv.ParseInt(scope.TaskID, 10, 64)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _, _ = d.Exec(`DELETE FROM target_pins WHERE task_id=$1`, taskID) })
}
// callWebFetch runs the WebFetch tool the agents build, with proxy.
func callWebFetch(t *testing.T, ctx context.Context, proxy, u string) (string, bool) {
t.Helper()
tools := withWebFetch(nil, proxy, "")
tools := withWebFetch(ctx, nil, proxy, "")
if len(tools) != 1 || tools[0].Name() != "WebFetch" {
t.Fatalf("withWebFetch built %d tools", len(tools))
}
+1 -1
View File
@@ -482,7 +482,7 @@ func (w *Worker) execute(ctx context.Context, name string, taskID int64, as *db.
Provider: w.prov,
SystemPrompt: system,
DynamicBoundary: boundary,
Tools: withWebFetch(tools, w.proxyAddr, w.proxyCACert),
Tools: withWebFetch(ctx, tools, w.proxyAddr, w.proxyCACert),
DeferredTools: def.Deferred,
UnlockSet: def.Unlock,
PermissionMode: permission.ModeBypass,
+1 -1
View File
@@ -61,4 +61,4 @@ require (
modernc.org/memory v1.11.0 // indirect
)
replace github.com/Autumn-27/norma => github.com/BaaRai/Norma v0.4.7
replace github.com/Autumn-27/norma => github.com/BaaRai/Norma v0.4.8
+2 -2
View File
@@ -1,5 +1,5 @@
github.com/BaaRai/Norma v0.4.7 h1:MJTKF7Dpuh3hjRJpmKtLJ8RknX30FEntDz08Q2y4iDs=
github.com/BaaRai/Norma v0.4.7/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw=
github.com/BaaRai/Norma v0.4.8 h1:k3at2aYSJ6aIDjEo7tC4VDSLBHXw2HobYjpKChVe6hI=
github.com/BaaRai/Norma v0.4.8/go.mod h1:BudWrf7iHl96FgHqFr/HwN3mKYmGjJDt+xL2ZHXfFZw=
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
+5 -1
View File
@@ -16,6 +16,7 @@ import (
"encoding/json"
"errors"
"fmt"
"net"
"regexp"
"strconv"
"strings"
@@ -190,6 +191,9 @@ type Interceptor struct {
targets targetStore
resolve *resolveCache
firstUse sync.Map
// dial is the base dialer of PinnedDialer: every connection it lets through
// is opened here. See SetBaseDialer.
dial func(ctx context.Context, network, addr string) (net.Conn, error)
}
// SetUpstreamProxyCheck installs the function that reports whether an upstream
@@ -220,7 +224,7 @@ var defaultEnabledTools = []string{
// first use.
func New(d *db.DB) *Interceptor {
return &Interceptor{db: d, pending: newPendingManager(),
targets: dbTargetStore{d}, resolve: newResolveCache(systemResolver{})}
targets: dbTargetStore{d}, resolve: newResolveCache(systemResolver{}), dial: baseDial}
}
// Invalidate clears the in-memory rule cache and the enabled-tools cache.
+180
View File
@@ -0,0 +1,180 @@
package intercept
import (
"context"
"encoding/json"
"fmt"
"net"
"net/netip"
"strconv"
"strings"
"time"
)
// The HTTP clients the platform owns (WebFetch and the custom http tool)
// connect through PinnedDialer. The hook, the rendered-URL gate and the
// redirect policy decide on a URL, and the client then resolves the name again
// on its own when it connects: a name can answer differently in between. In a
// gated scope the dialer takes a second, independent decision on the address
// it is asked to connect to (the target of the call, of a redirect hop, or a
// proxy). It resolves the name afresh, compares the answer with the pins, and
// connects only to the addresses it compared, never to the name. It never
// waits: an address no pin covers finds or opens the approval request and
// fails the connection at once with TARGET_APPROVAL_PENDING.
// baseDial opens a connection with the dialer settings of http.DefaultTransport.
var baseDial = (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext
// PinnedDialer returns the DialContext of a platform-owned HTTP client run by
// the tool toolName. The scope is read from the context net/http passes to the
// dial, which carries the values of the request's context: the scope the hook,
// the rendered-URL gate and the redirect policy read.
//
// scoped is whether the client was built for a session with a target scope.
// Such a client never dials without one: a scope missing at the dial means the
// context value was lost on the way, and the dial fails with
// TARGET_GATE_UNAVAILABLE. A client built without a scope (the operator's test
// endpoint) dials as before, and so does every client while the gate is off.
func PinnedDialer(toolName string, scoped bool) func(ctx context.Context, network, addr string) (net.Conn, error) {
return func(ctx context.Context, network, addr string) (net.Conn, error) {
scope, ok := TargetScopeFromContext(ctx)
switch {
case !ok && scoped:
return nil, &dialError{code: TargetGateUnavailable, target: addr, reason: "the target scope of the session did not reach the connection"}
case !ok:
return baseDial(ctx, network, addr)
case scope.Enabled == nil || !scope.Enabled():
if scope.Interceptor != nil {
return scope.Interceptor.baseDialer()(ctx, network, addr)
}
return baseDial(ctx, network, addr)
case scope.Interceptor == nil:
return nil, &dialError{code: TargetGateUnavailable, target: addr, reason: "the session's target scope carries no gate"}
}
return scope.Interceptor.dialTarget(ctx, scope, toolName, network, addr)
}
}
// GateOn reports whether ctx carries a target scope whose gate is on.
func GateOn(ctx context.Context) bool {
_, on := renderedScope(ctx)
return on
}
func (i *Interceptor) baseDialer() func(ctx context.Context, network, addr string) (net.Conn, error) {
i.mu.RLock()
defer i.mu.RUnlock()
return i.dial
}
// dialTarget decides one connection of a gated scope and opens it. The asset
// block rules, exemptions and pins are those of the hook, on the target the
// address names, with a fresh lookup. URL rules are not matched: they ran on
// the URL at the hook and on each hop in RedirectCheck.
//
// A derived exemption, and a pin that is name-only for this connection, dial
// the address as received. A stored exemption that holds, and a pin that
// holds every address of the lookup, dial those addresses and nothing else.
func (i *Interceptor) dialTarget(ctx context.Context, scope TargetScope, toolName, network, addr string) (net.Conn, error) {
input, _ := json.Marshal(map[string]string{"dial": addr})
ctx = WithCall(ctx, toolName, input)
t, err := addrTarget(network, addr)
if err != nil {
res := undeterminedTarget(ctx, scope, toolName, input, "the address of the connection could not be read ("+err.Error()+")")
return nil, &dialError{code: res.Code, target: addr, reason: res.Reason}
}
blocked := func(res TargetResult) error {
return &dialError{code: res.Code, target: t.String(), reason: res.Reason}
}
c := &targetCheck{i: i, ctx: ctx, scope: scope, tool: toolName, input: input, fresh: true}
unpinned, res := c.run([]Target{t})
if res.Code != "" {
return nil, blocked(res)
}
if len(unpinned) > 0 {
// Wait bound 0: the request is found or created, and the dial does not wait.
scope.WaitBound = 0
res = i.RequestTargets(ctx, scope, toolName, input, unpinned, nil)
if res.Code == TargetApprovalPending {
res.Reason = notCovered(unpinned[0], scope)
}
return nil, blocked(res)
}
switch ct := c.checked[0]; ct.cover {
case coverDerived, coverNameOnly:
return i.baseDialer()(ctx, network, addr)
default:
return i.dialAddrs(ctx, network, ct.ips, t.Port)
}
}
// addrTarget reads the address net/http dials, host:port with the host in
// ASCII form and the port always written, as the target urlTarget reads from
// the URL.
func addrTarget(network, addr string) (Target, error) {
switch network {
case "tcp", "tcp4", "tcp6":
default:
return Target{}, fmt.Errorf("network %q is not tcp", network)
}
host, port, err := net.SplitHostPort(addr)
if err != nil {
return Target{}, err
}
kind, h, ok := normalizeHost(host, true)
if !ok || kind == TargetCIDR {
return Target{}, fmt.Errorf("host %q", host)
}
t := Target{Kind: kind, Host: h, Port: validPort(port)}
if t.Port == 0 {
return Target{}, fmt.Errorf("port %q", port)
}
return t, nil
}
// dialAddrs connects to the addresses the decision compared, in their canonical
// order: the first that accepts wins. An address of the wrong family for tcp4
// or tcp6 is skipped. When every attempt fails, the last error is returned.
func (i *Interceptor) dialAddrs(ctx context.Context, network string, ips []string, port int) (net.Conn, error) {
dial := i.baseDialer()
var last error = &net.AddrError{Err: "no suitable address found", Addr: strings.Join(ips, ", ")}
for _, ip := range ips {
a, err := netip.ParseAddr(ip)
if err != nil || network == "tcp4" && !a.Is4() || network == "tcp6" && !a.Is6() {
continue
}
conn, err := dial(ctx, network, net.JoinHostPort(ip, strconv.Itoa(port)))
if err == nil {
return conn, nil
}
last = err
}
return nil, last
}
// notCovered says why a connection needs an approval: the addresses the dial
// resolved, and the pin they fall outside of.
func notCovered(r TargetRequest, scope TargetScope) string {
if r.Pin != nil {
return fmt.Sprintf("%s now resolves to %s, outside pin v%d (%s) of %s",
r.Host, strings.Join(r.ResolvedIPs, ", "), r.Pin.Version, strings.Join(r.Pin.PinnedIPs, ", "), scope.scopeName())
}
if r.Kind != TargetDomain {
return fmt.Sprintf("%s is not approved for %s", r.Host, scope.scopeName())
}
return fmt.Sprintf("%s resolves to %s and is not approved for %s", r.Host, strings.Join(r.ResolvedIPs, ", "), scope.scopeName())
}
// dialError is what PinnedDialer returns for a connection it does not open.
// Both clients hand the error to the model as the tool result.
type dialError struct {
code, target, reason string
}
func (e *dialError) Error() string {
msg := "[ARTEX platform control, not target defense] " + e.code + ": the connection to " + e.target + " was not opened: " + e.reason + "."
if e.code == TargetApprovalPending {
msg += " An approval was requested. Retry the same call after the approval."
}
return msg + " Do not fetch the target with another tool or by its address."
}
+336
View File
@@ -0,0 +1,336 @@
package intercept
import (
"context"
"net"
"net/http/httptest"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/Autumn-27/artex/db"
)
// Step 2.1 against PostgreSQL: a connection no pin covers at the dial opens or
// finds the durable request, connects nowhere, and fails at once.
// pgDialGate is a gate on d with a fake resolver and a recording base dialer.
func pgDialGate(t *testing.T, d *db.DB) (*Interceptor, *fakeResolver, *dialRecorder) {
t.Helper()
ic, res := pgGate(t, d)
rec := newDialRecorder()
ic.SetBaseDialer(rec.dial)
return ic, res, rec
}
// openTarget returns the audit target of the single open request of scope.
func openTarget(t *testing.T, d *db.DB, scope TargetScope) (*db.InterceptDetail, db.InterceptTarget) {
t.Helper()
ids := openRows(t, d, scope)
if len(ids) != 1 {
t.Fatalf("open requests = %v, want one", ids)
}
detail, err := d.GetInterceptDetail(ids[0])
if err != nil || detail == nil || detail.Audit == nil || len(detail.Audit.Targets) != 1 {
t.Fatalf("request detail = %+v, %v", detail, err)
}
return detail, detail.Audit.Targets[0]
}
// The failure test: the check passes on the pinned address, the name answers
// another one before the dial, and the call connects nowhere: not to the old
// address, not to the new one. One new_ip request shows both; once approved,
// the same call connects to the new address.
func TestPinAddressChangedBetweenCheckAndDialNotReachable(t *testing.T) {
d := targetTestDB(t)
ic, res, rec := pgDialGate(t, d)
srv, hits, port := pinnedServer(t, "rebound")
rec.route("203.0.113.7:"+port, srv.Listener.Addr().String())
rec.route("203.0.113.9:"+port, srv.Listener.Addr().String())
res.set("app.example.com", "203.0.113.7")
scope := taskScope(t, d, ic, time.Minute)
ctx := WithTargetScope(t.Context(), scope)
u := "http://app.example.com:" + port + "/"
pinURL(t, d, ic, scope, u)
mustPass(t, "the check", ic, scope, u)
res.set("app.example.com", "203.0.113.9")
began := time.Now()
_, err := get(ctx, pinnedClient(true), u)
want := "TARGET_APPROVAL_PENDING: the connection to app.example.com:" + port + " was not opened: app.example.com now resolves to 203.0.113.9, outside pin v1 (203.0.113.7) of task " + scope.TaskID
if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "Retry the same call after the approval") {
t.Fatalf("dial after the rebind: %v", err)
}
if elapsed := time.Since(began); elapsed > 10*time.Second {
t.Fatalf("the dial waited %v", elapsed)
}
if got := rec.calls(); len(got) != 0 || hits.Load() != 0 {
t.Fatalf("dialled %v, hits %d; want nothing", got, hits.Load())
}
detail, tgt := openTarget(t, d, scope)
if tgt.Reason != "new_ip" || !slices.Equal(tgt.ResolvedIPs, []string{"203.0.113.9"}) || tgt.PinVersion != 1 ||
!slices.Equal(tgt.PinnedIPs, []string{"203.0.113.7"}) || detail.ToolName != "WebFetch" || !strings.Contains(string(detail.ToolInput), `"dial": "app.example.com:`+port+`"`) {
t.Fatalf("request = %+v, target %+v", detail, tgt)
}
approveAll(t, d, ic, scope)
if body, err := get(ctx, pinnedClient(true), u); err != nil || body != "rebound" {
t.Fatalf("after the approval: %q, %v", body, err)
}
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.9:" + port}) {
t.Fatalf("dialled %v, want the new address", got)
}
if ps := pins(t, d, scope); len(ps) != 2 || ps[0].Version != 2 || !slices.Equal(ps[0].PinnedIPs, []string{"203.0.113.7", "203.0.113.9"}) {
t.Fatalf("pins = %+v, want v2 with both addresses", ps)
}
}
// One address outside the pin is enough: nothing is dialled, not even the
// pinned one.
func TestPinnedDialPartialOverlapNotReachable(t *testing.T) {
d := targetTestDB(t)
ic, res, rec := pgDialGate(t, d)
res.set("app.example.com", "203.0.113.7")
scope := taskScope(t, d, ic, time.Minute)
pinURL(t, d, ic, scope, "https://app.example.com/")
res.set("app.example.com", "203.0.113.7", "203.0.113.8")
_, err := PinnedDialer("WebFetch", true)(WithTargetScope(t.Context(), scope), "tcp", "app.example.com:443")
if err == nil || !strings.Contains(err.Error(), TargetApprovalPending) || len(rec.calls()) != 0 {
t.Fatalf("dial: %v, dialled %v", err, rec.calls())
}
if _, tgt := openTarget(t, d, scope); tgt.Reason != "new_ip" || !slices.Equal(tgt.ResolvedIPs, []string{"203.0.113.7", "203.0.113.8"}) {
t.Fatalf("request target %+v", tgt)
}
}
// A pin revoked between the check and the dial connects nowhere and asks again.
func TestPinnedDialRevokedBetweenCheckAndDial(t *testing.T) {
d := targetTestDB(t)
ic, res, rec := pgDialGate(t, d)
res.set("app.example.com", "203.0.113.7")
scope := taskScope(t, d, ic, time.Minute)
pinURL(t, d, ic, scope, "https://app.example.com/")
mustPass(t, "the check", ic, scope, "https://app.example.com/")
ps := pins(t, d, scope)
if _, err := d.RevokeTargetPin(scope.dbRef(t), ps[0].ID); err != nil {
t.Fatal(err)
}
_, err := PinnedDialer("WebFetch", true)(WithTargetScope(t.Context(), scope), "tcp", "app.example.com:443")
if err == nil || !strings.Contains(err.Error(), TargetApprovalPending) || !strings.Contains(err.Error(), "is not approved for task") || len(rec.calls()) != 0 {
t.Fatalf("dial: %v, dialled %v", err, rec.calls())
}
if _, tgt := openTarget(t, d, scope); tgt.Reason != "no_pin" {
t.Fatalf("request target %+v", tgt)
}
}
// A stored exemption is dialled at the addresses the dial resolved and found
// public; the same name on a private address is void at the dial, and with no
// pin it asks, connecting nowhere.
func TestPinnedDialStoredExemptionDialsCheckedAddresses(t *testing.T) {
d := targetTestDB(t)
ic, res, rec := pgDialGate(t, d)
srv, _, port := pinnedServer(t, "package")
rec.route("198.51.100.20:"+port, srv.Listener.Addr().String())
rec.route("10.0.0.5:"+port, srv.Listener.Addr().String())
storedExemptions(t, d, db.TargetExemption{Host: "pkg.example", Port: atoiPort(t, port)})
scope := taskScope(t, d, ic, time.Minute)
ctx := WithTargetScope(t.Context(), scope)
u := "http://pkg.example:" + port + "/"
res.set("pkg.example", "198.51.100.20")
if body, err := get(ctx, pinnedClient(true), u); err != nil || body != "package" {
t.Fatalf("exempt fetch: %q, %v", body, err)
}
res.set("pkg.example", "10.0.0.5")
if _, err := get(ctx, pinnedClient(true), u); err == nil || !strings.Contains(err.Error(), TargetApprovalPending) {
t.Fatalf("void exemption: %v", err)
}
if got := rec.calls(); !slices.Equal(got, []string{"198.51.100.20:" + port}) {
t.Fatalf("dialled %v, want the checked public address only", got)
}
if _, tgt := openTarget(t, d, scope); !tgt.ExemptVoid || tgt.Reason != "no_pin" {
t.Fatalf("request target %+v, want a void exemption", tgt)
}
}
// A pinned IP literal is dialled as written, never resolved; a name that
// resolves to the same address is another target, not covered (D22).
func TestPinnedDialIPLiteralPin(t *testing.T) {
d := targetTestDB(t)
ic, res, rec := pgDialGate(t, d)
srv, _, port := pinnedServer(t, "literal")
rec.route("198.51.100.7:"+port, srv.Listener.Addr().String())
scope := taskScope(t, d, ic, time.Minute)
ctx := WithTargetScope(t.Context(), scope)
pinURL(t, d, ic, scope, "http://198.51.100.7:"+port+"/")
if body, err := get(ctx, pinnedClient(true), "http://198.51.100.7:"+port+"/"); err != nil || body != "literal" {
t.Fatalf("literal: %q, %v", body, err)
}
if res.total() != 0 {
t.Fatalf("%d lookups for an IP literal", res.total())
}
res.set("lit.example", "198.51.100.7")
if _, err := get(ctx, pinnedClient(true), "http://lit.example:"+port+"/"); err == nil || !strings.Contains(err.Error(), TargetApprovalPending) {
t.Fatalf("a name on the pinned address: %v", err)
}
if got := rec.calls(); !slices.Equal(got, []string{"198.51.100.7:" + port}) {
t.Fatalf("dialled %v", got)
}
}
// Port precedence as at the check: a (host, 0) pin covers any port; a
// (host, 443) pin does not cover port 80.
func TestPinnedDialPortPrecedence(t *testing.T) {
d := targetTestDB(t)
ic, res, rec := pgDialGate(t, d)
srv, _, _ := pinnedServer(t, "any-port")
rec.route("203.0.113.7:8443", srv.Listener.Addr().String())
res.set("app.example.com", "203.0.113.7")
res.set("web.example.com", "203.0.113.8")
scope := taskScope(t, d, ic, time.Minute)
ctx := WithTargetScope(t.Context(), scope)
// A URL of a scheme without a default port declares no port: its pin is (host, 0).
pinURL(t, d, ic, scope, "ssh://app.example.com/")
if ps := pins(t, d, scope); len(ps) != 1 || ps[0].Port != 0 {
t.Fatalf("pins = %+v, want one on (app.example.com, 0)", ps)
}
pinURL(t, d, ic, scope, "https://web.example.com/")
conn, err := PinnedDialer("WebFetch", true)(ctx, "tcp", "app.example.com:8443")
if err != nil {
t.Fatalf("(host, 0) pin, port 8443: %v", err)
}
conn.Close()
if _, err := PinnedDialer("WebFetch", true)(ctx, "tcp", "web.example.com:80"); err == nil || !strings.Contains(err.Error(), TargetApprovalPending) {
t.Fatalf("(host, 443) pin, port 80: %v", err)
}
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.7:8443"}) {
t.Fatalf("dialled %v", got)
}
if _, tgt := openTarget(t, d, scope); tgt.Host != "web.example.com" || tgt.Port != 80 || tgt.Reason != "no_pin" {
t.Fatalf("request target %+v, want web.example.com:80", tgt)
}
}
// D-5: a redirect hop is checked by the redirect policy and then dialled
// against the pin of the hop's (host, port), never the origin's.
func TestRedirectHopDialledAgainstHopPinNotOrigin(t *testing.T) {
d := targetTestDB(t)
final, finalHits, finalPort := pinnedServer(t, "hop-b")
start, _, startPort := hopRedirect(t, "http://b.example:"+finalPort+"/final")
aURL, bURL := "http://a.example:"+startPort+"/start", "http://b.example:"+finalPort+"/"
setup := func(t *testing.T, pinB bool) (context.Context, TargetScope, *fakeResolver, *dialRecorder) {
ic, res, rec := pgDialGate(t, d)
rec.route("203.0.113.1:"+startPort, start.Listener.Addr().String())
rec.route("203.0.113.2:"+finalPort, final.Listener.Addr().String())
// Were the hop dialled at the origin's address, it would reach b as well.
rec.route("203.0.113.1:"+finalPort, final.Listener.Addr().String())
res.set("a.example", "203.0.113.1")
res.set("b.example", "203.0.113.2")
scope := taskScope(t, d, ic, time.Minute)
pinURL(t, d, ic, scope, aURL)
if pinB {
pinURL(t, d, ic, scope, bURL)
}
return WithTargetScope(t.Context(), scope), scope, res, rec
}
t.Run("each hop at its own pin", func(t *testing.T) {
ctx, _, _, rec := setup(t, true)
before := finalHits.Load()
if body, err := get(ctx, pinnedClient(true), aURL); err != nil || body != "hop-b" || finalHits.Load() != before+1 {
t.Fatalf("redirect: %q, %v", body, err)
}
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.1:" + startPort, "203.0.113.2:" + finalPort}) {
t.Fatalf("dialled %v", got)
}
})
t.Run("hop resolving to the origin's pinned address", func(t *testing.T) {
ctx, scope, res, rec := setup(t, true)
// The cache still holds b's pinned answer: the redirect policy passes.
res.set("b.example", "203.0.113.1")
before := finalHits.Load()
_, err := get(ctx, pinnedClient(true), aURL)
if err == nil || !strings.Contains(err.Error(), "TARGET_APPROVAL_PENDING: the connection to b.example:"+finalPort+" was not opened") {
t.Fatalf("hop: %v", err)
}
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.1:" + startPort}) || finalHits.Load() != before {
t.Fatalf("dialled %v, b hits %d; want the origin only", got, finalHits.Load()-before)
}
if _, tgt := openTarget(t, d, scope); tgt.Host != "b.example" || tgt.Port != atoiPort(t, finalPort) || tgt.Reason != "new_ip" {
t.Fatalf("request target %+v, want new_ip on b.example", tgt)
}
})
t.Run("hop without a pin", func(t *testing.T) {
ctx, scope, _, rec := setup(t, false)
before := finalHits.Load()
_, err := get(ctx, pinnedClient(true), aURL)
if err == nil || !strings.Contains(err.Error(), "redirects to b.example:"+finalPort) || !strings.Contains(err.Error(), "not followed") {
t.Fatalf("hop: %v", err)
}
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.1:" + startPort}) || finalHits.Load() != before {
t.Fatalf("dialled %v; want the dialer never called for b", got)
}
if _, tgt := openTarget(t, d, scope); tgt.Host != "b.example" || tgt.Reason != "no_pin" {
t.Fatalf("request target %+v", tgt)
}
})
}
// hopRedirect is a server that redirects every request to location.
func hopRedirect(t *testing.T, location string) (*httptest.Server, *atomic.Int64, string) {
t.Helper()
srv, hits, key := hopServer(t, location)
_, port, _ := net.SplitHostPort(key)
return srv, hits, port
}
// Two concurrent dials of one host whose address changed share one request:
// both fail at once, one approval extends the pin, both retries connect.
func TestDialMismatchOpensOneNewIPRequest(t *testing.T) {
d := targetTestDB(t)
ic, res, rec := pgDialGate(t, d)
srv, _, port := pinnedServer(t, "shared")
rec.route("203.0.113.9:"+port, srv.Listener.Addr().String())
res.set("app.example.com", "203.0.113.7")
scope := taskScope(t, d, ic, time.Minute)
ctx := WithTargetScope(t.Context(), scope)
u := "http://app.example.com:" + port + "/"
pinURL(t, d, ic, scope, u)
res.set("app.example.com", "203.0.113.9")
fetchBoth := func() []error {
var wg sync.WaitGroup
errs := make([]error, 2)
for n := range errs {
wg.Go(func() { _, errs[n] = get(ctx, pinnedClient(true), u) })
}
wg.Wait()
return errs
}
for n, err := range fetchBoth() {
if err == nil || !strings.Contains(err.Error(), TargetApprovalPending) {
t.Fatalf("dial %d: %v", n, err)
}
}
if ids := openRows(t, d, scope); len(ids) != 1 || allRows(t, d, scope) != 2 {
t.Fatalf("open %v, %d request rows; want the approved one and one new_ip", ids, allRows(t, d, scope))
}
if len(rec.calls()) != 0 {
t.Fatalf("dialled %v", rec.calls())
}
approveAll(t, d, ic, scope)
for n, err := range fetchBoth() {
if err != nil {
t.Fatalf("retry %d: %v", n, err)
}
}
if got := rec.calls(); len(got) != 2 || got[0] != "203.0.113.9:"+port || got[1] != got[0] {
t.Fatalf("dialled %v", got)
}
}
+537
View File
@@ -0,0 +1,537 @@
package intercept
import (
"context"
"crypto/tls"
"errors"
"io"
"net"
"net/http"
"net/http/httptest"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/Autumn-27/artex/db"
)
// Step 2.1: the pinned dialer of the platform-owned HTTP clients. Without a
// database: a dial that would open a request is covered by the PostgreSQL
// tests in target_dial_pg_test.go.
// dialRecorder is a base dialer for tests. It records every address it is
// asked to connect to, connects the routed ones (TEST-NET addresses) to local
// listeners and loopback addresses to themselves, and refuses the rest.
type dialRecorder struct {
mu sync.Mutex
routes map[string]string
refuse map[string]bool
dialed []string
}
func newDialRecorder() *dialRecorder {
return &dialRecorder{routes: map[string]string{}, refuse: map[string]bool{}}
}
func (d *dialRecorder) route(addr, to string) {
d.mu.Lock()
d.routes[addr] = to
d.mu.Unlock()
}
func (d *dialRecorder) dial(ctx context.Context, network, addr string) (net.Conn, error) {
d.mu.Lock()
d.dialed = append(d.dialed, addr)
to, ok := d.routes[addr]
refused := d.refuse[addr]
d.mu.Unlock()
if !ok && strings.HasPrefix(addr, "127.0.0.1:") {
to, ok = addr, true
}
if refused || !ok {
return nil, &net.OpError{Op: "dial", Net: network, Err: errors.New("connection refused by the test dialer")}
}
return (&net.Dialer{}).DialContext(ctx, network, to)
}
func (d *dialRecorder) calls() []string {
d.mu.Lock()
defer d.mu.Unlock()
return slices.Clone(d.dialed)
}
// pinnedServer is a listener that counts its requests; port is its port.
func pinnedServer(t *testing.T, body string) (srv *httptest.Server, hits *atomic.Int64, port string) {
t.Helper()
hits = &atomic.Int64{}
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
_, _ = w.Write([]byte(body))
}))
t.Cleanup(srv.Close)
_, port, _ = net.SplitHostPort(srv.Listener.Addr().String())
return srv, hits, port
}
// dialGate is a gate fixture whose base dialer is a dialRecorder.
func dialGate(t *testing.T) (*gateFixture, *dialRecorder) {
t.Helper()
g := newGate(t)
rec := newDialRecorder()
g.ic.SetBaseDialer(rec.dial)
return g, rec
}
func (g *gateFixture) pin(host string, port int, ips ...string) *db.TargetPin {
g.store.mu.Lock()
defer g.store.mu.Unlock()
p := &db.TargetPin{ID: int64(len(g.store.pins) + 1), Host: host, Port: port, Version: 1, Status: db.TargetPinActive, PinnedIPs: ips}
g.store.pins = append(g.store.pins, p)
return p
}
// pinnedClient is a client built like the platform's: the redirect policy and
// the pinned dialer, keep-alive off.
func pinnedClient(scoped bool) *http.Client {
tr := http.DefaultTransport.(*http.Transport).Clone()
tr.Proxy = nil
tr.DialContext = PinnedDialer("WebFetch", scoped)
tr.DisableKeepAlives = true
return &http.Client{Transport: tr, CheckRedirect: RedirectCheck("WebFetch")}
}
func get(ctx context.Context, client *http.Client, rawURL string) (string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return "", err
}
resp, err := client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
b, err := io.ReadAll(resp.Body)
return string(b), err
}
// With the gate off, or with no scope in a client built without one, the dial
// is the dial of today: the address as received, no lookup, no store access.
func TestPinnedDialGateOffIsPlainDial(t *testing.T) {
g, rec := dialGate(t)
srv, hits, port := pinnedServer(t, "plain")
rec.route("app.example.com:"+port, srv.Listener.Addr().String())
g.res.set("app.example.com", "203.0.113.7")
conn, err := PinnedDialer("WebFetch", false)(context.Background(), "tcp", srv.Listener.Addr().String())
if err != nil {
t.Fatalf("no scope: %v", err)
}
conn.Close()
for name, scope := range map[string]TargetScope{
"no flag": {TaskID: "7", WaitBound: time.Minute, Interceptor: g.ic},
"flag off": {TaskID: "7", Enabled: func() bool { return false }, WaitBound: time.Minute, Interceptor: g.ic},
} {
if body, err := get(WithTargetScope(context.Background(), scope), pinnedClient(true), "http://app.example.com:"+port+"/"); err != nil || body != "plain" {
t.Fatalf("%s: %q, %v", name, body, err)
}
}
if got := rec.calls(); len(got) != 2 || got[0] != "app.example.com:"+port || got[1] != got[0] {
t.Fatalf("dialled %v, want the address as received", got)
}
if hits.Load() != 2 || g.store.reads != 0 || len(g.store.logged) != 0 || g.res.total() != 0 {
t.Fatalf("hits=%d, %d store reads, %d rows, %d lookups; want none but the hits", hits.Load(), g.store.reads, len(g.store.logged), g.res.total())
}
}
// A client built for a scope never dials without one: a lost context value
// fails closed, nothing is dialled.
func TestPinnedDialScopeLostFailsClosed(t *testing.T) {
srv, hits, _ := pinnedServer(t, "unbound")
_, err := PinnedDialer("WebFetch", true)(context.Background(), "tcp", srv.Listener.Addr().String())
if err == nil || !strings.Contains(err.Error(), TargetGateUnavailable) || !strings.Contains(err.Error(), "did not reach the connection") {
t.Fatalf("dial without the scope: %v", err)
}
if _, err := get(context.Background(), pinnedClient(true), srv.URL); err == nil || !strings.Contains(err.Error(), TargetGateUnavailable) {
t.Fatalf("fetch without the scope: %v", err)
}
if hits.Load() != 0 {
t.Fatalf("%d requests reached the server", hits.Load())
}
}
// Every way the gate cannot decide fails closed with TARGET_GATE_UNAVAILABLE,
// dials nothing, and logs one decided row where there is a store and a scope.
func TestPinnedDialScopeUnreadableFailsClosed(t *testing.T) {
for _, tc := range []struct {
name string
setup func(g *gateFixture)
network, addr string
rows int
}{
{"nil interceptor", func(g *gateFixture) { g.scope.Interceptor = nil }, "tcp", "app.example.com:443", 0},
{"task id", func(g *gateFixture) { g.scope.TaskID = "seven" }, "tcp", "app.example.com:443", 0},
{"rules", func(g *gateFixture) { g.store.fail["BlockRules"] = errors.New("down") }, "tcp", "app.example.com:443", 1},
{"pins", func(g *gateFixture) { g.store.fail["ActivePins"] = errors.New("down") }, "tcp", "app.example.com:443", 1},
{"exemptions", func(g *gateFixture) { g.store.fail["Exemptions"] = errors.New("down") }, "tcp", "app.example.com:443", 1},
{"address", nil, "tcp", "app.example.com", 1},
{"port", nil, "tcp", "app.example.com:0", 1},
{"network", nil, "udp", "app.example.com:443", 1},
} {
t.Run(tc.name, func(t *testing.T) {
g, rec := dialGate(t)
g.res.set("app.example.com", "203.0.113.7")
g.pin("app.example.com", 443, "203.0.113.7")
if tc.setup != nil {
tc.setup(g)
}
_, err := PinnedDialer("WebFetch", true)(g.ctx(), tc.network, tc.addr)
if err == nil || !strings.Contains(err.Error(), TargetGateUnavailable) {
t.Fatalf("dial: %v, want TARGET_GATE_UNAVAILABLE", err)
}
if got := rec.calls(); len(got) != 0 {
t.Fatalf("dialled %v", got)
}
if got := g.store.outcomes(); len(got) != tc.rows || tc.rows == 1 && got[0] != "denied/gate_unavailable" {
t.Fatalf("rows = %v, want %d gate_unavailable", got, tc.rows)
}
})
}
}
// A pinned name is dialled at the address the lookup returned and the pin
// holds, never by the name.
func TestPinnedDialConnectsOnlyToPinnedAddresses(t *testing.T) {
g, rec := dialGate(t)
srv, hits, port := pinnedServer(t, "pinned")
rec.route("203.0.113.7:"+port, srv.Listener.Addr().String())
g.res.set("app.example.com", "203.0.113.7")
g.pin("app.example.com", atoiPort(t, port), "203.0.113.7")
if body, err := get(g.ctx(), pinnedClient(true), "http://app.example.com:"+port+"/"); err != nil || body != "pinned" {
t.Fatalf("pinned fetch: %q, %v", body, err)
}
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.7:" + port}) || hits.Load() != 1 {
t.Fatalf("dialled %v, hits %d; want the pinned address only", got, hits.Load())
}
}
// A name that does not resolve at the dial is blocked without a request (the
// fixture has no database: a request would fail the test), nothing is dialled.
func TestPinnedDialResolveFailureBlocksWithoutRequest(t *testing.T) {
g, rec := dialGate(t)
g.pin("app.example.com", 443, "203.0.113.7")
g.res.mu.Lock()
g.res.errs["app.example.com"] = errors.New("no such host")
g.res.mu.Unlock()
_, err := PinnedDialer("WebFetch", true)(g.ctx(), "tcp", "app.example.com:443")
if err == nil || !strings.Contains(err.Error(), TargetResolveFailed) || !strings.Contains(err.Error(), "app.example.com:443") {
t.Fatalf("dial: %v", err)
}
if got := g.store.outcomes(); len(got) != 1 || got[0] != "denied/resolve_failed" || len(rec.calls()) != 0 {
t.Fatalf("rows %v, dialled %v", got, rec.calls())
}
}
// An asset block rule on an address the dial resolves wins over the pin.
func TestPinnedDialBlockRuleOnDialAddress(t *testing.T) {
g, rec := dialGate(t)
g.pin("app.example.com", 443, "203.0.113.7", "203.0.113.8")
g.res.set("app.example.com", "203.0.113.8")
g.store.rules = []db.AssetInterceptRule{{Enabled: true, Kind: "exact_ip", Pattern: "203.0.113.8"}}
_, err := PinnedDialer("WebFetch", true)(g.ctx(), "tcp", "app.example.com:443")
if err == nil || !strings.Contains(err.Error(), TargetRuleBlocked) {
t.Fatalf("dial: %v", err)
}
if got := g.store.outcomes(); len(got) != 1 || got[0] != "denied/rule_blocked" || len(rec.calls()) != 0 {
t.Fatalf("rows %v, dialled %v", got, rec.calls())
}
}
// A derived exemption is dialled as received and never resolved, wherever its
// name points: an LLM gateway on an internal address, the recording proxy on
// loopback.
func TestPinnedDialDerivedExemptionDialsByName(t *testing.T) {
g, rec := dialGate(t)
srv, hits, port := pinnedServer(t, "gateway")
rec.route("llm.example:"+port, srv.Listener.Addr().String())
g.res.set("llm.example", "10.0.0.5")
proxy := srv.Listener.Addr().String()
g.ic.SetDerivedExemptions(func() []string { return []string{"https://llm.example:" + port, "http://" + proxy} })
for _, addr := range []string{"llm.example:" + port, proxy} {
conn, err := PinnedDialer("WebFetch", true)(g.ctx(), "tcp", addr)
if err != nil {
t.Fatalf("%s: %v", addr, err)
}
conn.Close()
}
if got := rec.calls(); !slices.Equal(got, []string{"llm.example:" + port, proxy}) || g.res.total() != 0 || hits.Load() != 0 {
t.Fatalf("dialled %v, %d lookups; want the addresses as received and no lookup", got, g.res.total())
}
}
// A pin that is name-only for this connection, by its mark or because the
// upstream proxy is set now, is dialled by name: the platform does not bind it.
func TestPinnedDialNameOnlyPinDialsByName(t *testing.T) {
for _, tc := range []struct {
name string
marked bool
proxied bool
}{{"marked pin", true, false}, {"upstream proxy now", false, true}} {
t.Run(tc.name, func(t *testing.T) {
g, rec := dialGate(t)
srv, _, port := pinnedServer(t, "name-only")
rec.route("app.example.com:"+port, srv.Listener.Addr().String())
g.res.set("app.example.com", "203.0.113.9")
g.pin("app.example.com", atoiPort(t, port), "203.0.113.7").NameOnlyEffective = tc.marked
g.ic.SetUpstreamProxyCheck(func() bool { return tc.proxied })
if body, err := get(g.ctx(), pinnedClient(true), "http://app.example.com:"+port+"/"); err != nil || body != "name-only" {
t.Fatalf("fetch: %q, %v", body, err)
}
if got := rec.calls(); !slices.Equal(got, []string{"app.example.com:" + port}) {
t.Fatalf("dialled %v, want the name", got)
}
})
}
}
// The dial reads its address as the check reads the URL: same key for case,
// trailing dot, Unicode, IPv6 brackets and IPv4-mapped IPv6. net/http dials the
// ASCII form of a Unicode name with the scheme's port written (S3).
func TestPinnedDialNormalisesLikeTheCheck(t *testing.T) {
for _, tc := range []struct{ rawURL, addr string }{
{"https://App.Example.COM./x", "App.Example.COM.:443"},
{"http://bücher.example/", "xn--bcher-kva.example:80"},
{"https://[2001:DB8::1]:8443/", "[2001:db8::1]:8443"},
{"http://[::ffff:192.0.2.1]/", "[::ffff:192.0.2.1]:80"},
} {
want, err := urlTarget(tc.rawURL)
if err != nil {
t.Fatal(err)
}
got, err := addrTarget("tcp", tc.addr)
if err != nil || got != want {
t.Errorf("%s: dial key %+v (%v), check key %+v", tc.addr, got, err, want)
}
}
var dialled string
tr := &http.Transport{DialContext: func(_ context.Context, _, addr string) (net.Conn, error) {
dialled = addr
return nil, errors.New("recorded")
}}
_, _ = (&http.Client{Transport: tr}).Get("http://bücher.example/")
if dialled != "xn--bcher-kva.example:80" {
t.Fatalf("net/http dialled %q, want the ASCII name and the scheme's port", dialled)
}
}
// Concurrent dials of one host share one lookup; the cached answer is not
// used, and the cache holds the new answer afterwards.
func TestPinnedDialFreshLookupCollapsed(t *testing.T) {
g, rec := dialGate(t)
srv, _, port := pinnedServer(t, "fresh")
rec.route("203.0.113.7:"+port, srv.Listener.Addr().String())
g.pin("app.example.com", atoiPort(t, port), "203.0.113.7")
g.res.set("app.example.com", "203.0.113.9")
if _, err := g.ic.resolve.lookup(context.Background(), "app.example.com", false); err != nil {
t.Fatal(err)
}
// The cache now holds 203.0.113.9, which no pin covers: a dial that used it
// would open a request, which this fixture cannot.
g.res.set("app.example.com", "203.0.113.7")
hold := make(chan struct{})
g.res.mu.Lock()
g.res.hold = hold
g.res.mu.Unlock()
var wg sync.WaitGroup
errs := make(chan error, 2)
for range 2 {
wg.Go(func() {
conn, err := PinnedDialer("WebFetch", true)(g.ctx(), "tcp", "app.example.com:"+port)
if err == nil {
conn.Close()
}
errs <- err
})
}
eventually(t, "the shared lookup", func() bool { return g.res.lookups("app.example.com") == 2 })
time.Sleep(100 * time.Millisecond)
close(hold)
wg.Wait()
close(errs)
for err := range errs {
if err != nil {
t.Fatalf("dial: %v", err)
}
}
if n := g.res.lookups("app.example.com"); n != 2 {
t.Fatalf("%d lookups, want the cached one and one shared by both dials", n)
}
g.ic.resolve.mu.Lock()
cached := g.ic.resolve.entries["app.example.com"].ips
g.ic.resolve.mu.Unlock()
if !slices.Equal(cached, []string{"203.0.113.7"}) || !slices.Equal(rec.calls(), []string{"203.0.113.7:" + port, "203.0.113.7:" + port}) {
t.Fatalf("cache %v, dialled %v", cached, rec.calls())
}
}
// The compared addresses are tried in canonical order, the first that accepts
// wins; tcp4 skips the IPv6 ones.
func TestPinnedDialTriesAddressesInOrder(t *testing.T) {
g, rec := dialGate(t)
srv, _, port := pinnedServer(t, "second")
rec.refuse["203.0.113.7:"+port] = true
rec.route("203.0.113.8:"+port, srv.Listener.Addr().String())
g.pin("app.example.com", atoiPort(t, port), "203.0.113.7", "203.0.113.8")
g.res.set("app.example.com", "203.0.113.8", "203.0.113.7")
if body, err := get(g.ctx(), pinnedClient(true), "http://app.example.com:"+port+"/"); err != nil || body != "second" {
t.Fatalf("fetch: %q, %v", body, err)
}
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.7:" + port, "203.0.113.8:" + port}) {
t.Fatalf("dialled %v", got)
}
g, rec = dialGate(t)
rec.route("203.0.113.8:"+port, srv.Listener.Addr().String())
g.pin("app.example.com", atoiPort(t, port), "2001:db8::1", "203.0.113.8")
g.res.set("app.example.com", "2001:db8::1", "203.0.113.8")
conn, err := PinnedDialer("WebFetch", true)(g.ctx(), "tcp4", "app.example.com:"+port)
if err != nil {
t.Fatalf("tcp4: %v", err)
}
conn.Close()
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.8:" + port}) {
t.Fatalf("tcp4 dialled %v, want the IPv4 address only", got)
}
}
// S1: the context net/http passes to DialContext carries the values of the
// request's context. The client is built without a scope (scoped=false), so
// only the request's context can send the dial down the pinned path.
func TestPinnedDialSeesScopeInDialContext(t *testing.T) {
g, rec := dialGate(t)
srv, _, port := pinnedServer(t, "scoped")
rec.route("203.0.113.7:"+port, srv.Listener.Addr().String())
g.pin("app.example.com", atoiPort(t, port), "203.0.113.7")
g.res.set("app.example.com", "203.0.113.7")
if body, err := get(g.ctx(), pinnedClient(false), "http://app.example.com:"+port+"/"); err != nil || body != "scoped" {
t.Fatalf("fetch: %q, %v", body, err)
}
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.7:" + port}) || g.res.lookups("app.example.com") != 1 {
t.Fatalf("dialled %v after %d lookups, want the pinned path", got, g.res.lookups("app.example.com"))
}
}
// S4: dialling the pinned address keeps HTTPS verified against the name of
// the request. A name the certificate does not hold fails verification.
func TestPinnedDialHTTPSVerifiesName(t *testing.T) {
g, rec := dialGate(t)
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("verified")) }))
t.Cleanup(srv.Close)
_, port, _ := net.SplitHostPort(srv.Listener.Addr().String())
rec.route("203.0.113.7:"+port, srv.Listener.Addr().String())
// httptest's certificate holds example.com.
for _, host := range []string{"example.com", "other.example"} {
g.pin(host, atoiPort(t, port), "203.0.113.7")
g.res.set(host, "203.0.113.7")
}
client := pinnedClient(true)
client.Transport.(*http.Transport).TLSClientConfig = &tls.Config{RootCAs: srv.Client().Transport.(*http.Transport).TLSClientConfig.RootCAs}
if body, err := get(g.ctx(), client, "https://example.com:"+port+"/"); err != nil || body != "verified" {
t.Fatalf("https by the pinned address: %q, %v", body, err)
}
_, err := get(g.ctx(), client, "https://other.example:"+port+"/")
var verr *tls.CertificateVerificationError
if !errors.As(err, &verr) {
t.Fatalf("a name the certificate does not hold: %v, want a verification error", err)
}
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.7:" + port, "203.0.113.7:" + port}) {
t.Fatalf("dialled %v", got)
}
}
// S6: with keep-alive off (as Norma sets it on a transport with a dialer, and
// the custom tool on its own), a second fetch of one client dials again: a pin
// changed in between applies to it.
func TestPinnedDialNoReuseAcrossCalls(t *testing.T) {
g, rec := dialGate(t)
srv, _, port := pinnedServer(t, "again")
rec.route("203.0.113.7:"+port, srv.Listener.Addr().String())
rec.route("203.0.113.8:"+port, srv.Listener.Addr().String())
p := atoiPort(t, port)
g.pin("app.example.com", p, "203.0.113.7")
g.res.set("app.example.com", "203.0.113.7")
client := pinnedClient(true)
u := "http://app.example.com:" + port + "/"
if _, err := get(g.ctx(), client, u); err != nil {
t.Fatal(err)
}
g.store.revoke("app.example.com", p)
g.pin("app.example.com", p, "203.0.113.8")
g.res.set("app.example.com", "203.0.113.8")
if _, err := get(g.ctx(), client, u); err != nil {
t.Fatal(err)
}
if got := rec.calls(); !slices.Equal(got, []string{"203.0.113.7:" + port, "203.0.113.8:" + port}) {
t.Fatalf("dialled %v, want a new dial under the new pin", got)
}
}
// S5: the redirected request keeps the original request's context, so the
// hop's redirect check and the hop's dial read the same target scope.
func TestRedirectContextCarriesScope(t *testing.T) {
g, rec := dialGate(t)
final, finalHits, finalKey := hopServer(t, "")
start, _, startKey := hopServer(t, final.URL+"/final")
for _, key := range []string{startKey, finalKey} {
host, port, _ := net.SplitHostPort(key)
g.pin(host, atoiPort(t, port), host)
}
var hopScope TargetScope
var hopOK bool
client := pinnedClient(true)
policy := client.CheckRedirect
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
hopScope, hopOK = TargetScopeFromContext(req.Context())
return policy(req, via)
}
if body, err := get(g.ctx(), client, start.URL+"/start"); err != nil || body != "hop-reached" || finalHits.Load() != 1 {
t.Fatalf("redirect: %q, %v", body, err)
}
if !hopOK || hopScope.TaskID != g.scope.TaskID || hopScope.Interceptor != g.ic {
t.Fatalf("hop context scope = %+v (%v), want the scope of the call", hopScope, hopOK)
}
if got := rec.calls(); !slices.Equal(got, []string{startKey, finalKey}) {
t.Fatalf("dialled %v, want the origin then the hop through the pinned path", got)
}
}
// D-2: a request raised while the upstream proxy is set says the platform does
// not bind the connection.
func TestNameOnlyReasonSaysConnectionNotBound(t *testing.T) {
scope := TargetScope{TaskID: "7"}
r := target("app.example.com", 443, "203.0.113.7")
const sentence = "This approval is name-only. The platform does not bind the connection to these addresses."
if got := targetReason(scope, r, nil, true); !strings.Contains(got, sentence) {
t.Fatalf("name-only reason = %q", got)
}
if got := targetReason(scope, r, nil, false); strings.Contains(got, "does not bind") {
t.Fatalf("reason without the upstream proxy = %q", got)
}
}
// D-2: the first-use row of an unmarked pin used while the upstream proxy is
// set records the pin as name-only.
func TestPinUsedRecordsEffectiveNameOnly(t *testing.T) {
g := newGate(t)
g.res.set("app.example.com", "203.0.113.7")
g.pin("app.example.com", 443, "203.0.113.7")
g.ic.SetUpstreamProxyCheck(func() bool { return true })
g.passes("pinned", g.fetch(appURL))
if got := g.store.outcomes(); len(got) != 1 || got[0] != "allowed/pin_match" || !g.store.logged[0].Audit.Targets[0].NameOnlyEffective {
t.Fatalf("rows %v, target %+v; want a name-only first use", got, g.store.logged[0].Audit.Targets[0])
}
}
+9 -1
View File
@@ -73,8 +73,13 @@ func (i *Interceptor) SetDerivedExemptions(f func() []string) {
i.mu.Unlock()
}
// proxyPorts are the ports net/http dials for a proxy URL without one, for the
// schemes schemePorts has no default for.
var proxyPorts = map[string]int{"socks5": 1080, "socks5h": 1080}
// DerivedExemptions returns the exemptions derived from the configuration, as
// exact host and port. An endpoint without a port is not listed.
// exact host and port. An endpoint without a port gets the one net/http dials
// (the scheme's default, 1080 for socks5); one with neither is not listed.
func (i *Interceptor) DerivedExemptions() []db.TargetExemption {
i.mu.RLock()
f := i.derivedExempt
@@ -86,6 +91,9 @@ func (i *Interceptor) DerivedExemptions() []db.TargetExemption {
seen := map[string]bool{}
for _, raw := range f() {
t, ok := NormalizeTarget(raw)
if scheme, _, found := strings.Cut(strings.TrimSpace(raw), "://"); ok && found && t.Port == 0 {
t.Port = proxyPorts[strings.ToLower(scheme)]
}
if !ok || t.Port == 0 || t.Kind == TargetCIDR || seen[db.TargetKey(t.Host, t.Port)] {
continue
}
+23 -1
View File
@@ -1,6 +1,11 @@
package intercept
import "testing"
import (
"slices"
"testing"
"github.com/Autumn-27/artex/db"
)
// An operator may exempt a private unicast IP literal, never a loopback,
// link-local (cloud metadata), unspecified or multicast one.
@@ -27,3 +32,20 @@ func TestValidateExemptionHost(t *testing.T) {
}
}
}
// A proxy endpoint configured without a port is exempt on the port net/http
// dials for it (OPEN-9): the scheme's default, 1080 for socks5. An endpoint
// with no port and no known scheme is not listed.
func TestDerivedExemptionsDefaultProxyPort(t *testing.T) {
ic := New(nil)
ic.SetDerivedExemptions(func() []string {
return []string{"socks5://egress.example.net", "SOCKS5H://dns.example.net", "http://proxy.example.net", "https://gw.example.net", "bare.example.net", "socks5://set.example.net:9050"}
})
want := []db.TargetExemption{
{Host: "egress.example.net", Port: 1080}, {Host: "dns.example.net", Port: 1080}, {Host: "proxy.example.net", Port: 80},
{Host: "gw.example.net", Port: 443}, {Host: "set.example.net", Port: 9050},
}
if got := ic.DerivedExemptions(); !slices.Equal(got, want) {
t.Fatalf("derived exemptions = %+v, want %+v", got, want)
}
}
+30 -6
View File
@@ -112,8 +112,21 @@ type targetCheck struct {
urls []string // the URLs the call declares, as written, for the URL block rules
// redirectFrom is the origin (host:port) of a redirect hop; empty for a call.
redirectFrom string
// fresh makes every lookup skip the cache: the decision of a connection
// (dialTarget) compares the answer it is about to dial.
fresh bool
// checked is what the run decided about each target, read by dialTarget.
checked []*checkedTarget
}
// What let a target pass, recorded in checkedTarget.cover.
const (
coverDerived = "derived" // an exemption derived from the configuration
coverStored = "stored" // a stored exemption that holds
coverPin = "pin" // a pin that holds every resolved address
coverNameOnly = "name_only" // a pin that is name-only for this call
)
// checkedTarget is one target of the call and what the gate learned about it.
type checkedTarget struct {
Target
@@ -122,6 +135,7 @@ type checkedTarget struct {
resolvedAt time.Time
err error // the name did not resolve
exemptVoid bool
cover string // what let the target pass; empty when nothing did
}
func (t Target) String() string {
@@ -169,6 +183,7 @@ func (c *targetCheck) run(targets []Target) ([]TargetRequest, TargetResult) {
// Then the declared form of every target: the name as written, the IP
// literal, the range.
checked := make([]*checkedTarget, len(targets))
c.checked = checked
for n, t := range targets {
ct := &checkedTarget{Target: t}
checked[n] = ct
@@ -185,7 +200,7 @@ func (c *targetCheck) run(targets []Target) ([]TargetRequest, TargetResult) {
if ct.Kind != TargetDomain || ex.derived[db.TargetKey(ct.Host, ct.Port)] {
continue
}
c.resolve(ct, false)
c.resolve(ct, c.fresh)
if rule, hit := blockRule(rules, ct); hit {
return c.block(TargetRuleBlocked, "rule_blocked", ct, fmt.Sprintf("%s is blocked by an asset rule. %s", ct.Target, rule.Reason()))
}
@@ -204,11 +219,13 @@ func (c *targetCheck) run(targets []Target) ([]TargetRequest, TargetResult) {
for _, ct := range checked {
key := db.TargetKey(ct.Host, ct.Port)
if ex.derived[key] {
ct.cover = coverDerived
uses = append(uses, func() { c.exemptUsed(ct, ex.digest) })
continue
}
if ex.stored[key] {
if exemptionHolds(ct) {
ct.cover = coverStored
uses = append(uses, func() { c.exemptUsed(ct, ex.digest) })
continue
}
@@ -223,9 +240,10 @@ func (c *targetCheck) run(targets []Target) ([]TargetRequest, TargetResult) {
return c.block(TargetGateUnavailable, "gate_unavailable", ct, "the pins could not be read: "+err.Error())
}
pin := coveringPin(pins, ct.ips, nameOnly)
if pin == nil && ct.Kind == TargetDomain {
if pin == nil && ct.Kind == TargetDomain && !c.fresh {
// A request is about to be raised: the addresses a human reads and
// pins are resolved now, never taken from the cache.
// pins are resolved now, never taken from the cache. A fresh run
// already has them.
if c.resolve(ct, true); ct.err != nil {
return c.block(TargetResolveFailed, "resolve_failed", ct, c.resolveFailure(ct))
}
@@ -235,7 +253,11 @@ func (c *targetCheck) run(targets []Target) ([]TargetRequest, TargetResult) {
pin = coveringPin(pins, ct.ips, nameOnly)
}
if pin != nil {
uses = append(uses, func() { c.pinUsed(ct, pin) })
ct.cover = coverPin
if nameOnly || pin.NameOnlyEffective {
ct.cover = coverNameOnly
}
uses = append(uses, func() { c.pinUsed(ct, pin, nameOnly) })
continue
}
req := TargetRequest{Host: ct.Host, Port: ct.Port, Kind: ct.Kind, ResolvedIPs: ct.ips,
@@ -326,12 +348,14 @@ func (c *targetCheck) exemptUsed(ct *checkedTarget, digest string) {
}
// pinUsed logs the first call of this process that a pin version covers.
func (c *targetCheck) pinUsed(ct *checkedTarget, pin *db.TargetPin) {
// nameOnly is whether the upstream proxy made every pin name-only for the call:
// the row records the effective state, not only the pin's mark.
func (c *targetCheck) pinUsed(ct *checkedTarget, pin *db.TargetPin, nameOnly bool) {
if _, seen := c.i.firstUse.LoadOrStore(fmt.Sprintf("pin|%d", pin.ID), true); seen {
return
}
t := ct.audit("pin_match")
t.PinID, t.PinVersion, t.PinnedIPs, t.NameOnlyEffective = pin.ID, pin.Version, pin.PinnedIPs, pin.NameOnlyEffective
t.PinID, t.PinVersion, t.PinnedIPs, t.NameOnlyEffective = pin.ID, pin.Version, pin.PinnedIPs, pin.NameOnlyEffective || nameOnly
c.log("allowed", fmt.Sprintf("Target %s is covered by pin v%d of %s.", ct.Target, pin.Version, c.scope.scopeName()), "", t)
}
+2 -1
View File
@@ -384,7 +384,8 @@ func targetReason(scope TargetScope, t TargetRequest, ask *Decision, nameOnly bo
fmt.Fprintf(&b, " Approving pins these addresses for %s until revoked.", bound)
if nameOnly {
b.WriteString(" Egress goes through the upstream proxy, which chooses the address it connects to." +
" The addresses listed are what the platform resolved. This approval is name-only.")
" The addresses listed are what the platform resolved. This approval is name-only." +
" The platform does not bind the connection to these addresses.")
}
return b.String()
}
+10
View File
@@ -24,6 +24,16 @@ type Resolver interface {
// resolver unless one is set.
func (i *Interceptor) SetResolver(r Resolver) { i.resolve.use(r) }
// SetBaseDialer replaces the dialer that opens the connections PinnedDialer
// lets through, which is a net.Dialer with the settings of
// http.DefaultTransport unless one is set. Tests use it to map addresses to
// local listeners and to see what was dialled.
func (i *Interceptor) SetBaseDialer(f func(ctx context.Context, network, addr string) (net.Conn, error)) {
i.mu.Lock()
i.dial = f
i.mu.Unlock()
}
// systemResolver is the resolver of the ARTEX process. It reads the same
// resolver configuration as the tools that run in the same container.
type systemResolver struct{}
+39 -4
View File
@@ -457,7 +457,7 @@ func (s *Server) runHTTPTool(ctx context.Context, key string, execRaw json.RawMe
client := &http.Client{Timeout: timeoutOr(spec.TimeoutMs, 30000)}
// Every redirect hop goes through the target gate, as the rendered URL did.
client.CheckRedirect = intercept.RedirectCheck(key)
if tr := s.httpProxyTransport(spec); tr != nil {
if tr := s.httpToolTransport(ctx, key, spec); tr != nil {
client.Transport = tr
}
resp, err := client.Do(req)
@@ -471,14 +471,38 @@ func (s *Server) runHTTPTool(ctx context.Context, key string, execRaw json.RawMe
return actool.Text(actool.Capture(tc, string(b))), nil
}
// httpToolTransport is the transport of one http tool call, or nil
// (http.DefaultTransport). A call through a proxy keeps httpProxyTransport: it
// dials the proxy, which makes the connection to the target. A direct call in
// a gated scope connects through the pinned dialer, to addresses a pin covers.
func (s *Server) httpToolTransport(ctx context.Context, key string, spec httpExec) *http.Transport {
if tr := s.httpProxyTransport(spec); tr != nil {
return tr
}
if !intercept.GateOn(ctx) {
return nil
}
base, ok := http.DefaultTransport.(*http.Transport)
if !ok {
base = &http.Transport{}
}
tr := base.Clone()
// An environment proxy would be dialled instead of the target.
tr.Proxy = nil
tr.DialContext = intercept.PinnedDialer(key, true)
// One request per call: no idle connection outlives the call.
tr.DisableKeepAlives = true
return tr
}
// httpProxyTransport builds a Transport for the http tool's proxy config, or nil
// (direct). use_recording_proxy routes through the recording proxy + trusts its CA.
func (s *Server) httpProxyTransport(spec httpExec) *http.Transport {
proxyStr := strings.TrimSpace(spec.Proxy)
proxyStr := proxyURLString(spec.Proxy)
var caFile string
if spec.UseRecordingProxy {
if addr := s.m.ProxyAddr(); addr != "" {
proxyStr = "http://" + addr
if addr := proxyURLString(s.m.ProxyAddr()); addr != "" {
proxyStr = addr
caFile = s.m.ProxyCACert()
}
}
@@ -501,6 +525,17 @@ func (s *Server) httpProxyTransport(spec httpExec) *http.Transport {
return tr
}
// proxyURLString returns a proxy address as a URL: a bare host:port gets
// "http://", an address that already has a scheme (as ProxyAddr returns it) is
// kept as is.
func proxyURLString(s string) string {
s = strings.TrimSpace(s)
if s == "" || strings.Contains(s, "://") {
return s
}
return "http://" + s
}
// ---------- helpers ----------
func identity(s string) string { return s }
+358
View File
@@ -0,0 +1,358 @@
package server
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"slices"
"strings"
"sync"
"testing"
"time"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/intercept"
)
// Step 2.1: the custom http tool connects directly through the pinned dialer
// in a gated scope, and a gated worker's WebFetch end to end.
// routeDialer is the base dialer of the gate in these tests: it records every
// address, connects the routed TEST-NET addresses to local listeners, and
// refuses the rest.
type routeDialer struct {
mu sync.Mutex
routes map[string]string
dialed []string
}
func newRouteDialer(g *gateServer) *routeDialer {
d := &routeDialer{routes: map[string]string{}}
g.m.interceptor.SetBaseDialer(d.dial)
return d
}
func (d *routeDialer) route(addr, to string) {
d.mu.Lock()
d.routes[addr] = to
d.mu.Unlock()
}
func (d *routeDialer) dial(ctx context.Context, network, addr string) (net.Conn, error) {
d.mu.Lock()
d.dialed = append(d.dialed, addr)
to, ok := d.routes[addr]
d.mu.Unlock()
if !ok {
return nil, &net.OpError{Op: "dial", Net: network, Err: errors.New("no route in the test dialer")}
}
return (&net.Dialer{}).DialContext(ctx, network, to)
}
func (d *routeDialer) calls() []string {
d.mu.Lock()
defer d.mu.Unlock()
return slices.Clone(d.dialed)
}
func (r *countingResolver) set(host string, ips ...string) {
r.mu.Lock()
r.answers[host] = ips
r.mu.Unlock()
}
// approveTarget opens the request of a gated http tool call on key and approves it.
func approveTarget(t *testing.T, g *gateServer, task *Task, tool interface {
Name() string
}, call func() (string, bool), key string) {
t.Helper()
if out, isErr := call(); !isErr || !strings.Contains(out, "TARGET_APPROVAL_PENDING") {
t.Fatalf("%s before the approval: error=%v out=%q", tool.Name(), isErr, out)
}
id, _, _, ok := openTargetRequest(t, g.m.pg, task.ID, key)
if !ok {
t.Fatalf("no request on %s", key)
}
decide(t, g, id, "allowed")
}
// A gated direct call reaches its target at the pinned address; once the name
// answers another address, the rendered-URL gate passes on the cached answer
// and the dial connects nowhere.
func TestCustomHTTPDirectDialPinned(t *testing.T) {
g := newGateServer(t)
on := true
task := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
srv, hits, _, port := countingServer(t)
res := &countingResolver{answers: map[string][]string{"api.example.com": {"203.0.113.7"}}}
g.m.interceptor.SetResolver(res)
dialer := newRouteDialer(g)
dialer.route("203.0.113.7:"+port, srv.Listener.Addr().String())
dialer.route("203.0.113.9:"+port, srv.Listener.Addr().String())
key := "api.example.com:" + port
tool := g.s.buildCustomTool(&db.Tool{Key: "probe_dial", Kind: "http",
Exec: execJSON(t, map[string]any{"url": "http://" + key + "/dial"})})
call := func() (string, bool) { return callTool(t, tool, gatedCtx(task, 0), `{}`, nil) }
approveTarget(t, g, task, tool, call, key)
if out, isErr := call(); isErr || !strings.Contains(out, "probe-reached") || hits.Load() != 1 {
t.Fatalf("pinned call: error=%v out=%q hits=%d", isErr, out, hits.Load())
}
res.set("api.example.com", "203.0.113.9")
out, isErr := call()
if !isErr || !strings.Contains(out, "TARGET_APPROVAL_PENDING: the connection to "+key+" was not opened") || hits.Load() != 1 {
t.Fatalf("rebound name: error=%v out=%q hits=%d", isErr, out, hits.Load())
}
if got := dialer.calls(); !slices.Equal(got, []string{"203.0.113.7:" + port}) {
t.Fatalf("dialled %v, want the pinned address once", got)
}
if _, toolName, input, ok := openTargetRequest(t, g.m.pg, task.ID, key); !ok || toolName != "probe_dial" || !strings.Contains(input, "dial") {
t.Fatalf("request at the dial: ok=%v tool=%q input=%s", ok, toolName, input)
}
}
// With the gate off (and without a scope) the call keeps http.DefaultTransport:
// no transport is built. A spec with a proxy keeps its proxy transport, without
// a dialer. Gate on and direct: a transport with the pinned dialer, no
// environment proxy, keep-alive off.
func TestCustomHTTPGateOffTransportUnchanged(t *testing.T) {
s := &Server{}
direct := httpExec{URL: "http://api.example.com/"}
scope := func(on bool) context.Context {
return intercept.WithTargetScope(context.Background(), intercept.TargetScope{TaskID: "7", Enabled: func() bool { return on }})
}
for name, ctx := range map[string]context.Context{"no scope": context.Background(), "flag off": scope(false)} {
if tr := s.httpToolTransport(ctx, "probe", direct); tr != nil {
t.Fatalf("%s: transport %+v, want nil (http.DefaultTransport)", name, tr)
}
}
if tr := s.httpToolTransport(scope(true), "probe", httpExec{URL: direct.URL, Proxy: "http://127.0.0.1:1"}); tr == nil || tr.Proxy == nil || tr.DialContext != nil {
t.Fatalf("operator proxy: transport %+v, want the proxy transport without a dialer", tr)
}
if tr := s.httpToolTransport(scope(true), "probe", direct); tr == nil || tr.Proxy != nil || tr.DialContext == nil || !tr.DisableKeepAlives {
t.Fatalf("gated direct call: transport %+v", tr)
}
}
// The proxy URL of the http tool's transport: an address with a scheme is kept
// as is, a bare host:port gets "http://"; never a doubled scheme.
func TestHTTPProxyTransportProxyScheme(t *testing.T) {
req := httptest.NewRequest("GET", "http://api.example.com/", nil)
for _, c := range []struct {
name, globalProxy string
spec httpExec
want string
}{
{"recording proxy, global proxy with a scheme", "http://127.0.0.1:3128", httpExec{UseRecordingProxy: true}, "http://127.0.0.1:3128"},
{"operator proxy with a scheme", "", httpExec{Proxy: "http://10.0.0.5:8080"}, "http://10.0.0.5:8080"},
{"operator proxy as bare host:port", "", httpExec{Proxy: " 10.0.0.5:8080 "}, "http://10.0.0.5:8080"},
} {
s := &Server{m: &Manager{globalProxy: c.globalProxy}}
tr := s.httpProxyTransport(c.spec)
if tr == nil || tr.Proxy == nil {
t.Fatalf("%s: transport %+v, want a proxy transport", c.name, tr)
}
u, err := tr.Proxy(req)
if err != nil || u == nil || u.String() != c.want {
t.Fatalf("%s: Proxy(req) = %v, %v; want %s", c.name, u, err, c.want)
}
}
}
// passRoundTripper stands in for an http.DefaultTransport that is not an
// *http.Transport; it hands every request to the original transport.
type passRoundTripper struct{ next http.RoundTripper }
func (p passRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
return p.next.RoundTrip(r)
}
// With http.DefaultTransport replaced by another RoundTripper, a gated direct
// call still gets a transport with the pinned dialer instead of a panic.
func TestHTTPToolTransportDefaultTransportNotStd(t *testing.T) {
orig := http.DefaultTransport
http.DefaultTransport = passRoundTripper{next: orig}
defer func() { http.DefaultTransport = orig }()
ctx := intercept.WithTargetScope(context.Background(), intercept.TargetScope{TaskID: "7", Enabled: func() bool { return true }})
tr := (&Server{}).httpToolTransport(ctx, "probe", httpExec{URL: "http://api.example.com/"})
if tr == nil || tr.DialContext == nil {
t.Fatalf("transport %+v, want one with the pinned dialer", tr)
}
}
// A redirect of a pinned origin to a pinned hop: the hop is dialled at the
// hop's pinned address.
func TestCustomHTTPRedirectHopDialPinned(t *testing.T) {
g := newGateServer(t)
on := true
task := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
final, finalHits, _, finalPort := countingServer(t)
hopKey := "hop.example.com:" + finalPort
start, _, startAddr := redirectingServer(t, "http://"+hopKey+"/final")
_, startPort, _ := net.SplitHostPort(startAddr)
originKey := "api.example.com:" + startPort
res := &countingResolver{answers: map[string][]string{"api.example.com": {"203.0.113.7"}, "hop.example.com": {"203.0.113.8"}}}
g.m.interceptor.SetResolver(res)
dialer := newRouteDialer(g)
dialer.route("203.0.113.7:"+startPort, start.Listener.Addr().String())
dialer.route("203.0.113.8:"+finalPort, final.Listener.Addr().String())
// Were the hop dialled at the origin's address, it would still reach it.
dialer.route("203.0.113.7:"+finalPort, final.Listener.Addr().String())
tool := g.s.buildCustomTool(&db.Tool{Key: "probe_hop", Kind: "http",
Exec: execJSON(t, map[string]any{"url": "http://" + originKey + "/start"})})
call := func() (string, bool) { return callTool(t, tool, gatedCtx(task, 0), `{}`, nil) }
approveTarget(t, g, task, tool, call, originKey)
approveTarget(t, g, task, tool, call, hopKey)
before := len(dialer.calls())
if out, isErr := call(); isErr || !strings.Contains(out, "probe-reached") || finalHits.Load() != 1 {
t.Fatalf("pinned hop: error=%v out=%q", isErr, out)
}
if got := dialer.calls()[before:]; !slices.Equal(got, []string{"203.0.113.7:" + startPort, "203.0.113.8:" + finalPort}) {
t.Fatalf("dialled %v, want the origin's then the hop's pinned address", got)
}
}
// The custom-tool test endpoint has no scope (D14): its connection never goes
// through the gate's dialer, and nothing is written.
func TestCustomToolTestEndpointDialUngated(t *testing.T) {
g := newGateServer(t)
on := true
g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
dialer := newRouteDialer(g)
_, hits, host, port := countingServer(t)
before := 0
if err := g.m.pg.QueryRow(`SELECT count(*) FROM intercept_pending WHERE decision_source='target' OR target_key IS NOT NULL`).Scan(&before); err != nil {
t.Fatal(err)
}
code, out := g.do(t, "POST", "/api/tools/custom/test", fmt.Sprintf(`{"kind":"http","exec":{"url":"http://%s:%s/test"}}`, host, port))
if code != 200 || out["is_error"] != false || hits.Load() != 1 || len(dialer.calls()) != 0 {
t.Fatalf("test endpoint: %d %v hits=%d dialled %v", code, out, hits.Load(), dialer.calls())
}
after := 0
if err := g.m.pg.QueryRow(`SELECT count(*) FROM intercept_pending WHERE decision_source='target' OR target_key IS NOT NULL`).Scan(&after); err != nil || after != before {
t.Fatalf("the test endpoint wrote %d target rows (%v)", after-before, err)
}
}
// Documents a known limit: a custom http tool that goes through a proxy is not
// bound to pins. The pinned dialer covers direct connections only; through a
// proxy the connection to the target is the proxy's, and the proxy here
// connects to a peer nobody pinned. Step 2.2 binds the recording proxy (and
// flips its half of this test); an operator-written proxy stays unbound
// (OPEN-6).
func TestBypass_CustomHTTPProxyPeerNotPinned(t *testing.T) {
g := newGateServer(t)
on := true
task := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
_, hits, _, port := countingServer(t)
peer := "127.0.0.1:" + port
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Whatever the request names, the proxy connects to peer.
resp, err := http.Get("http://" + peer + r.URL.Path)
if err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
defer resp.Body.Close()
_, _ = io.Copy(w, resp.Body)
}))
t.Cleanup(proxy.Close)
res := &countingResolver{answers: map[string][]string{"api.example.com": {"203.0.113.7"}}}
g.m.interceptor.SetResolver(res)
dialer := newRouteDialer(g)
key := "api.example.com:" + port
url := "http://" + key + "/bypass"
direct := g.s.buildCustomTool(&db.Tool{Key: "probe_bypass", Kind: "http", Exec: execJSON(t, map[string]any{"url": url})})
approveTarget(t, g, task, direct, func() (string, bool) { return callTool(t, direct, gatedCtx(task, 0), `{}`, nil) }, key)
// The global proxy carries a scheme, as in production (ProxyAddr returns it
// as is with capture off); httpProxyTransport must not write a second one.
g.m.mu.Lock()
g.m.globalProxy = proxy.URL
g.m.mu.Unlock()
t.Cleanup(func() {
g.m.mu.Lock()
g.m.globalProxy = ""
g.m.mu.Unlock()
})
for name, exec := range map[string]map[string]any{
"operator proxy": {"url": url, "proxy": proxy.URL},
"use_recording_proxy": {"url": url, "use_recording_proxy": true},
} {
before := hits.Load()
tool := g.s.buildCustomTool(&db.Tool{Key: "probe_bypass", Kind: "http", Exec: execJSON(t, exec)})
if out, isErr := callTool(t, tool, gatedCtx(task, 0), `{}`, nil); isErr || !strings.Contains(out, "probe-reached") || hits.Load() != before+1 {
t.Fatalf("%s: error=%v out=%q", name, isErr, out)
}
}
if got := dialer.calls(); len(got) != 0 {
t.Fatalf("dialled %v; the pinned address 203.0.113.7 is never dialled through a proxy", got)
}
}
// A gated worker's WebFetch end to end: approved, the next call passes; the
// name then answers another address from the next lookup on, so the following
// call passes the hook on the cached answer and fails at the dial with
// TARGET_APPROVAL_PENDING. The new_ip request is approved through the API and
// the call after passes.
func TestTargetGateEndToEndDialRebindBlocked(t *testing.T) {
g := newGateServer(t)
on := true
task := g.newTask(t, db.TaskCreateOptions{TargetGate: &on})
pg := g.m.pg
srv, hits, _, port := countingServer(t)
res := &countingResolver{answers: map[string][]string{"api.example.com": {"203.0.113.7"}}}
g.m.interceptor.SetResolver(res)
dialer := newRouteDialer(g)
dialer.route("203.0.113.7:"+port, srv.Listener.Addr().String())
dialer.route("203.0.113.9:"+port, srv.Listener.Addr().String())
key := "api.example.com:" + port
target := "http://" + key + "/e2e"
p := &toolScript{calls: fetches(target, target, target), results: make(chan string, 16),
hold: map[int64]chan struct{}{2: make(chan struct{}), 3: make(chan struct{})}}
done := runScriptedWorker(t, g, task, p)
var id int64
waitUntil(t, "the first request", func() bool {
var ok bool
id, _, _, ok = openTargetRequest(t, pg, task.ID, key)
return ok
})
decide(t, g, id, "allowed")
if r := p.next(t); !strings.Contains(r, "probe-reached") || hits.Load() != 1 {
t.Fatalf("first call after the approval: %q hits=%d", r, hits.Load())
}
res.set("api.example.com", "203.0.113.9")
close(p.hold[2])
if r := p.next(t); !strings.Contains(r, "TARGET_APPROVAL_PENDING: the connection to "+key+" was not opened") || hits.Load() != 1 {
t.Fatalf("second call, rebound at the dial: %q hits=%d", r, hits.Load())
}
newIP, toolName, _, ok := openTargetRequest(t, pg, task.ID, key)
if !ok || newIP == id || toolName != "WebFetch" {
t.Fatalf("new_ip request: ok=%v id=%d tool=%q", ok, newIP, toolName)
}
detail, err := pg.GetInterceptDetail(newIP)
if err != nil || detail == nil || detail.Audit == nil || detail.Audit.Targets[0].Reason != "new_ip" ||
!slices.Equal(detail.Audit.Targets[0].ResolvedIPs, []string{"203.0.113.9"}) {
t.Fatalf("new_ip request detail = %+v, %v", detail, err)
}
decide(t, g, newIP, "allowed")
close(p.hold[3])
if r := p.next(t); !strings.Contains(r, "probe-reached") || hits.Load() != 2 {
t.Fatalf("third call after the new_ip approval: %q hits=%d", r, hits.Load())
}
if got := dialer.calls(); !slices.Equal(got, []string{"203.0.113.7:" + port, "203.0.113.9:" + port}) {
t.Fatalf("dialled %v", got)
}
select {
case <-done:
case <-time.After(60 * time.Second):
t.Fatal("the worker run did not end")
}
}