diff --git a/server/finding_verification.go b/server/finding_verification.go index a1e8de3..b26094c 100644 --- a/server/finding_verification.go +++ b/server/finding_verification.go @@ -2,6 +2,9 @@ package server import ( "bytes" + "compress/flate" + "compress/gzip" + "compress/zlib" "context" "encoding/json" "fmt" @@ -40,8 +43,60 @@ func normalizeEvidenceText(s string) string { return strings.Join(strings.Fields(s), " ") } +// decodeEvidenceBody undoes the HTTP Content-Encoding declared in respHead on a +// captured response body, returning at most limit decoded bytes. Only identity +// and one layer of gzip or deflate are handled. Any other coding, a stacked +// coding or a stream that does not decode is an error: the raw bytes are never +// returned in place of a failed decode. +func decodeEvidenceBody(respHead string, raw []byte, limit int) (out []byte, truncated bool, err error) { + coding := "" + for _, line := range strings.Split(respHead, "\n") { + name, value, ok := strings.Cut(line, ":") + if !ok || !strings.EqualFold(strings.TrimSpace(name), "Content-Encoding") { + continue + } + for _, v := range strings.Split(value, ",") { + switch v = strings.ToLower(strings.TrimSpace(v)); v { + case "", "identity": + case "gzip", "deflate": + if coding != "" { + return nil, false, fmt.Errorf("unsupported content-encoding: more than one compressed layer") + } + coding = v + default: + return nil, false, fmt.Errorf("unsupported content-encoding: %q", v) + } + } + } + if coding == "" || len(raw) == 0 { + return raw, false, nil + } + var zr io.Reader + switch { + case coding == "gzip": + zr, err = gzip.NewReader(bytes.NewReader(raw)) + case len(raw) >= 2 && raw[0]&0x0f == 8 && raw[0]>>4 <= 7 && (uint(raw[0])<<8|uint(raw[1]))%31 == 0: + // HTTP "deflate" is the zlib format (RFC 9110); some servers send a bare + // deflate stream instead, told apart by the zlib header. + zr, err = zlib.NewReader(bytes.NewReader(raw)) + default: + zr = flate.NewReader(bytes.NewReader(raw)) + } + if err == nil { + out, err = io.ReadAll(io.LimitReader(zr, int64(limit)+1)) + } + if err != nil { + return nil, false, fmt.Errorf("%s body: %w", coding, err) + } + if len(out) > limit { + return out[:limit], true, nil + } + return out, false, nil +} + // readBoundResponse reads the response side of a bound snapshot through the -// evidence store (same bytes the UI shows), capped at verificationReadCap. +// evidence store (same bytes the UI shows), capped at verificationReadCap, and +// decodes its Content-Encoding under the same cap. func readBoundResponse(store *evidence.Store, snap db.TrafficEvidenceSnapshot) (boundBody, error) { f, _, err := store.OpenBody(snap, "response") if err != nil { @@ -56,6 +111,15 @@ func readBoundResponse(store *evidence.Store, snap db.TrafficEvidenceSnapshot) ( if len(raw) > verificationReadCap { bb.truncated = true raw = raw[:verificationReadCap] + } + raw, cut, err := decodeEvidenceBody(snap.RespHead, raw, verificationReadCap) + if err != nil { + return boundBody{}, err + } + if cut { + bb.truncated = true + } + if bb.truncated { // The cap may cut a multi-byte rune: drop at most UTFMax-1 trailing bytes. for i := 0; i < utf8.UTFMax-1 && !utf8.Valid(raw) && len(raw) > 0; i++ { raw = raw[:len(raw)-1] diff --git a/server/finding_verification_decode_test.go b/server/finding_verification_decode_test.go new file mode 100644 index 0000000..2666744 --- /dev/null +++ b/server/finding_verification_decode_test.go @@ -0,0 +1,285 @@ +package server + +import ( + "bytes" + "compress/flate" + "compress/gzip" + "compress/zlib" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Autumn-27/artex/db" + "github.com/Autumn-27/artex/evidence" +) + +const ( + decodeGzipHead = "HTTP 200\nContent-Type: text/html\nContent-Encoding: gzip\n" + decodeHTML = "\n user admin:\tx7\n end of dump\n" + decodeQuote = "user admin: x7 end of dump" +) + +// compressBody runs data through the compressor built by newWriter. +func compressBody(t *testing.T, newWriter func(io.Writer) io.WriteCloser, data []byte) []byte { + t.Helper() + var buf bytes.Buffer + w := newWriter(&buf) + if _, err := w.Write(data); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func gzipBody(t *testing.T, data []byte) []byte { + return compressBody(t, func(w io.Writer) io.WriteCloser { return gzip.NewWriter(w) }, data) +} + +// decodeSnapshot stores raw as a response body in a database-less evidence +// store (same layout as evidence.hashPath) and returns the snapshot bound to it. +func decodeSnapshot(t *testing.T, respHead string, raw []byte) (*evidence.Store, db.TrafficEvidenceSnapshot) { + t.Helper() + sum := sha256.Sum256(raw) + hash := hex.EncodeToString(sum[:]) + dir := t.TempDir() + path := filepath.Join(dir, "blobs", hash[:2], hash+".bin") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, raw, 0o644); err != nil { + t.Fatal(err) + } + return &evidence.Store{Dir: dir}, db.TrafficEvidenceSnapshot{RespHead: respHead, RespHash: hash, RespLen: int64(len(raw))} +} + +func TestDecodeEvidenceBodyGzipQuoteFound(t *testing.T) { + raw := gzipBody(t, []byte(decodeHTML)) + if strings.Contains(string(raw), "admin") { + t.Fatal("fixture is not compressed") + } + out, truncated, err := decodeEvidenceBody(decodeGzipHead, raw, verificationReadCap) + if err != nil || truncated { + t.Fatalf("decode: truncated=%v err=%v", truncated, err) + } + if string(out) != decodeHTML { + t.Fatalf("decoded %q, want %q", out, decodeHTML) + } + if !strings.Contains(normalizeEvidenceText(string(out)), normalizeEvidenceText(decodeQuote)) { + t.Fatalf("quote %q not found in decoded body", decodeQuote) + } +} + +func TestDecodeEvidenceBodyTruncatedGzipFails(t *testing.T) { + raw := gzipBody(t, []byte(strings.Repeat(decodeHTML, 50))) + for _, cut := range []int{5, len(raw) / 2, len(raw) - 1} { + out, truncated, err := decodeEvidenceBody(decodeGzipHead, raw[:cut], verificationReadCap) + if err == nil || out != nil || truncated { + t.Fatalf("cut at %d: out=%d bytes truncated=%v err=%v, want an error and no bytes", cut, len(out), truncated, err) + } + } + // Not gzip at all: the raw bytes must not come back either. + out, _, err := decodeEvidenceBody(decodeGzipHead, []byte(decodeHTML), verificationReadCap) + if err == nil || out != nil { + t.Fatalf("plain text declared as gzip: out=%q err=%v, want an error and no bytes", out, err) + } +} + +func TestDecodeEvidenceBodyIdentityAndNoHeaderUnchanged(t *testing.T) { + raw := []byte(decodeHTML) + for _, head := range []string{ + "", + "HTTP 200\nContent-Type: text/html\n", + "HTTP 200\nContent-Encoding: identity\n", + "HTTP 200\r\ncontent-encoding: Identity \r\n", + } { + out, truncated, err := decodeEvidenceBody(head, raw, verificationReadCap) + if err != nil || truncated || !bytes.Equal(out, raw) { + t.Fatalf("head %q: out=%q truncated=%v err=%v, want the raw bytes", head, out, truncated, err) + } + } +} + +func TestDecodeEvidenceBodyUnknownEncodingFails(t *testing.T) { + raw := gzipBody(t, []byte(decodeHTML)) + for _, head := range []string{ + "HTTP 200\nContent-Encoding: br\n", + "HTTP 200\nContent-Encoding: gzip, br\n", + "HTTP 200\nContent-Encoding: gzip\nContent-Encoding: zstd\n", + } { + out, _, err := decodeEvidenceBody(head, raw, verificationReadCap) + if err == nil || !strings.Contains(err.Error(), "unsupported content-encoding") || out != nil { + t.Fatalf("head %q: out=%d bytes err=%v, want unsupported content-encoding", head, len(out), err) + } + } +} + +func TestDecodeEvidenceBodyTwoLayersFails(t *testing.T) { + raw := gzipBody(t, gzipBody(t, []byte(decodeHTML))) + for _, head := range []string{ + "HTTP 200\nContent-Encoding: gzip, gzip\n", + "HTTP 200\nContent-Encoding: gzip\nContent-Encoding: deflate\n", + } { + out, _, err := decodeEvidenceBody(head, raw, verificationReadCap) + if err == nil || !strings.Contains(err.Error(), "unsupported content-encoding") || out != nil { + t.Fatalf("head %q: out=%d bytes err=%v, want unsupported content-encoding", head, len(out), err) + } + } +} + +func TestDecodeEvidenceBodyOverLimitTruncated(t *testing.T) { + const limit = 1000 + plain := []byte(strings.Repeat("0123456789", 500)) + out, truncated, err := decodeEvidenceBody(decodeGzipHead, gzipBody(t, plain), limit) + if err != nil || !truncated || len(out) != limit || !bytes.Equal(out, plain[:limit]) { + t.Fatalf("over limit: len=%d truncated=%v err=%v, want %d bytes truncated", len(out), truncated, err, limit) + } + // Exactly at the limit is complete, not truncated. + out, truncated, err = decodeEvidenceBody(decodeGzipHead, gzipBody(t, plain[:limit]), limit) + if err != nil || truncated || len(out) != limit { + t.Fatalf("at limit: len=%d truncated=%v err=%v, want %d bytes not truncated", len(out), truncated, err, limit) + } +} + +func TestDecodeEvidenceBodyUppercaseHeader(t *testing.T) { + raw := gzipBody(t, []byte(decodeHTML)) + for _, head := range []string{ + "HTTP 200\nContent-Encoding: GZIP\n", + "HTTP 200\r\nCONTENT-ENCODING: GZIP\r\n", + } { + out, truncated, err := decodeEvidenceBody(head, raw, verificationReadCap) + if err != nil || truncated || string(out) != decodeHTML { + t.Fatalf("head %q: out=%q truncated=%v err=%v", head, out, truncated, err) + } + } +} + +func TestDecodeEvidenceBodyDeflate(t *testing.T) { + const head = "HTTP 200\nContent-Encoding: deflate\n" + bodies := map[string][]byte{ + "zlib": compressBody(t, func(w io.Writer) io.WriteCloser { return zlib.NewWriter(w) }, []byte(decodeHTML)), + "raw": compressBody(t, func(w io.Writer) io.WriteCloser { + fw, err := flate.NewWriter(w, flate.DefaultCompression) + if err != nil { + t.Fatal(err) + } + return fw + }, []byte(decodeHTML)), + } + for name, raw := range bodies { + out, truncated, err := decodeEvidenceBody(head, raw, verificationReadCap) + if err != nil || truncated || string(out) != decodeHTML { + t.Fatalf("%s deflate: out=%q truncated=%v err=%v", name, out, truncated, err) + } + if out, _, err := decodeEvidenceBody(head, raw[:len(raw)-1], verificationReadCap); err == nil || out != nil { + t.Fatalf("truncated %s deflate: out=%d bytes err=%v, want an error and no bytes", name, len(out), err) + } + } +} + +func TestDecodeBoundResponseGzipTextIsSearchable(t *testing.T) { + store, snap := decodeSnapshot(t, decodeGzipHead, gzipBody(t, []byte(decodeHTML))) + bb, err := readBoundResponse(store, snap) + if err != nil { + t.Fatal(err) + } + if bb.nonText || bb.truncated { + t.Fatalf("gzip html: nonText=%v truncated=%v", bb.nonText, bb.truncated) + } + if !quoteInBodies(normalizeEvidenceText(decodeQuote), []boundBody{bb}) { + t.Fatalf("quote %q not found in %q", decodeQuote, bb.text) + } +} + +func TestDecodeBoundResponseGzipBinaryStaysNonText(t *testing.T) { + binary := append([]byte("PNG\x00\x01\x02 user admin: x7 end of dump "), 0xff, 0xfe) + store, snap := decodeSnapshot(t, decodeGzipHead, gzipBody(t, binary)) + bb, err := readBoundResponse(store, snap) + if err != nil { + t.Fatal(err) + } + if !bb.nonText || bb.text != "" { + t.Fatalf("gzip binary: nonText=%v text=%q, want non-text", bb.nonText, bb.text) + } + if quoteInBodies(normalizeEvidenceText(decodeQuote), []boundBody{bb}) { + t.Fatal("quote matched inside a non-text body") + } +} + +func TestDecodeBoundResponseBadGzipIsAnError(t *testing.T) { + raw := gzipBody(t, []byte(decodeHTML)) + store, snap := decodeSnapshot(t, decodeGzipHead, raw[:len(raw)-4]) + if bb, err := readBoundResponse(store, snap); err == nil { + t.Fatalf("truncated gzip: got %+v, want an error", bb) + } + store, snap = decodeSnapshot(t, "HTTP 200\nContent-Encoding: br\n", raw) + if bb, err := readBoundResponse(store, snap); err == nil || !strings.Contains(err.Error(), "unsupported content-encoding") { + t.Fatalf("br: got %+v err=%v, want unsupported content-encoding", bb, err) + } +} + +// A response captured with Content-Encoding: gzip and no body at all (HEAD, +// 304) is not a decode failure: it stays an empty body, without error. +func TestDecodeEvidenceBodyEmptyBodyWithEncodingStaysEmpty(t *testing.T) { + for _, head := range []string{ + decodeGzipHead, + "HTTP 304\r\nCONTENT-ENCODING: GZIP\r\n", + "HTTP 200\nContent-Encoding: deflate\n", + } { + for _, raw := range [][]byte{nil, {}} { + out, truncated, err := decodeEvidenceBody(head, raw, verificationReadCap) + if err != nil || truncated || len(out) != 0 { + t.Fatalf("head %q: out=%q truncated=%v err=%v, want an empty body and no error", head, out, truncated, err) + } + } + } +} + +// An empty bound body, compressed or not, is read without error but can never +// satisfy a quote. In assessFindingVerification a quote that matches no body +// only leads to unverified or not_assessed, never verified, so this is what +// keeps an empty response (HEAD, 304) from verifying a finding. +func TestDecodeBoundResponseEmptyBodyNeverMatchesQuote(t *testing.T) { + cases := []struct { + name string + head string + raw []byte + }{ + {"gzip header, no bytes", decodeGzipHead, nil}, + {"deflate header, no bytes", "HTTP 200\nContent-Encoding: deflate\n", nil}, + {"identity header, no bytes", "HTTP 200\nContent-Encoding: identity\n", nil}, + {"no encoding header, no bytes", "HTTP 304\nContent-Type: text/html\n", nil}, + {"gzip stream of an empty payload", decodeGzipHead, gzipBody(t, nil)}, + } + quotes := []string{ + decodeQuote, + "x", + // Present in the response head: the head is not searchable text. + "Content-Encoding: gzip", + "HTTP 200", + } + for _, c := range cases { + store, snap := decodeSnapshot(t, c.head, c.raw) + bb, err := readBoundResponse(store, snap) + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + if bb.text != "" || bb.nonText || bb.truncated { + t.Fatalf("%s: got %+v, want an empty text body", c.name, bb) + } + for _, q := range quotes { + nq := normalizeEvidenceText(q) + if nq == "" { + t.Fatalf("quote %q normalizes to nothing", q) + } + if quoteInBodies(nq, []boundBody{bb}) { + t.Fatalf("%s: quote %q matched an empty body", c.name, q) + } + } + } +}