mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-20 07:00:58 +08:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
01a9dae49e | ||
|
|
4e126a8524 | ||
|
|
abb6cb177d | ||
|
|
5122baa77f | ||
|
|
fcee6af45e | ||
|
|
b74ff21e11 |
@@ -2,6 +2,9 @@ name: release
|
|||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
tags: ["v*"]
|
tags: ["v*"]
|
||||||
|
# Dry run: builds, SBOM, attestations and workflow artifacts only. The GitHub
|
||||||
|
# Release and the image push are gated on github.event_name == 'push'.
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: write # required to create the Release
|
contents: write # required to create the Release
|
||||||
@@ -81,6 +84,10 @@ jobs:
|
|||||||
release:
|
release:
|
||||||
needs: binaries
|
needs: binaries
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
contents: write # create the Release (softprops/action-gh-release)
|
||||||
|
id-token: write # Sigstore signing of the provenance attestation
|
||||||
|
attestations: write # store the attestation on the repository
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4 # CHANGELOG.md and the extractor script
|
- uses: actions/checkout@v4 # CHANGELOG.md and the extractor script
|
||||||
- uses: actions/download-artifact@v4
|
- uses: actions/download-artifact@v4
|
||||||
@@ -91,6 +98,39 @@ jobs:
|
|||||||
- name: Generate checksums
|
- name: Generate checksums
|
||||||
working-directory: dist
|
working-directory: dist
|
||||||
run: sha256sum *.zip > SHA256SUMS
|
run: sha256sum *.zip > SHA256SUMS
|
||||||
|
# SBOM of what ships: the Go modules embedded in the linux/amd64 binary, plus
|
||||||
|
# the frontend lockfile and the Go module files.
|
||||||
|
- uses: actions/download-artifact@v4
|
||||||
|
with:
|
||||||
|
name: bin-linux-amd64
|
||||||
|
path: ${{ runner.temp }}/sbom-src/bin
|
||||||
|
- uses: anchore/sbom-action/download-syft@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c # v0.24.3
|
||||||
|
id: syft
|
||||||
|
with:
|
||||||
|
syft-version: v1.54.0
|
||||||
|
- name: Generate CycloneDX SBOM
|
||||||
|
env:
|
||||||
|
SYFT: ${{ steps.syft.outputs.cmd }}
|
||||||
|
SRC: ${{ runner.temp }}/sbom-src
|
||||||
|
TAG: ${{ github.ref_name }}
|
||||||
|
run: |
|
||||||
|
mkdir -p "$SRC/web"
|
||||||
|
cp go.mod go.sum "$SRC/"
|
||||||
|
cp web/package-lock.json "$SRC/web/"
|
||||||
|
sbom="dist/artex-${TAG}.cdx.json"
|
||||||
|
"$SYFT" scan "dir:$SRC" --source-name artex --source-version "$TAG" -o "cyclonedx-json=$sbom"
|
||||||
|
grep -q '"pkg:golang/' "$sbom" || { echo "::error::SBOM has no pkg:golang entries"; exit 1; }
|
||||||
|
grep -q '"pkg:npm/' "$sbom" || { echo "::error::SBOM has no pkg:npm entries"; exit 1; }
|
||||||
|
echo "SBOM_PATH=$sbom" >> "$GITHUB_ENV"
|
||||||
|
- uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
|
||||||
|
with:
|
||||||
|
subject-path: dist/*.zip
|
||||||
|
- uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: release-metadata
|
||||||
|
path: |
|
||||||
|
dist/SHA256SUMS
|
||||||
|
${{ env.SBOM_PATH }}
|
||||||
- name: Release body from CHANGELOG.md
|
- name: Release body from CHANGELOG.md
|
||||||
env:
|
env:
|
||||||
TAG: ${{ github.ref_name }}
|
TAG: ${{ github.ref_name }}
|
||||||
@@ -103,10 +143,12 @@ jobs:
|
|||||||
echo "Release $TAG. See CHANGELOG.md for the list of changes in this release." > "$RUNNER_TEMP/release-body.md"
|
echo "Release $TAG. See CHANGELOG.md for the list of changes in this release." > "$RUNNER_TEMP/release-body.md"
|
||||||
fi
|
fi
|
||||||
- uses: softprops/action-gh-release@v2
|
- uses: softprops/action-gh-release@v2
|
||||||
|
if: github.event_name == 'push'
|
||||||
with:
|
with:
|
||||||
files: |
|
files: |
|
||||||
dist/*.zip
|
dist/*.zip
|
||||||
dist/SHA256SUMS
|
dist/SHA256SUMS
|
||||||
|
${{ env.SBOM_PATH }}
|
||||||
generate_release_notes: false
|
generate_release_notes: false
|
||||||
body_path: ${{ runner.temp }}/release-body.md
|
body_path: ${{ runner.temp }}/release-body.md
|
||||||
|
|
||||||
@@ -118,7 +160,7 @@ jobs:
|
|||||||
# and provide DOCKERHUB_USERNAME / DOCKERHUB_TOKEN secrets plus DOCKERHUB_IMAGE).
|
# and provide DOCKERHUB_USERNAME / DOCKERHUB_TOKEN secrets plus DOCKERHUB_IMAGE).
|
||||||
docker:
|
docker:
|
||||||
needs: binaries
|
needs: binaries
|
||||||
if: ${{ vars.DOCKERHUB_ENABLED == 'true' }}
|
if: ${{ vars.DOCKERHUB_ENABLED == 'true' && github.event_name == 'push' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4 # fetch the Dockerfile and skills/
|
- uses: actions/checkout@v4 # fetch the Dockerfile and skills/
|
||||||
|
|||||||
@@ -134,6 +134,15 @@ cp config.example.json config.json # fill in the database connection
|
|||||||
> Please start with `start.sh` / `start.bat` instead of running `./artex` directly. It is a daemon script: if the process crashes, it restarts it after a backoff.
|
> Please start with `start.sh` / `start.bat` instead of running `./artex` directly. It is a daemon script: if the process crashes, it restarts it after a backoff.
|
||||||
> To keep it running in the background: `nohup ./start.sh >artex.log 2>&1 &`.
|
> To keep it running in the background: `nohup ./start.sh >artex.log 2>&1 &`.
|
||||||
|
|
||||||
|
#### Verifying a release
|
||||||
|
|
||||||
|
Each release attaches `SHA256SUMS` and a CycloneDX SBOM, `artex-<tag>.cdx.json` (Go modules embedded in the linux/amd64 binary, the frontend `package-lock.json`, and `go.mod`/`go.sum`), next to the zips. The zips also carry a GitHub build provenance attestation. Releases published before this was added have neither the SBOM nor the attestation.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sha256sum -c SHA256SUMS # add --ignore-missing if you downloaded only some zips
|
||||||
|
gh attestation verify <zip> --repo BaaRai/ARTEX # provenance: built by this repository's release workflow
|
||||||
|
```
|
||||||
|
|
||||||
### Option 4: build the single binary from source
|
### Option 4: build the single binary from source
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package db
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math/rand/v2"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -331,19 +332,36 @@ func TestCompanyScopeAttribution(t *testing.T) {
|
|||||||
d.Close()
|
d.Close()
|
||||||
t.Fatalf("startMax: %v", err)
|
t.Fatalf("startMax: %v", err)
|
||||||
}
|
}
|
||||||
|
// Remove everything the test creates, or the next pass on the same database
|
||||||
|
// finds this company's scope and attributes its assets to it. Assets first
|
||||||
|
// (they reference the company), then the company; its scope rows cascade.
|
||||||
|
var cid int64
|
||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
_, _ = d.Exec(`DELETE FROM assets WHERE id > $1`, startMax)
|
defer d.Close()
|
||||||
d.Close()
|
if _, err := d.Exec(`DELETE FROM assets WHERE id > $1`, startMax); err != nil {
|
||||||
|
t.Errorf("cleanup assets: %v", err)
|
||||||
|
}
|
||||||
|
if cid != 0 {
|
||||||
|
if _, err := d.Exec(`DELETE FROM companies WHERE id = $1`, cid); err != nil {
|
||||||
|
t.Errorf("cleanup company %d: %v", cid, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
uniq := startMax + 1
|
// Unique per run: MAX(assets.id) comes back to the same value after cleanup,
|
||||||
root := fmt.Sprintf("scopetest%d.com", uniq)
|
// so it cannot tell one pass from the next. The in-scope /24 is picked per
|
||||||
|
// run from 198.18.0.0/15 so a scope left by an interrupted run cannot claim
|
||||||
|
// this run's IP.
|
||||||
|
uniq := fmt.Sprintf("%d%d", time.Now().UnixNano(), rand.Uint32())
|
||||||
|
n := rand.IntN(512)
|
||||||
|
cidr := fmt.Sprintf("198.%d.%d.0/24", 18+n/256, n%256)
|
||||||
|
root := fmt.Sprintf("scopetest%s.com", uniq)
|
||||||
sub := "api." + root
|
sub := "api." + root
|
||||||
ipIn := "198.51.100.9"
|
ipIn := fmt.Sprintf("198.%d.%d.9", 18+n/256, n%256)
|
||||||
ipOut := "203.0.113.9"
|
ipOut := "203.0.113.9"
|
||||||
outDomain := fmt.Sprintf("other%d.net", uniq)
|
outDomain := fmt.Sprintf("other%s.net", uniq)
|
||||||
|
|
||||||
cid, _, err := cs.UpsertCompany(fmt.Sprintf("ScopeCo %d", uniq), "")
|
cid, _, err = cs.UpsertCompany(fmt.Sprintf("ScopeCo %s", uniq), "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("UpsertCompany: %v", err)
|
t.Fatalf("UpsertCompany: %v", err)
|
||||||
}
|
}
|
||||||
@@ -359,7 +377,7 @@ func TestCompanyScopeAttribution(t *testing.T) {
|
|||||||
t.Fatalf("pre-scope asset should be unattributed, got %v", *preCompanyID)
|
t.Fatalf("pre-scope asset should be unattributed, got %v", *preCompanyID)
|
||||||
}
|
}
|
||||||
|
|
||||||
cs.AddScope(cid, []string{root, "198.51.100.0/24"}, "unit test")
|
cs.AddScope(cid, []string{root, cidr}, "unit test")
|
||||||
|
|
||||||
mustCid := func(id int64, want int64, label string) {
|
mustCid := func(id int64, want int64, label string) {
|
||||||
var cID *int64
|
var cID *int64
|
||||||
|
|||||||
+22
-6
@@ -4,7 +4,9 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestPoolProfilesOrder pins the failover chain query: keyless profiles can't
|
// TestPoolProfilesOrder pins the failover chain query: keyless profiles can't
|
||||||
@@ -17,17 +19,22 @@ func TestPoolProfilesOrder(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||||
}
|
}
|
||||||
defer d.Close()
|
// Close via Cleanup, registered FIRST so it runs LAST: a plain defer would
|
||||||
|
// close the pool before the row-deleting cleanups below run.
|
||||||
|
t.Cleanup(func() { d.Close() })
|
||||||
|
|
||||||
|
// llm_profiles.name is unique, so a fixed name collides with a row left on
|
||||||
|
// the same database by an earlier run.
|
||||||
|
sfx := fmt.Sprintf("-%d", time.Now().UnixNano())
|
||||||
mk := func(name string, priority int, exclude bool, key string) int64 {
|
mk := func(name string, priority int, exclude bool, key string) int64 {
|
||||||
id, err := d.SaveProfile(&LLMProfile{
|
id, err := d.SaveProfile(&LLMProfile{
|
||||||
Name: name, Format: "openai", Model: "m", APIKey: key,
|
Name: name + sfx, Format: "openai", Model: "m", APIKey: key,
|
||||||
Priority: priority, PoolExclude: exclude,
|
Priority: priority, PoolExclude: exclude,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
t.Cleanup(func() { d.Exec(`DELETE FROM llm_profiles WHERE id=$1`, id) })
|
t.Cleanup(func() { deleteTestProfile(t, d, id) })
|
||||||
return id
|
return id
|
||||||
}
|
}
|
||||||
lo := mk("t-pool-lo", 1, false, "k1")
|
lo := mk("t-pool-lo", 1, false, "k1")
|
||||||
@@ -42,11 +49,11 @@ func TestPoolProfilesOrder(t *testing.T) {
|
|||||||
var got []int64
|
var got []int64
|
||||||
for _, p := range chain {
|
for _, p := range chain {
|
||||||
switch p.Name {
|
switch p.Name {
|
||||||
case "t-pool-lo", "t-pool-hi":
|
case "t-pool-lo" + sfx, "t-pool-hi" + sfx:
|
||||||
got = append(got, p.ID)
|
got = append(got, p.ID)
|
||||||
case "t-pool-excluded":
|
case "t-pool-excluded" + sfx:
|
||||||
t.Fatal("pool_exclude profile entered the failover chain")
|
t.Fatal("pool_exclude profile entered the failover chain")
|
||||||
case "t-pool-nokey":
|
case "t-pool-nokey" + sfx:
|
||||||
t.Fatal("keyless profile entered the failover chain")
|
t.Fatal("keyless profile entered the failover chain")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -55,6 +62,15 @@ func TestPoolProfilesOrder(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// deleteTestProfile removes a profile a test created and reports a failed delete
|
||||||
|
// instead of silently leaving the row for the next run.
|
||||||
|
func deleteTestProfile(t *testing.T, d *DB, id int64) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := d.Exec(`DELETE FROM llm_profiles WHERE id=$1`, id); err != nil {
|
||||||
|
t.Errorf("cleanup profile %d: %v", id, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDeleteProfileContextHonorsCancellation(t *testing.T) {
|
func TestDeleteProfileContextHonorsCancellation(t *testing.T) {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
|
|||||||
@@ -0,0 +1,245 @@
|
|||||||
|
package db
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
evCookie = "session=ev-cookie-secret"
|
||||||
|
evBearer = "Bearer ev-auth-secret"
|
||||||
|
)
|
||||||
|
|
||||||
|
// evidenceFixture records a finding. With legacy=false the snapshot goes through
|
||||||
|
// the normal insert path; with legacy=true it is written in clear by raw SQL.
|
||||||
|
func evidenceFixture(t *testing.T, d *DB, tag string, legacy bool) (*Task, int64, TrafficEvidenceSnapshot) {
|
||||||
|
t.Helper()
|
||||||
|
task, err := d.CreateTask("evidence heads at rest "+tag, "goal", nil, 0, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s := TrafficEvidenceSnapshot{SourceTrafficID: "t-sec-ev-" + tag, CapturedAt: 1, URL: "http://ev.example/" + tag, Method: "GET", Status: 200,
|
||||||
|
ReqHead: "GET /" + tag + " HTTP/1.1\nCookie: " + evCookie + "\nAuthorization: " + evBearer + "\n",
|
||||||
|
RespHead: "HTTP/1.1 200 OK\nSet-Cookie: " + evCookie + "\n", ReqHash: "rh-" + tag, RespHash: "ph-" + tag}
|
||||||
|
s.ID = TrafficSnapshotID(s)
|
||||||
|
t.Cleanup(func() {
|
||||||
|
d.Exec(`DELETE FROM task_archives WHERE task_id=$1`, task.ID)
|
||||||
|
d.DeleteTaskCascade(task.ID, false, true) // findings and their bindings too
|
||||||
|
if _, err := d.Exec(`DELETE FROM traffic_evidence_snapshots WHERE id=$1`, s.ID); err != nil {
|
||||||
|
t.Errorf("cleanup snapshot: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
f, err := d.Exploration(task.ExplorationID).RecordFinding(t.Context(), RecordFindingInput{TaskID: task.ID, ExplorationID: task.ExplorationID, Summary: "evidence " + tag, Severity: "low"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if legacy {
|
||||||
|
_, err = d.Exec(`INSERT INTO traffic_evidence_snapshots(id,source_traffic_id,captured_at,url,method,status,req_head,resp_head,req_hash,resp_hash,req_len,resp_len)
|
||||||
|
VALUES($1,$2,1,$3,'GET',200,$4,$5,$6,$7,0,0)`, s.ID, s.SourceTrafficID, s.URL, s.ReqHead, s.RespHead, s.ReqHash, s.RespHash)
|
||||||
|
if err == nil {
|
||||||
|
_, err = d.Exec(`INSERT INTO finding_traffic_bindings(finding_id,snapshot_id,position) VALUES($1,$2,0)`, f.FindingID, s.ID)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
err = d.WithEvidenceTx(t.Context(), func(tx *sql.Tx) error {
|
||||||
|
return AddFindingTrafficTx(tx, f.FindingID, []PreparedTrafficEvidence{{Ref: TrafficRef{TrafficID: s.SourceTrafficID, Role: "proof"}, Snapshot: s}})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return task, f.FindingID, s
|
||||||
|
}
|
||||||
|
|
||||||
|
// rawHeads reads the stored heads; both must be sealed and hide the secrets.
|
||||||
|
func rawHeads(t *testing.T, d *DB, id string) (req, resp string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := d.QueryRow(`SELECT req_head,resp_head FROM traffic_evidence_snapshots WHERE id=$1`, id).Scan(&req, &resp); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, v := range []string{req, resp} {
|
||||||
|
if !isEncrypted(v) || strings.Contains(v, evCookie) || strings.Contains(v, evBearer) {
|
||||||
|
t.Fatalf("head stored in clear: %q", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return req, resp
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertClearHeads checks the application read and that the stored value is
|
||||||
|
// sealed exactly once.
|
||||||
|
func assertClearHeads(t *testing.T, d *DB, findingID int64, want TrafficEvidenceSnapshot) {
|
||||||
|
t.Helper()
|
||||||
|
got, err := d.GetFindingTraffic(t.Context(), findingID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got.Bindings) != 1 || got.Bindings[0].Snapshot.ReqHead != want.ReqHead || got.Bindings[0].Snapshot.RespHead != want.RespHead || got.Bindings[0].SnapshotID != want.ID {
|
||||||
|
t.Fatalf("read back: %+v", got.Bindings)
|
||||||
|
}
|
||||||
|
req, resp := rawHeads(t, d, want.ID)
|
||||||
|
box := activeSecretBox.Load()
|
||||||
|
if p, err := box.decrypt(colEvReqHead.aad(), req); err != nil || p != want.ReqHead {
|
||||||
|
t.Fatalf("req_head not sealed once: %q %v", p, err)
|
||||||
|
}
|
||||||
|
if p, err := box.decrypt(colEvRespHead.aad(), resp); err != nil || p != want.RespHead {
|
||||||
|
t.Fatalf("resp_head not sealed once: %q %v", p, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvidenceHeadsSealedAtInsert(t *testing.T) {
|
||||||
|
d, err := Open(testDSN(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { d.Close() })
|
||||||
|
_, fid, s := evidenceFixture(t, d, "insert", false)
|
||||||
|
assertClearHeads(t, d, fid, s)
|
||||||
|
var url, method string
|
||||||
|
if err := d.QueryRow(`SELECT url,method FROM traffic_evidence_snapshots WHERE id=$1`, s.ID).Scan(&url, &method); err != nil || url != s.URL || method != "GET" {
|
||||||
|
t.Fatalf("metadata must stay in clear: %q %q %v", url, method, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvidenceHeadsBackfillAndKeyChecks(t *testing.T) {
|
||||||
|
d, err := Open(testDSN(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { d.Close() })
|
||||||
|
_, fid, s := evidenceFixture(t, d, "legacy", true)
|
||||||
|
t.Setenv(secretKeyEnv, base64.StdEncoding.EncodeToString(testSecretKey))
|
||||||
|
if err := d.InitSecrets(t.TempDir()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
assertClearHeads(t, d, fid, s)
|
||||||
|
req, resp := rawHeads(t, d, s.ID)
|
||||||
|
if err := d.InitSecrets(t.TempDir()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if r2, p2 := rawHeads(t, d, s.ID); r2 != req || p2 != resp {
|
||||||
|
t.Fatal("second backfill must leave the ciphertexts untouched")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A head sealed under another key: the read and the startup check fail and
|
||||||
|
// name the column. The startup check samples one sealed row per column, so
|
||||||
|
// this row must be the only sealed snapshot.
|
||||||
|
var sealedRows int
|
||||||
|
if err := d.QueryRow(`SELECT count(*) FROM traffic_evidence_snapshots WHERE req_head LIKE 'enc:v1:%' OR resp_head LIKE 'enc:v1:%'`).Scan(&sealedRows); err != nil || sealedRows != 1 {
|
||||||
|
t.Fatalf("want only this fixture's snapshot sealed, got %d (%v)", sealedRows, err)
|
||||||
|
}
|
||||||
|
other, _ := newSecretBox([]byte("ffffffffffffffffffffffffffffffff"))
|
||||||
|
for _, c := range []secretColumn{colEvReqHead, colEvRespHead} {
|
||||||
|
bad, _ := other.encrypt(c.aad(), "GET / HTTP/1.1\nCookie: other\n")
|
||||||
|
if _, err := d.Exec(`UPDATE traffic_evidence_snapshots SET `+c.col+`=$1 WHERE id=$2`, bad, s.ID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := d.GetFindingTraffic(t.Context(), fid); err == nil || !strings.Contains(err.Error(), c.aad()) {
|
||||||
|
t.Fatalf("want a read failure naming %s, got %v", c.aad(), err)
|
||||||
|
}
|
||||||
|
if err := d.InitSecrets(t.TempDir()); err == nil || !strings.Contains(err.Error(), "secret key check") || !strings.Contains(err.Error(), c.aad()) {
|
||||||
|
t.Fatalf("want a key-check failure naming %s, got %v", c.aad(), err)
|
||||||
|
}
|
||||||
|
if _, err := d.Exec(`UPDATE traffic_evidence_snapshots SET req_head=$1,resp_head=$2 WHERE id=$3`, req, resp, s.ID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertClearHeads(t, d, fid, s)
|
||||||
|
|
||||||
|
// No key with sealed heads: refuse, and each head counts. One head stays
|
||||||
|
// sealed so the refusal never turns into a new key.
|
||||||
|
t.Setenv(secretKeyEnv, "")
|
||||||
|
dir := t.TempDir()
|
||||||
|
withHeads := encryptedCount(t, d, dir)
|
||||||
|
for _, c := range []secretColumn{colEvReqHead, colEvRespHead} {
|
||||||
|
if _, err := d.Exec(`UPDATE traffic_evidence_snapshots SET req_head=$1,resp_head=$2 WHERE id=$3`, req, resp, s.ID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := d.Exec(`UPDATE traffic_evidence_snapshots SET `+c.col+`='' WHERE id=$1`, s.ID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n := encryptedCount(t, d, dir); withHeads-n != 1 {
|
||||||
|
t.Fatalf("no-key refusal counted %d then %d encrypted values; %s must count", withHeads, n, c.aad())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Export keeps the heads sealed; import opens them to check the snapshot ID,
|
||||||
|
// then the insert seals them once. A legacy clear archive is sealed on import.
|
||||||
|
func TestEvidenceHeadsArchiveRoundTrip(t *testing.T) {
|
||||||
|
d, err := Open(testDSN(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { d.Close() })
|
||||||
|
if err = d.EnsureLLMRecordsTable(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = d.EnsureLLMUsageTable(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, legacy := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprint("legacy=", legacy), func(t *testing.T) {
|
||||||
|
task, fid, s := evidenceFixture(t, d, fmt.Sprint("archive-", legacy), false)
|
||||||
|
if err := d.SetPaused(task.ID, true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
archive, err := d.QueueTaskArchive(task.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = d.ClaimTaskArchiveJob(t.Context()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
snapshot, err := d.SnapshotTaskArchive(task.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
exported := string(snapshot.Tables["traffic_evidence_snapshots"])
|
||||||
|
if !strings.Contains(exported, secretPrefix) || strings.Contains(exported, evCookie) || strings.Contains(exported, evBearer) {
|
||||||
|
t.Fatalf("archive must keep the heads sealed: %s", exported)
|
||||||
|
}
|
||||||
|
if err = d.CompleteTaskArchive(archive.ID, snapshot, "/tmp/evidence-heads-test.tar.zst", "fixture", 1, 1); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var rows []map[string]any
|
||||||
|
if err = json.Unmarshal(snapshot.Tables["traffic_evidence_snapshots"], &rows); err != nil || len(rows) != 1 {
|
||||||
|
t.Fatal(rows, err)
|
||||||
|
}
|
||||||
|
if legacy { // an archive written before the heads were sealed
|
||||||
|
rows[0]["req_head"], rows[0]["resp_head"] = s.ReqHead, s.RespHead
|
||||||
|
snapshot.Tables["traffic_evidence_snapshots"], _ = json.Marshal(rows)
|
||||||
|
} else { // a head sealed under another key fails the import, naming the column
|
||||||
|
other, _ := newSecretBox([]byte("ffffffffffffffffffffffffffffffff"))
|
||||||
|
bad, _ := other.encrypt(colEvReqHead.aad(), s.ReqHead)
|
||||||
|
good := rows[0]["req_head"]
|
||||||
|
rows[0]["req_head"] = bad
|
||||||
|
raw, _ := json.Marshal(rows)
|
||||||
|
if _, err := ArchiveEvidenceSnapshots(&TaskArchiveSnapshot{FormatVersion: snapshot.FormatVersion, Tables: map[string]json.RawMessage{"traffic_evidence_snapshots": raw}}); err == nil || !strings.Contains(err.Error(), colEvReqHead.aad()) {
|
||||||
|
t.Fatalf("want an import failure naming %s, got %v", colEvReqHead.aad(), err)
|
||||||
|
}
|
||||||
|
rows[0]["req_head"] = good
|
||||||
|
}
|
||||||
|
list, err := ArchiveEvidenceSnapshots(snapshot)
|
||||||
|
if err != nil || len(list) != 1 || list[0].ReqHead != s.ReqHead || list[0].RespHead != s.RespHead {
|
||||||
|
t.Fatalf("archive snapshots: %+v %v", list, err)
|
||||||
|
}
|
||||||
|
// The hot row is gone (GC), so the restore inserts it again.
|
||||||
|
if _, err = d.Exec(`DELETE FROM traffic_evidence_snapshots WHERE id=$1`, s.ID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = d.QueueTaskArchiveRestore(archive.ID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = d.ClaimTaskArchiveJob(t.Context()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = d.RestoreTaskArchive(archive.ID, snapshot, 0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
assertClearHeads(t, d, fid, s)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
+24
-2
@@ -195,10 +195,29 @@ func InsertEvidenceSnapshotTx(tx *sql.Tx, s TrafficEvidenceSnapshot) error {
|
|||||||
if s.CreatedAt.IsZero() {
|
if s.CreatedAt.IsZero() {
|
||||||
s.CreatedAt = time.Now().UTC()
|
s.CreatedAt = time.Now().UTC()
|
||||||
}
|
}
|
||||||
_, err := tx.Exec(`INSERT INTO traffic_evidence_snapshots
|
// The ID hashes the clear heads; only the stored heads are sealed.
|
||||||
|
reqHead, err := sealSecret(colEvReqHead, s.ReqHead)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
respHead, err := sealSecret(colEvRespHead, s.RespHead)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = tx.Exec(`INSERT INTO traffic_evidence_snapshots
|
||||||
(id,source_traffic_id,captured_at,url,method,status,content_type,req_head,resp_head,req_hash,resp_hash,req_len,resp_len,created_at)
|
(id,source_traffic_id,captured_at,url,method,status,content_type,req_head,resp_head,req_hash,resp_hash,req_len,resp_len,created_at)
|
||||||
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT(id) DO NOTHING`,
|
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT(id) DO NOTHING`,
|
||||||
s.ID, s.SourceTrafficID, s.CapturedAt, s.URL, s.Method, s.Status, s.ContentType, s.ReqHead, s.RespHead, s.ReqHash, s.RespHash, s.ReqLen, s.RespLen, s.CreatedAt)
|
s.ID, s.SourceTrafficID, s.CapturedAt, s.URL, s.Method, s.Status, s.ContentType, reqHead, respHead, s.ReqHash, s.RespHash, s.ReqLen, s.RespLen, s.CreatedAt)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// openHeads decrypts the heads of a snapshot read from the table or an archive.
|
||||||
|
// Clear legacy values pass through; a value that fails to decrypt is an error.
|
||||||
|
func (s *TrafficEvidenceSnapshot) openHeads() (err error) {
|
||||||
|
if s.ReqHead, err = openSecret(colEvReqHead, s.ReqHead); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.RespHead, err = openSecret(colEvRespHead, s.RespHead)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -263,6 +282,9 @@ FROM finding_traffic_bindings b JOIN traffic_evidence_snapshots s ON s.id=b.snap
|
|||||||
if err := json.Unmarshal(raw, &b.Snapshot); err != nil {
|
if err := json.Unmarshal(raw, &b.Snapshot); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if err := b.Snapshot.openHeads(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
out.Bindings = append(out.Bindings, b)
|
out.Bindings = append(out.Bindings, b)
|
||||||
}
|
}
|
||||||
return out, rows.Err()
|
return out, rows.Err()
|
||||||
|
|||||||
@@ -15,8 +15,17 @@ func ArchiveEvidenceSnapshots(snapshot *TaskArchiveSnapshot) ([]TrafficEvidenceS
|
|||||||
if snapshot.FormatVersion < 3 {
|
if snapshot.FormatVersion < 3 {
|
||||||
return nil, ErrTaskArchiveFormatMismatch
|
return nil, ErrTaskArchiveFormatMismatch
|
||||||
}
|
}
|
||||||
err := json.Unmarshal(snapshot.Tables["traffic_evidence_snapshots"], &out)
|
// The archive keeps the heads as stored (sealed). They are opened here so the
|
||||||
return out, err
|
// restore can check each content-addressed ID; the insert then seals them once.
|
||||||
|
if err := json.Unmarshal(snapshot.Tables["traffic_evidence_snapshots"], &out); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for i := range out {
|
||||||
|
if err := out[i].openHeads(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func restoreFindingTrafficTx(tx *sql.Tx, snapshot *TaskArchiveSnapshot) error {
|
func restoreFindingTrafficTx(tx *sql.Tx, snapshot *TaskArchiveSnapshot) error {
|
||||||
|
|||||||
+17
-9
@@ -1,6 +1,7 @@
|
|||||||
package db
|
package db
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -14,7 +15,8 @@ func TestProfileRetryRoundTrip(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||||
}
|
}
|
||||||
defer d.Close()
|
// Registered FIRST so it runs LAST, after the row cleanup below.
|
||||||
|
t.Cleanup(func() { d.Close() })
|
||||||
|
|
||||||
want := RetryOverride{
|
want := RetryOverride{
|
||||||
Connect: RetryRule{Attempts: 5, IntervalMS: 2000},
|
Connect: RetryRule{Attempts: 5, IntervalMS: 2000},
|
||||||
@@ -22,12 +24,12 @@ func TestProfileRetryRoundTrip(t *testing.T) {
|
|||||||
Stream: RetryRule{IntervalMS: 250},
|
Stream: RetryRule{IntervalMS: 250},
|
||||||
}
|
}
|
||||||
id, err := d.SaveProfile(&LLMProfile{
|
id, err := d.SaveProfile(&LLMProfile{
|
||||||
Name: "t-retry-roundtrip", Format: "openai", Model: "m", APIKey: "k", Retry: want,
|
Name: fmt.Sprintf("t-retry-roundtrip-%d", time.Now().UnixNano()), Format: "openai", Model: "m", APIKey: "k", Retry: want,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
t.Cleanup(func() { d.Exec(`DELETE FROM llm_profiles WHERE id=$1`, id) })
|
t.Cleanup(func() { deleteTestProfile(t, d, id) })
|
||||||
|
|
||||||
got, err := d.ProfileByID(id)
|
got, err := d.ProfileByID(id)
|
||||||
if err != nil || got == nil {
|
if err != nil || got == nil {
|
||||||
@@ -71,16 +73,17 @@ func TestProfileRetryClampedOnSave(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||||
}
|
}
|
||||||
defer d.Close()
|
// Registered FIRST so it runs LAST, after the row cleanup below.
|
||||||
|
t.Cleanup(func() { d.Close() })
|
||||||
|
|
||||||
id, err := d.SaveProfile(&LLMProfile{
|
id, err := d.SaveProfile(&LLMProfile{
|
||||||
Name: "t-retry-clamp", Format: "openai", Model: "m", APIKey: "k",
|
Name: fmt.Sprintf("t-retry-clamp-%d", time.Now().UnixNano()), Format: "openai", Model: "m", APIKey: "k",
|
||||||
Retry: RetryOverride{Connect: RetryRule{Attempts: -99, IntervalMS: -5}},
|
Retry: RetryOverride{Connect: RetryRule{Attempts: -99, IntervalMS: -5}},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
t.Cleanup(func() { d.Exec(`DELETE FROM llm_profiles WHERE id=$1`, id) })
|
t.Cleanup(func() { deleteTestProfile(t, d, id) })
|
||||||
got, err := d.ProfileByID(id)
|
got, err := d.ProfileByID(id)
|
||||||
if err != nil || got == nil {
|
if err != nil || got == nil {
|
||||||
t.Fatalf("ProfileByID: %v", err)
|
t.Fatalf("ProfileByID: %v", err)
|
||||||
@@ -97,17 +100,22 @@ func TestLLMRetryPolicyRoundTrip(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||||
}
|
}
|
||||||
defer d.Close()
|
// Registered FIRST so it runs LAST, after the restore below.
|
||||||
|
t.Cleanup(func() { d.Close() })
|
||||||
|
|
||||||
before, hadBefore, err := d.GetSetting(settingLLMRetryPolicy)
|
before, hadBefore, err := d.GetSetting(settingLLMRetryPolicy)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
|
var err error
|
||||||
if hadBefore {
|
if hadBefore {
|
||||||
d.SetSetting(settingLLMRetryPolicy, before)
|
err = d.SetSetting(settingLLMRetryPolicy, before)
|
||||||
} else {
|
} else {
|
||||||
d.Exec(`DELETE FROM settings WHERE key=$1`, settingLLMRetryPolicy)
|
_, err = d.Exec(`DELETE FROM settings WHERE key=$1`, settingLLMRetryPolicy)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("restore %s: %v", settingLLMRetryPolicy, err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
+4
-1
@@ -81,7 +81,10 @@ var (
|
|||||||
colCredPass = secretColumn{"task_credentials", "password", false, ""}
|
colCredPass = secretColumn{"task_credentials", "password", false, ""}
|
||||||
colNotifConfig = secretColumn{"notification_channels", "config", true, ""}
|
colNotifConfig = secretColumn{"notification_channels", "config", true, ""}
|
||||||
colToolExec = secretColumn{"tools", "exec", true, ""} // system=false rows: headers, proxy, body, code, command
|
colToolExec = secretColumn{"tools", "exec", true, ""} // system=false rows: headers, proxy, body, code, command
|
||||||
secretColumns = append([]secretColumn{colLLMKey, colLLMProxy, colMCPEnv, colCredPass, colNotifConfig, colToolExec}, secretSettingColumns()...)
|
// Evidence head blocks carry the captured Cookie and Authorization headers.
|
||||||
|
colEvReqHead = secretColumn{"traffic_evidence_snapshots", "req_head", false, ""}
|
||||||
|
colEvRespHead = secretColumn{"traffic_evidence_snapshots", "resp_head", false, ""}
|
||||||
|
secretColumns = append([]secretColumn{colLLMKey, colLLMProxy, colMCPEnv, colCredPass, colNotifConfig, colToolExec, colEvReqHead, colEvRespHead}, secretSettingColumns()...)
|
||||||
)
|
)
|
||||||
|
|
||||||
// secretSettingKeys are the settings stored encrypted, AAD "settings.<key>".
|
// secretSettingKeys are the settings stored encrypted, AAD "settings.<key>".
|
||||||
|
|||||||
@@ -34,5 +34,10 @@ func runEvidenceSuite(m *testing.M) int {
|
|||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
defer conn.ExecContext(context.Background(), `SELECT pg_advisory_unlock(7337741002)`)
|
defer conn.ExecContext(context.Background(), `SELECT pg_advisory_unlock(7337741002)`)
|
||||||
|
// Snapshot inserts seal the heads: load a fixed test key (env, no key file).
|
||||||
|
os.Setenv("ARTEX_SECRET_KEY", "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=")
|
||||||
|
if err = pg.InitSecrets(""); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
return m.Run()
|
return m.Run()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -172,6 +172,11 @@ func TestFindingTrafficAPIAndExport(t *testing.T) {
|
|||||||
if w = req("GET", fmt.Sprintf("%s/%d", base, first), ""); w.Code != 200 || !strings.Contains(w.Body.String(), `"binary":true`) {
|
if w = req("GET", fmt.Sprintf("%s/%d", base, first), ""); w.Code != 200 || !strings.Contains(w.Body.String(), `"binary":true`) {
|
||||||
t.Fatal(w.Code, w.Body)
|
t.Fatal(w.Code, w.Body)
|
||||||
}
|
}
|
||||||
|
// The viewer reads the heads in clear; the table stores them sealed.
|
||||||
|
var rawHead string
|
||||||
|
if err := s.m.pg.QueryRow(`SELECT req_head FROM traffic_evidence_snapshots WHERE id=$1`, list.Bindings[0].SnapshotID).Scan(&rawHead); err != nil || !strings.HasPrefix(rawHead, "enc:v1:") || !strings.Contains(w.Body.String(), `"req_head":"POST / HTTP/1.1"`) {
|
||||||
|
t.Fatalf("head at rest %q (%v), viewer %s", rawHead, err, w.Body)
|
||||||
|
}
|
||||||
if w = req("GET", fmt.Sprintf("%s/%d/body?side=response&offset=20001", base, first), ""); w.Code != 422 {
|
if w = req("GET", fmt.Sprintf("%s/%d/body?side=response&offset=20001", base, first), ""); w.Code != 422 {
|
||||||
t.Fatal(w.Code, w.Body)
|
t.Fatal(w.Code, w.Body)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user