mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-14 12:10:54 +08:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
01a9dae49e | ||
|
|
4e126a8524 | ||
|
|
abb6cb177d | ||
|
|
5122baa77f | ||
|
|
fcee6af45e | ||
|
|
b74ff21e11 |
@@ -2,6 +2,9 @@ name: release
|
||||
on:
|
||||
push:
|
||||
tags: ["v*"]
|
||||
# Dry run: builds, SBOM, attestations and workflow artifacts only. The GitHub
|
||||
# Release and the image push are gated on github.event_name == 'push'.
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write # required to create the Release
|
||||
@@ -81,6 +84,10 @@ jobs:
|
||||
release:
|
||||
needs: binaries
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write # create the Release (softprops/action-gh-release)
|
||||
id-token: write # Sigstore signing of the provenance attestation
|
||||
attestations: write # store the attestation on the repository
|
||||
steps:
|
||||
- uses: actions/checkout@v4 # CHANGELOG.md and the extractor script
|
||||
- uses: actions/download-artifact@v4
|
||||
@@ -91,6 +98,39 @@ jobs:
|
||||
- name: Generate checksums
|
||||
working-directory: dist
|
||||
run: sha256sum *.zip > SHA256SUMS
|
||||
# SBOM of what ships: the Go modules embedded in the linux/amd64 binary, plus
|
||||
# the frontend lockfile and the Go module files.
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: bin-linux-amd64
|
||||
path: ${{ runner.temp }}/sbom-src/bin
|
||||
- uses: anchore/sbom-action/download-syft@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c # v0.24.3
|
||||
id: syft
|
||||
with:
|
||||
syft-version: v1.54.0
|
||||
- name: Generate CycloneDX SBOM
|
||||
env:
|
||||
SYFT: ${{ steps.syft.outputs.cmd }}
|
||||
SRC: ${{ runner.temp }}/sbom-src
|
||||
TAG: ${{ github.ref_name }}
|
||||
run: |
|
||||
mkdir -p "$SRC/web"
|
||||
cp go.mod go.sum "$SRC/"
|
||||
cp web/package-lock.json "$SRC/web/"
|
||||
sbom="dist/artex-${TAG}.cdx.json"
|
||||
"$SYFT" scan "dir:$SRC" --source-name artex --source-version "$TAG" -o "cyclonedx-json=$sbom"
|
||||
grep -q '"pkg:golang/' "$sbom" || { echo "::error::SBOM has no pkg:golang entries"; exit 1; }
|
||||
grep -q '"pkg:npm/' "$sbom" || { echo "::error::SBOM has no pkg:npm entries"; exit 1; }
|
||||
echo "SBOM_PATH=$sbom" >> "$GITHUB_ENV"
|
||||
- uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
|
||||
with:
|
||||
subject-path: dist/*.zip
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: release-metadata
|
||||
path: |
|
||||
dist/SHA256SUMS
|
||||
${{ env.SBOM_PATH }}
|
||||
- name: Release body from CHANGELOG.md
|
||||
env:
|
||||
TAG: ${{ github.ref_name }}
|
||||
@@ -103,10 +143,12 @@ jobs:
|
||||
echo "Release $TAG. See CHANGELOG.md for the list of changes in this release." > "$RUNNER_TEMP/release-body.md"
|
||||
fi
|
||||
- uses: softprops/action-gh-release@v2
|
||||
if: github.event_name == 'push'
|
||||
with:
|
||||
files: |
|
||||
dist/*.zip
|
||||
dist/SHA256SUMS
|
||||
${{ env.SBOM_PATH }}
|
||||
generate_release_notes: false
|
||||
body_path: ${{ runner.temp }}/release-body.md
|
||||
|
||||
@@ -118,7 +160,7 @@ jobs:
|
||||
# and provide DOCKERHUB_USERNAME / DOCKERHUB_TOKEN secrets plus DOCKERHUB_IMAGE).
|
||||
docker:
|
||||
needs: binaries
|
||||
if: ${{ vars.DOCKERHUB_ENABLED == 'true' }}
|
||||
if: ${{ vars.DOCKERHUB_ENABLED == 'true' && github.event_name == 'push' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4 # fetch the Dockerfile and skills/
|
||||
|
||||
@@ -134,6 +134,15 @@ cp config.example.json config.json # fill in the database connection
|
||||
> Please start with `start.sh` / `start.bat` instead of running `./artex` directly. It is a daemon script: if the process crashes, it restarts it after a backoff.
|
||||
> To keep it running in the background: `nohup ./start.sh >artex.log 2>&1 &`.
|
||||
|
||||
#### Verifying a release
|
||||
|
||||
Each release attaches `SHA256SUMS` and a CycloneDX SBOM, `artex-<tag>.cdx.json` (Go modules embedded in the linux/amd64 binary, the frontend `package-lock.json`, and `go.mod`/`go.sum`), next to the zips. The zips also carry a GitHub build provenance attestation. Releases published before this was added have neither the SBOM nor the attestation.
|
||||
|
||||
```bash
|
||||
sha256sum -c SHA256SUMS # add --ignore-missing if you downloaded only some zips
|
||||
gh attestation verify <zip> --repo BaaRai/ARTEX # provenance: built by this repository's release workflow
|
||||
```
|
||||
|
||||
### Option 4: build the single binary from source
|
||||
|
||||
```bash
|
||||
|
||||
@@ -2,6 +2,7 @@ package db
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -331,19 +332,36 @@ func TestCompanyScopeAttribution(t *testing.T) {
|
||||
d.Close()
|
||||
t.Fatalf("startMax: %v", err)
|
||||
}
|
||||
// Remove everything the test creates, or the next pass on the same database
|
||||
// finds this company's scope and attributes its assets to it. Assets first
|
||||
// (they reference the company), then the company; its scope rows cascade.
|
||||
var cid int64
|
||||
t.Cleanup(func() {
|
||||
_, _ = d.Exec(`DELETE FROM assets WHERE id > $1`, startMax)
|
||||
d.Close()
|
||||
defer d.Close()
|
||||
if _, err := d.Exec(`DELETE FROM assets WHERE id > $1`, startMax); err != nil {
|
||||
t.Errorf("cleanup assets: %v", err)
|
||||
}
|
||||
if cid != 0 {
|
||||
if _, err := d.Exec(`DELETE FROM companies WHERE id = $1`, cid); err != nil {
|
||||
t.Errorf("cleanup company %d: %v", cid, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
uniq := startMax + 1
|
||||
root := fmt.Sprintf("scopetest%d.com", uniq)
|
||||
// Unique per run: MAX(assets.id) comes back to the same value after cleanup,
|
||||
// so it cannot tell one pass from the next. The in-scope /24 is picked per
|
||||
// run from 198.18.0.0/15 so a scope left by an interrupted run cannot claim
|
||||
// this run's IP.
|
||||
uniq := fmt.Sprintf("%d%d", time.Now().UnixNano(), rand.Uint32())
|
||||
n := rand.IntN(512)
|
||||
cidr := fmt.Sprintf("198.%d.%d.0/24", 18+n/256, n%256)
|
||||
root := fmt.Sprintf("scopetest%s.com", uniq)
|
||||
sub := "api." + root
|
||||
ipIn := "198.51.100.9"
|
||||
ipIn := fmt.Sprintf("198.%d.%d.9", 18+n/256, n%256)
|
||||
ipOut := "203.0.113.9"
|
||||
outDomain := fmt.Sprintf("other%d.net", uniq)
|
||||
outDomain := fmt.Sprintf("other%s.net", uniq)
|
||||
|
||||
cid, _, err := cs.UpsertCompany(fmt.Sprintf("ScopeCo %d", uniq), "")
|
||||
cid, _, err = cs.UpsertCompany(fmt.Sprintf("ScopeCo %s", uniq), "")
|
||||
if err != nil {
|
||||
t.Fatalf("UpsertCompany: %v", err)
|
||||
}
|
||||
@@ -359,7 +377,7 @@ func TestCompanyScopeAttribution(t *testing.T) {
|
||||
t.Fatalf("pre-scope asset should be unattributed, got %v", *preCompanyID)
|
||||
}
|
||||
|
||||
cs.AddScope(cid, []string{root, "198.51.100.0/24"}, "unit test")
|
||||
cs.AddScope(cid, []string{root, cidr}, "unit test")
|
||||
|
||||
mustCid := func(id int64, want int64, label string) {
|
||||
var cID *int64
|
||||
|
||||
+22
-6
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestPoolProfilesOrder pins the failover chain query: keyless profiles can't
|
||||
@@ -17,17 +19,22 @@ func TestPoolProfilesOrder(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
defer d.Close()
|
||||
// Close via Cleanup, registered FIRST so it runs LAST: a plain defer would
|
||||
// close the pool before the row-deleting cleanups below run.
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
// llm_profiles.name is unique, so a fixed name collides with a row left on
|
||||
// the same database by an earlier run.
|
||||
sfx := fmt.Sprintf("-%d", time.Now().UnixNano())
|
||||
mk := func(name string, priority int, exclude bool, key string) int64 {
|
||||
id, err := d.SaveProfile(&LLMProfile{
|
||||
Name: name, Format: "openai", Model: "m", APIKey: key,
|
||||
Name: name + sfx, Format: "openai", Model: "m", APIKey: key,
|
||||
Priority: priority, PoolExclude: exclude,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { d.Exec(`DELETE FROM llm_profiles WHERE id=$1`, id) })
|
||||
t.Cleanup(func() { deleteTestProfile(t, d, id) })
|
||||
return id
|
||||
}
|
||||
lo := mk("t-pool-lo", 1, false, "k1")
|
||||
@@ -42,11 +49,11 @@ func TestPoolProfilesOrder(t *testing.T) {
|
||||
var got []int64
|
||||
for _, p := range chain {
|
||||
switch p.Name {
|
||||
case "t-pool-lo", "t-pool-hi":
|
||||
case "t-pool-lo" + sfx, "t-pool-hi" + sfx:
|
||||
got = append(got, p.ID)
|
||||
case "t-pool-excluded":
|
||||
case "t-pool-excluded" + sfx:
|
||||
t.Fatal("pool_exclude profile entered the failover chain")
|
||||
case "t-pool-nokey":
|
||||
case "t-pool-nokey" + sfx:
|
||||
t.Fatal("keyless profile entered the failover chain")
|
||||
}
|
||||
}
|
||||
@@ -55,6 +62,15 @@ func TestPoolProfilesOrder(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// deleteTestProfile removes a profile a test created and reports a failed delete
|
||||
// instead of silently leaving the row for the next run.
|
||||
func deleteTestProfile(t *testing.T, d *DB, id int64) {
|
||||
t.Helper()
|
||||
if _, err := d.Exec(`DELETE FROM llm_profiles WHERE id=$1`, id); err != nil {
|
||||
t.Errorf("cleanup profile %d: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteProfileContextHonorsCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const (
|
||||
evCookie = "session=ev-cookie-secret"
|
||||
evBearer = "Bearer ev-auth-secret"
|
||||
)
|
||||
|
||||
// evidenceFixture records a finding. With legacy=false the snapshot goes through
|
||||
// the normal insert path; with legacy=true it is written in clear by raw SQL.
|
||||
func evidenceFixture(t *testing.T, d *DB, tag string, legacy bool) (*Task, int64, TrafficEvidenceSnapshot) {
|
||||
t.Helper()
|
||||
task, err := d.CreateTask("evidence heads at rest "+tag, "goal", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := TrafficEvidenceSnapshot{SourceTrafficID: "t-sec-ev-" + tag, CapturedAt: 1, URL: "http://ev.example/" + tag, Method: "GET", Status: 200,
|
||||
ReqHead: "GET /" + tag + " HTTP/1.1\nCookie: " + evCookie + "\nAuthorization: " + evBearer + "\n",
|
||||
RespHead: "HTTP/1.1 200 OK\nSet-Cookie: " + evCookie + "\n", ReqHash: "rh-" + tag, RespHash: "ph-" + tag}
|
||||
s.ID = TrafficSnapshotID(s)
|
||||
t.Cleanup(func() {
|
||||
d.Exec(`DELETE FROM task_archives WHERE task_id=$1`, task.ID)
|
||||
d.DeleteTaskCascade(task.ID, false, true) // findings and their bindings too
|
||||
if _, err := d.Exec(`DELETE FROM traffic_evidence_snapshots WHERE id=$1`, s.ID); err != nil {
|
||||
t.Errorf("cleanup snapshot: %v", err)
|
||||
}
|
||||
})
|
||||
f, err := d.Exploration(task.ExplorationID).RecordFinding(t.Context(), RecordFindingInput{TaskID: task.ID, ExplorationID: task.ExplorationID, Summary: "evidence " + tag, Severity: "low"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if legacy {
|
||||
_, err = d.Exec(`INSERT INTO traffic_evidence_snapshots(id,source_traffic_id,captured_at,url,method,status,req_head,resp_head,req_hash,resp_hash,req_len,resp_len)
|
||||
VALUES($1,$2,1,$3,'GET',200,$4,$5,$6,$7,0,0)`, s.ID, s.SourceTrafficID, s.URL, s.ReqHead, s.RespHead, s.ReqHash, s.RespHash)
|
||||
if err == nil {
|
||||
_, err = d.Exec(`INSERT INTO finding_traffic_bindings(finding_id,snapshot_id,position) VALUES($1,$2,0)`, f.FindingID, s.ID)
|
||||
}
|
||||
} else {
|
||||
err = d.WithEvidenceTx(t.Context(), func(tx *sql.Tx) error {
|
||||
return AddFindingTrafficTx(tx, f.FindingID, []PreparedTrafficEvidence{{Ref: TrafficRef{TrafficID: s.SourceTrafficID, Role: "proof"}, Snapshot: s}})
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return task, f.FindingID, s
|
||||
}
|
||||
|
||||
// rawHeads reads the stored heads; both must be sealed and hide the secrets.
|
||||
func rawHeads(t *testing.T, d *DB, id string) (req, resp string) {
|
||||
t.Helper()
|
||||
if err := d.QueryRow(`SELECT req_head,resp_head FROM traffic_evidence_snapshots WHERE id=$1`, id).Scan(&req, &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, v := range []string{req, resp} {
|
||||
if !isEncrypted(v) || strings.Contains(v, evCookie) || strings.Contains(v, evBearer) {
|
||||
t.Fatalf("head stored in clear: %q", v)
|
||||
}
|
||||
}
|
||||
return req, resp
|
||||
}
|
||||
|
||||
// assertClearHeads checks the application read and that the stored value is
|
||||
// sealed exactly once.
|
||||
func assertClearHeads(t *testing.T, d *DB, findingID int64, want TrafficEvidenceSnapshot) {
|
||||
t.Helper()
|
||||
got, err := d.GetFindingTraffic(t.Context(), findingID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Bindings) != 1 || got.Bindings[0].Snapshot.ReqHead != want.ReqHead || got.Bindings[0].Snapshot.RespHead != want.RespHead || got.Bindings[0].SnapshotID != want.ID {
|
||||
t.Fatalf("read back: %+v", got.Bindings)
|
||||
}
|
||||
req, resp := rawHeads(t, d, want.ID)
|
||||
box := activeSecretBox.Load()
|
||||
if p, err := box.decrypt(colEvReqHead.aad(), req); err != nil || p != want.ReqHead {
|
||||
t.Fatalf("req_head not sealed once: %q %v", p, err)
|
||||
}
|
||||
if p, err := box.decrypt(colEvRespHead.aad(), resp); err != nil || p != want.RespHead {
|
||||
t.Fatalf("resp_head not sealed once: %q %v", p, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidenceHeadsSealedAtInsert(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
_, fid, s := evidenceFixture(t, d, "insert", false)
|
||||
assertClearHeads(t, d, fid, s)
|
||||
var url, method string
|
||||
if err := d.QueryRow(`SELECT url,method FROM traffic_evidence_snapshots WHERE id=$1`, s.ID).Scan(&url, &method); err != nil || url != s.URL || method != "GET" {
|
||||
t.Fatalf("metadata must stay in clear: %q %q %v", url, method, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvidenceHeadsBackfillAndKeyChecks(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
_, fid, s := evidenceFixture(t, d, "legacy", true)
|
||||
t.Setenv(secretKeyEnv, base64.StdEncoding.EncodeToString(testSecretKey))
|
||||
if err := d.InitSecrets(t.TempDir()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertClearHeads(t, d, fid, s)
|
||||
req, resp := rawHeads(t, d, s.ID)
|
||||
if err := d.InitSecrets(t.TempDir()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r2, p2 := rawHeads(t, d, s.ID); r2 != req || p2 != resp {
|
||||
t.Fatal("second backfill must leave the ciphertexts untouched")
|
||||
}
|
||||
|
||||
// A head sealed under another key: the read and the startup check fail and
|
||||
// name the column. The startup check samples one sealed row per column, so
|
||||
// this row must be the only sealed snapshot.
|
||||
var sealedRows int
|
||||
if err := d.QueryRow(`SELECT count(*) FROM traffic_evidence_snapshots WHERE req_head LIKE 'enc:v1:%' OR resp_head LIKE 'enc:v1:%'`).Scan(&sealedRows); err != nil || sealedRows != 1 {
|
||||
t.Fatalf("want only this fixture's snapshot sealed, got %d (%v)", sealedRows, err)
|
||||
}
|
||||
other, _ := newSecretBox([]byte("ffffffffffffffffffffffffffffffff"))
|
||||
for _, c := range []secretColumn{colEvReqHead, colEvRespHead} {
|
||||
bad, _ := other.encrypt(c.aad(), "GET / HTTP/1.1\nCookie: other\n")
|
||||
if _, err := d.Exec(`UPDATE traffic_evidence_snapshots SET `+c.col+`=$1 WHERE id=$2`, bad, s.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.GetFindingTraffic(t.Context(), fid); err == nil || !strings.Contains(err.Error(), c.aad()) {
|
||||
t.Fatalf("want a read failure naming %s, got %v", c.aad(), err)
|
||||
}
|
||||
if err := d.InitSecrets(t.TempDir()); err == nil || !strings.Contains(err.Error(), "secret key check") || !strings.Contains(err.Error(), c.aad()) {
|
||||
t.Fatalf("want a key-check failure naming %s, got %v", c.aad(), err)
|
||||
}
|
||||
if _, err := d.Exec(`UPDATE traffic_evidence_snapshots SET req_head=$1,resp_head=$2 WHERE id=$3`, req, resp, s.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
assertClearHeads(t, d, fid, s)
|
||||
|
||||
// No key with sealed heads: refuse, and each head counts. One head stays
|
||||
// sealed so the refusal never turns into a new key.
|
||||
t.Setenv(secretKeyEnv, "")
|
||||
dir := t.TempDir()
|
||||
withHeads := encryptedCount(t, d, dir)
|
||||
for _, c := range []secretColumn{colEvReqHead, colEvRespHead} {
|
||||
if _, err := d.Exec(`UPDATE traffic_evidence_snapshots SET req_head=$1,resp_head=$2 WHERE id=$3`, req, resp, s.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d.Exec(`UPDATE traffic_evidence_snapshots SET `+c.col+`='' WHERE id=$1`, s.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := encryptedCount(t, d, dir); withHeads-n != 1 {
|
||||
t.Fatalf("no-key refusal counted %d then %d encrypted values; %s must count", withHeads, n, c.aad())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Export keeps the heads sealed; import opens them to check the snapshot ID,
|
||||
// then the insert seals them once. A legacy clear archive is sealed on import.
|
||||
func TestEvidenceHeadsArchiveRoundTrip(t *testing.T) {
|
||||
d, err := Open(testDSN(t))
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
t.Cleanup(func() { d.Close() })
|
||||
if err = d.EnsureLLMRecordsTable(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = d.EnsureLLMUsageTable(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, legacy := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint("legacy=", legacy), func(t *testing.T) {
|
||||
task, fid, s := evidenceFixture(t, d, fmt.Sprint("archive-", legacy), false)
|
||||
if err := d.SetPaused(task.ID, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
archive, err := d.QueueTaskArchive(task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = d.ClaimTaskArchiveJob(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snapshot, err := d.SnapshotTaskArchive(task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exported := string(snapshot.Tables["traffic_evidence_snapshots"])
|
||||
if !strings.Contains(exported, secretPrefix) || strings.Contains(exported, evCookie) || strings.Contains(exported, evBearer) {
|
||||
t.Fatalf("archive must keep the heads sealed: %s", exported)
|
||||
}
|
||||
if err = d.CompleteTaskArchive(archive.ID, snapshot, "/tmp/evidence-heads-test.tar.zst", "fixture", 1, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var rows []map[string]any
|
||||
if err = json.Unmarshal(snapshot.Tables["traffic_evidence_snapshots"], &rows); err != nil || len(rows) != 1 {
|
||||
t.Fatal(rows, err)
|
||||
}
|
||||
if legacy { // an archive written before the heads were sealed
|
||||
rows[0]["req_head"], rows[0]["resp_head"] = s.ReqHead, s.RespHead
|
||||
snapshot.Tables["traffic_evidence_snapshots"], _ = json.Marshal(rows)
|
||||
} else { // a head sealed under another key fails the import, naming the column
|
||||
other, _ := newSecretBox([]byte("ffffffffffffffffffffffffffffffff"))
|
||||
bad, _ := other.encrypt(colEvReqHead.aad(), s.ReqHead)
|
||||
good := rows[0]["req_head"]
|
||||
rows[0]["req_head"] = bad
|
||||
raw, _ := json.Marshal(rows)
|
||||
if _, err := ArchiveEvidenceSnapshots(&TaskArchiveSnapshot{FormatVersion: snapshot.FormatVersion, Tables: map[string]json.RawMessage{"traffic_evidence_snapshots": raw}}); err == nil || !strings.Contains(err.Error(), colEvReqHead.aad()) {
|
||||
t.Fatalf("want an import failure naming %s, got %v", colEvReqHead.aad(), err)
|
||||
}
|
||||
rows[0]["req_head"] = good
|
||||
}
|
||||
list, err := ArchiveEvidenceSnapshots(snapshot)
|
||||
if err != nil || len(list) != 1 || list[0].ReqHead != s.ReqHead || list[0].RespHead != s.RespHead {
|
||||
t.Fatalf("archive snapshots: %+v %v", list, err)
|
||||
}
|
||||
// The hot row is gone (GC), so the restore inserts it again.
|
||||
if _, err = d.Exec(`DELETE FROM traffic_evidence_snapshots WHERE id=$1`, s.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = d.QueueTaskArchiveRestore(archive.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = d.ClaimTaskArchiveJob(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = d.RestoreTaskArchive(archive.ID, snapshot, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertClearHeads(t, d, fid, s)
|
||||
})
|
||||
}
|
||||
}
|
||||
+24
-2
@@ -195,10 +195,29 @@ func InsertEvidenceSnapshotTx(tx *sql.Tx, s TrafficEvidenceSnapshot) error {
|
||||
if s.CreatedAt.IsZero() {
|
||||
s.CreatedAt = time.Now().UTC()
|
||||
}
|
||||
_, err := tx.Exec(`INSERT INTO traffic_evidence_snapshots
|
||||
// The ID hashes the clear heads; only the stored heads are sealed.
|
||||
reqHead, err := sealSecret(colEvReqHead, s.ReqHead)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
respHead, err := sealSecret(colEvRespHead, s.RespHead)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(`INSERT INTO traffic_evidence_snapshots
|
||||
(id,source_traffic_id,captured_at,url,method,status,content_type,req_head,resp_head,req_hash,resp_hash,req_len,resp_len,created_at)
|
||||
VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT(id) DO NOTHING`,
|
||||
s.ID, s.SourceTrafficID, s.CapturedAt, s.URL, s.Method, s.Status, s.ContentType, s.ReqHead, s.RespHead, s.ReqHash, s.RespHash, s.ReqLen, s.RespLen, s.CreatedAt)
|
||||
s.ID, s.SourceTrafficID, s.CapturedAt, s.URL, s.Method, s.Status, s.ContentType, reqHead, respHead, s.ReqHash, s.RespHash, s.ReqLen, s.RespLen, s.CreatedAt)
|
||||
return err
|
||||
}
|
||||
|
||||
// openHeads decrypts the heads of a snapshot read from the table or an archive.
|
||||
// Clear legacy values pass through; a value that fails to decrypt is an error.
|
||||
func (s *TrafficEvidenceSnapshot) openHeads() (err error) {
|
||||
if s.ReqHead, err = openSecret(colEvReqHead, s.ReqHead); err != nil {
|
||||
return err
|
||||
}
|
||||
s.RespHead, err = openSecret(colEvRespHead, s.RespHead)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -263,6 +282,9 @@ FROM finding_traffic_bindings b JOIN traffic_evidence_snapshots s ON s.id=b.snap
|
||||
if err := json.Unmarshal(raw, &b.Snapshot); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := b.Snapshot.openHeads(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out.Bindings = append(out.Bindings, b)
|
||||
}
|
||||
return out, rows.Err()
|
||||
|
||||
@@ -15,8 +15,17 @@ func ArchiveEvidenceSnapshots(snapshot *TaskArchiveSnapshot) ([]TrafficEvidenceS
|
||||
if snapshot.FormatVersion < 3 {
|
||||
return nil, ErrTaskArchiveFormatMismatch
|
||||
}
|
||||
err := json.Unmarshal(snapshot.Tables["traffic_evidence_snapshots"], &out)
|
||||
return out, err
|
||||
// The archive keeps the heads as stored (sealed). They are opened here so the
|
||||
// restore can check each content-addressed ID; the insert then seals them once.
|
||||
if err := json.Unmarshal(snapshot.Tables["traffic_evidence_snapshots"], &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range out {
|
||||
if err := out[i].openHeads(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func restoreFindingTrafficTx(tx *sql.Tx, snapshot *TaskArchiveSnapshot) error {
|
||||
|
||||
+17
-9
@@ -1,6 +1,7 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -14,7 +15,8 @@ func TestProfileRetryRoundTrip(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
defer d.Close()
|
||||
// Registered FIRST so it runs LAST, after the row cleanup below.
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
want := RetryOverride{
|
||||
Connect: RetryRule{Attempts: 5, IntervalMS: 2000},
|
||||
@@ -22,12 +24,12 @@ func TestProfileRetryRoundTrip(t *testing.T) {
|
||||
Stream: RetryRule{IntervalMS: 250},
|
||||
}
|
||||
id, err := d.SaveProfile(&LLMProfile{
|
||||
Name: "t-retry-roundtrip", Format: "openai", Model: "m", APIKey: "k", Retry: want,
|
||||
Name: fmt.Sprintf("t-retry-roundtrip-%d", time.Now().UnixNano()), Format: "openai", Model: "m", APIKey: "k", Retry: want,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { d.Exec(`DELETE FROM llm_profiles WHERE id=$1`, id) })
|
||||
t.Cleanup(func() { deleteTestProfile(t, d, id) })
|
||||
|
||||
got, err := d.ProfileByID(id)
|
||||
if err != nil || got == nil {
|
||||
@@ -71,16 +73,17 @@ func TestProfileRetryClampedOnSave(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
defer d.Close()
|
||||
// Registered FIRST so it runs LAST, after the row cleanup below.
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
id, err := d.SaveProfile(&LLMProfile{
|
||||
Name: "t-retry-clamp", Format: "openai", Model: "m", APIKey: "k",
|
||||
Name: fmt.Sprintf("t-retry-clamp-%d", time.Now().UnixNano()), Format: "openai", Model: "m", APIKey: "k",
|
||||
Retry: RetryOverride{Connect: RetryRule{Attempts: -99, IntervalMS: -5}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { d.Exec(`DELETE FROM llm_profiles WHERE id=$1`, id) })
|
||||
t.Cleanup(func() { deleteTestProfile(t, d, id) })
|
||||
got, err := d.ProfileByID(id)
|
||||
if err != nil || got == nil {
|
||||
t.Fatalf("ProfileByID: %v", err)
|
||||
@@ -97,17 +100,22 @@ func TestLLMRetryPolicyRoundTrip(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Skipf("postgres unavailable (%v) — skipping", err)
|
||||
}
|
||||
defer d.Close()
|
||||
// Registered FIRST so it runs LAST, after the restore below.
|
||||
t.Cleanup(func() { d.Close() })
|
||||
|
||||
before, hadBefore, err := d.GetSetting(settingLLMRetryPolicy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
var err error
|
||||
if hadBefore {
|
||||
d.SetSetting(settingLLMRetryPolicy, before)
|
||||
err = d.SetSetting(settingLLMRetryPolicy, before)
|
||||
} else {
|
||||
d.Exec(`DELETE FROM settings WHERE key=$1`, settingLLMRetryPolicy)
|
||||
_, err = d.Exec(`DELETE FROM settings WHERE key=$1`, settingLLMRetryPolicy)
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("restore %s: %v", settingLLMRetryPolicy, err)
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
+4
-1
@@ -81,7 +81,10 @@ var (
|
||||
colCredPass = secretColumn{"task_credentials", "password", false, ""}
|
||||
colNotifConfig = secretColumn{"notification_channels", "config", true, ""}
|
||||
colToolExec = secretColumn{"tools", "exec", true, ""} // system=false rows: headers, proxy, body, code, command
|
||||
secretColumns = append([]secretColumn{colLLMKey, colLLMProxy, colMCPEnv, colCredPass, colNotifConfig, colToolExec}, secretSettingColumns()...)
|
||||
// Evidence head blocks carry the captured Cookie and Authorization headers.
|
||||
colEvReqHead = secretColumn{"traffic_evidence_snapshots", "req_head", false, ""}
|
||||
colEvRespHead = secretColumn{"traffic_evidence_snapshots", "resp_head", false, ""}
|
||||
secretColumns = append([]secretColumn{colLLMKey, colLLMProxy, colMCPEnv, colCredPass, colNotifConfig, colToolExec, colEvReqHead, colEvRespHead}, secretSettingColumns()...)
|
||||
)
|
||||
|
||||
// secretSettingKeys are the settings stored encrypted, AAD "settings.<key>".
|
||||
|
||||
@@ -34,5 +34,10 @@ func runEvidenceSuite(m *testing.M) int {
|
||||
panic(err)
|
||||
}
|
||||
defer conn.ExecContext(context.Background(), `SELECT pg_advisory_unlock(7337741002)`)
|
||||
// Snapshot inserts seal the heads: load a fixed test key (env, no key file).
|
||||
os.Setenv("ARTEX_SECRET_KEY", "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=")
|
||||
if err = pg.InitSecrets(""); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return m.Run()
|
||||
}
|
||||
|
||||
@@ -172,6 +172,11 @@ func TestFindingTrafficAPIAndExport(t *testing.T) {
|
||||
if w = req("GET", fmt.Sprintf("%s/%d", base, first), ""); w.Code != 200 || !strings.Contains(w.Body.String(), `"binary":true`) {
|
||||
t.Fatal(w.Code, w.Body)
|
||||
}
|
||||
// The viewer reads the heads in clear; the table stores them sealed.
|
||||
var rawHead string
|
||||
if err := s.m.pg.QueryRow(`SELECT req_head FROM traffic_evidence_snapshots WHERE id=$1`, list.Bindings[0].SnapshotID).Scan(&rawHead); err != nil || !strings.HasPrefix(rawHead, "enc:v1:") || !strings.Contains(w.Body.String(), `"req_head":"POST / HTTP/1.1"`) {
|
||||
t.Fatalf("head at rest %q (%v), viewer %s", rawHead, err, w.Body)
|
||||
}
|
||||
if w = req("GET", fmt.Sprintf("%s/%d/body?side=response&offset=20001", base, first), ""); w.Code != 422 {
|
||||
t.Fatal(w.Code, w.Body)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user