Compare commits

...
Author SHA1 Message Date
BaaRai f3b9f6f688 db: refuse a pending migration older than the newest applied one
applyMigrations now fails closed, before any write, when a pending
migration has a version below the highest version recorded in
schema_migrations: migrations must be appended, never inserted.
2026-11-14 13:10:57 +01:00
BaaRai 1da0d07222 db: versioned migrations runner with schema_migrations
Move the llm_records and llm_usage DDL out of Go constants into embedded
files db/migrations/0001_llm_records.sql and 0002_llm_usage.sql (byte-for-byte
the old constants). Open now applies pending migrations after schema.sql and
before seedBuiltins, under the existing schema advisory lock and on the
lock-holding connection. Each migration runs in its own transaction with its
schema_migrations row (version, name, SHA-256 checksum, applied_at).

Open fails closed when the database has a version missing from the binary, or
an applied migration whose file name or checksum changed. A failing migration
is rolled back and blocks startup; earlier ones stay applied.

EnsureLLMRecordsTable and EnsureLLMUsageTable re-run the embedded files and
keep their transaction-level schema lock; Open never calls them.
2026-11-14 12:52:34 +01:00
BaaRai 8f3f0c54f8 Merge pull request #224 from BaaRai/fix/archive-restore-normalize-url
db: normalise archived asset URLs on restore
2026-11-14 12:27:39 +01:00
BaaRai 2e43b423ee db: normalise archived asset URLs on restore
restoreArchiveAssets inserted archived service and endpoint rows with
their raw url, and findArchiveAssetNaturalID matched on that raw value.
An archive taken before normalizeURL dropped userinfo therefore restored
http://user:pass@host in clear and duplicated the existing clean row.

Pass row["url"] through normalizeURL for service and endpoint rows
before the natural-identity lookup, so the lookup reuses the clean row
and an insert stores the normalised URL. Other columns keep their
archived values.
2026-11-14 12:17:33 +01:00
BaaRai 8bb4ca0745 Merge pull request #223 from BaaRai/fix/store-normalize-userinfo
db: drop URL userinfo in normalizeURL before storing assets.url
2026-11-14 12:06:44 +01:00
10 changed files with 1014 additions and 88 deletions
+6 -50
View File
@@ -153,57 +153,13 @@ type LLMRecord struct {
RawResponse string `json:"raw_response,omitempty"`
}
const llmRecordsSchema = `
CREATE TABLE IF NOT EXISTS llm_records (
id BIGSERIAL PRIMARY KEY,
ts TIMESTAMPTZ DEFAULT now(),
model TEXT,
profile_name TEXT,
session_id TEXT,
task_id TEXT,
worker TEXT,
latency_ms INTEGER,
input_tokens INTEGER,
output_tokens INTEGER,
cache_read INTEGER,
cache_write INTEGER,
status TEXT,
error TEXT,
request_body TEXT,
response_body TEXT,
raw_request TEXT,
raw_response TEXT
);
CREATE INDEX IF NOT EXISTS idx_llm_records_ts ON llm_records(ts);
CREATE INDEX IF NOT EXISTS idx_llm_records_session ON llm_records(session_id);
`
// llmRecordsMigrate adds new columns to existing tables.
const llmRecordsMigrate = `
ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS task_id TEXT;
ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS worker TEXT;
ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS profile_name TEXT;
ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS raw_request TEXT;
ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS raw_response TEXT;
`
// EnsureLLMRecordsTable creates the llm_records table if it does not exist.
// EnsureLLMRecordsTable creates the llm_records table if it does not exist, by
// re-running the idempotent migration 0001_llm_records.sql. Open already applies
// it through the migration runner; this stays for callers that want the table
// guaranteed. It takes the schema lock in its own transaction, so it must not be
// called while Open holds that lock.
func (d *DB) EnsureLLMRecordsTable() error {
tx, err := d.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
if err := coordinateWithSchemaMigration(tx); err != nil {
return err
}
if _, err := tx.Exec(llmRecordsSchema); err != nil {
return err
}
if _, err := tx.Exec(llmRecordsMigrate); err != nil {
return err
}
return tx.Commit()
return d.execEmbeddedMigration(llmRecordsMigration)
}
// InsertLLMRecord stores one LLM call record.
+23 -1
View File
@@ -1,6 +1,15 @@
// Package db is the PostgreSQL data source for ARTEX (replacing the old graph
// single-file SQLite). It opens the connection, applies the schema, and seeds
// the built-in agents and variable catalog.
//
// Schema changes:
// - schema.sql remains the idempotent base, executed on every startup.
// - Any new DDL goes in a numbered migration, migrations/NNNN_slug.sql.
// Open applies pending migrations once, in version order, and records
// each in schema_migrations with the SHA-256 of its file.
// - A published migration is never edited: Open refuses to start when an
// applied migration's checksum no longer matches its file. Fix forward
// with a new migration instead.
package db
import (
@@ -9,6 +18,7 @@ import (
_ "embed"
"errors"
"fmt"
"io/fs"
"net/url"
"os"
"strings"
@@ -153,8 +163,15 @@ func ensureDatabase(dsn string) error {
return nil
}
// Open connects, applies the schema (idempotent), and seeds builtin rows.
// Open connects, applies the schema (idempotent) and the embedded versioned
// migrations, and seeds builtin rows.
func Open(dsn string) (*DB, error) {
return openWithMigrations(dsn, migrationsFS())
}
// openWithMigrations is Open with the migration set as a parameter, so tests
// can pass an fstest.MapFS.
func openWithMigrations(dsn string, migrations fs.FS) (*DB, error) {
if err := ensureDatabase(dsn); err != nil {
return nil, err
}
@@ -187,6 +204,11 @@ func Open(dsn string) (*DB, error) {
if err := applySchemaWithRetry(context.Background(), conn, time.Sleep); err != nil {
return fmt.Errorf("apply schema: %w", err)
}
// Runs on conn, the lock holder; it must not call EnsureLLM*Table,
// whose own lock request would wait on conn forever.
if err := applyMigrations(context.Background(), conn, migrations); err != nil {
return fmt.Errorf("apply migrations: %w", err)
}
if err := d.seedBuiltins(); err != nil {
return fmt.Errorf("seed builtins: %w", err)
}
+9 -3
View File
@@ -1,11 +1,12 @@
package db
import (
"io/fs"
"testing"
)
// The raw-body columns were added after release, so existing installs only get
// them through llmRecordsMigrate. This runs the real migration against the dev
// them through the ALTERs in 0001_llm_records.sql. This runs the real migration against the dev
// PG on a table stripped back to its pre-upgrade shape, inside a transaction
// that is always rolled back — PG does transactional DDL, so nothing persists.
func TestLLMRecordsMigrateAddsRawColumnsToOldTable(t *testing.T) {
@@ -17,6 +18,11 @@ func TestLLMRecordsMigrateAddsRawColumnsToOldTable(t *testing.T) {
if err := d.EnsureLLMRecordsTable(); err != nil {
t.Fatalf("ensure: %v", err)
}
content, err := fs.ReadFile(migrationsFS(), llmRecordsMigration)
if err != nil {
t.Fatal(err)
}
llmRecordsMigrateSQL := string(content)
tx, err := d.Begin()
if err != nil {
@@ -28,7 +34,7 @@ func TestLLMRecordsMigrateAddsRawColumnsToOldTable(t *testing.T) {
if _, err := tx.Exec(`ALTER TABLE llm_records DROP COLUMN IF EXISTS raw_request, DROP COLUMN IF EXISTS raw_response`); err != nil {
t.Fatalf("simulate old table: %v", err)
}
if _, err := tx.Exec(llmRecordsMigrate); err != nil {
if _, err := tx.Exec(llmRecordsMigrateSQL); err != nil {
t.Fatalf("migrate: %v", err)
}
@@ -45,7 +51,7 @@ func TestLLMRecordsMigrateAddsRawColumnsToOldTable(t *testing.T) {
}
// Re-running must stay a no-op (the migration runs on every startup).
if _, err := tx.Exec(llmRecordsMigrate); err != nil {
if _, err := tx.Exec(llmRecordsMigrateSQL); err != nil {
t.Fatalf("migrate is not idempotent: %v", err)
}
+4 -34
View File
@@ -26,41 +26,11 @@ type LLMUsage struct {
Status string `json:"status"` // ok | error
}
const llmUsageSchema = `
CREATE TABLE IF NOT EXISTS llm_usage (
id BIGSERIAL PRIMARY KEY,
ts TIMESTAMPTZ NOT NULL DEFAULT now(),
task_id TEXT,
exploration_id BIGINT,
worker TEXT,
model TEXT,
profile_name TEXT,
latency_ms INTEGER,
input_tokens INTEGER NOT NULL DEFAULT 0,
output_tokens INTEGER NOT NULL DEFAULT 0,
cache_read INTEGER NOT NULL DEFAULT 0,
cache_write INTEGER NOT NULL DEFAULT 0,
status TEXT
);
CREATE INDEX IF NOT EXISTS idx_llm_usage_task ON llm_usage(task_id);
CREATE INDEX IF NOT EXISTS idx_llm_usage_model ON llm_usage(task_id, model);
CREATE INDEX IF NOT EXISTS idx_llm_usage_exp ON llm_usage(exploration_id);
`
// EnsureLLMUsageTable creates the llm_usage metering table if it does not exist.
// EnsureLLMUsageTable creates the llm_usage metering table if it does not exist,
// by re-running the idempotent migration 0002_llm_usage.sql (see
// EnsureLLMRecordsTable for the locking rule).
func (d *DB) EnsureLLMUsageTable() error {
tx, err := d.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
if err := coordinateWithSchemaMigration(tx); err != nil {
return err
}
if _, err := tx.Exec(llmUsageSchema); err != nil {
return err
}
return tx.Commit()
return d.execEmbeddedMigration(llmUsageMigration)
}
// InsertLLMUsage appends one metering row. Best-effort: callers log and continue on
+217
View File
@@ -0,0 +1,217 @@
package db
import (
"context"
"crypto/sha256"
"database/sql"
"embed"
"encoding/hex"
"fmt"
"io/fs"
"path"
"regexp"
"sort"
"strconv"
)
// Versioned migrations, applied by Open after schema.sql (rules in the package
// comment, db.go).
//
//go:embed migrations/*.sql
var embeddedMigrationFiles embed.FS
// Embedded migration files that EnsureLLMRecordsTable / EnsureLLMUsageTable
// re-run on demand.
const (
llmRecordsMigration = "0001_llm_records.sql"
llmUsageMigration = "0002_llm_usage.sql"
)
// migrationsFS is the embedded migrations directory, rooted so file names are
// the bare NNNN_slug.sql names.
func migrationsFS() fs.FS {
sub, err := fs.Sub(embeddedMigrationFiles, "migrations")
if err != nil {
panic(err) // the embed pattern guarantees the directory exists
}
return sub
}
const schemaMigrationsTable = `
CREATE TABLE IF NOT EXISTS schema_migrations (
version INTEGER PRIMARY KEY,
name TEXT,
checksum TEXT,
applied_at TIMESTAMPTZ DEFAULT now()
);
`
var migrationNameRE = regexp.MustCompile(`^([0-9]{4})_[a-z0-9_]+\.sql$`)
type migration struct {
version int
name string // file name, e.g. 0001_llm_records.sql
sql string
checksum string // hex SHA-256 of the file content
}
// loadMigrations reads every *.sql file at the root of fsys, sorted by version.
// A .sql file that does not follow NNNN_slug.sql, or a duplicated version, is
// an error so a typo cannot silently skip a migration.
func loadMigrations(fsys fs.FS) ([]migration, error) {
entries, err := fs.ReadDir(fsys, ".")
if err != nil {
return nil, fmt.Errorf("read migrations: %w", err)
}
var out []migration
seen := map[int]string{}
for _, e := range entries {
if e.IsDir() || path.Ext(e.Name()) != ".sql" {
continue
}
m := migrationNameRE.FindStringSubmatch(e.Name())
if m == nil {
return nil, fmt.Errorf("migration file %s: name must be NNNN_slug.sql", e.Name())
}
version, _ := strconv.Atoi(m[1])
if version == 0 {
return nil, fmt.Errorf("migration file %s: version must be > 0", e.Name())
}
if prev, dup := seen[version]; dup {
return nil, fmt.Errorf("migration version %04d is used by both %s and %s", version, prev, e.Name())
}
seen[version] = e.Name()
content, err := fs.ReadFile(fsys, e.Name())
if err != nil {
return nil, fmt.Errorf("read migration %s: %w", e.Name(), err)
}
sum := sha256.Sum256(content)
out = append(out, migration{version: version, name: e.Name(), sql: string(content), checksum: hex.EncodeToString(sum[:])})
}
sort.Slice(out, func(i, j int) bool { return out[i].version < out[j].version })
return out, nil
}
// applyMigrations brings the database up to the migrations in fsys. conn must
// be the connection that holds the schema advisory lock (see Open); every
// statement runs on it, so nothing here waits on that lock again.
//
// It fails closed before applying anything when the database records a version
// this binary does not ship, or an applied migration whose name or checksum
// differs from the file, or a pending migration is older than the newest
// applied one. Each pending migration then runs in its own
// transaction together with its schema_migrations row; a failure rolls that one
// back and stops, leaving earlier migrations applied.
func applyMigrations(ctx context.Context, conn *sql.Conn, fsys fs.FS) error {
migrations, err := loadMigrations(fsys)
if err != nil {
return err
}
if _, err := conn.ExecContext(ctx, schemaMigrationsTable); err != nil {
return fmt.Errorf("create schema_migrations: %w", err)
}
type appliedRow struct{ name, checksum string }
applied := map[int]appliedRow{}
rows, err := conn.QueryContext(ctx, `SELECT version, COALESCE(name,''), COALESCE(checksum,'') FROM schema_migrations ORDER BY version`)
if err != nil {
return fmt.Errorf("read schema_migrations: %w", err)
}
var appliedVersions []int
for rows.Next() {
var v int
var r appliedRow
if err := rows.Scan(&v, &r.name, &r.checksum); err != nil {
rows.Close()
return fmt.Errorf("read schema_migrations: %w", err)
}
applied[v] = r
appliedVersions = append(appliedVersions, v)
}
if err := rows.Close(); err != nil {
return fmt.Errorf("read schema_migrations: %w", err)
}
if err := rows.Err(); err != nil {
return fmt.Errorf("read schema_migrations: %w", err)
}
byVersion := map[int]migration{}
for _, m := range migrations {
byVersion[m.version] = m
}
for _, v := range appliedVersions {
r := applied[v]
m, ok := byVersion[v]
if !ok {
return fmt.Errorf("migration %04d (%s) is applied in the database but missing from this binary", v, r.name)
}
if r.name != m.name {
return fmt.Errorf("migration %04d: database records file %s but this binary has %s", v, r.name, m.name)
}
if r.checksum != m.checksum {
return fmt.Errorf("migration %04d (%s): checksum changed since it was applied (database %s, file %s); published migrations must not be edited", v, m.name, r.checksum, m.checksum)
}
}
// A pending version below the newest applied one would run out of order on
// this database: refuse it before writing anything.
if len(appliedVersions) > 0 {
maxVersion := appliedVersions[len(appliedVersions)-1]
for _, m := range migrations {
if _, done := applied[m.version]; !done && m.version < maxVersion {
return fmt.Errorf("migration %s (%04d) is older than the applied migration %s (%04d); migrations must be appended, never inserted",
m.name, m.version, applied[maxVersion].name, maxVersion)
}
}
}
for _, m := range migrations {
if _, done := applied[m.version]; done {
continue
}
if err := applyMigration(ctx, conn, m); err != nil {
return fmt.Errorf("migration %04d (%s): %w", m.version, m.name, err)
}
}
return nil
}
func applyMigration(ctx context.Context, conn *sql.Conn, m migration) error {
tx, err := conn.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck // no-op after Commit
// No args: pgx sends it over the simple protocol, which allows several statements.
if _, err := tx.ExecContext(ctx, m.sql); err != nil {
return err
}
if _, err := tx.ExecContext(ctx, `INSERT INTO schema_migrations(version, name, checksum) VALUES ($1, $2, $3)`, m.version, m.name, m.checksum); err != nil {
return fmt.Errorf("record: %w", err)
}
return tx.Commit()
}
// execEmbeddedMigration re-runs one embedded (idempotent) migration file in its
// own transaction under the schema lock. It does not touch schema_migrations.
// The lock is taken on a pooled connection, so this must never be called from
// inside Open's withSchemaMigrationLock callback: it would wait on the lock
// that callback's connection holds.
func (d *DB) execEmbeddedMigration(name string) error {
content, err := fs.ReadFile(migrationsFS(), name)
if err != nil {
return err
}
tx, err := d.Begin()
if err != nil {
return err
}
defer tx.Rollback() //nolint:errcheck
if err := coordinateWithSchemaMigration(tx); err != nil {
return err
}
if _, err := tx.Exec(string(content)); err != nil {
return err
}
return tx.Commit()
}
+415
View File
@@ -0,0 +1,415 @@
package db
import (
"crypto/sha256"
"database/sql"
"encoding/hex"
"io/fs"
"net/url"
"strings"
"testing"
"testing/fstest"
"time"
)
func TestLoadMigrationsValidation(t *testing.T) {
ms, err := loadMigrations(fstest.MapFS{
"0010_b.sql": {Data: []byte("SELECT 10;")},
"0002_a.sql": {Data: []byte("SELECT 2;")},
"README.md": {Data: []byte("not a migration")},
})
if err != nil {
t.Fatal(err)
}
if len(ms) != 2 || ms[0].version != 2 || ms[1].version != 10 {
t.Fatalf("want versions [2 10], got %+v", ms)
}
sum := sha256.Sum256([]byte("SELECT 2;"))
if ms[0].checksum != hex.EncodeToString(sum[:]) || ms[0].name != "0002_a.sql" {
t.Fatalf("migration 0002 = %+v", ms[0])
}
for name, fsys := range map[string]fstest.MapFS{
"bad name": {"2_a.sql": {Data: []byte("SELECT 1;")}},
"duplicate version": {"0001_a.sql": {Data: []byte("SELECT 1;")}, "0001_b.sql": {Data: []byte("SELECT 1;")}},
} {
if _, err := loadMigrations(fsys); err == nil {
t.Errorf("%s: want error", name)
}
}
}
// scratchDSN returns the DSN of a new, empty database named after the test
// database plus suffix, dropped again when the test ends.
func scratchDSN(t *testing.T, suffix string) string {
t.Helper()
u, err := url.Parse(testDSN(t))
if err != nil || strings.Trim(u.Path, "/") == "" {
t.Skipf("DSN has no database name — skipping")
}
name := strings.TrimPrefix(u.Path, "/") + "_mig_" + suffix
admin := *u
admin.Path = "/postgres"
conn, err := sql.Open("pgx", admin.String())
if err != nil {
t.Skipf("postgres unavailable (%v) — skipping", err)
}
if err := conn.Ping(); err != nil {
conn.Close()
t.Skipf("postgres unavailable (%v) — skipping", err)
}
drop := func() error {
_, err := conn.Exec(`DROP DATABASE IF EXISTS "` + name + `" WITH (FORCE)`)
return err
}
if err := drop(); err != nil {
t.Fatalf("drop leftover %s: %v", name, err)
}
t.Cleanup(func() {
if err := drop(); err != nil {
t.Errorf("drop %s: %v", name, err)
}
conn.Close()
})
u.Path = "/" + name
return u.String()
}
// withEmbedded returns the embedded migrations plus extra files.
func withEmbedded(t *testing.T, extra fstest.MapFS) fstest.MapFS {
t.Helper()
out := fstest.MapFS{}
for _, name := range []string{llmRecordsMigration, llmUsageMigration} {
content, err := fs.ReadFile(migrationsFS(), name)
if err != nil {
t.Fatal(err)
}
out[name] = &fstest.MapFile{Data: content}
}
for name, f := range extra {
out[name] = f
}
return out
}
type appliedMigration struct {
version int
name, checksum string
appliedAt time.Time
}
func readApplied(t *testing.T, d *DB) []appliedMigration {
t.Helper()
rows, err := d.Query(`SELECT version, name, checksum, applied_at FROM schema_migrations ORDER BY version`)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
var out []appliedMigration
for rows.Next() {
var m appliedMigration
if err := rows.Scan(&m.version, &m.name, &m.checksum, &m.appliedAt); err != nil {
t.Fatal(err)
}
out = append(out, m)
}
if err := rows.Err(); err != nil {
t.Fatal(err)
}
return out
}
// assertEmbeddedApplied checks that both LLM tables exist and that 0001 and
// 0002 are recorded with the SHA-256 of their embedded files.
func assertEmbeddedApplied(t *testing.T, d *DB) []appliedMigration {
t.Helper()
for _, table := range []string{"llm_records", "llm_usage"} {
var exists bool
if err := d.QueryRow(`SELECT to_regclass($1) IS NOT NULL`, table).Scan(&exists); err != nil {
t.Fatal(err)
}
if !exists {
t.Errorf("table %s missing after Open", table)
}
}
applied := readApplied(t, d)
want := []struct {
version int
name string
}{{1, llmRecordsMigration}, {2, llmUsageMigration}}
if len(applied) != len(want) {
t.Fatalf("schema_migrations = %+v, want versions 1 and 2", applied)
}
for i, w := range want {
content, err := fs.ReadFile(migrationsFS(), w.name)
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(content)
got := applied[i]
if got.version != w.version || got.name != w.name || got.checksum != hex.EncodeToString(sum[:]) {
t.Errorf("schema_migrations row %d = %+v, want version %d name %s checksum %x", i, got, w.version, w.name, sum)
}
}
return applied
}
// assertReopenIsNoop opens dsn again and checks that no migration row was added
// or re-applied, then that both Ensure functions stay idempotent.
func assertReopenIsNoop(t *testing.T, dsn string, before []appliedMigration) {
t.Helper()
d, err := Open(dsn)
if err != nil {
t.Fatalf("second Open: %v", err)
}
defer d.Close()
after := assertEmbeddedApplied(t, d)
for i := range before {
if !after[i].appliedAt.Equal(before[i].appliedAt) {
t.Errorf("migration %04d applied_at changed: %v -> %v", before[i].version, before[i].appliedAt, after[i].appliedAt)
}
}
for range 2 {
if err := d.EnsureLLMRecordsTable(); err != nil {
t.Fatalf("EnsureLLMRecordsTable after Open: %v", err)
}
if err := d.EnsureLLMUsageTable(); err != nil {
t.Fatalf("EnsureLLMUsageTable after Open: %v", err)
}
}
if got := readApplied(t, d); len(got) != len(before) {
t.Errorf("Ensure functions changed schema_migrations: %+v", got)
}
}
// TestOpenRecordsEmbeddedMigrations runs against the package test database:
// fresh on a first run, already migrated on a second run.
func TestOpenRecordsEmbeddedMigrations(t *testing.T) {
dsn := testDSN(t)
d, err := Open(dsn)
if err != nil {
t.Skipf("postgres unavailable (%v) — skipping", err)
}
applied := assertEmbeddedApplied(t, d)
d.Close()
assertReopenIsNoop(t, dsn, applied)
}
func TestOpenMigratesFreshDatabase(t *testing.T) {
dsn := scratchDSN(t, "fresh")
d, err := Open(dsn)
if err != nil {
t.Fatalf("Open: %v", err)
}
applied := assertEmbeddedApplied(t, d)
d.Close()
assertReopenIsNoop(t, dsn, applied)
}
// An install that predates schema_migrations already has both tables (created
// by the Ensure functions, possibly without the later llm_records columns).
// Open must adopt them: run 0001/0002 over them and record both.
func TestOpenAdoptsExistingTablesWithoutSchemaMigrations(t *testing.T) {
dsn := scratchDSN(t, "legacy")
d, err := openWithMigrations(dsn, fstest.MapFS{})
if err != nil {
t.Fatalf("Open without migrations: %v", err)
}
if err := d.EnsureLLMRecordsTable(); err != nil {
t.Fatal(err)
}
if err := d.EnsureLLMUsageTable(); err != nil {
t.Fatal(err)
}
for _, q := range []string{
`ALTER TABLE llm_records DROP COLUMN raw_request, DROP COLUMN raw_response`,
`INSERT INTO llm_records(model) VALUES ('legacy')`,
`INSERT INTO llm_usage(model, status) VALUES ('legacy', 'ok')`,
`DROP TABLE schema_migrations`,
} {
if _, err := d.Exec(q); err != nil {
t.Fatalf("%s: %v", q, err)
}
}
d.Close()
d, err = Open(dsn)
if err != nil {
t.Fatalf("Open on existing tables: %v", err)
}
applied := assertEmbeddedApplied(t, d)
var records, usage, rawCols int
if err := d.QueryRow(`SELECT count(*) FROM llm_records WHERE model='legacy'`).Scan(&records); err != nil {
t.Fatal(err)
}
if err := d.QueryRow(`SELECT count(*) FROM llm_usage WHERE model='legacy'`).Scan(&usage); err != nil {
t.Fatal(err)
}
if err := d.QueryRow(`SELECT count(*) FROM information_schema.columns
WHERE table_name='llm_records' AND column_name IN ('raw_request','raw_response')`).Scan(&rawCols); err != nil {
t.Fatal(err)
}
if records != 1 || usage != 1 || rawCols != 2 {
t.Errorf("after adopt: llm_records legacy rows=%d llm_usage legacy rows=%d raw columns=%d, want 1 1 2", records, usage, rawCols)
}
d.Close()
assertReopenIsNoop(t, dsn, applied)
}
// probeMigration records how many migrations were already applied when it
// ran, and fails if it ever runs twice (CREATE TABLE without IF NOT EXISTS).
var probeMigration = &fstest.MapFile{Data: []byte(`
CREATE TABLE migration_probe (applied_before INTEGER);
INSERT INTO migration_probe SELECT count(*) FROM schema_migrations;
`)}
func TestMigrationsApplyNewVersionOnceInOrder(t *testing.T) {
dsn := scratchDSN(t, "order")
fsys := withEmbedded(t, fstest.MapFS{"0003_probe.sql": probeMigration})
for pass := 1; pass <= 2; pass++ {
d, err := openWithMigrations(dsn, fsys)
if err != nil {
t.Fatalf("pass %d: %v", pass, err)
}
applied := readApplied(t, d)
if len(applied) != 3 || applied[2].version != 3 || applied[2].name != "0003_probe.sql" {
t.Fatalf("pass %d: schema_migrations = %+v", pass, applied)
}
var runs, before int
if err := d.QueryRow(`SELECT count(*), max(applied_before) FROM migration_probe`).Scan(&runs, &before); err != nil {
t.Fatal(err)
}
if runs != 1 || before != 2 {
t.Fatalf("pass %d: probe ran %d times with %d migrations before it, want 1 and 2", pass, runs, before)
}
d.Close()
}
}
func TestMigrationsFailClosed(t *testing.T) {
dsn := scratchDSN(t, "closed")
d, err := openWithMigrations(dsn, withEmbedded(t, fstest.MapFS{"0003_probe.sql": probeMigration}))
if err != nil {
t.Fatal(err)
}
d.Close()
cases := []struct {
name string
fsys fs.FS
want []string
}{
{"checksum changed", withEmbedded(t, fstest.MapFS{"0003_probe.sql": {Data: append([]byte("-- edited\n"), probeMigration.Data...)}}),
[]string{"0003", "0003_probe.sql", "checksum"}},
{"version missing from binary", migrationsFS(),
[]string{"0003", "0003_probe.sql", "missing from this binary"}},
{"name changed", withEmbedded(t, fstest.MapFS{"0003_renamed.sql": probeMigration}),
[]string{"0003", "0003_probe.sql", "0003_renamed.sql"}},
}
for _, c := range cases {
d, err := openWithMigrations(dsn, c.fsys)
if err == nil {
d.Close()
t.Errorf("%s: Open succeeded, want error", c.name)
continue
}
for _, w := range c.want {
if !strings.Contains(err.Error(), w) {
t.Errorf("%s: error %q does not mention %q", c.name, err, w)
}
}
}
}
func TestFailingMigrationRollsBackAndBlocksOpen(t *testing.T) {
dsn := scratchDSN(t, "fail")
fsys := withEmbedded(t, fstest.MapFS{
"0003_probe.sql": probeMigration,
"0004_broken.sql": {Data: []byte(`
CREATE TABLE migration_broken (x INTEGER);
SELECT 1/0;
`)},
})
d, err := openWithMigrations(dsn, fsys)
if err == nil {
d.Close()
t.Fatal("Open succeeded with a failing migration")
}
if !strings.Contains(err.Error(), "0004") || !strings.Contains(err.Error(), "0004_broken.sql") {
t.Errorf("error %q does not name the failing migration", err)
}
// Inspect with the migrations that succeeded: earlier ones stay applied,
// the failing one left nothing behind.
d, err = openWithMigrations(dsn, withEmbedded(t, fstest.MapFS{"0003_probe.sql": probeMigration}))
if err != nil {
t.Fatal(err)
}
defer d.Close()
applied := readApplied(t, d)
if len(applied) != 3 || applied[2].version != 3 {
t.Fatalf("schema_migrations = %+v, want versions 1..3", applied)
}
var broken bool
if err := d.QueryRow(`SELECT to_regclass('migration_broken') IS NOT NULL`).Scan(&broken); err != nil {
t.Fatal(err)
}
if broken {
t.Error("migration_broken exists: the failed migration was not rolled back")
}
}
// A database already at 0003 must refuse a binary that adds 0002 below it,
// before writing anything.
func TestMigrationsRefuseOutOfOrderVersion(t *testing.T) {
dsn := scratchDSN(t, "outoforder")
records, err := fs.ReadFile(migrationsFS(), llmRecordsMigration)
if err != nil {
t.Fatal(err)
}
d, err := openWithMigrations(dsn, fstest.MapFS{
llmRecordsMigration: {Data: records},
"0003_probe.sql": probeMigration,
})
if err != nil {
t.Fatal(err)
}
before := readApplied(t, d)
d.Close()
if len(before) != 2 || before[0].version != 1 || before[1].version != 3 {
t.Fatalf("setup schema_migrations = %+v, want versions 1 and 3", before)
}
d, err = openWithMigrations(dsn, withEmbedded(t, fstest.MapFS{"0003_probe.sql": probeMigration}))
if err == nil {
d.Close()
t.Fatal("Open succeeded with 0002 pending below applied 0003")
}
for _, w := range []string{llmUsageMigration, "(0002)", "0003_probe.sql", "(0003)", "never inserted"} {
if !strings.Contains(err.Error(), w) {
t.Errorf("error %q does not mention %q", err, w)
}
}
raw, err := sql.Open("pgx", dsn)
if err != nil {
t.Fatal(err)
}
defer raw.Close()
after := readApplied(t, &DB{raw})
if len(after) != len(before) {
t.Fatalf("schema_migrations changed: before %+v, after %+v", before, after)
}
for i := range before {
if after[i].version != before[i].version || after[i].checksum != before[i].checksum || !after[i].appliedAt.Equal(before[i].appliedAt) {
t.Errorf("schema_migrations row %d changed: before %+v, after %+v", i, before[i], after[i])
}
}
var usage bool
if err := raw.QueryRow(`SELECT to_regclass('llm_usage') IS NOT NULL`).Scan(&usage); err != nil {
t.Fatal(err)
}
if usage {
t.Error("llm_usage exists: the refused 0002 migration ran")
}
}
+29
View File
@@ -0,0 +1,29 @@
CREATE TABLE IF NOT EXISTS llm_records (
id BIGSERIAL PRIMARY KEY,
ts TIMESTAMPTZ DEFAULT now(),
model TEXT,
profile_name TEXT,
session_id TEXT,
task_id TEXT,
worker TEXT,
latency_ms INTEGER,
input_tokens INTEGER,
output_tokens INTEGER,
cache_read INTEGER,
cache_write INTEGER,
status TEXT,
error TEXT,
request_body TEXT,
response_body TEXT,
raw_request TEXT,
raw_response TEXT
);
CREATE INDEX IF NOT EXISTS idx_llm_records_ts ON llm_records(ts);
CREATE INDEX IF NOT EXISTS idx_llm_records_session ON llm_records(session_id);
ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS task_id TEXT;
ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS worker TEXT;
ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS profile_name TEXT;
ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS raw_request TEXT;
ALTER TABLE llm_records ADD COLUMN IF NOT EXISTS raw_response TEXT;
+19
View File
@@ -0,0 +1,19 @@
CREATE TABLE IF NOT EXISTS llm_usage (
id BIGSERIAL PRIMARY KEY,
ts TIMESTAMPTZ NOT NULL DEFAULT now(),
task_id TEXT,
exploration_id BIGINT,
worker TEXT,
model TEXT,
profile_name TEXT,
latency_ms INTEGER,
input_tokens INTEGER NOT NULL DEFAULT 0,
output_tokens INTEGER NOT NULL DEFAULT 0,
cache_read INTEGER NOT NULL DEFAULT 0,
cache_write INTEGER NOT NULL DEFAULT 0,
status TEXT
);
CREATE INDEX IF NOT EXISTS idx_llm_usage_task ON llm_usage(task_id);
CREATE INDEX IF NOT EXISTS idx_llm_usage_model ON llm_usage(task_id, model);
CREATE INDEX IF NOT EXISTS idx_llm_usage_exp ON llm_usage(exploration_id);
+8
View File
@@ -447,6 +447,14 @@ func restoreArchiveAssets(tx *sql.Tx, taskID int64, raw json.RawMessage) (map[in
row["company_source"] = "explicit"
warnings = append(warnings, fmt.Sprintf("asset %d's company %d has been deleted; restored as unaffiliated", oldID, companyID))
}
// Archives taken before normalizeURL stripped userinfo can hold
// http://user:pass@host URLs. Normalise like the live upserts so the
// restore neither stores the credential nor misses the clean row.
if typeName, _ := row["type"].(string); typeName == "service" || typeName == "endpoint" {
if rawURL, ok := row["url"].(string); ok {
row["url"] = normalizeURL(rawURL)
}
}
if existing, found, err := findArchiveAssetNaturalID(tx, row); err != nil {
return nil, nil, err
} else if found {
+284
View File
@@ -0,0 +1,284 @@
package db
import (
"encoding/json"
"fmt"
"strings"
"testing"
)
const restoreURLHost = "restore-userinfo.example"
// bigintArray renders ids as a PostgreSQL array literal for a $n::bigint[] cast.
func bigintArray(ids ...int64) string {
parts := make([]string, len(ids))
for i, id := range ids {
parts[i] = fmt.Sprint(id)
}
return "{" + strings.Join(parts, ",") + "}"
}
// archivedAssetRows exports live asset rows the way SnapshotTaskArchive does,
// so the restore tests start from realistic archive JSON.
func archivedAssetRows(t *testing.T, d *DB, ids ...int64) []map[string]any {
t.Helper()
raw, _, err := queryArchiveRows(d, `SELECT asset.* FROM assets asset WHERE asset.id = ANY($1::bigint[]) ORDER BY asset.id`, bigintArray(ids...))
if err != nil {
t.Fatal(err)
}
rows, err := decodeArchiveRows(raw)
if err != nil {
t.Fatal(err)
}
if len(rows) != len(ids) {
t.Fatalf("exported %d archive rows, want %d", len(rows), len(ids))
}
return rows
}
// restoreArchivedAssets runs restoreArchiveAssets on rows in a committed tx.
func restoreArchivedAssets(t *testing.T, d *DB, taskID int64, rows []map[string]any) map[int64]int64 {
t.Helper()
raw, err := json.Marshal(rows)
if err != nil {
t.Fatal(err)
}
tx, err := d.Begin()
if err != nil {
t.Fatal(err)
}
defer tx.Rollback() //nolint:errcheck
mapping, _, err := restoreArchiveAssets(tx, taskID, raw)
if err != nil {
t.Fatal(err)
}
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
return mapping
}
func restoreURLCleanup(d *DB) {
d.Exec(`DELETE FROM assets WHERE url LIKE '%` + restoreURLHost + `%' OR domain LIKE '%` + restoreURLHost + `' OR root_domain = '` + restoreURLHost + `' OR ip = '198.51.100.231'`) //nolint:errcheck
}
func countRestoreURLRows(t *testing.T, d *DB, assetType, cleanURL string) int {
t.Helper()
var n int
if err := d.QueryRow(`SELECT count(*) FROM assets WHERE type=$1 AND (url=$2 OR url LIKE '%@%' || $3::text || '%')`, assetType, cleanURL, restoreURLHost).Scan(&n); err != nil {
t.Fatal(err)
}
return n
}
func assertNoRestoreURLCredential(t *testing.T, d *DB, id int64) {
t.Helper()
var row string
if err := d.QueryRow(`SELECT row_to_json(a)::text FROM assets a WHERE id=$1`, id).Scan(&row); err != nil {
t.Fatal(err)
}
for _, secret := range []string{"alice", "bob", "s3cr3t"} {
if strings.Contains(row, secret) {
t.Fatalf("restored asset %d still carries %q: %s", id, secret, row)
}
}
}
func TestRestoreArchiveAssetsStripsURLUserinfo(t *testing.T) {
d, av2, _ := testSetup(t)
defer d.Close()
restoreURLCleanup(d)
defer restoreURLCleanup(d)
const taskID = int64(880000001)
svcURL := "http://" + restoreURLHost + ":8080/admin"
epURL := "https://api." + restoreURLHost + "/users?id=1"
svcID, err := av2.UpsertHTTPService(UpsertHTTPServiceReq{URL: svcURL, TaskID: taskID})
if err != nil {
t.Fatal(err)
}
epID, err := av2.UpsertEndpoint(UpsertEndpointReq{URL: epURL, Method: "GET", TaskID: taskID})
if err != nil {
t.Fatal(err)
}
rows := archivedAssetRows(t, d, svcID, epID)
// Simulate an archive taken before normalizeURL dropped userinfo.
rows[0]["url"] = "http://alice:s3cr3t-one@" + restoreURLHost + ":8080/admin"
rows[1]["url"] = "https://bob:s3cr3t-two@api." + restoreURLHost + "/users?id=1"
archived := map[string]map[string]any{"service": {}, "endpoint": {}}
for _, row := range rows {
typeName := row["type"].(string)
for _, key := range []string{"domain", "root_domain", "port", "service_name", "method"} {
archived[typeName][key] = row[key]
}
}
if _, err := d.Exec(`DELETE FROM assets WHERE id = ANY($1::bigint[])`, bigintArray(svcID, epID)); err != nil {
t.Fatal(err)
}
mapping := restoreArchivedAssets(t, d, taskID, rows)
for oldID, wantURL := range map[int64]string{svcID: svcURL, epID: epURL} {
newID := mapping[oldID]
if newID == 0 {
t.Fatalf("asset %d was not restored: %v", oldID, mapping)
}
var gotURL, typeName string
var domain, rootDomain, serviceName, method *string
var port *int64
if err := d.QueryRow(`SELECT url,type,domain,root_domain,port,service_name,method FROM assets WHERE id=$1`, newID).
Scan(&gotURL, &typeName, &domain, &rootDomain, &port, &serviceName, &method); err != nil {
t.Fatal(err)
}
if gotURL != wantURL {
t.Fatalf("restored %s url=%q, want %q", typeName, gotURL, wantURL)
}
assertNoRestoreURLCredential(t, d, newID)
// Only url is normalised: the other columns keep the archived values.
want := archived[typeName]
got := map[string]any{"domain": domain, "root_domain": rootDomain, "port": port, "service_name": serviceName, "method": method}
for key, value := range got {
gotJSON, _ := json.Marshal(value)
wantJSON, _ := json.Marshal(want[key])
if string(gotJSON) != string(wantJSON) {
t.Errorf("%s %s=%s, want archived %s", typeName, key, gotJSON, wantJSON)
}
}
if n := countRestoreURLRows(t, d, typeName, wantURL); n != 1 {
t.Fatalf("%s rows for %q = %d, want 1", typeName, wantURL, n)
}
}
}
func TestRestoreArchiveAssetsReusesCleanURLRow(t *testing.T) {
d, av2, _ := testSetup(t)
defer d.Close()
restoreURLCleanup(d)
defer restoreURLCleanup(d)
const liveTaskID, archivedTaskID = int64(880000002), int64(880000003)
svcURL := "http://" + restoreURLHost + ":8080/admin"
epURL := "https://api." + restoreURLHost + "/users?id=1"
svcID, err := av2.UpsertHTTPService(UpsertHTTPServiceReq{URL: svcURL, TaskID: liveTaskID})
if err != nil {
t.Fatal(err)
}
epID, err := av2.UpsertEndpoint(UpsertEndpointReq{URL: epURL, Method: "GET", TaskID: liveTaskID})
if err != nil {
t.Fatal(err)
}
rows := archivedAssetRows(t, d, svcID, epID)
// The archived rows are other, older rows whose url still carries userinfo.
rows[0]["id"], rows[0]["url"] = int64(990000101), "http://alice:s3cr3t-one@"+restoreURLHost+":8080/admin"
rows[1]["id"], rows[1]["url"] = int64(990000102), "https://bob:s3cr3t-two@api."+restoreURLHost+"/users?id=1"
for _, row := range rows {
row["task_ids"] = []any{archivedTaskID}
}
mapping := restoreArchivedAssets(t, d, archivedTaskID, rows)
for oldID, existingID := range map[int64]int64{990000101: svcID, 990000102: epID} {
if mapping[oldID] != existingID {
t.Fatalf("archived asset %d mapped to %d, want existing clean row %d", oldID, mapping[oldID], existingID)
}
var taskIDs string
if err := d.QueryRow(`SELECT task_ids::text FROM assets WHERE id=$1`, existingID).Scan(&taskIDs); err != nil {
t.Fatal(err)
}
if want := bigintArray(liveTaskID, archivedTaskID); taskIDs != want {
t.Fatalf("asset %d task_ids=%s, want %s", existingID, taskIDs, want)
}
assertNoRestoreURLCredential(t, d, existingID)
var leftover int
if err := d.QueryRow(`SELECT count(*) FROM assets WHERE id=$1`, oldID).Scan(&leftover); err != nil {
t.Fatal(err)
}
if leftover != 0 {
t.Fatalf("archived asset %d inserted beside the clean row", oldID)
}
}
if n := countRestoreURLRows(t, d, "service", svcURL); n != 1 {
t.Fatalf("service rows = %d, want 1", n)
}
if n := countRestoreURLRows(t, d, "endpoint", epURL); n != 1 {
t.Fatalf("endpoint rows = %d, want 1", n)
}
}
func TestRestoreArchiveAssetsDedupsURLAcrossCredentials(t *testing.T) {
d, av2, _ := testSetup(t)
defer d.Close()
restoreURLCleanup(d)
defer restoreURLCleanup(d)
const taskID = int64(880000004)
svcURL := "http://" + restoreURLHost + ":8080/admin"
svcID, err := av2.UpsertHTTPService(UpsertHTTPServiceReq{URL: svcURL, TaskID: taskID})
if err != nil {
t.Fatal(err)
}
first := archivedAssetRows(t, d, svcID)[0]
second := archivedAssetRows(t, d, svcID)[0]
if _, err := d.Exec(`DELETE FROM assets WHERE id=$1`, svcID); err != nil {
t.Fatal(err)
}
first["id"], first["url"] = int64(990000201), "http://alice:s3cr3t-one@"+restoreURLHost+":8080/admin"
second["id"], second["url"] = int64(990000202), "http://bob:s3cr3t-two@"+restoreURLHost+":8080/admin"
mapping := restoreArchivedAssets(t, d, taskID, []map[string]any{first, second})
if mapping[990000201] == 0 || mapping[990000201] != mapping[990000202] {
t.Fatalf("credential variants restored as separate rows: %v", mapping)
}
if n := countRestoreURLRows(t, d, "service", svcURL); n != 1 {
t.Fatalf("service rows = %d, want exactly 1", n)
}
assertNoRestoreURLCredential(t, d, mapping[990000201])
}
func TestRestoreArchiveAssetsLeavesDomainAndIPURLUntouched(t *testing.T) {
d, av2, _ := testSetup(t)
defer d.Close()
restoreURLCleanup(d)
defer restoreURLCleanup(d)
const taskID = int64(880000005)
domainID, err := av2.UpsertRootDomain(UpsertRootDomainReq{Domain: restoreURLHost, TaskID: taskID})
if err != nil {
t.Fatal(err)
}
ipID, err := av2.UpsertIP(UpsertIPReq{IP: "198.51.100.231", TaskID: taskID})
if err != nil {
t.Fatal(err)
}
rows := archivedAssetRows(t, d, domainID, ipID)
// A url normalizeURL would rewrite proves the normalisation is scoped to
// service and endpoint rows.
const oddURL = "HTTP://Restore-Userinfo.example/"
for _, row := range rows {
row["url"] = oddURL
}
want := map[int64][]any{}
for _, row := range rows {
id, _ := jsonInt64(row["id"])
want[id] = []any{row["type"], row["domain"], row["root_domain"], row["ip"]}
}
if _, err := d.Exec(`DELETE FROM assets WHERE id = ANY($1::bigint[])`, bigintArray(domainID, ipID)); err != nil {
t.Fatal(err)
}
mapping := restoreArchivedAssets(t, d, taskID, rows)
for oldID, fields := range want {
var typeName string
var domain, rootDomain, ip, gotURL *string
if err := d.QueryRow(`SELECT type,domain,root_domain,ip,url FROM assets WHERE id=$1`, mapping[oldID]).
Scan(&typeName, &domain, &rootDomain, &ip, &gotURL); err != nil {
t.Fatal(err)
}
got, _ := json.Marshal([]any{typeName, domain, rootDomain, ip})
wantJSON, _ := json.Marshal(fields)
if string(got) != string(wantJSON) {
t.Errorf("asset %d restored as %s, want %s", oldID, got, wantJSON)
}
if gotURL == nil || *gotURL != oddURL {
t.Errorf("%s asset url changed to %v, want %q", typeName, gotURL, oddURL)
}
}
}