feat: encrypt S3 archives and test credentials and interruption recovery

This commit is contained in:
Chubboy Ren
2026-11-18 02:05:17 +00:00
parent 3d44e9cd0d
commit da7cc62d4e
9 changed files with 892 additions and 207 deletions
+291
View File
@@ -0,0 +1,291 @@
package server
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"sync"
"testing"
"github.com/Autumn-27/artex/archivestore"
pgdb "github.com/Autumn-27/artex/db"
)
// Recreate persisted states at operation boundaries, then run the same recovery
// functions used at startup. PostgreSQL, files, and HTTP object storage are real
// isolated fixtures; no production failpoint hooks are needed.
func TestRemoteArchiveInterruptionRecovery(t *testing.T) {
if os.Getenv("ARTEX_PG_DSN") == "" {
t.Skip("requires isolated PostgreSQL")
}
phases := []string{
"staged_before_compression", "compressed_before_journal", "journal_before_upload",
"uploaded_before_verification", "verified_before_database",
"database_before_local_cleanup", "package_cleaned_before_stage_cleanup",
"restore_download_before_install", "restore_files_before_database", "restore_database_before_object_delete", "restore_object_deleted_before_stub",
"delete_journal_before_database", "delete_database_before_object",
}
for _, phase := range phases {
t.Run(phase, func(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
defer m.Close()
store, objects := newRecoveryObjectStore(t)
s := &Server{ctx: context.Background(), m: m, archiveStore: store}
task, err := m.CreateTask("interruption "+phase, "preserve source", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
id, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }()
if err := m.pg.SetPaused(id, true); err != nil {
t.Fatal(err)
}
job, err := m.pg.QueueTaskArchive(id)
if err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Archiving); err != nil {
t.Fatal(err)
}
source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt")
if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(source, []byte("original task data"), 0600); err != nil {
t.Fatal(err)
}
snapshot, err := m.pg.SnapshotTaskArchive(id)
if err != nil {
t.Fatal(err)
}
stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID)
if err != nil {
t.Fatal(err)
}
packagePath := taskArchivePath(m.dir, job.ID, task.ID)
var original, compressed int64
var checksum, uri, journal string
if phase != "staged_before_compression" {
original, compressed, checksum, err = writeTaskArchivePackage(packagePath, stage.payload, snapshot)
if err != nil {
t.Fatal(err)
}
uri = store.URI(id, job.ID, checksum)
}
if phase != "staged_before_compression" && phase != "compressed_before_journal" {
journal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, LocalPath: packagePath, Operation: "upload"})
if err != nil {
t.Fatal(err)
}
}
committed := phase != "staged_before_compression" && phase != "compressed_before_journal" &&
phase != "journal_before_upload" && phase != "uploaded_before_verification" && phase != "verified_before_database"
if phase == "uploaded_before_verification" {
data, err := os.ReadFile(packagePath)
if err != nil {
t.Fatal(err)
}
objects.put(uri, data)
} else if phase == "verified_before_database" || committed {
if err := store.Upload(s.ctx, uri, packagePath, checksum); err != nil {
t.Fatal(err)
}
}
if committed {
if err := m.pg.CompleteTaskArchive(job.ID, snapshot, uri, checksum, original, compressed); err != nil {
t.Fatal(err)
}
}
afterArchive := phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" ||
phase == "restore_object_deleted_before_stub" || phase == "delete_journal_before_database" || phase == "delete_database_before_object"
if phase == "package_cleaned_before_stage_cleanup" || afterArchive {
if err := os.Remove(packagePath); err != nil {
t.Fatal(err)
}
if err := os.Remove(journal); err != nil {
t.Fatal(err)
}
}
if afterArchive {
if err := stage.commit(); err != nil {
t.Fatal(err)
}
}
restored := false
remoteDeleted := false
if phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" {
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Restoring); err != nil {
t.Fatal(err)
}
parent := filepath.Join(taskArchiveRoot(m.dir), ".restore")
if err := os.MkdirAll(parent, 0700); err != nil {
t.Fatal(err)
}
extracted, err := os.MkdirTemp(parent, "interrupted-")
if err != nil {
t.Fatal(err)
}
downloaded := filepath.Join(extracted, "download.tar.zst")
if err := store.Download(s.ctx, uri, downloaded, checksum, compressed); err != nil {
t.Fatal(err)
}
if err := extractTaskArchivePackage(downloaded, checksum, extracted); err != nil {
t.Fatal(err)
}
if phase != "restore_download_before_install" {
if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID); err != nil {
t.Fatal(err)
}
}
if phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" {
if _, err := m.pg.RestoreTaskArchive(job.ID, snapshot, 0); err != nil {
t.Fatal(err)
}
restored = true
}
if phase == "restore_object_deleted_before_stub" {
if err := store.Delete(s.ctx, uri); err != nil {
t.Fatal(err)
}
remoteDeleted = true
}
}
deleting := phase == "delete_journal_before_database" || phase == "delete_database_before_object"
if deleting {
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Deleting); err != nil {
t.Fatal(err)
}
if _, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, Operation: "delete"}); err != nil {
t.Fatal(err)
}
if phase == "delete_database_before_object" {
if err := m.pg.DeleteTaskArchiveStub(job.ID); err != nil {
t.Fatal(err)
}
remoteDeleted = true
}
}
if err := m.pg.RecoverTaskArchiveJobs(); err != nil {
t.Fatal(err)
}
if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if err := s.recoverRemoteArchiveJournals(); err != nil {
t.Fatal(err)
}
wantSource := !committed || restored
data, fileErr := os.ReadFile(source)
if wantSource && (fileErr != nil || string(data) != "original task data") {
t.Fatalf("source data lost: %v", fileErr)
}
if !wantSource && !os.IsNotExist(fileErr) {
t.Fatalf("archived source resurrected: %v", fileErr)
}
if _, err := os.Stat(packagePath); !os.IsNotExist(err) {
t.Fatalf("temporary package remains: %v", err)
}
if got, want := objects.has(uri), committed && !remoteDeleted; got != want {
t.Fatalf("remote object exists=%v, want=%v", got, want)
}
if committed && !deleting {
actual, err := m.pg.IsTaskArchiveRestored(job.ID)
if err != nil || actual != restored {
t.Fatalf("restored=%v want=%v err=%v", actual, restored, err)
}
}
// A second startup must be harmless at every boundary.
if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if err := s.recoverRemoteArchiveJournals(); err != nil {
t.Fatal(err)
}
})
}
}
type recoveryObjects struct {
mu sync.Mutex
data map[string][]byte
}
func (o *recoveryObjects) put(uri string, data []byte) {
o.mu.Lock()
defer o.mu.Unlock()
// Test URIs always use the fixture bucket with path-style addressing.
o.data["/test/"+uri[len("s3://test/"):]] = append([]byte(nil), data...)
}
func (o *recoveryObjects) has(uri string) bool {
o.mu.Lock()
defer o.mu.Unlock()
if uri == "" {
return false
}
_, ok := o.data["/test/"+uri[len("s3://test/"):]]
return ok
}
func newRecoveryObjectStore(t *testing.T) (*archivestore.Store, *recoveryObjects) {
t.Helper()
objects := &recoveryObjects{data: map[string][]byte{}}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
objects.mu.Lock()
defer objects.mu.Unlock()
data, exists := objects.data[r.URL.Path]
w.Header().Set("ETag", `"recovery-etag"`)
switch r.Method {
case "PUT":
if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" {
t.Error("missing SSE")
}
objects.data[r.URL.Path], _ = io.ReadAll(r.Body)
case "HEAD":
if !exists {
w.WriteHeader(404)
return
}
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
w.Header().Set("X-Amz-Server-Side-Encryption", "AES256")
case "GET":
if !exists {
w.WriteHeader(404)
return
}
var start, end int
if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end >= len(data) || r.Header.Get("If-Match") != `"recovery-etag"` {
w.WriteHeader(416)
return
}
w.Header().Set("Content-Length", strconv.Itoa(end-start+1))
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
w.WriteHeader(206)
_, _ = w.Write(data[start : end+1])
case "DELETE":
delete(objects.data, r.URL.Path)
w.WriteHeader(204)
default:
t.Errorf("unexpected object request %s", r.Method)
w.WriteHeader(400)
}
}))
t.Cleanup(srv.Close)
store, err := archivestore.New(archivestore.Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
if err != nil {
t.Fatal(err)
}
return store, objects
}