package server import ( "context" "fmt" "io" "net/http" "net/http/httptest" "os" "path/filepath" "strconv" "sync" "testing" "github.com/Autumn-27/artex/archivestore" pgdb "github.com/Autumn-27/artex/db" ) // Recreate persisted states at operation boundaries, then run the same recovery // functions used at startup. PostgreSQL, files, and HTTP object storage are real // isolated fixtures; no production failpoint hooks are needed. func TestRemoteArchiveInterruptionRecovery(t *testing.T) { if os.Getenv("ARTEX_PG_DSN") == "" { t.Skip("requires isolated PostgreSQL") } phases := []string{ "staged_before_compression", "compressed_before_journal", "journal_before_upload", "uploaded_before_verification", "verified_before_database", "database_before_local_cleanup", "package_cleaned_before_stage_cleanup", "restore_download_before_install", "restore_files_before_database", "restore_database_before_object_delete", "restore_object_deleted_before_stub", "delete_journal_before_database", "delete_database_before_object", } for _, phase := range phases { t.Run(phase, func(t *testing.T) { m, err := NewManager(t.TempDir(), "") if err != nil { t.Fatal(err) } defer m.Close() store, objects := newRecoveryObjectStore(t) s := &Server{ctx: context.Background(), m: m, archiveStore: store} task, err := m.CreateTask("interruption "+phase, "preserve source", nil, 0, 0) if err != nil { t.Fatal(err) } id, _ := strconv.ParseInt(task.ID, 10, 64) defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }() if err := m.pg.SetPaused(id, true); err != nil { t.Fatal(err) } job, err := m.pg.QueueTaskArchive(id) if err != nil { t.Fatal(err) } if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Archiving); err != nil { t.Fatal(err) } source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt") if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil { t.Fatal(err) } if err := os.WriteFile(source, []byte("original task data"), 0600); err != nil { t.Fatal(err) } snapshot, err := m.pg.SnapshotTaskArchive(id) if err != nil { t.Fatal(err) } stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID) if err != nil { t.Fatal(err) } packagePath := taskArchivePath(m.dir, job.ID, task.ID) var original, compressed int64 var checksum, uri, journal string if phase != "staged_before_compression" { original, compressed, checksum, err = writeTaskArchivePackage(packagePath, stage.payload, snapshot) if err != nil { t.Fatal(err) } uri = store.URI(id, job.ID, checksum) } if phase != "staged_before_compression" && phase != "compressed_before_journal" { journal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, LocalPath: packagePath, Operation: "upload"}) if err != nil { t.Fatal(err) } } committed := phase != "staged_before_compression" && phase != "compressed_before_journal" && phase != "journal_before_upload" && phase != "uploaded_before_verification" && phase != "verified_before_database" if phase == "uploaded_before_verification" { data, err := os.ReadFile(packagePath) if err != nil { t.Fatal(err) } objects.put(uri, data) } else if phase == "verified_before_database" || committed { if err := store.Upload(s.ctx, uri, packagePath, checksum); err != nil { t.Fatal(err) } } if committed { if err := m.pg.CompleteTaskArchive(job.ID, snapshot, uri, checksum, original, compressed); err != nil { t.Fatal(err) } } afterArchive := phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" || phase == "delete_journal_before_database" || phase == "delete_database_before_object" if phase == "package_cleaned_before_stage_cleanup" || afterArchive { if err := os.Remove(packagePath); err != nil { t.Fatal(err) } if err := os.Remove(journal); err != nil { t.Fatal(err) } } if afterArchive { if err := stage.commit(); err != nil { t.Fatal(err) } } restored := false remoteDeleted := false if phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" { if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Restoring); err != nil { t.Fatal(err) } parent := filepath.Join(taskArchiveRoot(m.dir), ".restore") if err := os.MkdirAll(parent, 0700); err != nil { t.Fatal(err) } extracted, err := os.MkdirTemp(parent, "interrupted-") if err != nil { t.Fatal(err) } downloaded := filepath.Join(extracted, "download.tar.zst") if err := store.Download(s.ctx, uri, downloaded, checksum, compressed); err != nil { t.Fatal(err) } if err := extractTaskArchivePackage(downloaded, checksum, extracted); err != nil { t.Fatal(err) } if phase != "restore_download_before_install" { if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID); err != nil { t.Fatal(err) } } if phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" { if _, err := m.pg.RestoreTaskArchive(job.ID, snapshot, 0); err != nil { t.Fatal(err) } restored = true } if phase == "restore_object_deleted_before_stub" { if err := store.Delete(s.ctx, uri); err != nil { t.Fatal(err) } remoteDeleted = true } } deleting := phase == "delete_journal_before_database" || phase == "delete_database_before_object" if deleting { if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Deleting); err != nil { t.Fatal(err) } if _, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, Operation: "delete"}); err != nil { t.Fatal(err) } if phase == "delete_database_before_object" { if err := m.pg.DeleteTaskArchiveStub(job.ID); err != nil { t.Fatal(err) } remoteDeleted = true } } if err := m.pg.RecoverTaskArchiveJobs(); err != nil { t.Fatal(err) } if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { t.Fatal(err) } if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil { t.Fatal(err) } if err := s.recoverRemoteArchiveJournals(); err != nil { t.Fatal(err) } wantSource := !committed || restored data, fileErr := os.ReadFile(source) if wantSource && (fileErr != nil || string(data) != "original task data") { t.Fatalf("source data lost: %v", fileErr) } if !wantSource && !os.IsNotExist(fileErr) { t.Fatalf("archived source resurrected: %v", fileErr) } if _, err := os.Stat(packagePath); !os.IsNotExist(err) { t.Fatalf("temporary package remains: %v", err) } if got, want := objects.has(uri), committed && !remoteDeleted; got != want { t.Fatalf("remote object exists=%v, want=%v", got, want) } if committed && !deleting { actual, err := m.pg.IsTaskArchiveRestored(job.ID) if err != nil || actual != restored { t.Fatalf("restored=%v want=%v err=%v", actual, restored, err) } } // A second startup must be harmless at every boundary. if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { t.Fatal(err) } if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil { t.Fatal(err) } if err := s.recoverRemoteArchiveJournals(); err != nil { t.Fatal(err) } }) } } type recoveryObjects struct { mu sync.Mutex data map[string][]byte } func (o *recoveryObjects) put(uri string, data []byte) { o.mu.Lock() defer o.mu.Unlock() // Test URIs always use the fixture bucket with path-style addressing. o.data["/test/"+uri[len("s3://test/"):]] = append([]byte(nil), data...) } func (o *recoveryObjects) has(uri string) bool { o.mu.Lock() defer o.mu.Unlock() if uri == "" { return false } _, ok := o.data["/test/"+uri[len("s3://test/"):]] return ok } func newRecoveryObjectStore(t *testing.T) (*archivestore.Store, *recoveryObjects) { t.Helper() objects := &recoveryObjects{data: map[string][]byte{}} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { objects.mu.Lock() defer objects.mu.Unlock() data, exists := objects.data[r.URL.Path] w.Header().Set("ETag", `"recovery-etag"`) switch r.Method { case "PUT": if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" { t.Error("missing SSE") } objects.data[r.URL.Path], _ = io.ReadAll(r.Body) case "HEAD": if !exists { w.WriteHeader(404) return } w.Header().Set("Content-Length", strconv.Itoa(len(data))) w.Header().Set("X-Amz-Server-Side-Encryption", "AES256") case "GET": if !exists { w.WriteHeader(404) return } var start, end int if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end >= len(data) || r.Header.Get("If-Match") != `"recovery-etag"` { w.WriteHeader(416) return } w.Header().Set("Content-Length", strconv.Itoa(end-start+1)) w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) w.WriteHeader(206) _, _ = w.Write(data[start : end+1]) case "DELETE": delete(objects.data, r.URL.Path) w.WriteHeader(204) default: t.Errorf("unexpected object request %s", r.Method) w.WriteHeader(400) } })) t.Cleanup(srv.Close) store, err := archivestore.New(archivestore.Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) if err != nil { t.Fatal(err) } return store, objects }