diff --git a/README.md b/README.md index 2d91397..ad8efec 100644 --- a/README.md +++ b/README.md @@ -421,12 +421,6 @@ runtime supports it. `/chat/completions` (or `/responses` for `openai-responses`), so their base **does** end in `/v1` (for example `https://api.openai.com/v1`). -**Task archive storage**: local disk by default. Set `ARTEX_ARCHIVE_STORAGE=s3` to -store cold task packages in an S3-compatible bucket and release their local -package space after verification. Existing local archives remain usable. See -[Task archive storage](docs/task-archive-storage.md) for configuration, permissions, -integrity checks, temporary disk requirements and recovery. - **Concurrency**: work agents per task, in "System Settings" (default 3). **Untrusted-output boundary (on by default)**: tool results for the worker, planner, chat and main diff --git a/db/task_archives.go b/db/task_archives.go index 9f07341..bb9aae2 100644 --- a/db/task_archives.go +++ b/db/task_archives.go @@ -60,7 +60,6 @@ type TaskArchive struct { Warnings json.RawMessage `json:"warnings"` FormatVersion int `json:"format_version"` ArchivePath string `json:"-"` - Storage string `json:"storage"` SHA256 string `json:"sha256,omitempty"` OriginalSize int64 `json:"original_size"` CompressedSize int64 `json:"compressed_size"` @@ -162,10 +161,6 @@ func scanTaskArchive(sc interface{ Scan(...any) error }) (*TaskArchive, error) { if err != nil { return nil, err } - item.Storage = "local" - if strings.HasPrefix(item.ArchivePath, "s3://") { - item.Storage = "s3" - } if err := json.Unmarshal([]byte(sources), &item.SourceTaskIDs); err != nil { return nil, err } diff --git a/docs/task-archive-storage.md b/docs/task-archive-storage.md deleted file mode 100644 index 113921a..0000000 --- a/docs/task-archive-storage.md +++ /dev/null @@ -1,160 +0,0 @@ -# Task archive storage - -Task archives use local disk by default. The optional S3-compatible backend moves -cold task packages to object storage while retaining Archive, Restore and Delete. -Active workspaces and PostgreSQL remain local. This is task cold storage, not a -full-installation backup; a recovery plan must preserve the database, data -directory, bucket objects, and the credentials or roles needed to read them. - -## Configuration - -Use a private bucket and a separate prefix for each ARTEX installation. Configure -the bucket's default encryption to match the selected mode below, and enforce -that mode/key with a bucket policy. ARTEX does not provision or inspect bucket -encryption settings. - -Set the following server environment and restart ARTEX: - -```dotenv -ARTEX_ARCHIVE_STORAGE=s3 -ARTEX_ARCHIVE_S3_ENDPOINT=https://s3.us-east-1.amazonaws.com -ARTEX_ARCHIVE_S3_REGION=us-east-1 -ARTEX_ARCHIVE_S3_BUCKET=your-artex-archives -ARTEX_ARCHIVE_S3_PREFIX=artex/tasks -ARTEX_ARCHIVE_S3_SSE=AES256 -``` - -With an EC2 instance role, ECS task role or other SDK-supported credential source, -no archive-specific access keys are needed. - -| Variable | Meaning | -| --- | --- | -| `ARTEX_ARCHIVE_STORAGE` | `local` (default) or `s3`. | -| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local tests. | -| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. | -| `ARTEX_ARCHIVE_S3_BUCKET` | Required bucket name. | -| `ARTEX_ARCHIVE_S3_PREFIX` | Object prefix; defaults to `artex/tasks`. | -| `ARTEX_ARCHIVE_S3_SSE` | `AES256` (default, SSE-S3) or `aws:kms` (SSE-KMS). There is no unencrypted mode. | -| `ARTEX_ARCHIVE_S3_KMS_KEY_ID` | Optional KMS key ARN/ID/alias; valid only with `aws:kms`. Omit to use the provider's managed KMS key. | -| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Optional dedicated credential override; must be supplied with the secret key. | -| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Secret for the dedicated override. | -| `ARTEX_ARCHIVE_S3_SESSION_TOKEN` | Optional session token for dedicated temporary credentials. | -| `ARTEX_ARCHIVE_S3_PATH_STYLE` | `true` for path-style servers; otherwise virtual-hosted addressing is used. | - -An invalid mode or partial credential override fails configuration instead of -silently falling back to local storage or ambient credentials. Preserve the -bucket, endpoint, prefix and read/delete access while remote archives remain. -Older local archives remain usable; they are not uploaded automatically. - -## Encryption and permissions - -Every single-object upload and multipart initiation explicitly requests the -configured server-side encryption. Upload verification also requires HEAD to -report the requested mode, and a KMS key identifier for SSE-KMS, before accepting -the object. Size and full SHA-256 readback remain mandatory; encryption metadata -does not replace content verification. - -For SSE-KMS, set both the bucket's default encryption and ARTEX to `aws:kms`; -set `ARTEX_ARCHIVE_S3_KMS_KEY_ID` to the intended key when using a customer-managed -key. An explicit `AES256` request takes precedence over a bucket's KMS default, -so do not leave ARTEX on SSE-S3 when the bucket is intended to use KMS. - -The principal needs object upload, read (HEAD and ranged GET), delete and -multipart upload/abort permissions. KMS uploads/readback also need the applicable -key-policy and IAM permissions for `kms:GenerateDataKey` and `kms:Decrypt`. -Retain old KMS keys and decrypt permissions when changing the key or mode; restore -does not require an older object to use the current upload policy. - -Compatible providers must support the requested SSE mode and its HEAD metadata, -plus conditional ranged reads. A provider that ignores the encryption request -or cannot confirm it is rejected; the original task is retained. Local source and -temporary files keep their existing local-disk protection and restricted modes. - -See the official [PutObject encryption and permission contract](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html) -and [HEAD encryption response rules](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html). - -## Credential sources - -Without the archive-specific override, ARTEX uses the AWS SDK default credential -chain, including standard AWS environment variables, shared profiles, web -identity, ECS task credentials and EC2 instance credentials through IMDS. Role -credentials retain the SDK's automatic refresh behavior. A named shared profile -can be selected with `AWS_PROFILE`. - -For an external S3-compatible provider, dedicated keys can be supplied through -the archive-specific variables. Both key fields are required; a session token -is supported for temporary credentials. This explicit override takes precedence -over ambient AWS credentials. Operator-supplied temporary credentials must be -rotated by the operator; they are not an IAM role provider. - -Keep secrets in a protected service environment file outside the repository and -frontend. A systemd `EnvironmentFile=` can point to a root-owned mode-0600 file. -See the [AWS SDK credential-chain documentation](https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-gosdk.html). - -## Lifecycle - -1. Pause or finish a task and use its **Archive** action. -2. ARTEX stages the source files and builds a local `.tar.zst` package. Large - packages use multipart upload, initially 8 MiB parts with two upload workers; - the SDK increases the part size for exceptionally large seekable files. -3. ARTEX confirms encryption metadata and reads the complete object back, - checking size and SHA-256. ETags and uploaded checksum metadata alone are - insufficient. -4. **Only after verification and a successful PostgreSQL commit** does ARTEX - remove the local package and staged source files. The committed archive - location is an `s3://` URI; there is no schema or archive-format migration. -5. **Restore** downloads and verifies before installing files and database rows. - The remote package is consumed only after restoration commits. -6. **Delete** commits removal of the archive stub, then cleans up the object. - A durable journal retains failed cleanup work. - -The archive list identifies **Object storage** and **Local storage** and shows -upload/download verification phases. Reads use bounded 8 MiB ranges pinned to the -object's ETag, with retries for interrupted ranges rather than the entire object. - -## Interruption recovery - -Source staging and restore journals live with task archives. Remote cleanup -journals live under `data/archives/tasks/.remote`; they contain locations, not -credentials. Keep this directory with PostgreSQL and the installation data. - -| Interruption | Recovery | -| --- | --- | -| During staging/compression, before an upload journal exists | Roll staged source files back. Remove any temporary package only after rollback succeeds. The task remains in PostgreSQL. | -| After the upload journal, before successful verification | Roll source files back and clean up any uncommitted object/package. Failed cleanup retains its journal for retry. | -| After verification, before the database commit | Treat the object as uncommitted: retain the original task and clean up the orphan object. Verification alone never authorizes local deletion. | -| After the database commit, before package/staging cleanup | Keep the committed object and finish local cleanup. The persisted location and checksum establish commit, even if Restore or Delete has since been queued. | -| Download/extraction before restored files are installed | Remove temporary extraction data; keep the remote object and archive stub for another attempt. | -| Restored files installed, before database restoration commits | Roll back installed files using the restore journal. Retain the remote package. | -| Database restoration committed, before remote cleanup | Recognize the already-restored task and finish cleanup without replaying database rows. | -| Object consumed, before the restored archive stub is removed | Skip download/replay; repeat idempotent object deletion and remove the stub. | -| Delete journal written, before database deletion commits | Keep the object while its archive stub exists. The queued Delete operation can retry. | -| Database deletion committed, before object deletion succeeds | The durable outbox retains the URI and retries object deletion. | -| An old failed upload survives a retry using local storage | Remove the abandoned remote object, but retain a local package referenced by the committed archive row. | - -The staging recovery rule also applies to local archives. Dedicated tests cover -both storage modes with ready, restore-queued, delete-queued and uncommitted -states. Additional PostgreSQL/HTTP-S3 tests reconstruct interrupted operations at -13 persisted boundaries and repeat recovery to check idempotence. - -Handled multipart-upload failures are aborted by the SDK. Abrupt process death -may leave unfinished multipart uploads; configure bucket lifecycle expiration -for those parts. Existing object versions, if versioning is enabled, are governed -by the bucket's version-retention policy. - -## Disk and transfer requirements - -Archive creation still requires temporary local space for the compressed -package. Restore requires space for the download and extracted files. Full -upload readback and restores incur storage reads and applicable transfer charges. -Failed remote cleanup does not prevent unrelated local archive jobs from being -processed once the failed cleanup request returns. - -## Optional connectivity check - -Run `go run ./cmd/artex-archive-check` with the archive environment to upload, -verify, download and delete a small unique object. It performs real bucket -operations but does not touch task data, and reports cleanup failures. - -Automated tests use HTTP fixtures for S3 and metadata credentials, plus isolated -PostgreSQL. They do not establish compatibility with a live cloud bucket. diff --git a/server/server.go b/server/server.go index 7d750d3..baf4423 100644 --- a/server/server.go +++ b/server/server.go @@ -23,7 +23,6 @@ import ( "time" "github.com/Autumn-27/artex/agent" - "github.com/Autumn-27/artex/archivestore" "github.com/Autumn-27/artex/db" "github.com/Autumn-27/artex/egress" "github.com/Autumn-27/artex/intercept" @@ -145,11 +144,9 @@ type Server struct { // Cold task archives run through one persistent FIFO worker. The buffered wake // channel coalesces enqueue bursts; the database remains the source of truth. - archiveWake chan struct{} - archiveWG sync.WaitGroup - archiveStore *archivestore.Store - archiveStoreErr error - side *sideQuestionState + archiveWake chan struct{} + archiveWG sync.WaitGroup + side *sideQuestionState } // provEntry is a cached provider + its config for one LLM profile id. diff --git a/server/task_archive_interrupt_test.go b/server/task_archive_interrupt_test.go deleted file mode 100644 index 2f4a44e..0000000 --- a/server/task_archive_interrupt_test.go +++ /dev/null @@ -1,291 +0,0 @@ -package server - -import ( - "context" - "fmt" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strconv" - "sync" - "testing" - - "github.com/Autumn-27/artex/archivestore" - pgdb "github.com/Autumn-27/artex/db" -) - -// Recreate persisted states at operation boundaries, then run the same recovery -// functions used at startup. PostgreSQL, files, and HTTP object storage are real -// isolated fixtures; no production failpoint hooks are needed. -func TestRemoteArchiveInterruptionRecovery(t *testing.T) { - if os.Getenv("ARTEX_PG_DSN") == "" { - t.Skip("requires isolated PostgreSQL") - } - phases := []string{ - "staged_before_compression", "compressed_before_journal", "journal_before_upload", - "uploaded_before_verification", "verified_before_database", - "database_before_local_cleanup", "package_cleaned_before_stage_cleanup", - "restore_download_before_install", "restore_files_before_database", "restore_database_before_object_delete", "restore_object_deleted_before_stub", - "delete_journal_before_database", "delete_database_before_object", - } - for _, phase := range phases { - t.Run(phase, func(t *testing.T) { - m, err := NewManager(t.TempDir(), "") - if err != nil { - t.Fatal(err) - } - defer m.Close() - store, objects := newRecoveryObjectStore(t) - s := &Server{ctx: context.Background(), m: m, archiveStore: store} - task, err := m.CreateTask("interruption "+phase, "preserve source", nil, 0, 0) - if err != nil { - t.Fatal(err) - } - id, _ := strconv.ParseInt(task.ID, 10, 64) - defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }() - if err := m.pg.SetPaused(id, true); err != nil { - t.Fatal(err) - } - job, err := m.pg.QueueTaskArchive(id) - if err != nil { - t.Fatal(err) - } - if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Archiving); err != nil { - t.Fatal(err) - } - source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt") - if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(source, []byte("original task data"), 0600); err != nil { - t.Fatal(err) - } - snapshot, err := m.pg.SnapshotTaskArchive(id) - if err != nil { - t.Fatal(err) - } - stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID) - if err != nil { - t.Fatal(err) - } - packagePath := taskArchivePath(m.dir, job.ID, task.ID) - var original, compressed int64 - var checksum, uri, journal string - if phase != "staged_before_compression" { - original, compressed, checksum, err = writeTaskArchivePackage(packagePath, stage.payload, snapshot) - if err != nil { - t.Fatal(err) - } - uri = store.URI(id, job.ID, checksum) - } - if phase != "staged_before_compression" && phase != "compressed_before_journal" { - journal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, LocalPath: packagePath, Operation: "upload"}) - if err != nil { - t.Fatal(err) - } - } - committed := phase != "staged_before_compression" && phase != "compressed_before_journal" && - phase != "journal_before_upload" && phase != "uploaded_before_verification" && phase != "verified_before_database" - if phase == "uploaded_before_verification" { - data, err := os.ReadFile(packagePath) - if err != nil { - t.Fatal(err) - } - objects.put(uri, data) - } else if phase == "verified_before_database" || committed { - if err := store.Upload(s.ctx, uri, packagePath, checksum); err != nil { - t.Fatal(err) - } - } - if committed { - if err := m.pg.CompleteTaskArchive(job.ID, snapshot, uri, checksum, original, compressed); err != nil { - t.Fatal(err) - } - } - afterArchive := phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || - phase == "restore_object_deleted_before_stub" || phase == "delete_journal_before_database" || phase == "delete_database_before_object" - if phase == "package_cleaned_before_stage_cleanup" || afterArchive { - if err := os.Remove(packagePath); err != nil { - t.Fatal(err) - } - if err := os.Remove(journal); err != nil { - t.Fatal(err) - } - } - if afterArchive { - if err := stage.commit(); err != nil { - t.Fatal(err) - } - } - restored := false - remoteDeleted := false - if phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" { - if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Restoring); err != nil { - t.Fatal(err) - } - parent := filepath.Join(taskArchiveRoot(m.dir), ".restore") - if err := os.MkdirAll(parent, 0700); err != nil { - t.Fatal(err) - } - extracted, err := os.MkdirTemp(parent, "interrupted-") - if err != nil { - t.Fatal(err) - } - downloaded := filepath.Join(extracted, "download.tar.zst") - if err := store.Download(s.ctx, uri, downloaded, checksum, compressed); err != nil { - t.Fatal(err) - } - if err := extractTaskArchivePackage(downloaded, checksum, extracted); err != nil { - t.Fatal(err) - } - if phase != "restore_download_before_install" { - if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID); err != nil { - t.Fatal(err) - } - } - if phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" { - if _, err := m.pg.RestoreTaskArchive(job.ID, snapshot, 0); err != nil { - t.Fatal(err) - } - restored = true - } - if phase == "restore_object_deleted_before_stub" { - if err := store.Delete(s.ctx, uri); err != nil { - t.Fatal(err) - } - remoteDeleted = true - } - } - deleting := phase == "delete_journal_before_database" || phase == "delete_database_before_object" - if deleting { - if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Deleting); err != nil { - t.Fatal(err) - } - if _, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, Operation: "delete"}); err != nil { - t.Fatal(err) - } - if phase == "delete_database_before_object" { - if err := m.pg.DeleteTaskArchiveStub(job.ID); err != nil { - t.Fatal(err) - } - remoteDeleted = true - } - } - if err := m.pg.RecoverTaskArchiveJobs(); err != nil { - t.Fatal(err) - } - if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { - t.Fatal(err) - } - if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil { - t.Fatal(err) - } - if err := s.recoverRemoteArchiveJournals(); err != nil { - t.Fatal(err) - } - wantSource := !committed || restored - data, fileErr := os.ReadFile(source) - if wantSource && (fileErr != nil || string(data) != "original task data") { - t.Fatalf("source data lost: %v", fileErr) - } - if !wantSource && !os.IsNotExist(fileErr) { - t.Fatalf("archived source resurrected: %v", fileErr) - } - if _, err := os.Stat(packagePath); !os.IsNotExist(err) { - t.Fatalf("temporary package remains: %v", err) - } - if got, want := objects.has(uri), committed && !remoteDeleted; got != want { - t.Fatalf("remote object exists=%v, want=%v", got, want) - } - if committed && !deleting { - actual, err := m.pg.IsTaskArchiveRestored(job.ID) - if err != nil || actual != restored { - t.Fatalf("restored=%v want=%v err=%v", actual, restored, err) - } - } - // A second startup must be harmless at every boundary. - if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { - t.Fatal(err) - } - if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil { - t.Fatal(err) - } - if err := s.recoverRemoteArchiveJournals(); err != nil { - t.Fatal(err) - } - }) - } -} - -type recoveryObjects struct { - mu sync.Mutex - data map[string][]byte -} - -func (o *recoveryObjects) put(uri string, data []byte) { - o.mu.Lock() - defer o.mu.Unlock() - // Test URIs always use the fixture bucket with path-style addressing. - o.data["/test/"+uri[len("s3://test/"):]] = append([]byte(nil), data...) -} -func (o *recoveryObjects) has(uri string) bool { - o.mu.Lock() - defer o.mu.Unlock() - if uri == "" { - return false - } - _, ok := o.data["/test/"+uri[len("s3://test/"):]] - return ok -} -func newRecoveryObjectStore(t *testing.T) (*archivestore.Store, *recoveryObjects) { - t.Helper() - objects := &recoveryObjects{data: map[string][]byte{}} - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - objects.mu.Lock() - defer objects.mu.Unlock() - data, exists := objects.data[r.URL.Path] - w.Header().Set("ETag", `"recovery-etag"`) - switch r.Method { - case "PUT": - if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" { - t.Error("missing SSE") - } - objects.data[r.URL.Path], _ = io.ReadAll(r.Body) - case "HEAD": - if !exists { - w.WriteHeader(404) - return - } - w.Header().Set("Content-Length", strconv.Itoa(len(data))) - w.Header().Set("X-Amz-Server-Side-Encryption", "AES256") - case "GET": - if !exists { - w.WriteHeader(404) - return - } - var start, end int - if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end >= len(data) || r.Header.Get("If-Match") != `"recovery-etag"` { - w.WriteHeader(416) - return - } - w.Header().Set("Content-Length", strconv.Itoa(end-start+1)) - w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) - w.WriteHeader(206) - _, _ = w.Write(data[start : end+1]) - case "DELETE": - delete(objects.data, r.URL.Path) - w.WriteHeader(204) - default: - t.Errorf("unexpected object request %s", r.Method) - w.WriteHeader(400) - } - })) - t.Cleanup(srv.Close) - store, err := archivestore.New(archivestore.Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) - if err != nil { - t.Fatal(err) - } - return store, objects -} diff --git a/server/task_archive_package.go b/server/task_archive_package.go index e365ea3..f23c4c8 100644 --- a/server/task_archive_package.go +++ b/server/task_archive_package.go @@ -428,7 +428,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { errs = append(errs, getErr) continue } - if archive != nil && archive.ArchivePath != "" && archive.SHA256 != "" { + if archive != nil && archive.State == pgdb.ArchiveReady { if err := os.RemoveAll(root); err != nil { errs = append(errs, err) } @@ -437,16 +437,9 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { stage := &taskArchiveFileStage{root: root, payload: filepath.Join(root, "payload"), journal: journal} if err := stage.rollback(); err != nil { errs = append(errs, err) - continue } - // The process can exit after compression but before an upload journal or - // database location exists. Remove that temporary package only after the - // source-file rollback succeeds, for both local and remote backends. - packagePath := taskArchivePath(dataDir, journal.ArchiveID, journal.TaskID) - if err := validateArchivePath(dataDir, packagePath); err != nil { - errs = append(errs, err) - } else if err := removeLocalArchivePackage(packagePath); err != nil { - errs = append(errs, err) + if archive != nil && archive.ArchivePath != "" { + _ = os.Remove(archive.ArchivePath) } } return errors.Join(errs...) diff --git a/server/task_archive_remote.go b/server/task_archive_remote.go deleted file mode 100644 index 87e230f..0000000 --- a/server/task_archive_remote.go +++ /dev/null @@ -1,171 +0,0 @@ -package server - -import ( - "encoding/json" - "errors" - "fmt" - "log" - "os" - "path/filepath" - "strings" - - pgdb "github.com/Autumn-27/artex/db" -) - -// An outbox bridges the local PostgreSQL transaction and object storage. Never -// delete the only package before metadata permits it. Unfinished cleanup is -// retried after a restart (and before subsequent jobs). -type remoteArchiveJournal struct { - ArchiveID int64 `json:"archive_id"` - URI string `json:"uri"` - LocalPath string `json:"local_path,omitempty"` - Operation string `json:"operation"` -} - -func (s *Server) validateStoredArchive(uri string) error { - if strings.HasPrefix(uri, "s3://") { - return s.archiveStore.Validate(uri) - } - return validateArchivePath(s.m.dir, uri) -} - -func removeLocalArchivePackage(filename string) error { - if err := os.Remove(filename); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -func (s *Server) removeStoredArchive(uri string) error { - if err := s.validateStoredArchive(uri); err != nil { - return err - } - if strings.HasPrefix(uri, "s3://") { - return s.archiveStore.Delete(s.ctx, uri) - } - return removeLocalArchivePackage(uri) -} - -func (s *Server) writeRemoteArchiveJournal(j remoteArchiveJournal) (string, error) { - dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote") - if err := os.MkdirAll(dir, archiveDirMode); err != nil { - return "", err - } - f, err := os.CreateTemp(dir, ".journal-") - if err != nil { - return "", err - } - tmp := f.Name() - defer os.Remove(tmp) - encodeErr := json.NewEncoder(f).Encode(j) - syncErr := f.Sync() - if err := errors.Join(encodeErr, syncErr, f.Close()); err != nil { - return "", err - } - name := tmp + ".json" - if err := os.Rename(tmp, name); err != nil { - return "", err - } - // Persist the directory entry before an upload or database deletion starts. - d, err := os.Open(dir) - if err != nil { - return "", err - } - if err := errors.Join(d.Sync(), d.Close()); err != nil { - return "", err - } - // The .remote directory itself may have just been created. - parent, err := os.Open(filepath.Dir(dir)) - if err != nil { - return "", err - } - return name, errors.Join(parent.Sync(), parent.Close()) -} - -func (s *Server) recoverRemoteArchiveJournals() error { - dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote") - entries, err := os.ReadDir(dir) - if os.IsNotExist(err) { - return nil - } - if err != nil { - return err - } - var errs []error - for _, entry := range entries { - if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { - continue - } - name := filepath.Join(dir, entry.Name()) - if err := s.recoverRemoteArchiveJournal(name); err != nil { - errs = append(errs, err) - } - } - return errors.Join(errs...) -} - -func (s *Server) recoverRemoteArchiveJournal(name string) error { - raw, err := os.ReadFile(name) - if err != nil { - return err - } - var j remoteArchiveJournal - if err := json.Unmarshal(raw, &j); err != nil { - return err - } - if j.ArchiveID <= 0 { - return errors.New("invalid remote archive journal ID") - } - if err := s.archiveStore.Validate(j.URI); err != nil { - return err - } - item, err := s.m.pg.GetTaskArchive(j.ArchiveID) - if err != nil { - return err - } - switch j.Operation { - case "upload": - if item == nil || item.ArchivePath != j.URI { - if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil { - return err - } - } - if err := validateArchivePath(s.m.dir, j.LocalPath); err != nil { - return err - } - // An old failed upload may outlive a retry after switching back to - // local storage. The retry reuses this pathname: keep its committed - // package instead of treating it as the old upload's temporary copy. - if item == nil || item.ArchivePath != j.LocalPath { - if err := removeLocalArchivePackage(j.LocalPath); err != nil { - return err - } - } - case "delete": - if item == nil { - if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil { - return err - } - } - default: - return fmt.Errorf("unknown remote archive operation %q", j.Operation) - } - return os.Remove(name) -} - -func (s *Server) deleteRemoteTaskArchive(job *pgdb.TaskArchive) error { - journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, Operation: "delete"}) - if err != nil { - return err - } - if err := s.m.pg.DeleteTaskArchiveStub(job.ID); err != nil { - // A connection failure can leave the transaction's commit ambiguous. - // Recovery consults PostgreSQL before deciding whether to delete. - return err - } - if err := s.recoverRemoteArchiveJournal(journal); err != nil { - // The task is gone; the durable outbox retries the remaining object cleanup. - log.Printf("[task-archive] archive %d deleted; remote cleanup pending: %v", job.ID, err) - } - return nil -} diff --git a/server/task_archive_remote_test.go b/server/task_archive_remote_test.go deleted file mode 100644 index ac2a44d..0000000 --- a/server/task_archive_remote_test.go +++ /dev/null @@ -1,419 +0,0 @@ -package server - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strconv" - "strings" - "sync" - "sync/atomic" - "testing" - - "github.com/Autumn-27/artex/archivestore" - pgdb "github.com/Autumn-27/artex/db" -) - -func TestRemoteArchiveLifecycle(t *testing.T) { - if os.Getenv("ARTEX_PG_DSN") == "" { - t.Skip("requires isolated PostgreSQL") - } - t.Setenv("ARTEX_ARCHIVE_STORAGE", "local") - m, err := NewManager(t.TempDir(), "") - if err != nil { - t.Fatal(err) - } - defer m.Close() - ctx, cancel := context.WithCancel(context.Background()) - cancel() - s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir()) - s.archiveWG.Wait() - s.ctx = context.Background() - var mu sync.Mutex - objects := map[string][]byte{} - corrupt, failDelete := false, false - remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - mu.Lock() - defer mu.Unlock() - switch r.Method { - case "PUT": - objects[r.URL.Path], _ = io.ReadAll(r.Body) - w.Header().Set("ETag", `"test"`) - case "HEAD": - w.Header().Set("X-Amz-Server-Side-Encryption", "AES256") - w.Header().Set("ETag", `"test"`) - w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path]))) - case "GET": - data := objects[r.URL.Path] - if corrupt && len(data) > 0 { - data = bytes.Clone(data) - data[0] ^= 0xff - } - var start, end int64 - if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) || r.Header.Get("If-Match") != `"test"` { - w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) - return - } - w.Header().Set("ETag", `"test"`) - w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10)) - w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) - w.WriteHeader(http.StatusPartialContent) - _, _ = w.Write(data[start : end+1]) - case "DELETE": - if failDelete { - w.WriteHeader(403) - return - } - delete(objects, r.URL.Path) - w.WriteHeader(204) - default: - w.WriteHeader(400) - } - })) - defer remote.Close() - s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) - if err != nil { - t.Fatal(err) - } - task, err := m.CreateTask("S3 round trip", "preserve data", nil, 0, 0) - if err != nil { - t.Fatal(err) - } - id, _ := strconv.ParseInt(task.ID, 10, 64) - defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }() - if err := m.pg.SetPaused(id, true); err != nil { - t.Fatal(err) - } - file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "important.txt") - if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(file, []byte("keep this content"), 0600); err != nil { - t.Fatal(err) - } - pgTask, err := m.pg.GetTask(id) - if err != nil { - t.Fatal(err) - } - if _, err := m.pg.Exploration(pgTask.ExplorationID).AppendActivity(pgdb.Activity{Worker: "system", Kind: "system", Summary: "archive proof"}); err != nil { - t.Fatal(err) - } - job, err := m.pg.QueueTaskArchive(id) - if err != nil { - t.Fatal(err) - } - // A configured-but-invalid backend must fail before moving any source data. - s.archiveStoreErr = errors.New("invalid fixture configuration") - if err := s.runOneTaskArchiveJob(); !errors.Is(err, s.archiveStoreErr) { - t.Fatalf("configuration error was not returned: %v", err) - } - if got, err := os.ReadFile(file); err != nil || string(got) != "keep this content" { - t.Fatalf("bad configuration lost source: %v", err) - } - if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil { - t.Fatal("bad configuration lost task", err) - } - s.archiveStoreErr = nil - if _, err := m.pg.QueueTaskArchive(id); err != nil { - t.Fatal(err) - } - // A corrupt readback must roll back filesystem staging and leave PG intact. - mu.Lock() - corrupt = true - mu.Unlock() - if err := s.runOneTaskArchiveJob(); err == nil { - t.Fatal("expected integrity failure") - } - if got, _ := os.ReadFile(file); string(got) != "keep this content" { - t.Fatal("upload failure lost source") - } - if task, err := m.pg.GetTask(id); err != nil || task == nil { - t.Fatal("upload failure lost task") - } - mu.Lock() - corrupt = false - mu.Unlock() - if err := s.recoverRemoteArchiveJournals(); err != nil { - t.Fatal(err) - } - if _, err := m.pg.QueueTaskArchive(id); err != nil { - t.Fatal(err) - } - if err := s.runOneTaskArchiveJob(); err != nil { - t.Fatal(err) - } - job, err = m.pg.GetTaskArchive(job.ID) - if err != nil { - t.Fatal(err) - } - if job.State != pgdb.ArchiveReady || job.Storage != "s3" || !strings.HasPrefix(job.ArchivePath, "s3://") { - t.Fatalf("unexpected archive %+v", job) - } - if _, err := os.Stat(file); !os.IsNotExist(err) { - t.Fatal("workspace not released") - } - if _, err := os.Stat(taskArchivePath(m.dir, job.ID, task.ID)); !os.IsNotExist(err) { - t.Fatal("local package retained") - } - // Simulate the crash window after commit: cleanup keeps the remote object. - journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, LocalPath: taskArchivePath(m.dir, job.ID, task.ID), Operation: "upload"}) - if err != nil { - t.Fatal(err) - } - if err := s.recoverRemoteArchiveJournal(journal); err != nil { - t.Fatal(err) - } - // A corrupted download must leave the archive and its remote package intact. - mu.Lock() - corrupt = true - mu.Unlock() - if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { - t.Fatal(err) - } - if err := s.runOneTaskArchiveJob(); err == nil { - t.Fatal("expected corrupted restore to fail") - } - if _, err := os.Stat(file); !os.IsNotExist(err) { - t.Fatal("corrupted archive installed workspace files") - } - if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || restored { - t.Fatalf("corrupted archive restored database: %v %v", restored, err) - } - mu.Lock() - if len(objects) == 0 { - t.Error("corrupted restore deleted the only remote package") - } - corrupt = false - mu.Unlock() - // Restoration commits files and PG first. Failed remote deletion is retryable. - mu.Lock() - failDelete = true - mu.Unlock() - if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { - t.Fatal(err) - } - if err := s.runOneTaskArchiveJob(); err == nil { - t.Fatal("expected remote delete failure") - } - if got, _ := os.ReadFile(file); string(got) != "keep this content" { - t.Fatal("restored content differs") - } - if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || !restored { - t.Fatalf("PG restore %v %v", restored, err) - } - mu.Lock() - failDelete = false - mu.Unlock() - if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { - t.Fatal(err) - } - if err := s.runOneTaskArchiveJob(); err != nil { - t.Fatal(err) - } - if item, err := m.pg.GetTaskArchive(job.ID); err != nil || item != nil { - t.Fatal("restore did not consume stub") - } - var count int - if err := m.pg.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='archive proof'`, pgTask.ExplorationID).Scan(&count); err != nil || count != 1 { - t.Fatalf("activity restore count=%d err=%v", count, err) - } - // Re-archive, then crash/retry the deletion outbox after PG removed its stub. - job, err = m.pg.QueueTaskArchive(id) - if err != nil { - t.Fatal(err) - } - if err := s.runOneTaskArchiveJob(); err != nil { - t.Fatal(err) - } - mu.Lock() - failDelete = true - mu.Unlock() - if _, err := m.pg.QueueTaskArchiveDelete(job.ID); err != nil { - t.Fatal(err) - } - if err := s.runOneTaskArchiveJob(); err != nil { - t.Fatal(err) - } - mu.Lock() - failDelete = false - mu.Unlock() - if err := s.recoverRemoteArchiveJournals(); err != nil { - t.Fatal(err) - } - mu.Lock() - remaining := len(objects) - mu.Unlock() - if remaining != 0 { - t.Fatalf("left %d objects", remaining) - } -} - -// Once PostgreSQL has committed an archive location and checksum, a subsequent -// queued operation must not make startup recovery resurrect the old workspace. -func TestTaskArchiveStageRecoveryAfterCommit(t *testing.T) { - if os.Getenv("ARTEX_PG_DSN") == "" { - t.Skip("requires isolated PostgreSQL") - } - m, err := NewManager(t.TempDir(), "") - if err != nil { - t.Fatal(err) - } - defer m.Close() - task, err := m.CreateTask("remote stage recovery", "test", nil, 0, 0) - if err != nil { - t.Fatal(err) - } - taskID, _ := strconv.ParseInt(task.ID, 10, 64) - defer func() { - _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID) - _ = m.pg.DeleteTask(taskID) - }() - if err := m.pg.SetPaused(taskID, true); err != nil { - t.Fatal(err) - } - job, err := m.pg.QueueTaskArchive(taskID) - if err != nil { - t.Fatal(err) - } - pgTask, err := m.pg.GetTask(taskID) - if err != nil { - t.Fatal(err) - } - for _, backend := range []string{"local", "s3"} { - for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} { - t.Run(backend+"/"+state, func(t *testing.T) { - committed := state != pgdb.ArchiveQueued - checksum := "" - if committed { - checksum = strings.Repeat("a", 64) - } - packagePath := taskArchivePath(m.dir, job.ID, task.ID) - if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(packagePath, []byte("verified package"), 0600); err != nil { - t.Fatal(err) - } - location := packagePath - if backend == "s3" { - location = "s3://test/tasks/package.tar.zst" - } - if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, location, checksum); err != nil { - t.Fatal(err) - } - file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt") - if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil { - t.Fatal(err) - } - stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID) - if err != nil { - t.Fatal(err) - } - if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(stage.root); !os.IsNotExist(err) { - t.Fatalf("staging cleanup incomplete: %v", err) - } - if backend == "local" && committed { - if data, err := os.ReadFile(packagePath); err != nil || string(data) != "verified package" { - t.Fatal("committed local package lost", err) - } - } else if !committed { - if _, err := os.Stat(packagePath); !os.IsNotExist(err) { - t.Fatal("uncommitted temporary package retained", err) - } - } - if committed { - if _, err := os.Stat(file); !os.IsNotExist(err) { - t.Fatalf("resurrected committed workspace: %v", err) - } - } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" { - t.Fatalf("uncommitted archive lost source: %v", err) - } - }) - } - } -} - -// Failed remote cleanup can survive switching to local storage and retrying the -// same archive ID. Its journal must never remove the newly committed local copy. -func TestRemoteArchiveJournalPreservesLocalRetry(t *testing.T) { - if os.Getenv("ARTEX_PG_DSN") == "" { - t.Skip("requires isolated PostgreSQL") - } - m, err := NewManager(t.TempDir(), "") - if err != nil { - t.Fatal(err) - } - defer m.Close() - task, err := m.CreateTask("local retry", "test", nil, 0, 0) - if err != nil { - t.Fatal(err) - } - taskID, _ := strconv.ParseInt(task.ID, 10, 64) - defer func() { - _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID) - _ = m.pg.DeleteTask(taskID) - }() - if err := m.pg.SetPaused(taskID, true); err != nil { - t.Fatal(err) - } - job, err := m.pg.QueueTaskArchive(taskID) - if err != nil { - t.Fatal(err) - } - filename := taskArchivePath(m.dir, job.ID, task.ID) - if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filename, []byte("committed local archive"), 0600); err != nil { - t.Fatal(err) - } - if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, pgdb.ArchiveReady, filename, strings.Repeat("a", 64)); err != nil { - t.Fatal(err) - } - var deletes atomic.Int32 - remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodDelete { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - deletes.Add(1) - w.WriteHeader(http.StatusNoContent) - })) - defer remote.Close() - store, err := archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) - if err != nil { - t.Fatal(err) - } - s := &Server{ctx: context.Background(), m: m, archiveStore: store} - journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ - ArchiveID: job.ID, URI: store.URI(taskID, job.ID, strings.Repeat("b", 64)), - LocalPath: filename, Operation: "upload", - }) - if err != nil { - t.Fatal(err) - } - if err := s.recoverRemoteArchiveJournal(journal); err != nil { - t.Fatal(err) - } - if data, err := os.ReadFile(filename); err != nil || string(data) != "committed local archive" { - t.Fatalf("old upload cleanup removed local retry: %v", err) - } - if deletes.Load() != 1 { - t.Fatal("uncommitted remote object was not removed") - } - if _, err := os.Stat(journal); !os.IsNotExist(err) { - t.Fatalf("finished journal remains: %v", err) - } -} diff --git a/server/task_archives.go b/server/task_archives.go index 9a7af52..836c2d8 100644 --- a/server/task_archives.go +++ b/server/task_archives.go @@ -14,7 +14,6 @@ import ( "time" "github.com/Autumn-27/artex/agent" - "github.com/Autumn-27/artex/archivestore" pgdb "github.com/Autumn-27/artex/db" "github.com/Autumn-27/artex/traffic" ) @@ -38,12 +37,6 @@ func (s *Server) startTaskArchiveWorker() { if s.m == nil || s.m.pg == nil { return } - s.archiveStore, s.archiveStoreErr = archivestore.FromEnv() - if s.archiveStoreErr != nil { - log.Printf("[task-archive] storage configuration: %v", s.archiveStoreErr) - } else if s.archiveStore != nil { - log.Printf("[task-archive] new archives use S3 object storage with full SHA-256 readback verification") - } if err := s.m.pg.RecoverTaskArchiveJobs(); err != nil { log.Printf("[task-archive] recover jobs: %v", err) } @@ -95,14 +88,6 @@ func (s *Server) notifyTaskArchiveWorker() { } func (s *Server) runOneTaskArchiveJob() error { - if s.ctx.Err() != nil { - return s.ctx.Err() - } - // Retry deferred remote cleanup before claiming more work. Failed cleanup - // keeps its durable journal and does not block unrelated local archives. - if err := s.recoverRemoteArchiveJournals(); err != nil { - log.Printf("[task-archive] deferred object cleanup: %v", err) - } job, err := s.m.pg.ClaimTaskArchiveJob(s.ctx) if err != nil || job == nil { return err @@ -130,9 +115,6 @@ func (s *Server) runOneTaskArchiveJob() error { } func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { - if s.archiveStoreErr != nil { - return s.archiveStoreErr - } taskID := strconv.FormatInt(job.TaskID, 10) if !s.beginTaskDelete(taskID) { return errors.New("task is already undergoing another archive or delete operation") @@ -217,21 +199,6 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { _ = os.Remove(archivePath) } }() - storedPath := archivePath - var remoteJournal string - if s.archiveStore != nil { - storedPath = s.archiveStore.URI(job.TaskID, job.ID, checksum) - remoteJournal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ - ArchiveID: job.ID, URI: storedPath, LocalPath: archivePath, Operation: "upload", - }) - if err != nil { - return err - } - _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "upload_and_verify", 65) - if err := s.archiveStore.Upload(s.ctx, storedPath, archivePath, checksum); err != nil { - return err - } - } var trafficStage *traffic.HostDeleteStage if s.m.traffic != nil && len(snapshot.ExclusiveHosts) > 0 { _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_traffic", 68) @@ -246,18 +213,11 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { }() } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_database", 78) - if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, storedPath, checksum, originalSize, compressedSize); err != nil { + if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, archivePath, checksum, originalSize, compressedSize); err != nil { return err } committed = true - removePackage = s.archiveStore != nil - if remoteJournal != "" { - // The journal also covers a crash after database commit but before local - // package removal. Keep it if cleanup fails, so the worker can retry. - if err := removeLocalArchivePackage(archivePath); err == nil { - _ = os.Remove(remoteJournal) - } - } + removePackage = false if trafficStage != nil { if err := trafficStage.Commit(); err != nil { warning := "archive completed, but cleaning the exclusive traffic hot store failed (the archive package can still restore it): " + err.Error() @@ -279,7 +239,7 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { } func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { - if err := s.validateStoredArchive(job.ArchivePath); err != nil { + if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil { return err } // A prior attempt may have committed PostgreSQL and files, then failed only @@ -287,7 +247,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { if restored, err := s.m.pg.IsTaskArchiveRestored(job.ID); err != nil { return err } else if restored { - if err := s.removeStoredArchive(job.ArchivePath); err != nil { + if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) { return err } if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil { @@ -306,15 +266,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { return err } defer func() { _ = os.RemoveAll(extracted) }() - packagePath := job.ArchivePath - if strings.HasPrefix(packagePath, "s3://") { - _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "download_and_verify", 12) - packagePath = filepath.Join(extracted, "download.tar.zst") - if err := s.archiveStore.Download(s.ctx, job.ArchivePath, packagePath, job.SHA256, job.CompressedSize); err != nil { - return err - } - } - if err := extractTaskArchivePackage(packagePath, job.SHA256, extracted); err != nil { + if err := extractTaskArchivePackage(job.ArchivePath, job.SHA256, extracted); err != nil { return err } manifestRaw, err := os.ReadFile(filepath.Join(extracted, "manifest.json")) @@ -404,7 +356,7 @@ func (s *Server) restoreTaskArchivePayload(job *pgdb.TaskArchive, snapshot *pgdb return err } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "consume_package", 95) - if err := s.removeStoredArchive(job.ArchivePath); err != nil { + if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) { return err } if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil { @@ -441,12 +393,9 @@ func (s *Server) loadRestoredTask(taskID int64) { } func (s *Server) deleteTaskArchive(job *pgdb.TaskArchive) error { - if err := s.validateStoredArchive(job.ArchivePath); err != nil { + if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil { return err } - if strings.HasPrefix(job.ArchivePath, "s3://") { - return s.deleteRemoteTaskArchive(job) - } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "stage_package_delete", 35) staged, moved, err := stageTaskArchivePackageDelete(job.ArchivePath, job.ID) if err != nil { diff --git a/web/src/app/(main)/function/tasks/page.tsx b/web/src/app/(main)/function/tasks/page.tsx index c2f9cc3..6b8ba8e 100644 --- a/web/src/app/(main)/function/tasks/page.tsx +++ b/web/src/app/(main)/function/tasks/page.tsx @@ -1494,7 +1494,7 @@ function ArchiveConfirmDialog({ {count === 1 ? "Archive task" : `Archive ${count} tasks`} - Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into the configured archive storage. Once archived, a task can be restored from "Archived". Restoring requires free space on this server. + Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into local cold storage. Once archived, a task can be restored from "Archived". @@ -1934,7 +1934,6 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) { title={`Before compression: ${formatArchiveBytes(archive.original_size)}`} > {archiveCompressionLabel(archive)} -
{archive.storage === "s3" ? "Object storage" : "Local storage"}
{archiveDataTotal(archive).toLocaleString()} @@ -1953,7 +1952,7 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {

))} - {archive.phase === "upload_and_verify" ? "Uploading and verifying" : archive.phase === "download_and_verify" ? "Downloading and verifying" : archive.phase} · {archive.progress}% + {archive.phase} · {archive.progress}%
diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index f31b18c..cd53b64 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -101,7 +101,6 @@ export interface TaskArchiveTokenStats { } export interface TaskArchive { - storage?: "local" | "s3"; id: number; task_id: number; state: TaskArchiveState;