From a40781ae1fc0d65d7045e669e5e4d87465e69e77 Mon Sep 17 00:00:00 2001
From: Chubboy Ren
Date: Wed, 18 Nov 2026 02:05:17 +0000
Subject: [PATCH] refactor: split archive backend from server and UI
integration
---
README.md | 6 -
db/task_archives.go | 5 -
docs/task-archive-storage.md | 160 --------
server/server.go | 9 +-
server/task_archive_interrupt_test.go | 291 --------------
server/task_archive_package.go | 13 +-
server/task_archive_remote.go | 171 ---------
server/task_archive_remote_test.go | 419 ---------------------
server/task_archives.go | 65 +---
web/src/app/(main)/function/tasks/page.tsx | 5 +-
web/src/lib/types.ts | 1 -
11 files changed, 15 insertions(+), 1130 deletions(-)
delete mode 100644 docs/task-archive-storage.md
delete mode 100644 server/task_archive_interrupt_test.go
delete mode 100644 server/task_archive_remote.go
delete mode 100644 server/task_archive_remote_test.go
diff --git a/README.md b/README.md
index 2d91397..ad8efec 100644
--- a/README.md
+++ b/README.md
@@ -421,12 +421,6 @@ runtime supports it.
`/chat/completions` (or `/responses` for `openai-responses`), so their base **does** end in `/v1`
(for example `https://api.openai.com/v1`).
-**Task archive storage**: local disk by default. Set `ARTEX_ARCHIVE_STORAGE=s3` to
-store cold task packages in an S3-compatible bucket and release their local
-package space after verification. Existing local archives remain usable. See
-[Task archive storage](docs/task-archive-storage.md) for configuration, permissions,
-integrity checks, temporary disk requirements and recovery.
-
**Concurrency**: work agents per task, in "System Settings" (default 3).
**Untrusted-output boundary (on by default)**: tool results for the worker, planner, chat and main
diff --git a/db/task_archives.go b/db/task_archives.go
index 9f07341..bb9aae2 100644
--- a/db/task_archives.go
+++ b/db/task_archives.go
@@ -60,7 +60,6 @@ type TaskArchive struct {
Warnings json.RawMessage `json:"warnings"`
FormatVersion int `json:"format_version"`
ArchivePath string `json:"-"`
- Storage string `json:"storage"`
SHA256 string `json:"sha256,omitempty"`
OriginalSize int64 `json:"original_size"`
CompressedSize int64 `json:"compressed_size"`
@@ -162,10 +161,6 @@ func scanTaskArchive(sc interface{ Scan(...any) error }) (*TaskArchive, error) {
if err != nil {
return nil, err
}
- item.Storage = "local"
- if strings.HasPrefix(item.ArchivePath, "s3://") {
- item.Storage = "s3"
- }
if err := json.Unmarshal([]byte(sources), &item.SourceTaskIDs); err != nil {
return nil, err
}
diff --git a/docs/task-archive-storage.md b/docs/task-archive-storage.md
deleted file mode 100644
index 113921a..0000000
--- a/docs/task-archive-storage.md
+++ /dev/null
@@ -1,160 +0,0 @@
-# Task archive storage
-
-Task archives use local disk by default. The optional S3-compatible backend moves
-cold task packages to object storage while retaining Archive, Restore and Delete.
-Active workspaces and PostgreSQL remain local. This is task cold storage, not a
-full-installation backup; a recovery plan must preserve the database, data
-directory, bucket objects, and the credentials or roles needed to read them.
-
-## Configuration
-
-Use a private bucket and a separate prefix for each ARTEX installation. Configure
-the bucket's default encryption to match the selected mode below, and enforce
-that mode/key with a bucket policy. ARTEX does not provision or inspect bucket
-encryption settings.
-
-Set the following server environment and restart ARTEX:
-
-```dotenv
-ARTEX_ARCHIVE_STORAGE=s3
-ARTEX_ARCHIVE_S3_ENDPOINT=https://s3.us-east-1.amazonaws.com
-ARTEX_ARCHIVE_S3_REGION=us-east-1
-ARTEX_ARCHIVE_S3_BUCKET=your-artex-archives
-ARTEX_ARCHIVE_S3_PREFIX=artex/tasks
-ARTEX_ARCHIVE_S3_SSE=AES256
-```
-
-With an EC2 instance role, ECS task role or other SDK-supported credential source,
-no archive-specific access keys are needed.
-
-| Variable | Meaning |
-| --- | --- |
-| `ARTEX_ARCHIVE_STORAGE` | `local` (default) or `s3`. |
-| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local tests. |
-| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. |
-| `ARTEX_ARCHIVE_S3_BUCKET` | Required bucket name. |
-| `ARTEX_ARCHIVE_S3_PREFIX` | Object prefix; defaults to `artex/tasks`. |
-| `ARTEX_ARCHIVE_S3_SSE` | `AES256` (default, SSE-S3) or `aws:kms` (SSE-KMS). There is no unencrypted mode. |
-| `ARTEX_ARCHIVE_S3_KMS_KEY_ID` | Optional KMS key ARN/ID/alias; valid only with `aws:kms`. Omit to use the provider's managed KMS key. |
-| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Optional dedicated credential override; must be supplied with the secret key. |
-| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Secret for the dedicated override. |
-| `ARTEX_ARCHIVE_S3_SESSION_TOKEN` | Optional session token for dedicated temporary credentials. |
-| `ARTEX_ARCHIVE_S3_PATH_STYLE` | `true` for path-style servers; otherwise virtual-hosted addressing is used. |
-
-An invalid mode or partial credential override fails configuration instead of
-silently falling back to local storage or ambient credentials. Preserve the
-bucket, endpoint, prefix and read/delete access while remote archives remain.
-Older local archives remain usable; they are not uploaded automatically.
-
-## Encryption and permissions
-
-Every single-object upload and multipart initiation explicitly requests the
-configured server-side encryption. Upload verification also requires HEAD to
-report the requested mode, and a KMS key identifier for SSE-KMS, before accepting
-the object. Size and full SHA-256 readback remain mandatory; encryption metadata
-does not replace content verification.
-
-For SSE-KMS, set both the bucket's default encryption and ARTEX to `aws:kms`;
-set `ARTEX_ARCHIVE_S3_KMS_KEY_ID` to the intended key when using a customer-managed
-key. An explicit `AES256` request takes precedence over a bucket's KMS default,
-so do not leave ARTEX on SSE-S3 when the bucket is intended to use KMS.
-
-The principal needs object upload, read (HEAD and ranged GET), delete and
-multipart upload/abort permissions. KMS uploads/readback also need the applicable
-key-policy and IAM permissions for `kms:GenerateDataKey` and `kms:Decrypt`.
-Retain old KMS keys and decrypt permissions when changing the key or mode; restore
-does not require an older object to use the current upload policy.
-
-Compatible providers must support the requested SSE mode and its HEAD metadata,
-plus conditional ranged reads. A provider that ignores the encryption request
-or cannot confirm it is rejected; the original task is retained. Local source and
-temporary files keep their existing local-disk protection and restricted modes.
-
-See the official [PutObject encryption and permission contract](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html)
-and [HEAD encryption response rules](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html).
-
-## Credential sources
-
-Without the archive-specific override, ARTEX uses the AWS SDK default credential
-chain, including standard AWS environment variables, shared profiles, web
-identity, ECS task credentials and EC2 instance credentials through IMDS. Role
-credentials retain the SDK's automatic refresh behavior. A named shared profile
-can be selected with `AWS_PROFILE`.
-
-For an external S3-compatible provider, dedicated keys can be supplied through
-the archive-specific variables. Both key fields are required; a session token
-is supported for temporary credentials. This explicit override takes precedence
-over ambient AWS credentials. Operator-supplied temporary credentials must be
-rotated by the operator; they are not an IAM role provider.
-
-Keep secrets in a protected service environment file outside the repository and
-frontend. A systemd `EnvironmentFile=` can point to a root-owned mode-0600 file.
-See the [AWS SDK credential-chain documentation](https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-gosdk.html).
-
-## Lifecycle
-
-1. Pause or finish a task and use its **Archive** action.
-2. ARTEX stages the source files and builds a local `.tar.zst` package. Large
- packages use multipart upload, initially 8 MiB parts with two upload workers;
- the SDK increases the part size for exceptionally large seekable files.
-3. ARTEX confirms encryption metadata and reads the complete object back,
- checking size and SHA-256. ETags and uploaded checksum metadata alone are
- insufficient.
-4. **Only after verification and a successful PostgreSQL commit** does ARTEX
- remove the local package and staged source files. The committed archive
- location is an `s3://` URI; there is no schema or archive-format migration.
-5. **Restore** downloads and verifies before installing files and database rows.
- The remote package is consumed only after restoration commits.
-6. **Delete** commits removal of the archive stub, then cleans up the object.
- A durable journal retains failed cleanup work.
-
-The archive list identifies **Object storage** and **Local storage** and shows
-upload/download verification phases. Reads use bounded 8 MiB ranges pinned to the
-object's ETag, with retries for interrupted ranges rather than the entire object.
-
-## Interruption recovery
-
-Source staging and restore journals live with task archives. Remote cleanup
-journals live under `data/archives/tasks/.remote`; they contain locations, not
-credentials. Keep this directory with PostgreSQL and the installation data.
-
-| Interruption | Recovery |
-| --- | --- |
-| During staging/compression, before an upload journal exists | Roll staged source files back. Remove any temporary package only after rollback succeeds. The task remains in PostgreSQL. |
-| After the upload journal, before successful verification | Roll source files back and clean up any uncommitted object/package. Failed cleanup retains its journal for retry. |
-| After verification, before the database commit | Treat the object as uncommitted: retain the original task and clean up the orphan object. Verification alone never authorizes local deletion. |
-| After the database commit, before package/staging cleanup | Keep the committed object and finish local cleanup. The persisted location and checksum establish commit, even if Restore or Delete has since been queued. |
-| Download/extraction before restored files are installed | Remove temporary extraction data; keep the remote object and archive stub for another attempt. |
-| Restored files installed, before database restoration commits | Roll back installed files using the restore journal. Retain the remote package. |
-| Database restoration committed, before remote cleanup | Recognize the already-restored task and finish cleanup without replaying database rows. |
-| Object consumed, before the restored archive stub is removed | Skip download/replay; repeat idempotent object deletion and remove the stub. |
-| Delete journal written, before database deletion commits | Keep the object while its archive stub exists. The queued Delete operation can retry. |
-| Database deletion committed, before object deletion succeeds | The durable outbox retains the URI and retries object deletion. |
-| An old failed upload survives a retry using local storage | Remove the abandoned remote object, but retain a local package referenced by the committed archive row. |
-
-The staging recovery rule also applies to local archives. Dedicated tests cover
-both storage modes with ready, restore-queued, delete-queued and uncommitted
-states. Additional PostgreSQL/HTTP-S3 tests reconstruct interrupted operations at
-13 persisted boundaries and repeat recovery to check idempotence.
-
-Handled multipart-upload failures are aborted by the SDK. Abrupt process death
-may leave unfinished multipart uploads; configure bucket lifecycle expiration
-for those parts. Existing object versions, if versioning is enabled, are governed
-by the bucket's version-retention policy.
-
-## Disk and transfer requirements
-
-Archive creation still requires temporary local space for the compressed
-package. Restore requires space for the download and extracted files. Full
-upload readback and restores incur storage reads and applicable transfer charges.
-Failed remote cleanup does not prevent unrelated local archive jobs from being
-processed once the failed cleanup request returns.
-
-## Optional connectivity check
-
-Run `go run ./cmd/artex-archive-check` with the archive environment to upload,
-verify, download and delete a small unique object. It performs real bucket
-operations but does not touch task data, and reports cleanup failures.
-
-Automated tests use HTTP fixtures for S3 and metadata credentials, plus isolated
-PostgreSQL. They do not establish compatibility with a live cloud bucket.
diff --git a/server/server.go b/server/server.go
index 7d750d3..baf4423 100644
--- a/server/server.go
+++ b/server/server.go
@@ -23,7 +23,6 @@ import (
"time"
"github.com/Autumn-27/artex/agent"
- "github.com/Autumn-27/artex/archivestore"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/egress"
"github.com/Autumn-27/artex/intercept"
@@ -145,11 +144,9 @@ type Server struct {
// Cold task archives run through one persistent FIFO worker. The buffered wake
// channel coalesces enqueue bursts; the database remains the source of truth.
- archiveWake chan struct{}
- archiveWG sync.WaitGroup
- archiveStore *archivestore.Store
- archiveStoreErr error
- side *sideQuestionState
+ archiveWake chan struct{}
+ archiveWG sync.WaitGroup
+ side *sideQuestionState
}
// provEntry is a cached provider + its config for one LLM profile id.
diff --git a/server/task_archive_interrupt_test.go b/server/task_archive_interrupt_test.go
deleted file mode 100644
index 2f4a44e..0000000
--- a/server/task_archive_interrupt_test.go
+++ /dev/null
@@ -1,291 +0,0 @@
-package server
-
-import (
- "context"
- "fmt"
- "io"
- "net/http"
- "net/http/httptest"
- "os"
- "path/filepath"
- "strconv"
- "sync"
- "testing"
-
- "github.com/Autumn-27/artex/archivestore"
- pgdb "github.com/Autumn-27/artex/db"
-)
-
-// Recreate persisted states at operation boundaries, then run the same recovery
-// functions used at startup. PostgreSQL, files, and HTTP object storage are real
-// isolated fixtures; no production failpoint hooks are needed.
-func TestRemoteArchiveInterruptionRecovery(t *testing.T) {
- if os.Getenv("ARTEX_PG_DSN") == "" {
- t.Skip("requires isolated PostgreSQL")
- }
- phases := []string{
- "staged_before_compression", "compressed_before_journal", "journal_before_upload",
- "uploaded_before_verification", "verified_before_database",
- "database_before_local_cleanup", "package_cleaned_before_stage_cleanup",
- "restore_download_before_install", "restore_files_before_database", "restore_database_before_object_delete", "restore_object_deleted_before_stub",
- "delete_journal_before_database", "delete_database_before_object",
- }
- for _, phase := range phases {
- t.Run(phase, func(t *testing.T) {
- m, err := NewManager(t.TempDir(), "")
- if err != nil {
- t.Fatal(err)
- }
- defer m.Close()
- store, objects := newRecoveryObjectStore(t)
- s := &Server{ctx: context.Background(), m: m, archiveStore: store}
- task, err := m.CreateTask("interruption "+phase, "preserve source", nil, 0, 0)
- if err != nil {
- t.Fatal(err)
- }
- id, _ := strconv.ParseInt(task.ID, 10, 64)
- defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }()
- if err := m.pg.SetPaused(id, true); err != nil {
- t.Fatal(err)
- }
- job, err := m.pg.QueueTaskArchive(id)
- if err != nil {
- t.Fatal(err)
- }
- if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Archiving); err != nil {
- t.Fatal(err)
- }
- source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt")
- if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil {
- t.Fatal(err)
- }
- if err := os.WriteFile(source, []byte("original task data"), 0600); err != nil {
- t.Fatal(err)
- }
- snapshot, err := m.pg.SnapshotTaskArchive(id)
- if err != nil {
- t.Fatal(err)
- }
- stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID)
- if err != nil {
- t.Fatal(err)
- }
- packagePath := taskArchivePath(m.dir, job.ID, task.ID)
- var original, compressed int64
- var checksum, uri, journal string
- if phase != "staged_before_compression" {
- original, compressed, checksum, err = writeTaskArchivePackage(packagePath, stage.payload, snapshot)
- if err != nil {
- t.Fatal(err)
- }
- uri = store.URI(id, job.ID, checksum)
- }
- if phase != "staged_before_compression" && phase != "compressed_before_journal" {
- journal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, LocalPath: packagePath, Operation: "upload"})
- if err != nil {
- t.Fatal(err)
- }
- }
- committed := phase != "staged_before_compression" && phase != "compressed_before_journal" &&
- phase != "journal_before_upload" && phase != "uploaded_before_verification" && phase != "verified_before_database"
- if phase == "uploaded_before_verification" {
- data, err := os.ReadFile(packagePath)
- if err != nil {
- t.Fatal(err)
- }
- objects.put(uri, data)
- } else if phase == "verified_before_database" || committed {
- if err := store.Upload(s.ctx, uri, packagePath, checksum); err != nil {
- t.Fatal(err)
- }
- }
- if committed {
- if err := m.pg.CompleteTaskArchive(job.ID, snapshot, uri, checksum, original, compressed); err != nil {
- t.Fatal(err)
- }
- }
- afterArchive := phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" ||
- phase == "restore_object_deleted_before_stub" || phase == "delete_journal_before_database" || phase == "delete_database_before_object"
- if phase == "package_cleaned_before_stage_cleanup" || afterArchive {
- if err := os.Remove(packagePath); err != nil {
- t.Fatal(err)
- }
- if err := os.Remove(journal); err != nil {
- t.Fatal(err)
- }
- }
- if afterArchive {
- if err := stage.commit(); err != nil {
- t.Fatal(err)
- }
- }
- restored := false
- remoteDeleted := false
- if phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" {
- if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Restoring); err != nil {
- t.Fatal(err)
- }
- parent := filepath.Join(taskArchiveRoot(m.dir), ".restore")
- if err := os.MkdirAll(parent, 0700); err != nil {
- t.Fatal(err)
- }
- extracted, err := os.MkdirTemp(parent, "interrupted-")
- if err != nil {
- t.Fatal(err)
- }
- downloaded := filepath.Join(extracted, "download.tar.zst")
- if err := store.Download(s.ctx, uri, downloaded, checksum, compressed); err != nil {
- t.Fatal(err)
- }
- if err := extractTaskArchivePackage(downloaded, checksum, extracted); err != nil {
- t.Fatal(err)
- }
- if phase != "restore_download_before_install" {
- if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID); err != nil {
- t.Fatal(err)
- }
- }
- if phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" {
- if _, err := m.pg.RestoreTaskArchive(job.ID, snapshot, 0); err != nil {
- t.Fatal(err)
- }
- restored = true
- }
- if phase == "restore_object_deleted_before_stub" {
- if err := store.Delete(s.ctx, uri); err != nil {
- t.Fatal(err)
- }
- remoteDeleted = true
- }
- }
- deleting := phase == "delete_journal_before_database" || phase == "delete_database_before_object"
- if deleting {
- if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Deleting); err != nil {
- t.Fatal(err)
- }
- if _, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, Operation: "delete"}); err != nil {
- t.Fatal(err)
- }
- if phase == "delete_database_before_object" {
- if err := m.pg.DeleteTaskArchiveStub(job.ID); err != nil {
- t.Fatal(err)
- }
- remoteDeleted = true
- }
- }
- if err := m.pg.RecoverTaskArchiveJobs(); err != nil {
- t.Fatal(err)
- }
- if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
- t.Fatal(err)
- }
- if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil {
- t.Fatal(err)
- }
- if err := s.recoverRemoteArchiveJournals(); err != nil {
- t.Fatal(err)
- }
- wantSource := !committed || restored
- data, fileErr := os.ReadFile(source)
- if wantSource && (fileErr != nil || string(data) != "original task data") {
- t.Fatalf("source data lost: %v", fileErr)
- }
- if !wantSource && !os.IsNotExist(fileErr) {
- t.Fatalf("archived source resurrected: %v", fileErr)
- }
- if _, err := os.Stat(packagePath); !os.IsNotExist(err) {
- t.Fatalf("temporary package remains: %v", err)
- }
- if got, want := objects.has(uri), committed && !remoteDeleted; got != want {
- t.Fatalf("remote object exists=%v, want=%v", got, want)
- }
- if committed && !deleting {
- actual, err := m.pg.IsTaskArchiveRestored(job.ID)
- if err != nil || actual != restored {
- t.Fatalf("restored=%v want=%v err=%v", actual, restored, err)
- }
- }
- // A second startup must be harmless at every boundary.
- if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
- t.Fatal(err)
- }
- if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil {
- t.Fatal(err)
- }
- if err := s.recoverRemoteArchiveJournals(); err != nil {
- t.Fatal(err)
- }
- })
- }
-}
-
-type recoveryObjects struct {
- mu sync.Mutex
- data map[string][]byte
-}
-
-func (o *recoveryObjects) put(uri string, data []byte) {
- o.mu.Lock()
- defer o.mu.Unlock()
- // Test URIs always use the fixture bucket with path-style addressing.
- o.data["/test/"+uri[len("s3://test/"):]] = append([]byte(nil), data...)
-}
-func (o *recoveryObjects) has(uri string) bool {
- o.mu.Lock()
- defer o.mu.Unlock()
- if uri == "" {
- return false
- }
- _, ok := o.data["/test/"+uri[len("s3://test/"):]]
- return ok
-}
-func newRecoveryObjectStore(t *testing.T) (*archivestore.Store, *recoveryObjects) {
- t.Helper()
- objects := &recoveryObjects{data: map[string][]byte{}}
- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- objects.mu.Lock()
- defer objects.mu.Unlock()
- data, exists := objects.data[r.URL.Path]
- w.Header().Set("ETag", `"recovery-etag"`)
- switch r.Method {
- case "PUT":
- if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" {
- t.Error("missing SSE")
- }
- objects.data[r.URL.Path], _ = io.ReadAll(r.Body)
- case "HEAD":
- if !exists {
- w.WriteHeader(404)
- return
- }
- w.Header().Set("Content-Length", strconv.Itoa(len(data)))
- w.Header().Set("X-Amz-Server-Side-Encryption", "AES256")
- case "GET":
- if !exists {
- w.WriteHeader(404)
- return
- }
- var start, end int
- if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end >= len(data) || r.Header.Get("If-Match") != `"recovery-etag"` {
- w.WriteHeader(416)
- return
- }
- w.Header().Set("Content-Length", strconv.Itoa(end-start+1))
- w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
- w.WriteHeader(206)
- _, _ = w.Write(data[start : end+1])
- case "DELETE":
- delete(objects.data, r.URL.Path)
- w.WriteHeader(204)
- default:
- t.Errorf("unexpected object request %s", r.Method)
- w.WriteHeader(400)
- }
- }))
- t.Cleanup(srv.Close)
- store, err := archivestore.New(archivestore.Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
- if err != nil {
- t.Fatal(err)
- }
- return store, objects
-}
diff --git a/server/task_archive_package.go b/server/task_archive_package.go
index e365ea3..f23c4c8 100644
--- a/server/task_archive_package.go
+++ b/server/task_archive_package.go
@@ -428,7 +428,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error {
errs = append(errs, getErr)
continue
}
- if archive != nil && archive.ArchivePath != "" && archive.SHA256 != "" {
+ if archive != nil && archive.State == pgdb.ArchiveReady {
if err := os.RemoveAll(root); err != nil {
errs = append(errs, err)
}
@@ -437,16 +437,9 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error {
stage := &taskArchiveFileStage{root: root, payload: filepath.Join(root, "payload"), journal: journal}
if err := stage.rollback(); err != nil {
errs = append(errs, err)
- continue
}
- // The process can exit after compression but before an upload journal or
- // database location exists. Remove that temporary package only after the
- // source-file rollback succeeds, for both local and remote backends.
- packagePath := taskArchivePath(dataDir, journal.ArchiveID, journal.TaskID)
- if err := validateArchivePath(dataDir, packagePath); err != nil {
- errs = append(errs, err)
- } else if err := removeLocalArchivePackage(packagePath); err != nil {
- errs = append(errs, err)
+ if archive != nil && archive.ArchivePath != "" {
+ _ = os.Remove(archive.ArchivePath)
}
}
return errors.Join(errs...)
diff --git a/server/task_archive_remote.go b/server/task_archive_remote.go
deleted file mode 100644
index 87e230f..0000000
--- a/server/task_archive_remote.go
+++ /dev/null
@@ -1,171 +0,0 @@
-package server
-
-import (
- "encoding/json"
- "errors"
- "fmt"
- "log"
- "os"
- "path/filepath"
- "strings"
-
- pgdb "github.com/Autumn-27/artex/db"
-)
-
-// An outbox bridges the local PostgreSQL transaction and object storage. Never
-// delete the only package before metadata permits it. Unfinished cleanup is
-// retried after a restart (and before subsequent jobs).
-type remoteArchiveJournal struct {
- ArchiveID int64 `json:"archive_id"`
- URI string `json:"uri"`
- LocalPath string `json:"local_path,omitempty"`
- Operation string `json:"operation"`
-}
-
-func (s *Server) validateStoredArchive(uri string) error {
- if strings.HasPrefix(uri, "s3://") {
- return s.archiveStore.Validate(uri)
- }
- return validateArchivePath(s.m.dir, uri)
-}
-
-func removeLocalArchivePackage(filename string) error {
- if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
- return err
- }
- return nil
-}
-
-func (s *Server) removeStoredArchive(uri string) error {
- if err := s.validateStoredArchive(uri); err != nil {
- return err
- }
- if strings.HasPrefix(uri, "s3://") {
- return s.archiveStore.Delete(s.ctx, uri)
- }
- return removeLocalArchivePackage(uri)
-}
-
-func (s *Server) writeRemoteArchiveJournal(j remoteArchiveJournal) (string, error) {
- dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote")
- if err := os.MkdirAll(dir, archiveDirMode); err != nil {
- return "", err
- }
- f, err := os.CreateTemp(dir, ".journal-")
- if err != nil {
- return "", err
- }
- tmp := f.Name()
- defer os.Remove(tmp)
- encodeErr := json.NewEncoder(f).Encode(j)
- syncErr := f.Sync()
- if err := errors.Join(encodeErr, syncErr, f.Close()); err != nil {
- return "", err
- }
- name := tmp + ".json"
- if err := os.Rename(tmp, name); err != nil {
- return "", err
- }
- // Persist the directory entry before an upload or database deletion starts.
- d, err := os.Open(dir)
- if err != nil {
- return "", err
- }
- if err := errors.Join(d.Sync(), d.Close()); err != nil {
- return "", err
- }
- // The .remote directory itself may have just been created.
- parent, err := os.Open(filepath.Dir(dir))
- if err != nil {
- return "", err
- }
- return name, errors.Join(parent.Sync(), parent.Close())
-}
-
-func (s *Server) recoverRemoteArchiveJournals() error {
- dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote")
- entries, err := os.ReadDir(dir)
- if os.IsNotExist(err) {
- return nil
- }
- if err != nil {
- return err
- }
- var errs []error
- for _, entry := range entries {
- if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
- continue
- }
- name := filepath.Join(dir, entry.Name())
- if err := s.recoverRemoteArchiveJournal(name); err != nil {
- errs = append(errs, err)
- }
- }
- return errors.Join(errs...)
-}
-
-func (s *Server) recoverRemoteArchiveJournal(name string) error {
- raw, err := os.ReadFile(name)
- if err != nil {
- return err
- }
- var j remoteArchiveJournal
- if err := json.Unmarshal(raw, &j); err != nil {
- return err
- }
- if j.ArchiveID <= 0 {
- return errors.New("invalid remote archive journal ID")
- }
- if err := s.archiveStore.Validate(j.URI); err != nil {
- return err
- }
- item, err := s.m.pg.GetTaskArchive(j.ArchiveID)
- if err != nil {
- return err
- }
- switch j.Operation {
- case "upload":
- if item == nil || item.ArchivePath != j.URI {
- if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil {
- return err
- }
- }
- if err := validateArchivePath(s.m.dir, j.LocalPath); err != nil {
- return err
- }
- // An old failed upload may outlive a retry after switching back to
- // local storage. The retry reuses this pathname: keep its committed
- // package instead of treating it as the old upload's temporary copy.
- if item == nil || item.ArchivePath != j.LocalPath {
- if err := removeLocalArchivePackage(j.LocalPath); err != nil {
- return err
- }
- }
- case "delete":
- if item == nil {
- if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil {
- return err
- }
- }
- default:
- return fmt.Errorf("unknown remote archive operation %q", j.Operation)
- }
- return os.Remove(name)
-}
-
-func (s *Server) deleteRemoteTaskArchive(job *pgdb.TaskArchive) error {
- journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, Operation: "delete"})
- if err != nil {
- return err
- }
- if err := s.m.pg.DeleteTaskArchiveStub(job.ID); err != nil {
- // A connection failure can leave the transaction's commit ambiguous.
- // Recovery consults PostgreSQL before deciding whether to delete.
- return err
- }
- if err := s.recoverRemoteArchiveJournal(journal); err != nil {
- // The task is gone; the durable outbox retries the remaining object cleanup.
- log.Printf("[task-archive] archive %d deleted; remote cleanup pending: %v", job.ID, err)
- }
- return nil
-}
diff --git a/server/task_archive_remote_test.go b/server/task_archive_remote_test.go
deleted file mode 100644
index ac2a44d..0000000
--- a/server/task_archive_remote_test.go
+++ /dev/null
@@ -1,419 +0,0 @@
-package server
-
-import (
- "bytes"
- "context"
- "errors"
- "fmt"
- "io"
- "net/http"
- "net/http/httptest"
- "os"
- "path/filepath"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
- "testing"
-
- "github.com/Autumn-27/artex/archivestore"
- pgdb "github.com/Autumn-27/artex/db"
-)
-
-func TestRemoteArchiveLifecycle(t *testing.T) {
- if os.Getenv("ARTEX_PG_DSN") == "" {
- t.Skip("requires isolated PostgreSQL")
- }
- t.Setenv("ARTEX_ARCHIVE_STORAGE", "local")
- m, err := NewManager(t.TempDir(), "")
- if err != nil {
- t.Fatal(err)
- }
- defer m.Close()
- ctx, cancel := context.WithCancel(context.Background())
- cancel()
- s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
- s.archiveWG.Wait()
- s.ctx = context.Background()
- var mu sync.Mutex
- objects := map[string][]byte{}
- corrupt, failDelete := false, false
- remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- mu.Lock()
- defer mu.Unlock()
- switch r.Method {
- case "PUT":
- objects[r.URL.Path], _ = io.ReadAll(r.Body)
- w.Header().Set("ETag", `"test"`)
- case "HEAD":
- w.Header().Set("X-Amz-Server-Side-Encryption", "AES256")
- w.Header().Set("ETag", `"test"`)
- w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path])))
- case "GET":
- data := objects[r.URL.Path]
- if corrupt && len(data) > 0 {
- data = bytes.Clone(data)
- data[0] ^= 0xff
- }
- var start, end int64
- if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) || r.Header.Get("If-Match") != `"test"` {
- w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
- return
- }
- w.Header().Set("ETag", `"test"`)
- w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10))
- w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
- w.WriteHeader(http.StatusPartialContent)
- _, _ = w.Write(data[start : end+1])
- case "DELETE":
- if failDelete {
- w.WriteHeader(403)
- return
- }
- delete(objects, r.URL.Path)
- w.WriteHeader(204)
- default:
- w.WriteHeader(400)
- }
- }))
- defer remote.Close()
- s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
- if err != nil {
- t.Fatal(err)
- }
- task, err := m.CreateTask("S3 round trip", "preserve data", nil, 0, 0)
- if err != nil {
- t.Fatal(err)
- }
- id, _ := strconv.ParseInt(task.ID, 10, 64)
- defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }()
- if err := m.pg.SetPaused(id, true); err != nil {
- t.Fatal(err)
- }
- file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "important.txt")
- if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
- t.Fatal(err)
- }
- if err := os.WriteFile(file, []byte("keep this content"), 0600); err != nil {
- t.Fatal(err)
- }
- pgTask, err := m.pg.GetTask(id)
- if err != nil {
- t.Fatal(err)
- }
- if _, err := m.pg.Exploration(pgTask.ExplorationID).AppendActivity(pgdb.Activity{Worker: "system", Kind: "system", Summary: "archive proof"}); err != nil {
- t.Fatal(err)
- }
- job, err := m.pg.QueueTaskArchive(id)
- if err != nil {
- t.Fatal(err)
- }
- // A configured-but-invalid backend must fail before moving any source data.
- s.archiveStoreErr = errors.New("invalid fixture configuration")
- if err := s.runOneTaskArchiveJob(); !errors.Is(err, s.archiveStoreErr) {
- t.Fatalf("configuration error was not returned: %v", err)
- }
- if got, err := os.ReadFile(file); err != nil || string(got) != "keep this content" {
- t.Fatalf("bad configuration lost source: %v", err)
- }
- if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil {
- t.Fatal("bad configuration lost task", err)
- }
- s.archiveStoreErr = nil
- if _, err := m.pg.QueueTaskArchive(id); err != nil {
- t.Fatal(err)
- }
- // A corrupt readback must roll back filesystem staging and leave PG intact.
- mu.Lock()
- corrupt = true
- mu.Unlock()
- if err := s.runOneTaskArchiveJob(); err == nil {
- t.Fatal("expected integrity failure")
- }
- if got, _ := os.ReadFile(file); string(got) != "keep this content" {
- t.Fatal("upload failure lost source")
- }
- if task, err := m.pg.GetTask(id); err != nil || task == nil {
- t.Fatal("upload failure lost task")
- }
- mu.Lock()
- corrupt = false
- mu.Unlock()
- if err := s.recoverRemoteArchiveJournals(); err != nil {
- t.Fatal(err)
- }
- if _, err := m.pg.QueueTaskArchive(id); err != nil {
- t.Fatal(err)
- }
- if err := s.runOneTaskArchiveJob(); err != nil {
- t.Fatal(err)
- }
- job, err = m.pg.GetTaskArchive(job.ID)
- if err != nil {
- t.Fatal(err)
- }
- if job.State != pgdb.ArchiveReady || job.Storage != "s3" || !strings.HasPrefix(job.ArchivePath, "s3://") {
- t.Fatalf("unexpected archive %+v", job)
- }
- if _, err := os.Stat(file); !os.IsNotExist(err) {
- t.Fatal("workspace not released")
- }
- if _, err := os.Stat(taskArchivePath(m.dir, job.ID, task.ID)); !os.IsNotExist(err) {
- t.Fatal("local package retained")
- }
- // Simulate the crash window after commit: cleanup keeps the remote object.
- journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, LocalPath: taskArchivePath(m.dir, job.ID, task.ID), Operation: "upload"})
- if err != nil {
- t.Fatal(err)
- }
- if err := s.recoverRemoteArchiveJournal(journal); err != nil {
- t.Fatal(err)
- }
- // A corrupted download must leave the archive and its remote package intact.
- mu.Lock()
- corrupt = true
- mu.Unlock()
- if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
- t.Fatal(err)
- }
- if err := s.runOneTaskArchiveJob(); err == nil {
- t.Fatal("expected corrupted restore to fail")
- }
- if _, err := os.Stat(file); !os.IsNotExist(err) {
- t.Fatal("corrupted archive installed workspace files")
- }
- if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || restored {
- t.Fatalf("corrupted archive restored database: %v %v", restored, err)
- }
- mu.Lock()
- if len(objects) == 0 {
- t.Error("corrupted restore deleted the only remote package")
- }
- corrupt = false
- mu.Unlock()
- // Restoration commits files and PG first. Failed remote deletion is retryable.
- mu.Lock()
- failDelete = true
- mu.Unlock()
- if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
- t.Fatal(err)
- }
- if err := s.runOneTaskArchiveJob(); err == nil {
- t.Fatal("expected remote delete failure")
- }
- if got, _ := os.ReadFile(file); string(got) != "keep this content" {
- t.Fatal("restored content differs")
- }
- if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || !restored {
- t.Fatalf("PG restore %v %v", restored, err)
- }
- mu.Lock()
- failDelete = false
- mu.Unlock()
- if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
- t.Fatal(err)
- }
- if err := s.runOneTaskArchiveJob(); err != nil {
- t.Fatal(err)
- }
- if item, err := m.pg.GetTaskArchive(job.ID); err != nil || item != nil {
- t.Fatal("restore did not consume stub")
- }
- var count int
- if err := m.pg.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='archive proof'`, pgTask.ExplorationID).Scan(&count); err != nil || count != 1 {
- t.Fatalf("activity restore count=%d err=%v", count, err)
- }
- // Re-archive, then crash/retry the deletion outbox after PG removed its stub.
- job, err = m.pg.QueueTaskArchive(id)
- if err != nil {
- t.Fatal(err)
- }
- if err := s.runOneTaskArchiveJob(); err != nil {
- t.Fatal(err)
- }
- mu.Lock()
- failDelete = true
- mu.Unlock()
- if _, err := m.pg.QueueTaskArchiveDelete(job.ID); err != nil {
- t.Fatal(err)
- }
- if err := s.runOneTaskArchiveJob(); err != nil {
- t.Fatal(err)
- }
- mu.Lock()
- failDelete = false
- mu.Unlock()
- if err := s.recoverRemoteArchiveJournals(); err != nil {
- t.Fatal(err)
- }
- mu.Lock()
- remaining := len(objects)
- mu.Unlock()
- if remaining != 0 {
- t.Fatalf("left %d objects", remaining)
- }
-}
-
-// Once PostgreSQL has committed an archive location and checksum, a subsequent
-// queued operation must not make startup recovery resurrect the old workspace.
-func TestTaskArchiveStageRecoveryAfterCommit(t *testing.T) {
- if os.Getenv("ARTEX_PG_DSN") == "" {
- t.Skip("requires isolated PostgreSQL")
- }
- m, err := NewManager(t.TempDir(), "")
- if err != nil {
- t.Fatal(err)
- }
- defer m.Close()
- task, err := m.CreateTask("remote stage recovery", "test", nil, 0, 0)
- if err != nil {
- t.Fatal(err)
- }
- taskID, _ := strconv.ParseInt(task.ID, 10, 64)
- defer func() {
- _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID)
- _ = m.pg.DeleteTask(taskID)
- }()
- if err := m.pg.SetPaused(taskID, true); err != nil {
- t.Fatal(err)
- }
- job, err := m.pg.QueueTaskArchive(taskID)
- if err != nil {
- t.Fatal(err)
- }
- pgTask, err := m.pg.GetTask(taskID)
- if err != nil {
- t.Fatal(err)
- }
- for _, backend := range []string{"local", "s3"} {
- for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} {
- t.Run(backend+"/"+state, func(t *testing.T) {
- committed := state != pgdb.ArchiveQueued
- checksum := ""
- if committed {
- checksum = strings.Repeat("a", 64)
- }
- packagePath := taskArchivePath(m.dir, job.ID, task.ID)
- if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil {
- t.Fatal(err)
- }
- if err := os.WriteFile(packagePath, []byte("verified package"), 0600); err != nil {
- t.Fatal(err)
- }
- location := packagePath
- if backend == "s3" {
- location = "s3://test/tasks/package.tar.zst"
- }
- if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, location, checksum); err != nil {
- t.Fatal(err)
- }
- file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt")
- if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
- t.Fatal(err)
- }
- if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil {
- t.Fatal(err)
- }
- stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID)
- if err != nil {
- t.Fatal(err)
- }
- if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
- t.Fatal(err)
- }
- if _, err := os.Stat(stage.root); !os.IsNotExist(err) {
- t.Fatalf("staging cleanup incomplete: %v", err)
- }
- if backend == "local" && committed {
- if data, err := os.ReadFile(packagePath); err != nil || string(data) != "verified package" {
- t.Fatal("committed local package lost", err)
- }
- } else if !committed {
- if _, err := os.Stat(packagePath); !os.IsNotExist(err) {
- t.Fatal("uncommitted temporary package retained", err)
- }
- }
- if committed {
- if _, err := os.Stat(file); !os.IsNotExist(err) {
- t.Fatalf("resurrected committed workspace: %v", err)
- }
- } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" {
- t.Fatalf("uncommitted archive lost source: %v", err)
- }
- })
- }
- }
-}
-
-// Failed remote cleanup can survive switching to local storage and retrying the
-// same archive ID. Its journal must never remove the newly committed local copy.
-func TestRemoteArchiveJournalPreservesLocalRetry(t *testing.T) {
- if os.Getenv("ARTEX_PG_DSN") == "" {
- t.Skip("requires isolated PostgreSQL")
- }
- m, err := NewManager(t.TempDir(), "")
- if err != nil {
- t.Fatal(err)
- }
- defer m.Close()
- task, err := m.CreateTask("local retry", "test", nil, 0, 0)
- if err != nil {
- t.Fatal(err)
- }
- taskID, _ := strconv.ParseInt(task.ID, 10, 64)
- defer func() {
- _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID)
- _ = m.pg.DeleteTask(taskID)
- }()
- if err := m.pg.SetPaused(taskID, true); err != nil {
- t.Fatal(err)
- }
- job, err := m.pg.QueueTaskArchive(taskID)
- if err != nil {
- t.Fatal(err)
- }
- filename := taskArchivePath(m.dir, job.ID, task.ID)
- if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil {
- t.Fatal(err)
- }
- if err := os.WriteFile(filename, []byte("committed local archive"), 0600); err != nil {
- t.Fatal(err)
- }
- if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, pgdb.ArchiveReady, filename, strings.Repeat("a", 64)); err != nil {
- t.Fatal(err)
- }
- var deletes atomic.Int32
- remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.Method != http.MethodDelete {
- w.WriteHeader(http.StatusMethodNotAllowed)
- return
- }
- deletes.Add(1)
- w.WriteHeader(http.StatusNoContent)
- }))
- defer remote.Close()
- store, err := archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
- if err != nil {
- t.Fatal(err)
- }
- s := &Server{ctx: context.Background(), m: m, archiveStore: store}
- journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{
- ArchiveID: job.ID, URI: store.URI(taskID, job.ID, strings.Repeat("b", 64)),
- LocalPath: filename, Operation: "upload",
- })
- if err != nil {
- t.Fatal(err)
- }
- if err := s.recoverRemoteArchiveJournal(journal); err != nil {
- t.Fatal(err)
- }
- if data, err := os.ReadFile(filename); err != nil || string(data) != "committed local archive" {
- t.Fatalf("old upload cleanup removed local retry: %v", err)
- }
- if deletes.Load() != 1 {
- t.Fatal("uncommitted remote object was not removed")
- }
- if _, err := os.Stat(journal); !os.IsNotExist(err) {
- t.Fatalf("finished journal remains: %v", err)
- }
-}
diff --git a/server/task_archives.go b/server/task_archives.go
index 9a7af52..836c2d8 100644
--- a/server/task_archives.go
+++ b/server/task_archives.go
@@ -14,7 +14,6 @@ import (
"time"
"github.com/Autumn-27/artex/agent"
- "github.com/Autumn-27/artex/archivestore"
pgdb "github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/traffic"
)
@@ -38,12 +37,6 @@ func (s *Server) startTaskArchiveWorker() {
if s.m == nil || s.m.pg == nil {
return
}
- s.archiveStore, s.archiveStoreErr = archivestore.FromEnv()
- if s.archiveStoreErr != nil {
- log.Printf("[task-archive] storage configuration: %v", s.archiveStoreErr)
- } else if s.archiveStore != nil {
- log.Printf("[task-archive] new archives use S3 object storage with full SHA-256 readback verification")
- }
if err := s.m.pg.RecoverTaskArchiveJobs(); err != nil {
log.Printf("[task-archive] recover jobs: %v", err)
}
@@ -95,14 +88,6 @@ func (s *Server) notifyTaskArchiveWorker() {
}
func (s *Server) runOneTaskArchiveJob() error {
- if s.ctx.Err() != nil {
- return s.ctx.Err()
- }
- // Retry deferred remote cleanup before claiming more work. Failed cleanup
- // keeps its durable journal and does not block unrelated local archives.
- if err := s.recoverRemoteArchiveJournals(); err != nil {
- log.Printf("[task-archive] deferred object cleanup: %v", err)
- }
job, err := s.m.pg.ClaimTaskArchiveJob(s.ctx)
if err != nil || job == nil {
return err
@@ -130,9 +115,6 @@ func (s *Server) runOneTaskArchiveJob() error {
}
func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
- if s.archiveStoreErr != nil {
- return s.archiveStoreErr
- }
taskID := strconv.FormatInt(job.TaskID, 10)
if !s.beginTaskDelete(taskID) {
return errors.New("task is already undergoing another archive or delete operation")
@@ -217,21 +199,6 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
_ = os.Remove(archivePath)
}
}()
- storedPath := archivePath
- var remoteJournal string
- if s.archiveStore != nil {
- storedPath = s.archiveStore.URI(job.TaskID, job.ID, checksum)
- remoteJournal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{
- ArchiveID: job.ID, URI: storedPath, LocalPath: archivePath, Operation: "upload",
- })
- if err != nil {
- return err
- }
- _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "upload_and_verify", 65)
- if err := s.archiveStore.Upload(s.ctx, storedPath, archivePath, checksum); err != nil {
- return err
- }
- }
var trafficStage *traffic.HostDeleteStage
if s.m.traffic != nil && len(snapshot.ExclusiveHosts) > 0 {
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_traffic", 68)
@@ -246,18 +213,11 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
}()
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_database", 78)
- if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, storedPath, checksum, originalSize, compressedSize); err != nil {
+ if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, archivePath, checksum, originalSize, compressedSize); err != nil {
return err
}
committed = true
- removePackage = s.archiveStore != nil
- if remoteJournal != "" {
- // The journal also covers a crash after database commit but before local
- // package removal. Keep it if cleanup fails, so the worker can retry.
- if err := removeLocalArchivePackage(archivePath); err == nil {
- _ = os.Remove(remoteJournal)
- }
- }
+ removePackage = false
if trafficStage != nil {
if err := trafficStage.Commit(); err != nil {
warning := "archive completed, but cleaning the exclusive traffic hot store failed (the archive package can still restore it): " + err.Error()
@@ -279,7 +239,7 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
}
func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
- if err := s.validateStoredArchive(job.ArchivePath); err != nil {
+ if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil {
return err
}
// A prior attempt may have committed PostgreSQL and files, then failed only
@@ -287,7 +247,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
if restored, err := s.m.pg.IsTaskArchiveRestored(job.ID); err != nil {
return err
} else if restored {
- if err := s.removeStoredArchive(job.ArchivePath); err != nil {
+ if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) {
return err
}
if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil {
@@ -306,15 +266,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
return err
}
defer func() { _ = os.RemoveAll(extracted) }()
- packagePath := job.ArchivePath
- if strings.HasPrefix(packagePath, "s3://") {
- _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "download_and_verify", 12)
- packagePath = filepath.Join(extracted, "download.tar.zst")
- if err := s.archiveStore.Download(s.ctx, job.ArchivePath, packagePath, job.SHA256, job.CompressedSize); err != nil {
- return err
- }
- }
- if err := extractTaskArchivePackage(packagePath, job.SHA256, extracted); err != nil {
+ if err := extractTaskArchivePackage(job.ArchivePath, job.SHA256, extracted); err != nil {
return err
}
manifestRaw, err := os.ReadFile(filepath.Join(extracted, "manifest.json"))
@@ -404,7 +356,7 @@ func (s *Server) restoreTaskArchivePayload(job *pgdb.TaskArchive, snapshot *pgdb
return err
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "consume_package", 95)
- if err := s.removeStoredArchive(job.ArchivePath); err != nil {
+ if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) {
return err
}
if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil {
@@ -441,12 +393,9 @@ func (s *Server) loadRestoredTask(taskID int64) {
}
func (s *Server) deleteTaskArchive(job *pgdb.TaskArchive) error {
- if err := s.validateStoredArchive(job.ArchivePath); err != nil {
+ if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil {
return err
}
- if strings.HasPrefix(job.ArchivePath, "s3://") {
- return s.deleteRemoteTaskArchive(job)
- }
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "stage_package_delete", 35)
staged, moved, err := stageTaskArchivePackageDelete(job.ArchivePath, job.ID)
if err != nil {
diff --git a/web/src/app/(main)/function/tasks/page.tsx b/web/src/app/(main)/function/tasks/page.tsx
index c2f9cc3..6b8ba8e 100644
--- a/web/src/app/(main)/function/tasks/page.tsx
+++ b/web/src/app/(main)/function/tasks/page.tsx
@@ -1494,7 +1494,7 @@ function ArchiveConfirmDialog({
{count === 1 ? "Archive task" : `Archive ${count} tasks`}
- Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into the configured archive storage. Once archived, a task can be restored from "Archived". Restoring requires free space on this server.
+ Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into local cold storage. Once archived, a task can be restored from "Archived".
@@ -1934,7 +1934,6 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {
title={`Before compression: ${formatArchiveBytes(archive.original_size)}`}
>
{archiveCompressionLabel(archive)}
- {archive.storage === "s3" ? "Object storage" : "Local storage"}
{archiveDataTotal(archive).toLocaleString()}
@@ -1953,7 +1952,7 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {
))}
- {archive.phase === "upload_and_verify" ? "Uploading and verifying" : archive.phase === "download_and_verify" ? "Downloading and verifying" : archive.phase} · {archive.progress}%
+ {archive.phase} · {archive.progress}%
diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts
index f31b18c..cd53b64 100644
--- a/web/src/lib/types.ts
+++ b/web/src/lib/types.ts
@@ -101,7 +101,6 @@ export interface TaskArchiveTokenStats {
}
export interface TaskArchive {
- storage?: "local" | "s3";
id: number;
task_id: number;
state: TaskArchiveState;