From da16b8d81a6946726da123a3bf69b9aae5375a5a Mon Sep 17 00:00:00 2001
From: Chubboy Ren
Date: Tue, 17 Nov 2026 11:56:15 +0000
Subject: [PATCH 1/4] feat: archive cold tasks to S3-compatible object storage
---
README.md | 6 +
archivestore/config_test.go | 62 ++++
archivestore/s3.go | 280 +++++++++++++++
archivestore/s3_test.go | 311 ++++++++++++++++
cmd/artex-archive-check/main.go | 64 ++++
db/task_archives.go | 5 +
docs/task-archive-storage.md | 104 ++++++
go.mod | 13 +
go.sum | 38 ++
server/server.go | 9 +-
server/task_archive_package.go | 4 +-
server/task_archive_remote.go | 171 +++++++++
server/task_archive_remote_test.go | 396 +++++++++++++++++++++
server/task_archives.go | 65 +++-
web/src/app/(main)/function/tasks/page.tsx | 5 +-
web/src/lib/types.ts | 1 +
16 files changed, 1520 insertions(+), 14 deletions(-)
create mode 100644 archivestore/config_test.go
create mode 100644 archivestore/s3.go
create mode 100644 archivestore/s3_test.go
create mode 100644 cmd/artex-archive-check/main.go
create mode 100644 docs/task-archive-storage.md
create mode 100644 server/task_archive_remote.go
create mode 100644 server/task_archive_remote_test.go
diff --git a/README.md b/README.md
index d15b112..7cc166c 100644
--- a/README.md
+++ b/README.md
@@ -418,6 +418,12 @@ runtime supports it.
`/chat/completions` (or `/responses` for `openai-responses`), so their base **does** end in `/v1`
(for example `https://api.openai.com/v1`).
+**Task archive storage**: local disk by default. Set `ARTEX_ARCHIVE_STORAGE=s3` to
+store cold task packages in an S3-compatible bucket and release their local
+package space after verification. Existing local archives remain usable. See
+[Task archive storage](docs/task-archive-storage.md) for configuration, permissions,
+integrity checks, temporary disk requirements and recovery.
+
**Concurrency**: work agents per task, in "System Settings" (default 3).
**Untrusted-output boundary (on by default)**: tool results for the worker, planner, chat and main
diff --git a/archivestore/config_test.go b/archivestore/config_test.go
new file mode 100644
index 0000000..cc12532
--- /dev/null
+++ b/archivestore/config_test.go
@@ -0,0 +1,62 @@
+package archivestore
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestArchiveStorageEnvironment(t *testing.T) {
+ for _, key := range []string{
+ "ARTEX_ARCHIVE_STORAGE", "ARTEX_ARCHIVE_S3_ENDPOINT", "ARTEX_ARCHIVE_S3_REGION",
+ "ARTEX_ARCHIVE_S3_BUCKET", "ARTEX_ARCHIVE_S3_PREFIX", "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID",
+ "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_PATH_STYLE",
+ } {
+ t.Setenv(key, "")
+ }
+ for _, backend := range []string{"", "local"} {
+ t.Setenv("ARTEX_ARCHIVE_STORAGE", backend)
+ if store, err := FromEnv(); err != nil || store != nil {
+ t.Fatalf("local default: %v %v", store, err)
+ }
+ }
+ for _, backend := range []string{"unknown", "s3"} {
+ t.Setenv("ARTEX_ARCHIVE_STORAGE", backend)
+ if _, err := FromEnv(); err == nil {
+ t.Fatalf("%q silently fell back to local", backend)
+ }
+ }
+ t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", "http://127.0.0.1:9000")
+ t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "test-bucket")
+ t.Setenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "test-key")
+ t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "test-secret")
+ t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true")
+ store, err := FromEnv()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if uri := store.URI(1, 2, "checksum"); uri != "s3://test-bucket/artex/tasks/task-1-2-checksum.tar.zst" {
+ t.Fatal("unexpected default object location", uri)
+ }
+ t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "")
+ if _, err := FromEnv(); err == nil {
+ t.Fatal("missing credentials accepted")
+ }
+}
+
+func TestArchiveEndpointValidation(t *testing.T) {
+ config := Config{Bucket: "test", AccessKey: "test-key", SecretKey: "test-secret"}
+ for _, endpoint := range []string{"https://objects.example.test", "http://127.0.0.1:9000", "http://127.0.0.2:9000", "http://[::1]:9000", "http://localhost:9000"} {
+ config.Endpoint = endpoint
+ if _, err := New(config); err != nil {
+ t.Errorf("rejected %s: %v", endpoint, err)
+ }
+ }
+ for _, endpoint := range []string{"http://objects.example.test", "https://user:test-secret@objects.example.test", "https://objects.example.test/bucket", "https://objects.example.test?token=test-secret", "https://objects.example.test#fragment"} {
+ config.Endpoint = endpoint
+ if _, err := New(config); err == nil {
+ t.Error("accepted endpoint with non-loopback HTTP, credentials, path, query or fragment")
+ } else if strings.Contains(err.Error(), "test-secret") {
+ t.Fatal("configuration error disclosed a secret")
+ }
+ }
+}
diff --git a/archivestore/s3.go b/archivestore/s3.go
new file mode 100644
index 0000000..edc1513
--- /dev/null
+++ b/archivestore/s3.go
@@ -0,0 +1,280 @@
+// Package archivestore stores immutable cold-task packages in S3-compatible storage.
+package archivestore
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/url"
+ "os"
+ "path"
+ "strings"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+type Config struct {
+ Endpoint, Region, Bucket, Prefix, AccessKey, SecretKey string
+ PathStyle bool
+}
+
+type Store struct {
+ client *s3.Client
+ bucket, prefix string
+}
+
+// FromEnv returns nil for the default local backend. A configured but invalid
+// backend is an error, never a silent fallback to the machine's local disk.
+func FromEnv() (*Store, error) {
+ switch os.Getenv("ARTEX_ARCHIVE_STORAGE") {
+ case "", "local":
+ return nil, nil
+ case "s3":
+ return New(Config{
+ Endpoint: os.Getenv("ARTEX_ARCHIVE_S3_ENDPOINT"), Region: os.Getenv("ARTEX_ARCHIVE_S3_REGION"),
+ Bucket: os.Getenv("ARTEX_ARCHIVE_S3_BUCKET"), Prefix: os.Getenv("ARTEX_ARCHIVE_S3_PREFIX"),
+ AccessKey: os.Getenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID"), SecretKey: os.Getenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY"),
+ PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true",
+ })
+ default:
+ return nil, errors.New("ARTEX_ARCHIVE_STORAGE must be local or s3")
+ }
+}
+
+func New(c Config) (*Store, error) {
+ u, err := url.Parse(c.Endpoint)
+ loopback := false
+ if err == nil {
+ loopback = u.Hostname() == "localhost"
+ if ip := net.ParseIP(u.Hostname()); ip != nil {
+ loopback = ip.IsLoopback()
+ }
+ }
+ if err != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") ||
+ (u.Scheme != "https" && !(u.Scheme == "http" && loopback)) {
+ return nil, errors.New("archive S3 endpoint must be an HTTPS origin (HTTP allowed only on loopback)")
+ }
+ c.Prefix = strings.Trim(c.Prefix, "/")
+ if c.Prefix == "" {
+ c.Prefix = "artex/tasks"
+ }
+ if path.Clean(c.Prefix) != c.Prefix || strings.Contains(c.Prefix, "..") || strings.ContainsAny(c.Prefix, "\\?#") {
+ return nil, errors.New("invalid archive S3 prefix")
+ }
+ if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") || c.AccessKey == "" || c.SecretKey == "" {
+ return nil, errors.New("archive S3 bucket and dedicated access credentials are required")
+ }
+ if c.Region == "" {
+ c.Region = "us-east-1"
+ }
+ transport := http.DefaultTransport.(*http.Transport).Clone()
+ transport.ResponseHeaderTimeout = 60 * time.Second
+ client := s3.New(s3.Options{
+ Region: c.Region, BaseEndpoint: aws.String(c.Endpoint), UsePathStyle: c.PathStyle,
+ Credentials: credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, ""),
+ HTTPClient: &http.Client{Transport: transport},
+ RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
+ ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired,
+ })
+ return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/"}, nil
+}
+
+func (s *Store) URI(taskID, archiveID int64, checksum string) string {
+ return "s3://" + s.bucket + "/" + s.prefix + fmt.Sprintf("task-%d-%d-%s.tar.zst", taskID, archiveID, checksum)
+}
+
+func (s *Store) key(uri string) (string, error) {
+ if s == nil {
+ return "", errors.New("S3 archive storage is not configured")
+ }
+ u, err := url.Parse(uri)
+ if err != nil || u.Scheme != "s3" || u.Host != s.bucket || u.User != nil || u.RawQuery != "" || u.Fragment != "" {
+ return "", errors.New("archive object is outside the configured bucket")
+ }
+ key := strings.TrimPrefix(u.Path, "/")
+ if !strings.HasPrefix(key, s.prefix) || len(key) <= len(s.prefix) || path.Clean(key) != key || strings.Contains(key, "\\") {
+ return "", errors.New("archive object is outside the configured prefix")
+ }
+ return key, nil
+}
+
+func (s *Store) Validate(uri string) error { _, err := s.key(uri); return err }
+
+// Upload verifies the entire uploaded object by reading it back, including
+// multipart uploads whose ETag is not a checksum. Callers retain source data
+// until verification succeeds and PostgreSQL commits the remote location.
+func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) error {
+ key, err := s.key(uri)
+ if err != nil {
+ return err
+ }
+ if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size {
+ return errors.New("invalid archive SHA-256")
+ }
+ f, err := os.Open(filename)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ info, err := f.Stat()
+ if err != nil {
+ return err
+ }
+ uploader := manager.NewUploader(s.client, func(u *manager.Uploader) {
+ u.PartSize = 8 << 20
+ u.Concurrency = 2
+ u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
+ })
+ _, err = uploader.Upload(ctx, &s3.PutObjectInput{
+ Bucket: aws.String(s.bucket), Key: aws.String(key), Body: f,
+ ContentType: aws.String("application/zstd"), Metadata: map[string]string{"sha256": checksum},
+ })
+ if err != nil {
+ return fmt.Errorf("upload archive: %w", err)
+ }
+ return s.readVerified(ctx, key, io.Discard, checksum, info.Size())
+}
+
+func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64) error {
+ if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size {
+ return errors.New("invalid archive SHA-256")
+ }
+ if expectedSize < 0 {
+ return errors.New("invalid archive size")
+ }
+ object, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)})
+ if err != nil {
+ return fmt.Errorf("inspect archive: %w", err)
+ }
+ if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" {
+ return errors.New("remote archive size mismatch or missing ETag; source data retained")
+ }
+ // Commit only complete, validated chunks to the output and hash. A broken
+ // connection can then retry its chunk without duplicating bytes or restarting
+ // a multi-gigabyte read. IfMatch pins every request to the same object.
+ h := sha256.New()
+ for offset := int64(0); offset < expectedSize; {
+ end := offset + min(archiveReadChunkSize, expectedSize-offset) - 1
+ data, err := s.readArchiveChunk(ctx, key, aws.ToString(object.ETag), offset, end, expectedSize)
+ if err != nil {
+ return err
+ }
+ n, err := out.Write(data)
+ if err != nil {
+ return err
+ }
+ if n != len(data) {
+ return io.ErrShortWrite
+ }
+ _, _ = h.Write(data)
+ offset += int64(len(data))
+ }
+ if !strings.EqualFold(hex.EncodeToString(h.Sum(nil)), checksum) {
+ return errors.New("remote archive size or SHA-256 mismatch; source data retained")
+ }
+ return nil
+}
+
+const (
+ archiveReadChunkSize = int64(8 << 20)
+ archiveReadAttempts = 5
+)
+
+func (s *Store) readArchiveChunk(ctx context.Context, key, etag string, start, end, total int64) ([]byte, error) {
+ var lastErr error
+ for attempt := 0; attempt < archiveReadAttempts; attempt++ {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ if attempt > 0 {
+ timer := time.NewTimer((250 * time.Millisecond) << (attempt - 1))
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return nil, ctx.Err()
+ case <-timer.C:
+ }
+ }
+ data, retry, err := s.readArchiveChunkOnce(ctx, key, etag, start, end, total)
+ if err == nil {
+ return data, nil
+ }
+ if !retry {
+ return nil, err
+ }
+ lastErr = err
+ }
+ return nil, fmt.Errorf("read archive bytes %d-%d after %d attempts: %w", start, end, archiveReadAttempts, lastErr)
+}
+
+func (s *Store) readArchiveChunkOnce(ctx context.Context, key, etag string, start, end, total int64) ([]byte, bool, error) {
+ requestCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
+ defer cancel()
+ object, err := s.client.GetObject(requestCtx, &s3.GetObjectInput{
+ Bucket: aws.String(s.bucket), Key: aws.String(key), IfMatch: aws.String(etag),
+ Range: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)),
+ })
+ if err != nil {
+ var networkError net.Error
+ retry := errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded)
+ return nil, retry, fmt.Errorf("read archive range: %w", err)
+ }
+ defer object.Body.Close()
+ want := end - start + 1
+ if aws.ToString(object.ETag) != etag || aws.ToString(object.ContentRange) != fmt.Sprintf("bytes %d-%d/%d", start, end, total) || object.ContentLength == nil || *object.ContentLength != want {
+ return nil, false, errors.New("remote archive changed or returned an invalid byte range; source data retained")
+ }
+ data, err := io.ReadAll(io.LimitReader(object.Body, want+1))
+ if err != nil {
+ return nil, true, err
+ }
+ if int64(len(data)) < want {
+ return nil, true, io.ErrUnexpectedEOF
+ }
+ if int64(len(data)) != want {
+ return nil, false, errors.New("remote archive range exceeds its declared size")
+ }
+ return data, false, nil
+}
+
+func (s *Store) Download(ctx context.Context, uri, filename, checksum string, size int64) (err error) {
+ key, err := s.key(uri)
+ if err != nil {
+ return err
+ }
+ if size <= 0 {
+ return errors.New("invalid archive size")
+ }
+ f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+ if err != nil {
+ return err
+ }
+ defer func() {
+ err = errors.Join(err, f.Close())
+ if err != nil {
+ _ = os.Remove(filename)
+ }
+ }()
+ if err = s.readVerified(ctx, key, f, checksum, size); err != nil {
+ return err
+ }
+ return f.Sync()
+}
+
+func (s *Store) Delete(ctx context.Context, uri string) error {
+ key, err := s.key(uri)
+ if err != nil {
+ return err
+ }
+ _, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)})
+ return err
+}
diff --git a/archivestore/s3_test.go b/archivestore/s3_test.go
new file mode 100644
index 0000000..316e770
--- /dev/null
+++ b/archivestore/s3_test.go
@@ -0,0 +1,311 @@
+package archivestore
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strconv"
+ "sync"
+ "testing"
+)
+
+func TestS3RoundTripAndIntegrity(t *testing.T) {
+ for _, size := range []int{1024, 10 << 20} {
+ t.Run(strconv.Itoa(size), func(t *testing.T) {
+ var mu sync.Mutex
+ var object []byte
+ parts := map[int][]byte{}
+ corrupt, multipart := false, false
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ defer mu.Unlock()
+ q := r.URL.Query()
+ w.Header().Set("ETag", `"test-etag"`)
+ switch {
+ case r.Method == "POST" && q.Has("uploads"):
+ multipart = true
+ fmt.Fprint(w, `test-upload`)
+ case r.Method == "PUT" && q.Has("partNumber"):
+ n, _ := strconv.Atoi(q.Get("partNumber"))
+ parts[n], _ = io.ReadAll(r.Body)
+ case r.Method == "POST" && q.Has("uploadId"):
+ for i := 1; i <= len(parts); i++ {
+ object = append(object, parts[i]...)
+ }
+ fmt.Fprint(w, `test-etag`)
+ case r.Method == "PUT":
+ object, _ = io.ReadAll(r.Body)
+ case r.Method == "HEAD":
+ w.Header().Set("Content-Length", strconv.Itoa(len(object)))
+ case r.Method == "GET":
+ data := object
+ if corrupt {
+ data = bytes.Clone(object)
+ data[0] ^= 0xff
+ }
+ serveArchiveRange(t, w, r, data)
+ case r.Method == "DELETE":
+ object = nil
+ w.WriteHeader(204)
+ default:
+ t.Errorf("unexpected request %s %s", r.Method, r.URL)
+ w.WriteHeader(400)
+ }
+ }))
+ defer srv.Close()
+ store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ data := bytes.Repeat([]byte{0x42}, size)
+ h := sha256.Sum256(data)
+ checksum := hex.EncodeToString(h[:])
+ root := t.TempDir()
+ source := filepath.Join(root, "source")
+ if err := os.WriteFile(source, data, 0600); err != nil {
+ t.Fatal(err)
+ }
+ uri := store.URI(1, 2, checksum)
+ ctx := context.Background()
+ if err := store.Upload(ctx, uri, source, checksum); err != nil {
+ t.Fatal(err)
+ }
+ if multipart != (size > 8<<20) {
+ t.Fatalf("multipart=%v size=%d", multipart, size)
+ }
+ dest := filepath.Join(root, "download")
+ if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil {
+ t.Fatal(err)
+ }
+ got, _ := os.ReadFile(dest)
+ if !bytes.Equal(got, data) {
+ t.Fatal("download differs")
+ }
+ mu.Lock()
+ corrupt = true
+ mu.Unlock()
+ bad := filepath.Join(root, "bad")
+ if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil {
+ t.Fatal("accepted corrupt object")
+ }
+ if _, err := os.Stat(bad); !os.IsNotExist(err) {
+ t.Fatal("left corrupt download")
+ }
+ if err := store.Upload(ctx, uri, source, checksum); err == nil {
+ t.Fatal("upload accepted failed readback")
+ }
+ if err := store.Delete(ctx, uri); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+}
+
+func serveArchiveRange(t *testing.T, w http.ResponseWriter, r *http.Request, data []byte) {
+ t.Helper()
+ var start, end int64
+ if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) {
+ t.Errorf("invalid range %q", r.Header.Get("Range"))
+ w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
+ return
+ }
+ if r.Header.Get("If-Match") != `"test-etag"` {
+ t.Errorf("missing object version condition: %q", r.Header.Get("If-Match"))
+ w.WriteHeader(http.StatusPreconditionFailed)
+ return
+ }
+ w.Header().Set("ETag", `"test-etag"`)
+ w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10))
+ w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
+ w.WriteHeader(http.StatusPartialContent)
+ _, _ = w.Write(data[start : end+1])
+}
+
+func TestS3ReadbackRetriesInterruptedRange(t *testing.T) {
+ data := append(bytes.Repeat([]byte{0x42}, int(archiveReadChunkSize)), bytes.Repeat([]byte{0x73}, 1024)...)
+ var mu sync.Mutex
+ var ranges []string
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ defer mu.Unlock()
+ w.Header().Set("ETag", `"test-etag"`)
+ if r.Method == "HEAD" {
+ w.Header().Set("Content-Length", strconv.Itoa(len(data)))
+ return
+ }
+ ranges = append(ranges, r.Header.Get("Range"))
+ if len(ranges) == 2 {
+ w.Header().Set("Content-Length", strconv.FormatInt(int64(len(data))-archiveReadChunkSize, 10))
+ w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", archiveReadChunkSize, len(data)-1, len(data)))
+ w.WriteHeader(http.StatusPartialContent)
+ _, _ = w.Write(data[archiveReadChunkSize : archiveReadChunkSize+128]) // Close during a later chunk.
+ return
+ }
+ serveArchiveRange(t, w, r, data)
+ }))
+ defer srv.Close()
+ store := testRangeStore(t, srv.URL)
+ h := sha256.Sum256(data)
+ checksum := hex.EncodeToString(h[:])
+ dest := filepath.Join(t.TempDir(), "archive")
+ if err := store.Download(context.Background(), store.URI(1, 2, checksum), dest, checksum, int64(len(data))); err != nil {
+ t.Fatal(err)
+ }
+ got, err := os.ReadFile(dest)
+ if err != nil || !bytes.Equal(got, data) {
+ t.Fatalf("retried download differs: %v", err)
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if len(ranges) != 3 || ranges[1] != ranges[2] || ranges[0] != fmt.Sprintf("bytes=0-%d", archiveReadChunkSize-1) || ranges[2] != fmt.Sprintf("bytes=%d-%d", archiveReadChunkSize, len(data)-1) {
+ t.Fatalf("retry did not stay within the interrupted range: %v", ranges)
+ }
+}
+
+func testRangeStore(t *testing.T, endpoint string) *Store {
+ t.Helper()
+ s, err := New(Config{Endpoint: endpoint, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return s
+}
+
+func TestS3ReadbackRejectsInvalidRangesAndExhaustedRetries(t *testing.T) {
+ for _, mode := range []string{"ignored-range", "changed-etag", "changed-object", "truncated"} {
+ t.Run(mode, func(t *testing.T) {
+ data := bytes.Repeat([]byte{0x42}, 1024)
+ var mu sync.Mutex
+ gets := 0
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ defer mu.Unlock()
+ w.Header().Set("ETag", `"test-etag"`)
+ if r.Method == "HEAD" {
+ w.Header().Set("Content-Length", strconv.Itoa(len(data)))
+ return
+ }
+ gets++
+ if mode == "changed-object" {
+ w.WriteHeader(http.StatusPreconditionFailed)
+ return
+ }
+ w.Header().Set("Content-Length", strconv.Itoa(len(data)))
+ if mode == "ignored-range" {
+ _, _ = w.Write(data)
+ return
+ }
+ if mode == "changed-etag" {
+ w.Header().Set("ETag", `"different-object"`)
+ }
+ w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", len(data)-1, len(data)))
+ w.WriteHeader(http.StatusPartialContent)
+ if mode == "truncated" {
+ _, _ = w.Write(data[:128])
+ } else {
+ _, _ = w.Write(data)
+ }
+ }))
+ defer srv.Close()
+ store := testRangeStore(t, srv.URL)
+ h := sha256.Sum256(data)
+ checksum := hex.EncodeToString(h[:])
+ dest := filepath.Join(t.TempDir(), "archive")
+ if err := store.Download(context.Background(), store.URI(1, 2, checksum), dest, checksum, int64(len(data))); err == nil {
+ t.Fatal("accepted invalid or incomplete readback")
+ }
+ if _, err := os.Stat(dest); !os.IsNotExist(err) {
+ t.Fatal("left a failed download on disk")
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ want := 1
+ if mode == "truncated" {
+ want = archiveReadAttempts
+ }
+ if gets != want {
+ t.Fatalf("GET count %d, want %d", gets, want)
+ }
+ })
+ }
+}
+
+type failedArchiveWriter struct{ err error }
+
+func (w failedArchiveWriter) Write([]byte) (int, error) { return 0, w.err }
+
+func TestS3ReadbackPropagatesOutputFailure(t *testing.T) {
+ data := []byte("test archive")
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("ETag", `"test-etag"`)
+ if r.Method == "HEAD" {
+ w.Header().Set("Content-Length", strconv.Itoa(len(data)))
+ return
+ }
+ serveArchiveRange(t, w, r, data)
+ }))
+ defer srv.Close()
+ store := testRangeStore(t, srv.URL)
+ h := sha256.Sum256(data)
+ outputError := errors.New("output disk full")
+ err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data)))
+ if !errors.Is(err, outputError) {
+ t.Fatalf("lost output error: %v", err)
+ }
+}
+
+func TestS3ReadbackStopsOnCancellation(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ data := bytes.Repeat([]byte{0x42}, 1024)
+ gets := make(chan struct{}, archiveReadAttempts+1)
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("ETag", `"test-etag"`)
+ w.Header().Set("Content-Length", strconv.Itoa(len(data)))
+ if r.Method == "HEAD" {
+ return
+ }
+ gets <- struct{}{}
+ w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", len(data)-1, len(data)))
+ w.WriteHeader(http.StatusPartialContent)
+ _, _ = w.Write(data[:128])
+ w.(http.Flusher).Flush()
+ cancel()
+ }))
+ defer srv.Close()
+ store := testRangeStore(t, srv.URL)
+ h := sha256.Sum256(data)
+ checksum := hex.EncodeToString(h[:])
+ dest := filepath.Join(t.TempDir(), "archive")
+ err := store.Download(ctx, store.URI(1, 2, checksum), dest, checksum, int64(len(data)))
+ if !errors.Is(err, context.Canceled) || len(gets) != 1 {
+ t.Fatalf("cancellation was not honored: error=%v GETs=%d", err, len(gets))
+ }
+ if _, err := os.Stat(dest); !os.IsNotExist(err) {
+ t.Fatal("left a cancelled download on disk")
+ }
+}
+
+func TestS3RejectsForeignLocations(t *testing.T) {
+ s, err := New(Config{Endpoint: "https://sfo3.digitaloceanspaces.com", Bucket: "allowed", Prefix: "archives/tasks", AccessKey: "test", SecretKey: "test"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, uri := range []string{"s3://other/archives/tasks/a", "s3://allowed/archives/other/a", "s3://allowed/archives/tasks/../a", "s3://allowed/archives/tasks/%2e%2e/a", "s3://allowed/archives/tasks/a?secret=x", "s3://user@allowed/archives/tasks/a", "file:///etc/passwd"} {
+ if err := s.Validate(uri); err == nil {
+ t.Errorf("accepted %s", uri)
+ }
+ }
+ var absent *Store
+ if absent.Validate("s3://allowed/archives/tasks/a") == nil {
+ t.Fatal("missing config accepted")
+ }
+}
diff --git a/cmd/artex-archive-check/main.go b/cmd/artex-archive-check/main.go
new file mode 100644
index 0000000..af0efd9
--- /dev/null
+++ b/cmd/artex-archive-check/main.go
@@ -0,0 +1,64 @@
+// artex-archive-check tests the configured object store without touching tasks.
+package main
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "log"
+ "os"
+ "path/filepath"
+ "time"
+
+ "github.com/Autumn-27/artex/archivestore"
+)
+
+func main() {
+ if err := check(); err != nil {
+ log.Fatal(err)
+ }
+}
+
+func check() error {
+ s, err := archivestore.FromEnv()
+ if err != nil {
+ return err
+ }
+ if s == nil {
+ return fmt.Errorf("S3 archive storage is not configured")
+ }
+ dir, err := os.MkdirTemp("", "artex-archive-check-")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(dir)
+ data := []byte("ARTEX object storage connectivity check " + time.Now().UTC().Format(time.RFC3339Nano))
+ h := sha256.Sum256(data)
+ checksum := hex.EncodeToString(h[:])
+ uri := s.URI(0, time.Now().UnixNano(), checksum)
+ file := filepath.Join(dir, "check")
+ if err := os.WriteFile(file, data, 0600); err != nil {
+ return err
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+ defer cancel()
+ defer func() {
+ cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ if err := s.Delete(cleanup, uri); err != nil {
+ log.Printf("test object cleanup failed for %s: %v", uri, err)
+ }
+ }()
+ if err := s.Upload(ctx, uri, file, checksum); err != nil {
+ return err
+ }
+ if err := s.Download(ctx, uri, filepath.Join(dir, "download"), checksum, int64(len(data))); err != nil {
+ return err
+ }
+ if err := s.Delete(ctx, uri); err != nil {
+ return err
+ }
+ fmt.Println("S3 upload, full SHA-256 verification, download and delete passed")
+ return nil
+}
diff --git a/db/task_archives.go b/db/task_archives.go
index f312000..676f1d2 100644
--- a/db/task_archives.go
+++ b/db/task_archives.go
@@ -60,6 +60,7 @@ type TaskArchive struct {
Warnings json.RawMessage `json:"warnings"`
FormatVersion int `json:"format_version"`
ArchivePath string `json:"-"`
+ Storage string `json:"storage"`
SHA256 string `json:"sha256,omitempty"`
OriginalSize int64 `json:"original_size"`
CompressedSize int64 `json:"compressed_size"`
@@ -161,6 +162,10 @@ func scanTaskArchive(sc interface{ Scan(...any) error }) (*TaskArchive, error) {
if err != nil {
return nil, err
}
+ item.Storage = "local"
+ if strings.HasPrefix(item.ArchivePath, "s3://") {
+ item.Storage = "s3"
+ }
if err := json.Unmarshal([]byte(sources), &item.SourceTaskIDs); err != nil {
return nil, err
}
diff --git a/docs/task-archive-storage.md b/docs/task-archive-storage.md
new file mode 100644
index 0000000..18bb3b4
--- /dev/null
+++ b/docs/task-archive-storage.md
@@ -0,0 +1,104 @@
+# Task archive storage
+
+Task archives use local disk by default. The optional S3-compatible backend moves
+cold task packages to object storage after verifying their contents, releasing
+local package space while retaining ARTEX's Archive, Restore and Delete actions.
+
+Active workspaces and the live PostgreSQL database stay local. This feature is
+task-level cold storage; use the existing backup workflow for a full installation
+backup. A complete recovery plan must also preserve the bucket objects and
+access to them.
+
+## Configuration
+
+Create a private bucket and a dedicated credential scoped to the archive prefix.
+The backend needs object upload, read (including HEAD and byte ranges), delete,
+and multipart upload/abort permissions. Use a separate prefix for each deployment.
+
+Set these environment variables for the ARTEX server, then restart it:
+
+```dotenv
+ARTEX_ARCHIVE_STORAGE=s3
+ARTEX_ARCHIVE_S3_ENDPOINT=https://s3.us-east-1.amazonaws.com
+ARTEX_ARCHIVE_S3_REGION=us-east-1
+ARTEX_ARCHIVE_S3_BUCKET=your-artex-archives
+ARTEX_ARCHIVE_S3_PREFIX=artex/tasks
+ARTEX_ARCHIVE_S3_ACCESS_KEY_ID=REPLACE
+ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY=REPLACE
+```
+
+Store credentials in the service's protected environment file, outside the
+repository and frontend. For systemd, a root-owned environment file with mode
+0600 can be referenced through `EnvironmentFile=`.
+
+| Variable | Meaning |
+| --- | --- |
+| `ARTEX_ARCHIVE_STORAGE` | `local` (default) or `s3`. |
+| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local testing. |
+| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. Match the provider's requirements. |
+| `ARTEX_ARCHIVE_S3_BUCKET` | Required bucket name. |
+| `ARTEX_ARCHIVE_S3_PREFIX` | Object prefix; defaults to `artex/tasks`. |
+| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Dedicated access key. |
+| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Dedicated secret key. |
+| `ARTEX_ARCHIVE_S3_PATH_STYLE` | Set to `true` for compatible path-style servers; otherwise virtual-hosted addressing is used. |
+
+For DigitalOcean Spaces, use its regional origin, such as
+`https://sfo3.digitaloceanspaces.com`, and the provider's signing region. For a
+local path-style test server, use an endpoint such as `http://127.0.0.1:9000` and
+set `ARTEX_ARCHIVE_S3_PATH_STYLE=true`.
+
+Invalid S3 configuration fails new archive jobs instead of silently falling back
+to local disk. Preserve the configured bucket, endpoint, prefix and read/delete
+access while remote archives remain. The same installation can still restore or
+delete older local archives.
+
+## Lifecycle and integrity
+
+1. Pause or finish a task and use its existing **Archive** action.
+2. ARTEX builds its `.tar.zst` package locally and uploads it. Large packages use
+ multipart upload, starting with 8 MiB parts and two upload workers (the SDK
+ increases the part size for exceptionally large seekable files).
+3. ARTEX reads the entire object back and checks its size and SHA-256. Multipart
+ ETags and uploaded checksum metadata alone are not treated as proof of content.
+4. Only after verification does PostgreSQL commit the `s3://` location and
+ compact the archived task. ARTEX then removes local source files and the
+ temporary package. The archive list shows **Object storage** or **Local storage**.
+5. **Restore** downloads and verifies the package before restoring task files and
+ database records. The remote package is consumed after restoration commits.
+ Failed remote cleanup can be retried without replaying restored records.
+6. **Delete** removes the archive stub and queues durable cleanup of its remote
+ object. Cleanup is retried after restart and before subsequent archive jobs.
+
+Reads use bounded 8 MiB ranges pinned to the object's ETag. Interrupted ranges
+are retried without duplicating output or restarting a multi-gigabyte download.
+An object change, checksum mismatch or invalid range fails verification. Failed
+downloads are removed, and upload/verification failures retain the original task.
+
+There is no database schema or archive format migration. Existing local archive
+packages remain compatible. Files outside ARTEX's task archive database are not
+automatically imported or uploaded.
+
+## Disk, transfer and recovery
+
+Archive creation still needs space for the temporary compressed package.
+Restoration needs space for the downloaded package and extracted files. Full
+upload readback and later restores incur object-storage reads and any applicable
+transfer charges. Configure expiration of abandoned multipart uploads for abrupt
+process termination.
+
+Recovery journals under `data/archives/tasks/.remote` contain object locations,
+not credentials. Preserve them with PostgreSQL and the installation's data
+directory. The worker uses these journals to clean up uploads that never
+committed, keep committed objects, and finish deferred deletions. Failed remote
+cleanup does not block unrelated local archive jobs.
+
+## Optional connectivity check
+
+Run `go run ./cmd/artex-archive-check` with the same archive environment as the
+server to upload, verify, download and delete a small uniquely named test object.
+This command performs real bucket operations but does not touch task data. It
+reports leftover test objects if cleanup fails.
+
+Automated tests use an HTTP S3 fixture and isolated PostgreSQL databases. They
+cover multipart transfers, corruption, interrupted range retries, object changes,
+cancellation, upload rollback, corrupted restores and persistent cleanup.
diff --git a/go.mod b/go.mod
index ba11f2c..7eb3dcf 100644
--- a/go.mod
+++ b/go.mod
@@ -4,6 +4,10 @@ go 1.26.10
require (
github.com/Autumn-27/norma v0.4.10
+ github.com/aws/aws-sdk-go-v2 v1.47.5
+ github.com/aws/aws-sdk-go-v2/credentials v1.20.13
+ github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19
+ github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.5.3
@@ -22,6 +26,15 @@ require (
require (
github.com/andybalholm/brotli v1.2.1 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect
+ github.com/aws/smithy-go v1.28.7 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/creack/pty v1.1.24 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
diff --git a/go.sum b/go.sum
index 15b752d..bb0b1af 100644
--- a/go.sum
+++ b/go.sum
@@ -4,6 +4,44 @@ github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eT
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
+github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4=
+github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 h1:MW0Iql1SOTgQPj8mCxrU/eH/K/AIDWi08ItOsXaWRLU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25/go.mod h1:+ucVYmHYB3JqVueY0E1RHahdXAnKQbIHqybSbvOUbOM=
+github.com/aws/aws-sdk-go-v2/config v1.33.13 h1:u52x6ISQOx172tpwM0VRxhJzEsTJYpMcEYUGM2A1wrw=
+github.com/aws/aws-sdk-go-v2/config v1.33.13/go.mod h1:/ixtz6XPHb42zekjQ0LqD9XjTpTysNWhIGwufY1OzI4=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.13 h1:j4z41OtspmQ3ftQycaDunJ26b69a9SHfaSVUPqgRv8k=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.13/go.mod h1:SgPA1oS3b+dlMxZb618jDmGNscpXSCaxz15igS52OlE=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19 h1:+JWfo9Mkg5HWwDwnpRPu7WNiiyEmiQRTaNSFsZ/1KIY=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19/go.mod h1:l4/tI+TfymypMqq2oRCkhMAB7Gi7nlt5JaoUUz+cUbI=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 h1:GACgECI422Xt5n4Pojr3O0aocPv8lfH4PJe2Aon5Wv8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9/go.mod h1:lPfnRkHpn1qWZJLbVEw10rIqSuCee1pNQTyGkUjAYh8=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 h1:RS1Wt+K0r4X0ci9Vf4JYTUfHzsQihYA2LgGJeEneKpk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8/go.mod h1:adMnTy5kyoUW86jhXnGSVc/mvMQI4GZd1qYWMeaS8M8=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 h1:to7+Vr3rLkpYRwVKJ5QRriI0eE306vrVN/6TnVHgxvY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6/go.mod h1:Jc89ASa2k7HMAskj3bSnt8bf+k+p2IApzkoXnEjkgUw=
+github.com/aws/aws-sdk-go-v2/service/signin v1.12.1 h1:WZLC+k/CZx11kZGQDqu9mpA8yhiXGX2I//jhuJE1FDo=
+github.com/aws/aws-sdk-go-v2/service/signin v1.12.1/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y=
+github.com/aws/aws-sdk-go-v2/service/sso v1.39.1 h1:4VB/dnDo7pmXSLQMszSoBzWndENuc7mzwTpD6lQn77A=
+github.com/aws/aws-sdk-go-v2/service/sso v1.39.1/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 h1:w8yui2ZqdHmGwVBDGvazoIvKw/DnLENBLkYnN8q8iqo=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps=
+github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw=
+github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE=
+github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
diff --git a/server/server.go b/server/server.go
index ac78a25..0d4134b 100644
--- a/server/server.go
+++ b/server/server.go
@@ -22,6 +22,7 @@ import (
"time"
"github.com/Autumn-27/artex/agent"
+ "github.com/Autumn-27/artex/archivestore"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/egress"
"github.com/Autumn-27/artex/intercept"
@@ -127,9 +128,11 @@ type Server struct {
// Cold task archives run through one persistent FIFO worker. The buffered wake
// channel coalesces enqueue bursts; the database remains the source of truth.
- archiveWake chan struct{}
- archiveWG sync.WaitGroup
- side *sideQuestionState
+ archiveWake chan struct{}
+ archiveWG sync.WaitGroup
+ archiveStore *archivestore.Store
+ archiveStoreErr error
+ side *sideQuestionState
}
// provEntry is a cached provider + its config for one LLM profile id.
diff --git a/server/task_archive_package.go b/server/task_archive_package.go
index f23c4c8..e9a451d 100644
--- a/server/task_archive_package.go
+++ b/server/task_archive_package.go
@@ -428,7 +428,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error {
errs = append(errs, getErr)
continue
}
- if archive != nil && archive.State == pgdb.ArchiveReady {
+ if archive != nil && archive.ArchivePath != "" && archive.SHA256 != "" {
if err := os.RemoveAll(root); err != nil {
errs = append(errs, err)
}
@@ -438,7 +438,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error {
if err := stage.rollback(); err != nil {
errs = append(errs, err)
}
- if archive != nil && archive.ArchivePath != "" {
+ if archive != nil && archive.ArchivePath != "" && !strings.HasPrefix(archive.ArchivePath, "s3://") {
_ = os.Remove(archive.ArchivePath)
}
}
diff --git a/server/task_archive_remote.go b/server/task_archive_remote.go
new file mode 100644
index 0000000..87e230f
--- /dev/null
+++ b/server/task_archive_remote.go
@@ -0,0 +1,171 @@
+package server
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+
+ pgdb "github.com/Autumn-27/artex/db"
+)
+
+// An outbox bridges the local PostgreSQL transaction and object storage. Never
+// delete the only package before metadata permits it. Unfinished cleanup is
+// retried after a restart (and before subsequent jobs).
+type remoteArchiveJournal struct {
+ ArchiveID int64 `json:"archive_id"`
+ URI string `json:"uri"`
+ LocalPath string `json:"local_path,omitempty"`
+ Operation string `json:"operation"`
+}
+
+func (s *Server) validateStoredArchive(uri string) error {
+ if strings.HasPrefix(uri, "s3://") {
+ return s.archiveStore.Validate(uri)
+ }
+ return validateArchivePath(s.m.dir, uri)
+}
+
+func removeLocalArchivePackage(filename string) error {
+ if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ return nil
+}
+
+func (s *Server) removeStoredArchive(uri string) error {
+ if err := s.validateStoredArchive(uri); err != nil {
+ return err
+ }
+ if strings.HasPrefix(uri, "s3://") {
+ return s.archiveStore.Delete(s.ctx, uri)
+ }
+ return removeLocalArchivePackage(uri)
+}
+
+func (s *Server) writeRemoteArchiveJournal(j remoteArchiveJournal) (string, error) {
+ dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote")
+ if err := os.MkdirAll(dir, archiveDirMode); err != nil {
+ return "", err
+ }
+ f, err := os.CreateTemp(dir, ".journal-")
+ if err != nil {
+ return "", err
+ }
+ tmp := f.Name()
+ defer os.Remove(tmp)
+ encodeErr := json.NewEncoder(f).Encode(j)
+ syncErr := f.Sync()
+ if err := errors.Join(encodeErr, syncErr, f.Close()); err != nil {
+ return "", err
+ }
+ name := tmp + ".json"
+ if err := os.Rename(tmp, name); err != nil {
+ return "", err
+ }
+ // Persist the directory entry before an upload or database deletion starts.
+ d, err := os.Open(dir)
+ if err != nil {
+ return "", err
+ }
+ if err := errors.Join(d.Sync(), d.Close()); err != nil {
+ return "", err
+ }
+ // The .remote directory itself may have just been created.
+ parent, err := os.Open(filepath.Dir(dir))
+ if err != nil {
+ return "", err
+ }
+ return name, errors.Join(parent.Sync(), parent.Close())
+}
+
+func (s *Server) recoverRemoteArchiveJournals() error {
+ dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote")
+ entries, err := os.ReadDir(dir)
+ if os.IsNotExist(err) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ var errs []error
+ for _, entry := range entries {
+ if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
+ continue
+ }
+ name := filepath.Join(dir, entry.Name())
+ if err := s.recoverRemoteArchiveJournal(name); err != nil {
+ errs = append(errs, err)
+ }
+ }
+ return errors.Join(errs...)
+}
+
+func (s *Server) recoverRemoteArchiveJournal(name string) error {
+ raw, err := os.ReadFile(name)
+ if err != nil {
+ return err
+ }
+ var j remoteArchiveJournal
+ if err := json.Unmarshal(raw, &j); err != nil {
+ return err
+ }
+ if j.ArchiveID <= 0 {
+ return errors.New("invalid remote archive journal ID")
+ }
+ if err := s.archiveStore.Validate(j.URI); err != nil {
+ return err
+ }
+ item, err := s.m.pg.GetTaskArchive(j.ArchiveID)
+ if err != nil {
+ return err
+ }
+ switch j.Operation {
+ case "upload":
+ if item == nil || item.ArchivePath != j.URI {
+ if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil {
+ return err
+ }
+ }
+ if err := validateArchivePath(s.m.dir, j.LocalPath); err != nil {
+ return err
+ }
+ // An old failed upload may outlive a retry after switching back to
+ // local storage. The retry reuses this pathname: keep its committed
+ // package instead of treating it as the old upload's temporary copy.
+ if item == nil || item.ArchivePath != j.LocalPath {
+ if err := removeLocalArchivePackage(j.LocalPath); err != nil {
+ return err
+ }
+ }
+ case "delete":
+ if item == nil {
+ if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil {
+ return err
+ }
+ }
+ default:
+ return fmt.Errorf("unknown remote archive operation %q", j.Operation)
+ }
+ return os.Remove(name)
+}
+
+func (s *Server) deleteRemoteTaskArchive(job *pgdb.TaskArchive) error {
+ journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, Operation: "delete"})
+ if err != nil {
+ return err
+ }
+ if err := s.m.pg.DeleteTaskArchiveStub(job.ID); err != nil {
+ // A connection failure can leave the transaction's commit ambiguous.
+ // Recovery consults PostgreSQL before deciding whether to delete.
+ return err
+ }
+ if err := s.recoverRemoteArchiveJournal(journal); err != nil {
+ // The task is gone; the durable outbox retries the remaining object cleanup.
+ log.Printf("[task-archive] archive %d deleted; remote cleanup pending: %v", job.ID, err)
+ }
+ return nil
+}
diff --git a/server/task_archive_remote_test.go b/server/task_archive_remote_test.go
new file mode 100644
index 0000000..34e0848
--- /dev/null
+++ b/server/task_archive_remote_test.go
@@ -0,0 +1,396 @@
+package server
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+
+ "github.com/Autumn-27/artex/archivestore"
+ pgdb "github.com/Autumn-27/artex/db"
+)
+
+func TestRemoteArchiveLifecycle(t *testing.T) {
+ if os.Getenv("ARTEX_PG_DSN") == "" {
+ t.Skip("requires isolated PostgreSQL")
+ }
+ t.Setenv("ARTEX_ARCHIVE_STORAGE", "local")
+ m, err := NewManager(t.TempDir(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
+ s.archiveWG.Wait()
+ s.ctx = context.Background()
+ var mu sync.Mutex
+ objects := map[string][]byte{}
+ corrupt, failDelete := false, false
+ remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ defer mu.Unlock()
+ switch r.Method {
+ case "PUT":
+ objects[r.URL.Path], _ = io.ReadAll(r.Body)
+ w.Header().Set("ETag", `"test"`)
+ case "HEAD":
+ w.Header().Set("ETag", `"test"`)
+ w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path])))
+ case "GET":
+ data := objects[r.URL.Path]
+ if corrupt && len(data) > 0 {
+ data = bytes.Clone(data)
+ data[0] ^= 0xff
+ }
+ var start, end int64
+ if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) || r.Header.Get("If-Match") != `"test"` {
+ w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
+ return
+ }
+ w.Header().Set("ETag", `"test"`)
+ w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10))
+ w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
+ w.WriteHeader(http.StatusPartialContent)
+ _, _ = w.Write(data[start : end+1])
+ case "DELETE":
+ if failDelete {
+ w.WriteHeader(403)
+ return
+ }
+ delete(objects, r.URL.Path)
+ w.WriteHeader(204)
+ default:
+ w.WriteHeader(400)
+ }
+ }))
+ defer remote.Close()
+ s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ task, err := m.CreateTask("S3 round trip", "preserve data", nil, 0, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ id, _ := strconv.ParseInt(task.ID, 10, 64)
+ defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }()
+ if err := m.pg.SetPaused(id, true); err != nil {
+ t.Fatal(err)
+ }
+ file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "important.txt")
+ if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(file, []byte("keep this content"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ pgTask, err := m.pg.GetTask(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.pg.Exploration(pgTask.ExplorationID).AppendActivity(pgdb.Activity{Worker: "system", Kind: "system", Summary: "archive proof"}); err != nil {
+ t.Fatal(err)
+ }
+ job, err := m.pg.QueueTaskArchive(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // A configured-but-invalid backend must fail before moving any source data.
+ s.archiveStoreErr = errors.New("invalid fixture configuration")
+ if err := s.runOneTaskArchiveJob(); !errors.Is(err, s.archiveStoreErr) {
+ t.Fatalf("configuration error was not returned: %v", err)
+ }
+ if got, err := os.ReadFile(file); err != nil || string(got) != "keep this content" {
+ t.Fatalf("bad configuration lost source: %v", err)
+ }
+ if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil {
+ t.Fatal("bad configuration lost task", err)
+ }
+ s.archiveStoreErr = nil
+ if _, err := m.pg.QueueTaskArchive(id); err != nil {
+ t.Fatal(err)
+ }
+ // A corrupt readback must roll back filesystem staging and leave PG intact.
+ mu.Lock()
+ corrupt = true
+ mu.Unlock()
+ if err := s.runOneTaskArchiveJob(); err == nil {
+ t.Fatal("expected integrity failure")
+ }
+ if got, _ := os.ReadFile(file); string(got) != "keep this content" {
+ t.Fatal("upload failure lost source")
+ }
+ if task, err := m.pg.GetTask(id); err != nil || task == nil {
+ t.Fatal("upload failure lost task")
+ }
+ mu.Lock()
+ corrupt = false
+ mu.Unlock()
+ if err := s.recoverRemoteArchiveJournals(); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.pg.QueueTaskArchive(id); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.runOneTaskArchiveJob(); err != nil {
+ t.Fatal(err)
+ }
+ job, err = m.pg.GetTaskArchive(job.ID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if job.State != pgdb.ArchiveReady || job.Storage != "s3" || !strings.HasPrefix(job.ArchivePath, "s3://") {
+ t.Fatalf("unexpected archive %+v", job)
+ }
+ if _, err := os.Stat(file); !os.IsNotExist(err) {
+ t.Fatal("workspace not released")
+ }
+ if _, err := os.Stat(taskArchivePath(m.dir, job.ID, task.ID)); !os.IsNotExist(err) {
+ t.Fatal("local package retained")
+ }
+ // Simulate the crash window after commit: cleanup keeps the remote object.
+ journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, LocalPath: taskArchivePath(m.dir, job.ID, task.ID), Operation: "upload"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := s.recoverRemoteArchiveJournal(journal); err != nil {
+ t.Fatal(err)
+ }
+ // A corrupted download must leave the archive and its remote package intact.
+ mu.Lock()
+ corrupt = true
+ mu.Unlock()
+ if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.runOneTaskArchiveJob(); err == nil {
+ t.Fatal("expected corrupted restore to fail")
+ }
+ if _, err := os.Stat(file); !os.IsNotExist(err) {
+ t.Fatal("corrupted archive installed workspace files")
+ }
+ if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || restored {
+ t.Fatalf("corrupted archive restored database: %v %v", restored, err)
+ }
+ mu.Lock()
+ if len(objects) == 0 {
+ t.Error("corrupted restore deleted the only remote package")
+ }
+ corrupt = false
+ mu.Unlock()
+ // Restoration commits files and PG first. Failed remote deletion is retryable.
+ mu.Lock()
+ failDelete = true
+ mu.Unlock()
+ if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.runOneTaskArchiveJob(); err == nil {
+ t.Fatal("expected remote delete failure")
+ }
+ if got, _ := os.ReadFile(file); string(got) != "keep this content" {
+ t.Fatal("restored content differs")
+ }
+ if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || !restored {
+ t.Fatalf("PG restore %v %v", restored, err)
+ }
+ mu.Lock()
+ failDelete = false
+ mu.Unlock()
+ if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.runOneTaskArchiveJob(); err != nil {
+ t.Fatal(err)
+ }
+ if item, err := m.pg.GetTaskArchive(job.ID); err != nil || item != nil {
+ t.Fatal("restore did not consume stub")
+ }
+ var count int
+ if err := m.pg.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='archive proof'`, pgTask.ExplorationID).Scan(&count); err != nil || count != 1 {
+ t.Fatalf("activity restore count=%d err=%v", count, err)
+ }
+ // Re-archive, then crash/retry the deletion outbox after PG removed its stub.
+ job, err = m.pg.QueueTaskArchive(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := s.runOneTaskArchiveJob(); err != nil {
+ t.Fatal(err)
+ }
+ mu.Lock()
+ failDelete = true
+ mu.Unlock()
+ if _, err := m.pg.QueueTaskArchiveDelete(job.ID); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.runOneTaskArchiveJob(); err != nil {
+ t.Fatal(err)
+ }
+ mu.Lock()
+ failDelete = false
+ mu.Unlock()
+ if err := s.recoverRemoteArchiveJournals(); err != nil {
+ t.Fatal(err)
+ }
+ mu.Lock()
+ remaining := len(objects)
+ mu.Unlock()
+ if remaining != 0 {
+ t.Fatalf("left %d objects", remaining)
+ }
+}
+
+// Once PostgreSQL has committed an archive location and checksum, a subsequent
+// queued operation must not make startup recovery resurrect the old workspace.
+func TestRemoteArchiveStageRecoveryAfterCommit(t *testing.T) {
+ if os.Getenv("ARTEX_PG_DSN") == "" {
+ t.Skip("requires isolated PostgreSQL")
+ }
+ m, err := NewManager(t.TempDir(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ task, err := m.CreateTask("remote stage recovery", "test", nil, 0, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ taskID, _ := strconv.ParseInt(task.ID, 10, 64)
+ defer func() {
+ _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID)
+ _ = m.pg.DeleteTask(taskID)
+ }()
+ if err := m.pg.SetPaused(taskID, true); err != nil {
+ t.Fatal(err)
+ }
+ job, err := m.pg.QueueTaskArchive(taskID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pgTask, err := m.pg.GetTask(taskID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} {
+ t.Run(state, func(t *testing.T) {
+ committed := state != pgdb.ArchiveQueued
+ checksum := ""
+ if committed {
+ checksum = strings.Repeat("a", 64)
+ }
+ if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, "s3://test/tasks/package.tar.zst", checksum); err != nil {
+ t.Fatal(err)
+ }
+ file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt")
+ if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(stage.root); !os.IsNotExist(err) {
+ t.Fatalf("staging cleanup incomplete: %v", err)
+ }
+ if committed {
+ if _, err := os.Stat(file); !os.IsNotExist(err) {
+ t.Fatalf("resurrected committed workspace: %v", err)
+ }
+ } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" {
+ t.Fatalf("uncommitted archive lost source: %v", err)
+ }
+ })
+ }
+}
+
+// Failed remote cleanup can survive switching to local storage and retrying the
+// same archive ID. Its journal must never remove the newly committed local copy.
+func TestRemoteArchiveJournalPreservesLocalRetry(t *testing.T) {
+ if os.Getenv("ARTEX_PG_DSN") == "" {
+ t.Skip("requires isolated PostgreSQL")
+ }
+ m, err := NewManager(t.TempDir(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ task, err := m.CreateTask("local retry", "test", nil, 0, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ taskID, _ := strconv.ParseInt(task.ID, 10, 64)
+ defer func() {
+ _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID)
+ _ = m.pg.DeleteTask(taskID)
+ }()
+ if err := m.pg.SetPaused(taskID, true); err != nil {
+ t.Fatal(err)
+ }
+ job, err := m.pg.QueueTaskArchive(taskID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ filename := taskArchivePath(m.dir, job.ID, task.ID)
+ if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filename, []byte("committed local archive"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, pgdb.ArchiveReady, filename, strings.Repeat("a", 64)); err != nil {
+ t.Fatal(err)
+ }
+ var deletes atomic.Int32
+ remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodDelete {
+ w.WriteHeader(http.StatusMethodNotAllowed)
+ return
+ }
+ deletes.Add(1)
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer remote.Close()
+ store, err := archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ s := &Server{ctx: context.Background(), m: m, archiveStore: store}
+ journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{
+ ArchiveID: job.ID, URI: store.URI(taskID, job.ID, strings.Repeat("b", 64)),
+ LocalPath: filename, Operation: "upload",
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := s.recoverRemoteArchiveJournal(journal); err != nil {
+ t.Fatal(err)
+ }
+ if data, err := os.ReadFile(filename); err != nil || string(data) != "committed local archive" {
+ t.Fatalf("old upload cleanup removed local retry: %v", err)
+ }
+ if deletes.Load() != 1 {
+ t.Fatal("uncommitted remote object was not removed")
+ }
+ if _, err := os.Stat(journal); !os.IsNotExist(err) {
+ t.Fatalf("finished journal remains: %v", err)
+ }
+}
diff --git a/server/task_archives.go b/server/task_archives.go
index b73dfb2..3ac6341 100644
--- a/server/task_archives.go
+++ b/server/task_archives.go
@@ -14,6 +14,7 @@ import (
"time"
"github.com/Autumn-27/artex/agent"
+ "github.com/Autumn-27/artex/archivestore"
pgdb "github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/traffic"
)
@@ -37,6 +38,12 @@ func (s *Server) startTaskArchiveWorker() {
if s.m == nil || s.m.pg == nil {
return
}
+ s.archiveStore, s.archiveStoreErr = archivestore.FromEnv()
+ if s.archiveStoreErr != nil {
+ log.Printf("[task-archive] storage configuration: %v", s.archiveStoreErr)
+ } else if s.archiveStore != nil {
+ log.Printf("[task-archive] new archives use S3 object storage with full SHA-256 readback verification")
+ }
if err := s.m.pg.RecoverTaskArchiveJobs(); err != nil {
log.Printf("[task-archive] recover jobs: %v", err)
}
@@ -88,6 +95,14 @@ func (s *Server) notifyTaskArchiveWorker() {
}
func (s *Server) runOneTaskArchiveJob() error {
+ if s.ctx.Err() != nil {
+ return s.ctx.Err()
+ }
+ // Retry deferred remote cleanup before claiming more work. Failed cleanup
+ // keeps its durable journal and does not block unrelated local archives.
+ if err := s.recoverRemoteArchiveJournals(); err != nil {
+ log.Printf("[task-archive] deferred object cleanup: %v", err)
+ }
job, err := s.m.pg.ClaimTaskArchiveJob(s.ctx)
if err != nil || job == nil {
return err
@@ -115,6 +130,9 @@ func (s *Server) runOneTaskArchiveJob() error {
}
func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
+ if s.archiveStoreErr != nil {
+ return s.archiveStoreErr
+ }
taskID := strconv.FormatInt(job.TaskID, 10)
if !s.beginTaskDelete(taskID) {
return errors.New("task is already undergoing another archive or delete operation")
@@ -199,6 +217,21 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
_ = os.Remove(archivePath)
}
}()
+ storedPath := archivePath
+ var remoteJournal string
+ if s.archiveStore != nil {
+ storedPath = s.archiveStore.URI(job.TaskID, job.ID, checksum)
+ remoteJournal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{
+ ArchiveID: job.ID, URI: storedPath, LocalPath: archivePath, Operation: "upload",
+ })
+ if err != nil {
+ return err
+ }
+ _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "upload_and_verify", 65)
+ if err := s.archiveStore.Upload(s.ctx, storedPath, archivePath, checksum); err != nil {
+ return err
+ }
+ }
var trafficStage *traffic.HostDeleteStage
if s.m.traffic != nil && len(snapshot.ExclusiveHosts) > 0 {
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_traffic", 68)
@@ -213,11 +246,18 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
}()
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_database", 78)
- if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, archivePath, checksum, originalSize, compressedSize); err != nil {
+ if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, storedPath, checksum, originalSize, compressedSize); err != nil {
return err
}
committed = true
- removePackage = false
+ removePackage = s.archiveStore != nil
+ if remoteJournal != "" {
+ // The journal also covers a crash after database commit but before local
+ // package removal. Keep it if cleanup fails, so the worker can retry.
+ if err := removeLocalArchivePackage(archivePath); err == nil {
+ _ = os.Remove(remoteJournal)
+ }
+ }
if trafficStage != nil {
if err := trafficStage.Commit(); err != nil {
warning := "archive completed, but cleaning the exclusive traffic hot store failed (the archive package can still restore it): " + err.Error()
@@ -239,7 +279,7 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
}
func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
- if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil {
+ if err := s.validateStoredArchive(job.ArchivePath); err != nil {
return err
}
// A prior attempt may have committed PostgreSQL and files, then failed only
@@ -247,7 +287,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
if restored, err := s.m.pg.IsTaskArchiveRestored(job.ID); err != nil {
return err
} else if restored {
- if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) {
+ if err := s.removeStoredArchive(job.ArchivePath); err != nil {
return err
}
if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil {
@@ -266,7 +306,15 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
return err
}
defer func() { _ = os.RemoveAll(extracted) }()
- if err := extractTaskArchivePackage(job.ArchivePath, job.SHA256, extracted); err != nil {
+ packagePath := job.ArchivePath
+ if strings.HasPrefix(packagePath, "s3://") {
+ _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "download_and_verify", 12)
+ packagePath = filepath.Join(extracted, "download.tar.zst")
+ if err := s.archiveStore.Download(s.ctx, job.ArchivePath, packagePath, job.SHA256, job.CompressedSize); err != nil {
+ return err
+ }
+ }
+ if err := extractTaskArchivePackage(packagePath, job.SHA256, extracted); err != nil {
return err
}
manifestRaw, err := os.ReadFile(filepath.Join(extracted, "manifest.json"))
@@ -356,7 +404,7 @@ func (s *Server) restoreTaskArchivePayload(job *pgdb.TaskArchive, snapshot *pgdb
return err
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "consume_package", 95)
- if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) {
+ if err := s.removeStoredArchive(job.ArchivePath); err != nil {
return err
}
if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil {
@@ -393,9 +441,12 @@ func (s *Server) loadRestoredTask(taskID int64) {
}
func (s *Server) deleteTaskArchive(job *pgdb.TaskArchive) error {
- if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil {
+ if err := s.validateStoredArchive(job.ArchivePath); err != nil {
return err
}
+ if strings.HasPrefix(job.ArchivePath, "s3://") {
+ return s.deleteRemoteTaskArchive(job)
+ }
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "stage_package_delete", 35)
staged, moved, err := stageTaskArchivePackageDelete(job.ArchivePath, job.ID)
if err != nil {
diff --git a/web/src/app/(main)/function/tasks/page.tsx b/web/src/app/(main)/function/tasks/page.tsx
index 6b8ba8e..c2f9cc3 100644
--- a/web/src/app/(main)/function/tasks/page.tsx
+++ b/web/src/app/(main)/function/tasks/page.tsx
@@ -1494,7 +1494,7 @@ function ArchiveConfirmDialog({
{count === 1 ? "Archive task" : `Archive ${count} tasks`}
- Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into local cold storage. Once archived, a task can be restored from "Archived".
+ Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into the configured archive storage. Once archived, a task can be restored from "Archived". Restoring requires free space on this server.
@@ -1934,6 +1934,7 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {
title={`Before compression: ${formatArchiveBytes(archive.original_size)}`}
>
{archiveCompressionLabel(archive)}
+ {archive.storage === "s3" ? "Object storage" : "Local storage"}
{archiveDataTotal(archive).toLocaleString()}
@@ -1952,7 +1953,7 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {
))}
- {archive.phase} · {archive.progress}%
+ {archive.phase === "upload_and_verify" ? "Uploading and verifying" : archive.phase === "download_and_verify" ? "Downloading and verifying" : archive.phase} · {archive.progress}%
diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts
index d098f9e..4a0b479 100644
--- a/web/src/lib/types.ts
+++ b/web/src/lib/types.ts
@@ -101,6 +101,7 @@ export interface TaskArchiveTokenStats {
}
export interface TaskArchive {
+ storage?: "local" | "s3";
id: number;
task_id: number;
state: TaskArchiveState;
From da7cc62d4eee283586fb3c0202467ec4e44b2fce Mon Sep 17 00:00:00 2001
From: Chubboy Ren
Date: Wed, 18 Nov 2026 02:05:17 +0000
Subject: [PATCH 2/4] feat: encrypt S3 archives and test credentials and
interruption recovery
---
archivestore/config_test.go | 20 ++
archivestore/credentials_test.go | 132 ++++++++++++
archivestore/s3.go | 73 +++++--
archivestore/s3_test.go | 279 ++++++++++++++++--------
docs/task-archive-storage.md | 194 +++++++++++------
go.mod | 6 +
server/task_archive_interrupt_test.go | 291 ++++++++++++++++++++++++++
server/task_archive_package.go | 11 +-
server/task_archive_remote_test.go | 93 ++++----
9 files changed, 892 insertions(+), 207 deletions(-)
create mode 100644 archivestore/credentials_test.go
create mode 100644 server/task_archive_interrupt_test.go
diff --git a/archivestore/config_test.go b/archivestore/config_test.go
index cc12532..73afc73 100644
--- a/archivestore/config_test.go
+++ b/archivestore/config_test.go
@@ -10,6 +10,7 @@ func TestArchiveStorageEnvironment(t *testing.T) {
"ARTEX_ARCHIVE_STORAGE", "ARTEX_ARCHIVE_S3_ENDPOINT", "ARTEX_ARCHIVE_S3_REGION",
"ARTEX_ARCHIVE_S3_BUCKET", "ARTEX_ARCHIVE_S3_PREFIX", "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID",
"ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_PATH_STYLE",
+ "ARTEX_ARCHIVE_S3_SESSION_TOKEN", "ARTEX_ARCHIVE_S3_SSE", "ARTEX_ARCHIVE_S3_KMS_KEY_ID",
} {
t.Setenv(key, "")
}
@@ -60,3 +61,22 @@ func TestArchiveEndpointValidation(t *testing.T) {
}
}
}
+
+func TestArchiveEncryptionConfiguration(t *testing.T) {
+ c := Config{Endpoint: "https://objects.example.test", Bucket: "test", AccessKey: "test", SecretKey: "test"}
+ for _, mode := range []string{"none", "invalid"} {
+ c.SSE = mode
+ if _, err := New(c); err == nil {
+ t.Fatal("unencrypted/invalid mode accepted")
+ }
+ }
+ c.SSE = "AES256"
+ c.KMSKeyID = "alias/archive"
+ if _, err := New(c); err == nil {
+ t.Fatal("KMS key accepted with SSE-S3")
+ }
+ c.SSE = "aws:kms"
+ if _, err := New(c); err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/archivestore/credentials_test.go b/archivestore/credentials_test.go
new file mode 100644
index 0000000..6c667ed
--- /dev/null
+++ b/archivestore/credentials_test.go
@@ -0,0 +1,132 @@
+package archivestore
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+)
+
+func isolateAWSCredentials(t *testing.T) {
+ t.Helper()
+ for _, key := range []string{
+ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AWS_PROFILE", "AWS_DEFAULT_PROFILE",
+ "AWS_ROLE_ARN", "AWS_ROLE_SESSION_NAME", "AWS_WEB_IDENTITY_TOKEN_FILE",
+ "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_CREDENTIALS_FULL_URI",
+ "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
+ "AWS_EC2_METADATA_SERVICE_ENDPOINT", "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE",
+ } {
+ t.Setenv(key, "")
+ }
+ dir := t.TempDir()
+ t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(dir, "credentials"))
+ t.Setenv("AWS_CONFIG_FILE", filepath.Join(dir, "config"))
+ t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
+}
+
+func TestArchiveCredentialChain(t *testing.T) {
+ config := Config{Endpoint: "https://objects.example.test", Bucket: "test"}
+ for _, mode := range []string{"explicit-session", "environment", "shared-profile", "ecs", "imds"} {
+ t.Run(mode, func(t *testing.T) {
+ isolateAWSCredentials(t)
+ c := config
+ switch mode {
+ case "explicit-session":
+ t.Setenv("AWS_ACCESS_KEY_ID", "other-key")
+ t.Setenv("AWS_SECRET_ACCESS_KEY", "other-secret")
+ c.AccessKey, c.SecretKey, c.SessionToken = "fixture-key", "fixture-secret", "fixture-session"
+ case "environment":
+ t.Setenv("AWS_ACCESS_KEY_ID", "fixture-key")
+ t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture-secret")
+ t.Setenv("AWS_SESSION_TOKEN", "fixture-session")
+ case "shared-profile":
+ t.Setenv("AWS_PROFILE", "archive")
+ data := `[archive]
+aws_access_key_id=fixture-key
+aws_secret_access_key=fixture-secret
+aws_session_token=fixture-session
+`
+ if err := os.WriteFile(os.Getenv("AWS_SHARED_CREDENTIALS_FILE"), []byte(data), 0600); err != nil {
+ t.Fatal(err)
+ }
+ case "ecs", "imds":
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if mode == "imds" {
+ switch r.URL.Path {
+ case "/latest/api/token":
+ if r.Method != "PUT" {
+ t.Error("IMDS did not use token request")
+ }
+ w.Header().Set("X-aws-ec2-metadata-token-ttl-seconds", "21600")
+ fmt.Fprint(w, "fixture-imds-token")
+ return
+ case "/latest/meta-data/iam/security-credentials/":
+ if r.Header.Get("X-aws-ec2-metadata-token") != "fixture-imds-token" {
+ t.Error("missing IMDSv2 token")
+ }
+ fmt.Fprint(w, "archive-role")
+ return
+ case "/latest/meta-data/iam/security-credentials/archive-role":
+ if r.Header.Get("X-aws-ec2-metadata-token") != "fixture-imds-token" {
+ t.Error("missing IMDSv2 token")
+ }
+ default:
+ t.Errorf("unexpected IMDS request %s", r.URL.Path)
+ w.WriteHeader(404)
+ return
+ }
+ } else if r.Header.Get("Authorization") != "fixture-container-token" {
+ t.Error("missing ECS authorization token")
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "Code": "Success", "AccessKeyId": "fixture-key", "SecretAccessKey": "fixture-secret",
+ "Token": "fixture-session", "Expiration": time.Now().Add(time.Hour).UTC().Format(time.RFC3339),
+ })
+ }))
+ defer srv.Close()
+ if mode == "imds" {
+ t.Setenv("AWS_EC2_METADATA_DISABLED", "false")
+ t.Setenv("AWS_EC2_METADATA_V1_DISABLED", "true")
+ t.Setenv("AWS_EC2_METADATA_SERVICE_ENDPOINT", srv.URL)
+ } else {
+ t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", srv.URL+"/credentials")
+ t.Setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN", "fixture-container-token")
+ }
+ }
+ store, err := New(c)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+ defer cancel()
+ credentials, err := store.client.Options().Credentials.Retrieve(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if credentials.AccessKeyID != "fixture-key" || credentials.SecretAccessKey != "fixture-secret" || credentials.SessionToken != "fixture-session" {
+ t.Fatal("wrong credential provider or session token")
+ }
+ if (mode == "imds" || mode == "ecs") && !credentials.CanExpire {
+ t.Fatal("role credentials must remain refreshable")
+ }
+ })
+ }
+}
+
+func TestArchiveIncompleteCredentialOverride(t *testing.T) {
+ isolateAWSCredentials(t)
+ t.Setenv("AWS_ACCESS_KEY_ID", "valid-ambient-key")
+ t.Setenv("AWS_SECRET_ACCESS_KEY", "valid-ambient-secret")
+ for _, c := range []Config{{AccessKey: "private-value"}, {SecretKey: "private-value"}, {SessionToken: "private-value"}} {
+ c.Endpoint, c.Bucket = "https://objects.example.test", "test"
+ if _, err := New(c); err == nil || strings.Contains(err.Error(), "private-value") {
+ t.Fatalf("partial override must fail without disclosing credentials: %v", err)
+ }
+ }
+}
diff --git a/archivestore/s3.go b/archivestore/s3.go
index edc1513..27c50af 100644
--- a/archivestore/s3.go
+++ b/archivestore/s3.go
@@ -17,19 +17,24 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/aws"
+ awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/aws-sdk-go-v2/service/s3/types"
)
type Config struct {
Endpoint, Region, Bucket, Prefix, AccessKey, SecretKey string
PathStyle bool
+ SessionToken, SSE, KMSKeyID string
}
type Store struct {
client *s3.Client
bucket, prefix string
+ encryption types.ServerSideEncryption
+ kmsKeyID string
}
// FromEnv returns nil for the default local backend. A configured but invalid
@@ -43,7 +48,9 @@ func FromEnv() (*Store, error) {
Endpoint: os.Getenv("ARTEX_ARCHIVE_S3_ENDPOINT"), Region: os.Getenv("ARTEX_ARCHIVE_S3_REGION"),
Bucket: os.Getenv("ARTEX_ARCHIVE_S3_BUCKET"), Prefix: os.Getenv("ARTEX_ARCHIVE_S3_PREFIX"),
AccessKey: os.Getenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID"), SecretKey: os.Getenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY"),
- PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true",
+ PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true",
+ SessionToken: os.Getenv("ARTEX_ARCHIVE_S3_SESSION_TOKEN"),
+ SSE: os.Getenv("ARTEX_ARCHIVE_S3_SSE"), KMSKeyID: os.Getenv("ARTEX_ARCHIVE_S3_KMS_KEY_ID"),
})
default:
return nil, errors.New("ARTEX_ARCHIVE_STORAGE must be local or s3")
@@ -70,22 +77,49 @@ func New(c Config) (*Store, error) {
if path.Clean(c.Prefix) != c.Prefix || strings.Contains(c.Prefix, "..") || strings.ContainsAny(c.Prefix, "\\?#") {
return nil, errors.New("invalid archive S3 prefix")
}
- if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") || c.AccessKey == "" || c.SecretKey == "" {
- return nil, errors.New("archive S3 bucket and dedicated access credentials are required")
+ if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") {
+ return nil, errors.New("a valid archive S3 bucket is required")
+ }
+ explicitCredentials := c.AccessKey != "" || c.SecretKey != "" || c.SessionToken != ""
+ if explicitCredentials && (c.AccessKey == "" || c.SecretKey == "") {
+ return nil, errors.New("archive S3 access key and secret key must both be set when overriding AWS credentials")
+ }
+ if c.SSE == "" {
+ c.SSE = "AES256"
+ }
+ if c.SSE != "AES256" && c.SSE != "aws:kms" {
+ return nil, errors.New("archive S3 SSE must be AES256 or aws:kms; encryption cannot be disabled")
+ }
+ if c.KMSKeyID != "" && c.SSE != "aws:kms" {
+ return nil, errors.New("archive S3 KMS key requires SSE=aws:kms")
}
if c.Region == "" {
c.Region = "us-east-1"
}
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.ResponseHeaderTimeout = 60 * time.Second
- client := s3.New(s3.Options{
- Region: c.Region, BaseEndpoint: aws.String(c.Endpoint), UsePathStyle: c.PathStyle,
- Credentials: credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, ""),
- HTTPClient: &http.Client{Transport: transport},
- RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
- ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired,
+ httpClient := &http.Client{Transport: transport}
+ cfg := aws.Config{Region: c.Region, HTTPClient: httpClient}
+ if explicitCredentials {
+ cfg.Credentials = credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken)
+ } else {
+ // Keep the SDK's refreshable default chain: standard AWS environment/shared
+ // profiles, web identity, ECS task roles and EC2 instance roles (IMDSv2).
+ var err error
+ cfg, err = awsconfig.LoadDefaultConfig(context.Background(),
+ awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient))
+ if err != nil {
+ return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err)
+ }
+ }
+ client := s3.NewFromConfig(cfg, func(o *s3.Options) {
+ o.BaseEndpoint = aws.String(c.Endpoint)
+ o.UsePathStyle = c.PathStyle
+ o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
+ o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
})
- return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/"}, nil
+ return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/",
+ encryption: types.ServerSideEncryption(c.SSE), kmsKeyID: c.KMSKeyID}, nil
}
func (s *Store) URI(taskID, archiveID int64, checksum string) string {
@@ -134,17 +168,22 @@ func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) erro
u.Concurrency = 2
u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
})
- _, err = uploader.Upload(ctx, &s3.PutObjectInput{
+ input := &s3.PutObjectInput{
Bucket: aws.String(s.bucket), Key: aws.String(key), Body: f,
ContentType: aws.String("application/zstd"), Metadata: map[string]string{"sha256": checksum},
- })
+ ServerSideEncryption: s.encryption,
+ }
+ if s.kmsKeyID != "" {
+ input.SSEKMSKeyId = aws.String(s.kmsKeyID)
+ }
+ _, err = uploader.Upload(ctx, input)
if err != nil {
return fmt.Errorf("upload archive: %w", err)
}
- return s.readVerified(ctx, key, io.Discard, checksum, info.Size())
+ return s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true)
}
-func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64) error {
+func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64, requireEncryption bool) error {
if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size {
return errors.New("invalid archive SHA-256")
}
@@ -155,6 +194,10 @@ func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, che
if err != nil {
return fmt.Errorf("inspect archive: %w", err)
}
+ if requireEncryption && (object.ServerSideEncryption != s.encryption ||
+ (s.encryption == types.ServerSideEncryptionAwsKms && aws.ToString(object.SSEKMSKeyId) == "")) {
+ return errors.New("remote archive did not confirm the requested server-side encryption; source data retained")
+ }
if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" {
return errors.New("remote archive size mismatch or missing ETag; source data retained")
}
@@ -264,7 +307,7 @@ func (s *Store) Download(ctx context.Context, uri, filename, checksum string, si
_ = os.Remove(filename)
}
}()
- if err = s.readVerified(ctx, key, f, checksum, size); err != nil {
+ if err = s.readVerified(ctx, key, f, checksum, size, false); err != nil {
return err
}
return f.Sync()
diff --git a/archivestore/s3_test.go b/archivestore/s3_test.go
index 316e770..dc618c1 100644
--- a/archivestore/s3_test.go
+++ b/archivestore/s3_test.go
@@ -13,99 +13,121 @@ import (
"os"
"path/filepath"
"strconv"
+ "strings"
"sync"
+ "sync/atomic"
"testing"
)
func TestS3RoundTripAndIntegrity(t *testing.T) {
- for _, size := range []int{1024, 10 << 20} {
- t.Run(strconv.Itoa(size), func(t *testing.T) {
- var mu sync.Mutex
- var object []byte
- parts := map[int][]byte{}
- corrupt, multipart := false, false
- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- mu.Lock()
- defer mu.Unlock()
- q := r.URL.Query()
- w.Header().Set("ETag", `"test-etag"`)
- switch {
- case r.Method == "POST" && q.Has("uploads"):
- multipart = true
- fmt.Fprint(w, `test-upload`)
- case r.Method == "PUT" && q.Has("partNumber"):
- n, _ := strconv.Atoi(q.Get("partNumber"))
- parts[n], _ = io.ReadAll(r.Body)
- case r.Method == "POST" && q.Has("uploadId"):
- for i := 1; i <= len(parts); i++ {
- object = append(object, parts[i]...)
+ for _, sse := range []string{"AES256", "aws:kms"} {
+ kms := ""
+ if sse == "aws:kms" {
+ kms = "arn:aws:kms:us-east-1:123456789012:key/test-key"
+ }
+ for _, size := range []int{1024, 10 << 20} {
+ t.Run(sse+"/"+strconv.Itoa(size), func(t *testing.T) {
+ var mu sync.Mutex
+ var object []byte
+ parts := map[int][]byte{}
+ corrupt, multipart := false, false
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mu.Lock()
+ defer mu.Unlock()
+ q := r.URL.Query()
+ if (r.Method == "PUT" && !q.Has("partNumber")) || (r.Method == "POST" && q.Has("uploads")) {
+ if r.Header.Get("X-Amz-Server-Side-Encryption") != sse || r.Header.Get("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id") != kms {
+ t.Error("upload did not explicitly request the configured encryption")
+ }
}
- fmt.Fprint(w, `test-etag`)
- case r.Method == "PUT":
- object, _ = io.ReadAll(r.Body)
- case r.Method == "HEAD":
- w.Header().Set("Content-Length", strconv.Itoa(len(object)))
- case r.Method == "GET":
- data := object
- if corrupt {
- data = bytes.Clone(object)
- data[0] ^= 0xff
+ if r.Method == "HEAD" {
+ if r.Header.Get("X-Amz-Server-Side-Encryption") != "" {
+ t.Error("encryption write header leaked into HEAD")
+ }
+ w.Header().Set("X-Amz-Server-Side-Encryption", sse)
+ if kms != "" {
+ w.Header().Set("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id", kms)
+ }
}
- serveArchiveRange(t, w, r, data)
- case r.Method == "DELETE":
- object = nil
- w.WriteHeader(204)
- default:
- t.Errorf("unexpected request %s %s", r.Method, r.URL)
- w.WriteHeader(400)
+ w.Header().Set("ETag", `"test-etag"`)
+ switch {
+ case r.Method == "POST" && q.Has("uploads"):
+ multipart = true
+ fmt.Fprint(w, `test-upload`)
+ case r.Method == "PUT" && q.Has("partNumber"):
+ n, _ := strconv.Atoi(q.Get("partNumber"))
+ parts[n], _ = io.ReadAll(r.Body)
+ case r.Method == "POST" && q.Has("uploadId"):
+ for i := 1; i <= len(parts); i++ {
+ object = append(object, parts[i]...)
+ }
+ fmt.Fprint(w, `test-etag`)
+ case r.Method == "PUT":
+ object, _ = io.ReadAll(r.Body)
+ case r.Method == "HEAD":
+ w.Header().Set("Content-Length", strconv.Itoa(len(object)))
+ case r.Method == "GET":
+ data := object
+ if corrupt {
+ data = bytes.Clone(object)
+ data[0] ^= 0xff
+ }
+ serveArchiveRange(t, w, r, data)
+ case r.Method == "DELETE":
+ object = nil
+ w.WriteHeader(204)
+ default:
+ t.Errorf("unexpected request %s %s", r.Method, r.URL)
+ w.WriteHeader(400)
+ }
+ }))
+ defer srv.Close()
+ store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true, SSE: sse, KMSKeyID: kms})
+ if err != nil {
+ t.Fatal(err)
}
- }))
- defer srv.Close()
- store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
- if err != nil {
- t.Fatal(err)
- }
- data := bytes.Repeat([]byte{0x42}, size)
- h := sha256.Sum256(data)
- checksum := hex.EncodeToString(h[:])
- root := t.TempDir()
- source := filepath.Join(root, "source")
- if err := os.WriteFile(source, data, 0600); err != nil {
- t.Fatal(err)
- }
- uri := store.URI(1, 2, checksum)
- ctx := context.Background()
- if err := store.Upload(ctx, uri, source, checksum); err != nil {
- t.Fatal(err)
- }
- if multipart != (size > 8<<20) {
- t.Fatalf("multipart=%v size=%d", multipart, size)
- }
- dest := filepath.Join(root, "download")
- if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil {
- t.Fatal(err)
- }
- got, _ := os.ReadFile(dest)
- if !bytes.Equal(got, data) {
- t.Fatal("download differs")
- }
- mu.Lock()
- corrupt = true
- mu.Unlock()
- bad := filepath.Join(root, "bad")
- if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil {
- t.Fatal("accepted corrupt object")
- }
- if _, err := os.Stat(bad); !os.IsNotExist(err) {
- t.Fatal("left corrupt download")
- }
- if err := store.Upload(ctx, uri, source, checksum); err == nil {
- t.Fatal("upload accepted failed readback")
- }
- if err := store.Delete(ctx, uri); err != nil {
- t.Fatal(err)
- }
- })
+ data := bytes.Repeat([]byte{0x42}, size)
+ h := sha256.Sum256(data)
+ checksum := hex.EncodeToString(h[:])
+ root := t.TempDir()
+ source := filepath.Join(root, "source")
+ if err := os.WriteFile(source, data, 0600); err != nil {
+ t.Fatal(err)
+ }
+ uri := store.URI(1, 2, checksum)
+ ctx := context.Background()
+ if err := store.Upload(ctx, uri, source, checksum); err != nil {
+ t.Fatal(err)
+ }
+ if multipart != (size > 8<<20) {
+ t.Fatalf("multipart=%v size=%d", multipart, size)
+ }
+ dest := filepath.Join(root, "download")
+ if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil {
+ t.Fatal(err)
+ }
+ got, _ := os.ReadFile(dest)
+ if !bytes.Equal(got, data) {
+ t.Fatal("download differs")
+ }
+ mu.Lock()
+ corrupt = true
+ mu.Unlock()
+ bad := filepath.Join(root, "bad")
+ if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil {
+ t.Fatal("accepted corrupt object")
+ }
+ if _, err := os.Stat(bad); !os.IsNotExist(err) {
+ t.Fatal("left corrupt download")
+ }
+ if err := store.Upload(ctx, uri, source, checksum); err == nil {
+ t.Fatal("upload accepted failed readback")
+ }
+ if err := store.Delete(ctx, uri); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
}
}
@@ -256,7 +278,7 @@ func TestS3ReadbackPropagatesOutputFailure(t *testing.T) {
store := testRangeStore(t, srv.URL)
h := sha256.Sum256(data)
outputError := errors.New("output disk full")
- err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data)))
+ err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data)), false)
if !errors.Is(err, outputError) {
t.Fatalf("lost output error: %v", err)
}
@@ -309,3 +331,88 @@ func TestS3RejectsForeignLocations(t *testing.T) {
t.Fatal("missing config accepted")
}
}
+
+func TestS3UploadRejectsUnconfirmedEncryption(t *testing.T) {
+ for _, mode := range []string{"missing", "different", "kms-without-key"} {
+ t.Run(mode, func(t *testing.T) {
+ data := []byte("private archive")
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.Method {
+ case "PUT":
+ _, _ = io.Copy(io.Discard, r.Body)
+ case "HEAD":
+ w.Header().Set("ETag", `"test-etag"`)
+ w.Header().Set("Content-Length", strconv.Itoa(len(data)))
+ if mode != "missing" {
+ w.Header().Set("X-Amz-Server-Side-Encryption", "aws:kms")
+ }
+ default:
+ t.Errorf("content must not be accepted before encryption confirmation: %s", r.Method)
+ w.WriteHeader(400)
+ }
+ }))
+ defer srv.Close()
+ c := Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}
+ if mode == "kms-without-key" {
+ c.SSE = "aws:kms"
+ }
+ store, err := New(c)
+ if err != nil {
+ t.Fatal(err)
+ }
+ filename := filepath.Join(t.TempDir(), "source")
+ if err := os.WriteFile(filename, data, 0600); err != nil {
+ t.Fatal(err)
+ }
+ sum := sha256.Sum256(data)
+ checksum := hex.EncodeToString(sum[:])
+ if err := store.Upload(context.Background(), store.URI(1, 2, checksum), filename, checksum); err == nil || !strings.Contains(err.Error(), "encryption") {
+ t.Fatalf("unconfirmed encryption accepted: %v", err)
+ }
+ if got, err := os.ReadFile(filename); err != nil || !bytes.Equal(got, data) {
+ t.Fatal("source was not retained", err)
+ }
+ })
+ }
+}
+
+func TestS3MultipartFailureAbortsUpload(t *testing.T) {
+ var aborted atomic.Bool
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ q := r.URL.Query()
+ switch {
+ case r.Method == "POST" && q.Has("uploads"):
+ if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" {
+ t.Error("multipart encryption missing")
+ }
+ fmt.Fprint(w, `failed-upload`)
+ case r.Method == "PUT" && q.Has("partNumber"):
+ _, _ = io.Copy(io.Discard, r.Body)
+ w.WriteHeader(403)
+ case r.Method == "DELETE" && q.Get("uploadId") == "failed-upload":
+ aborted.Store(true)
+ w.WriteHeader(204)
+ default:
+ t.Errorf("unexpected failed-upload request: %s", r.Method)
+ w.WriteHeader(400)
+ }
+ }))
+ defer srv.Close()
+ store := testRangeStore(t, srv.URL)
+ data := bytes.Repeat([]byte("x"), 9<<20)
+ checksum := sha256.Sum256(data)
+ sum := hex.EncodeToString(checksum[:])
+ filename := filepath.Join(t.TempDir(), "source")
+ if err := os.WriteFile(filename, data, 0600); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.Upload(context.Background(), store.URI(1, 2, sum), filename, sum); err == nil {
+ t.Fatal("failed part accepted")
+ }
+ if !aborted.Load() {
+ t.Fatal("failed multipart upload was not aborted")
+ }
+ if info, err := os.Stat(filename); err != nil || info.Size() != int64(len(data)) {
+ t.Fatal("source was not retained", err)
+ }
+}
diff --git a/docs/task-archive-storage.md b/docs/task-archive-storage.md
index 18bb3b4..113921a 100644
--- a/docs/task-archive-storage.md
+++ b/docs/task-archive-storage.md
@@ -1,21 +1,19 @@
# Task archive storage
Task archives use local disk by default. The optional S3-compatible backend moves
-cold task packages to object storage after verifying their contents, releasing
-local package space while retaining ARTEX's Archive, Restore and Delete actions.
-
-Active workspaces and the live PostgreSQL database stay local. This feature is
-task-level cold storage; use the existing backup workflow for a full installation
-backup. A complete recovery plan must also preserve the bucket objects and
-access to them.
+cold task packages to object storage while retaining Archive, Restore and Delete.
+Active workspaces and PostgreSQL remain local. This is task cold storage, not a
+full-installation backup; a recovery plan must preserve the database, data
+directory, bucket objects, and the credentials or roles needed to read them.
## Configuration
-Create a private bucket and a dedicated credential scoped to the archive prefix.
-The backend needs object upload, read (including HEAD and byte ranges), delete,
-and multipart upload/abort permissions. Use a separate prefix for each deployment.
+Use a private bucket and a separate prefix for each ARTEX installation. Configure
+the bucket's default encryption to match the selected mode below, and enforce
+that mode/key with a bucket policy. ARTEX does not provision or inspect bucket
+encryption settings.
-Set these environment variables for the ARTEX server, then restart it:
+Set the following server environment and restart ARTEX:
```dotenv
ARTEX_ARCHIVE_STORAGE=s3
@@ -23,82 +21,140 @@ ARTEX_ARCHIVE_S3_ENDPOINT=https://s3.us-east-1.amazonaws.com
ARTEX_ARCHIVE_S3_REGION=us-east-1
ARTEX_ARCHIVE_S3_BUCKET=your-artex-archives
ARTEX_ARCHIVE_S3_PREFIX=artex/tasks
-ARTEX_ARCHIVE_S3_ACCESS_KEY_ID=REPLACE
-ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY=REPLACE
+ARTEX_ARCHIVE_S3_SSE=AES256
```
-Store credentials in the service's protected environment file, outside the
-repository and frontend. For systemd, a root-owned environment file with mode
-0600 can be referenced through `EnvironmentFile=`.
+With an EC2 instance role, ECS task role or other SDK-supported credential source,
+no archive-specific access keys are needed.
| Variable | Meaning |
| --- | --- |
| `ARTEX_ARCHIVE_STORAGE` | `local` (default) or `s3`. |
-| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local testing. |
-| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. Match the provider's requirements. |
+| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local tests. |
+| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. |
| `ARTEX_ARCHIVE_S3_BUCKET` | Required bucket name. |
| `ARTEX_ARCHIVE_S3_PREFIX` | Object prefix; defaults to `artex/tasks`. |
-| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Dedicated access key. |
-| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Dedicated secret key. |
-| `ARTEX_ARCHIVE_S3_PATH_STYLE` | Set to `true` for compatible path-style servers; otherwise virtual-hosted addressing is used. |
+| `ARTEX_ARCHIVE_S3_SSE` | `AES256` (default, SSE-S3) or `aws:kms` (SSE-KMS). There is no unencrypted mode. |
+| `ARTEX_ARCHIVE_S3_KMS_KEY_ID` | Optional KMS key ARN/ID/alias; valid only with `aws:kms`. Omit to use the provider's managed KMS key. |
+| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Optional dedicated credential override; must be supplied with the secret key. |
+| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Secret for the dedicated override. |
+| `ARTEX_ARCHIVE_S3_SESSION_TOKEN` | Optional session token for dedicated temporary credentials. |
+| `ARTEX_ARCHIVE_S3_PATH_STYLE` | `true` for path-style servers; otherwise virtual-hosted addressing is used. |
-For DigitalOcean Spaces, use its regional origin, such as
-`https://sfo3.digitaloceanspaces.com`, and the provider's signing region. For a
-local path-style test server, use an endpoint such as `http://127.0.0.1:9000` and
-set `ARTEX_ARCHIVE_S3_PATH_STYLE=true`.
+An invalid mode or partial credential override fails configuration instead of
+silently falling back to local storage or ambient credentials. Preserve the
+bucket, endpoint, prefix and read/delete access while remote archives remain.
+Older local archives remain usable; they are not uploaded automatically.
-Invalid S3 configuration fails new archive jobs instead of silently falling back
-to local disk. Preserve the configured bucket, endpoint, prefix and read/delete
-access while remote archives remain. The same installation can still restore or
-delete older local archives.
+## Encryption and permissions
-## Lifecycle and integrity
+Every single-object upload and multipart initiation explicitly requests the
+configured server-side encryption. Upload verification also requires HEAD to
+report the requested mode, and a KMS key identifier for SSE-KMS, before accepting
+the object. Size and full SHA-256 readback remain mandatory; encryption metadata
+does not replace content verification.
-1. Pause or finish a task and use its existing **Archive** action.
-2. ARTEX builds its `.tar.zst` package locally and uploads it. Large packages use
- multipart upload, starting with 8 MiB parts and two upload workers (the SDK
- increases the part size for exceptionally large seekable files).
-3. ARTEX reads the entire object back and checks its size and SHA-256. Multipart
- ETags and uploaded checksum metadata alone are not treated as proof of content.
-4. Only after verification does PostgreSQL commit the `s3://` location and
- compact the archived task. ARTEX then removes local source files and the
- temporary package. The archive list shows **Object storage** or **Local storage**.
-5. **Restore** downloads and verifies the package before restoring task files and
- database records. The remote package is consumed after restoration commits.
- Failed remote cleanup can be retried without replaying restored records.
-6. **Delete** removes the archive stub and queues durable cleanup of its remote
- object. Cleanup is retried after restart and before subsequent archive jobs.
+For SSE-KMS, set both the bucket's default encryption and ARTEX to `aws:kms`;
+set `ARTEX_ARCHIVE_S3_KMS_KEY_ID` to the intended key when using a customer-managed
+key. An explicit `AES256` request takes precedence over a bucket's KMS default,
+so do not leave ARTEX on SSE-S3 when the bucket is intended to use KMS.
-Reads use bounded 8 MiB ranges pinned to the object's ETag. Interrupted ranges
-are retried without duplicating output or restarting a multi-gigabyte download.
-An object change, checksum mismatch or invalid range fails verification. Failed
-downloads are removed, and upload/verification failures retain the original task.
+The principal needs object upload, read (HEAD and ranged GET), delete and
+multipart upload/abort permissions. KMS uploads/readback also need the applicable
+key-policy and IAM permissions for `kms:GenerateDataKey` and `kms:Decrypt`.
+Retain old KMS keys and decrypt permissions when changing the key or mode; restore
+does not require an older object to use the current upload policy.
-There is no database schema or archive format migration. Existing local archive
-packages remain compatible. Files outside ARTEX's task archive database are not
-automatically imported or uploaded.
+Compatible providers must support the requested SSE mode and its HEAD metadata,
+plus conditional ranged reads. A provider that ignores the encryption request
+or cannot confirm it is rejected; the original task is retained. Local source and
+temporary files keep their existing local-disk protection and restricted modes.
-## Disk, transfer and recovery
+See the official [PutObject encryption and permission contract](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html)
+and [HEAD encryption response rules](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html).
-Archive creation still needs space for the temporary compressed package.
-Restoration needs space for the downloaded package and extracted files. Full
-upload readback and later restores incur object-storage reads and any applicable
-transfer charges. Configure expiration of abandoned multipart uploads for abrupt
-process termination.
+## Credential sources
-Recovery journals under `data/archives/tasks/.remote` contain object locations,
-not credentials. Preserve them with PostgreSQL and the installation's data
-directory. The worker uses these journals to clean up uploads that never
-committed, keep committed objects, and finish deferred deletions. Failed remote
-cleanup does not block unrelated local archive jobs.
+Without the archive-specific override, ARTEX uses the AWS SDK default credential
+chain, including standard AWS environment variables, shared profiles, web
+identity, ECS task credentials and EC2 instance credentials through IMDS. Role
+credentials retain the SDK's automatic refresh behavior. A named shared profile
+can be selected with `AWS_PROFILE`.
+
+For an external S3-compatible provider, dedicated keys can be supplied through
+the archive-specific variables. Both key fields are required; a session token
+is supported for temporary credentials. This explicit override takes precedence
+over ambient AWS credentials. Operator-supplied temporary credentials must be
+rotated by the operator; they are not an IAM role provider.
+
+Keep secrets in a protected service environment file outside the repository and
+frontend. A systemd `EnvironmentFile=` can point to a root-owned mode-0600 file.
+See the [AWS SDK credential-chain documentation](https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-gosdk.html).
+
+## Lifecycle
+
+1. Pause or finish a task and use its **Archive** action.
+2. ARTEX stages the source files and builds a local `.tar.zst` package. Large
+ packages use multipart upload, initially 8 MiB parts with two upload workers;
+ the SDK increases the part size for exceptionally large seekable files.
+3. ARTEX confirms encryption metadata and reads the complete object back,
+ checking size and SHA-256. ETags and uploaded checksum metadata alone are
+ insufficient.
+4. **Only after verification and a successful PostgreSQL commit** does ARTEX
+ remove the local package and staged source files. The committed archive
+ location is an `s3://` URI; there is no schema or archive-format migration.
+5. **Restore** downloads and verifies before installing files and database rows.
+ The remote package is consumed only after restoration commits.
+6. **Delete** commits removal of the archive stub, then cleans up the object.
+ A durable journal retains failed cleanup work.
+
+The archive list identifies **Object storage** and **Local storage** and shows
+upload/download verification phases. Reads use bounded 8 MiB ranges pinned to the
+object's ETag, with retries for interrupted ranges rather than the entire object.
+
+## Interruption recovery
+
+Source staging and restore journals live with task archives. Remote cleanup
+journals live under `data/archives/tasks/.remote`; they contain locations, not
+credentials. Keep this directory with PostgreSQL and the installation data.
+
+| Interruption | Recovery |
+| --- | --- |
+| During staging/compression, before an upload journal exists | Roll staged source files back. Remove any temporary package only after rollback succeeds. The task remains in PostgreSQL. |
+| After the upload journal, before successful verification | Roll source files back and clean up any uncommitted object/package. Failed cleanup retains its journal for retry. |
+| After verification, before the database commit | Treat the object as uncommitted: retain the original task and clean up the orphan object. Verification alone never authorizes local deletion. |
+| After the database commit, before package/staging cleanup | Keep the committed object and finish local cleanup. The persisted location and checksum establish commit, even if Restore or Delete has since been queued. |
+| Download/extraction before restored files are installed | Remove temporary extraction data; keep the remote object and archive stub for another attempt. |
+| Restored files installed, before database restoration commits | Roll back installed files using the restore journal. Retain the remote package. |
+| Database restoration committed, before remote cleanup | Recognize the already-restored task and finish cleanup without replaying database rows. |
+| Object consumed, before the restored archive stub is removed | Skip download/replay; repeat idempotent object deletion and remove the stub. |
+| Delete journal written, before database deletion commits | Keep the object while its archive stub exists. The queued Delete operation can retry. |
+| Database deletion committed, before object deletion succeeds | The durable outbox retains the URI and retries object deletion. |
+| An old failed upload survives a retry using local storage | Remove the abandoned remote object, but retain a local package referenced by the committed archive row. |
+
+The staging recovery rule also applies to local archives. Dedicated tests cover
+both storage modes with ready, restore-queued, delete-queued and uncommitted
+states. Additional PostgreSQL/HTTP-S3 tests reconstruct interrupted operations at
+13 persisted boundaries and repeat recovery to check idempotence.
+
+Handled multipart-upload failures are aborted by the SDK. Abrupt process death
+may leave unfinished multipart uploads; configure bucket lifecycle expiration
+for those parts. Existing object versions, if versioning is enabled, are governed
+by the bucket's version-retention policy.
+
+## Disk and transfer requirements
+
+Archive creation still requires temporary local space for the compressed
+package. Restore requires space for the download and extracted files. Full
+upload readback and restores incur storage reads and applicable transfer charges.
+Failed remote cleanup does not prevent unrelated local archive jobs from being
+processed once the failed cleanup request returns.
## Optional connectivity check
-Run `go run ./cmd/artex-archive-check` with the same archive environment as the
-server to upload, verify, download and delete a small uniquely named test object.
-This command performs real bucket operations but does not touch task data. It
-reports leftover test objects if cleanup fails.
+Run `go run ./cmd/artex-archive-check` with the archive environment to upload,
+verify, download and delete a small unique object. It performs real bucket
+operations but does not touch task data, and reports cleanup failures.
-Automated tests use an HTTP S3 fixture and isolated PostgreSQL databases. They
-cover multipart transfers, corruption, interrupted range retries, object changes,
-cancellation, upload rollback, corrupted restores and persistent cleanup.
+Automated tests use HTTP fixtures for S3 and metadata credentials, plus isolated
+PostgreSQL. They do not establish compatibility with a live cloud bucket.
diff --git a/go.mod b/go.mod
index 1ce0982..353f837 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.26.10
require (
github.com/Autumn-27/norma v0.4.11
github.com/aws/aws-sdk-go-v2 v1.47.5
+ github.com/aws/aws-sdk-go-v2/config v1.33.13
github.com/aws/aws-sdk-go-v2/credentials v1.20.13
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6
@@ -27,6 +28,7 @@ require (
github.com/andybalholm/brotli v1.2.1 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect
+ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect
@@ -34,6 +36,10 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect
+ github.com/aws/aws-sdk-go-v2/service/signin v1.12.1 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sso v1.39.1 // indirect
+ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect
github.com/aws/smithy-go v1.28.7 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/creack/pty v1.1.24 // indirect
diff --git a/server/task_archive_interrupt_test.go b/server/task_archive_interrupt_test.go
new file mode 100644
index 0000000..2f4a44e
--- /dev/null
+++ b/server/task_archive_interrupt_test.go
@@ -0,0 +1,291 @@
+package server
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strconv"
+ "sync"
+ "testing"
+
+ "github.com/Autumn-27/artex/archivestore"
+ pgdb "github.com/Autumn-27/artex/db"
+)
+
+// Recreate persisted states at operation boundaries, then run the same recovery
+// functions used at startup. PostgreSQL, files, and HTTP object storage are real
+// isolated fixtures; no production failpoint hooks are needed.
+func TestRemoteArchiveInterruptionRecovery(t *testing.T) {
+ if os.Getenv("ARTEX_PG_DSN") == "" {
+ t.Skip("requires isolated PostgreSQL")
+ }
+ phases := []string{
+ "staged_before_compression", "compressed_before_journal", "journal_before_upload",
+ "uploaded_before_verification", "verified_before_database",
+ "database_before_local_cleanup", "package_cleaned_before_stage_cleanup",
+ "restore_download_before_install", "restore_files_before_database", "restore_database_before_object_delete", "restore_object_deleted_before_stub",
+ "delete_journal_before_database", "delete_database_before_object",
+ }
+ for _, phase := range phases {
+ t.Run(phase, func(t *testing.T) {
+ m, err := NewManager(t.TempDir(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer m.Close()
+ store, objects := newRecoveryObjectStore(t)
+ s := &Server{ctx: context.Background(), m: m, archiveStore: store}
+ task, err := m.CreateTask("interruption "+phase, "preserve source", nil, 0, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ id, _ := strconv.ParseInt(task.ID, 10, 64)
+ defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }()
+ if err := m.pg.SetPaused(id, true); err != nil {
+ t.Fatal(err)
+ }
+ job, err := m.pg.QueueTaskArchive(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Archiving); err != nil {
+ t.Fatal(err)
+ }
+ source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt")
+ if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(source, []byte("original task data"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ snapshot, err := m.pg.SnapshotTaskArchive(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ packagePath := taskArchivePath(m.dir, job.ID, task.ID)
+ var original, compressed int64
+ var checksum, uri, journal string
+ if phase != "staged_before_compression" {
+ original, compressed, checksum, err = writeTaskArchivePackage(packagePath, stage.payload, snapshot)
+ if err != nil {
+ t.Fatal(err)
+ }
+ uri = store.URI(id, job.ID, checksum)
+ }
+ if phase != "staged_before_compression" && phase != "compressed_before_journal" {
+ journal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, LocalPath: packagePath, Operation: "upload"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ committed := phase != "staged_before_compression" && phase != "compressed_before_journal" &&
+ phase != "journal_before_upload" && phase != "uploaded_before_verification" && phase != "verified_before_database"
+ if phase == "uploaded_before_verification" {
+ data, err := os.ReadFile(packagePath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ objects.put(uri, data)
+ } else if phase == "verified_before_database" || committed {
+ if err := store.Upload(s.ctx, uri, packagePath, checksum); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if committed {
+ if err := m.pg.CompleteTaskArchive(job.ID, snapshot, uri, checksum, original, compressed); err != nil {
+ t.Fatal(err)
+ }
+ }
+ afterArchive := phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" ||
+ phase == "restore_object_deleted_before_stub" || phase == "delete_journal_before_database" || phase == "delete_database_before_object"
+ if phase == "package_cleaned_before_stage_cleanup" || afterArchive {
+ if err := os.Remove(packagePath); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Remove(journal); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if afterArchive {
+ if err := stage.commit(); err != nil {
+ t.Fatal(err)
+ }
+ }
+ restored := false
+ remoteDeleted := false
+ if phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" {
+ if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Restoring); err != nil {
+ t.Fatal(err)
+ }
+ parent := filepath.Join(taskArchiveRoot(m.dir), ".restore")
+ if err := os.MkdirAll(parent, 0700); err != nil {
+ t.Fatal(err)
+ }
+ extracted, err := os.MkdirTemp(parent, "interrupted-")
+ if err != nil {
+ t.Fatal(err)
+ }
+ downloaded := filepath.Join(extracted, "download.tar.zst")
+ if err := store.Download(s.ctx, uri, downloaded, checksum, compressed); err != nil {
+ t.Fatal(err)
+ }
+ if err := extractTaskArchivePackage(downloaded, checksum, extracted); err != nil {
+ t.Fatal(err)
+ }
+ if phase != "restore_download_before_install" {
+ if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" {
+ if _, err := m.pg.RestoreTaskArchive(job.ID, snapshot, 0); err != nil {
+ t.Fatal(err)
+ }
+ restored = true
+ }
+ if phase == "restore_object_deleted_before_stub" {
+ if err := store.Delete(s.ctx, uri); err != nil {
+ t.Fatal(err)
+ }
+ remoteDeleted = true
+ }
+ }
+ deleting := phase == "delete_journal_before_database" || phase == "delete_database_before_object"
+ if deleting {
+ if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Deleting); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, Operation: "delete"}); err != nil {
+ t.Fatal(err)
+ }
+ if phase == "delete_database_before_object" {
+ if err := m.pg.DeleteTaskArchiveStub(job.ID); err != nil {
+ t.Fatal(err)
+ }
+ remoteDeleted = true
+ }
+ }
+ if err := m.pg.RecoverTaskArchiveJobs(); err != nil {
+ t.Fatal(err)
+ }
+ if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
+ t.Fatal(err)
+ }
+ if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.recoverRemoteArchiveJournals(); err != nil {
+ t.Fatal(err)
+ }
+ wantSource := !committed || restored
+ data, fileErr := os.ReadFile(source)
+ if wantSource && (fileErr != nil || string(data) != "original task data") {
+ t.Fatalf("source data lost: %v", fileErr)
+ }
+ if !wantSource && !os.IsNotExist(fileErr) {
+ t.Fatalf("archived source resurrected: %v", fileErr)
+ }
+ if _, err := os.Stat(packagePath); !os.IsNotExist(err) {
+ t.Fatalf("temporary package remains: %v", err)
+ }
+ if got, want := objects.has(uri), committed && !remoteDeleted; got != want {
+ t.Fatalf("remote object exists=%v, want=%v", got, want)
+ }
+ if committed && !deleting {
+ actual, err := m.pg.IsTaskArchiveRestored(job.ID)
+ if err != nil || actual != restored {
+ t.Fatalf("restored=%v want=%v err=%v", actual, restored, err)
+ }
+ }
+ // A second startup must be harmless at every boundary.
+ if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
+ t.Fatal(err)
+ }
+ if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil {
+ t.Fatal(err)
+ }
+ if err := s.recoverRemoteArchiveJournals(); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+}
+
+type recoveryObjects struct {
+ mu sync.Mutex
+ data map[string][]byte
+}
+
+func (o *recoveryObjects) put(uri string, data []byte) {
+ o.mu.Lock()
+ defer o.mu.Unlock()
+ // Test URIs always use the fixture bucket with path-style addressing.
+ o.data["/test/"+uri[len("s3://test/"):]] = append([]byte(nil), data...)
+}
+func (o *recoveryObjects) has(uri string) bool {
+ o.mu.Lock()
+ defer o.mu.Unlock()
+ if uri == "" {
+ return false
+ }
+ _, ok := o.data["/test/"+uri[len("s3://test/"):]]
+ return ok
+}
+func newRecoveryObjectStore(t *testing.T) (*archivestore.Store, *recoveryObjects) {
+ t.Helper()
+ objects := &recoveryObjects{data: map[string][]byte{}}
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ objects.mu.Lock()
+ defer objects.mu.Unlock()
+ data, exists := objects.data[r.URL.Path]
+ w.Header().Set("ETag", `"recovery-etag"`)
+ switch r.Method {
+ case "PUT":
+ if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" {
+ t.Error("missing SSE")
+ }
+ objects.data[r.URL.Path], _ = io.ReadAll(r.Body)
+ case "HEAD":
+ if !exists {
+ w.WriteHeader(404)
+ return
+ }
+ w.Header().Set("Content-Length", strconv.Itoa(len(data)))
+ w.Header().Set("X-Amz-Server-Side-Encryption", "AES256")
+ case "GET":
+ if !exists {
+ w.WriteHeader(404)
+ return
+ }
+ var start, end int
+ if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end >= len(data) || r.Header.Get("If-Match") != `"recovery-etag"` {
+ w.WriteHeader(416)
+ return
+ }
+ w.Header().Set("Content-Length", strconv.Itoa(end-start+1))
+ w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
+ w.WriteHeader(206)
+ _, _ = w.Write(data[start : end+1])
+ case "DELETE":
+ delete(objects.data, r.URL.Path)
+ w.WriteHeader(204)
+ default:
+ t.Errorf("unexpected object request %s", r.Method)
+ w.WriteHeader(400)
+ }
+ }))
+ t.Cleanup(srv.Close)
+ store, err := archivestore.New(archivestore.Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return store, objects
+}
diff --git a/server/task_archive_package.go b/server/task_archive_package.go
index e9a451d..e365ea3 100644
--- a/server/task_archive_package.go
+++ b/server/task_archive_package.go
@@ -437,9 +437,16 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error {
stage := &taskArchiveFileStage{root: root, payload: filepath.Join(root, "payload"), journal: journal}
if err := stage.rollback(); err != nil {
errs = append(errs, err)
+ continue
}
- if archive != nil && archive.ArchivePath != "" && !strings.HasPrefix(archive.ArchivePath, "s3://") {
- _ = os.Remove(archive.ArchivePath)
+ // The process can exit after compression but before an upload journal or
+ // database location exists. Remove that temporary package only after the
+ // source-file rollback succeeds, for both local and remote backends.
+ packagePath := taskArchivePath(dataDir, journal.ArchiveID, journal.TaskID)
+ if err := validateArchivePath(dataDir, packagePath); err != nil {
+ errs = append(errs, err)
+ } else if err := removeLocalArchivePackage(packagePath); err != nil {
+ errs = append(errs, err)
}
}
return errors.Join(errs...)
diff --git a/server/task_archive_remote_test.go b/server/task_archive_remote_test.go
index 34e0848..ac2a44d 100644
--- a/server/task_archive_remote_test.go
+++ b/server/task_archive_remote_test.go
@@ -46,6 +46,7 @@ func TestRemoteArchiveLifecycle(t *testing.T) {
objects[r.URL.Path], _ = io.ReadAll(r.Body)
w.Header().Set("ETag", `"test"`)
case "HEAD":
+ w.Header().Set("X-Amz-Server-Side-Encryption", "AES256")
w.Header().Set("ETag", `"test"`)
w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path])))
case "GET":
@@ -255,7 +256,7 @@ func TestRemoteArchiveLifecycle(t *testing.T) {
// Once PostgreSQL has committed an archive location and checksum, a subsequent
// queued operation must not make startup recovery resurrect the old workspace.
-func TestRemoteArchiveStageRecoveryAfterCommit(t *testing.T) {
+func TestTaskArchiveStageRecoveryAfterCommit(t *testing.T) {
if os.Getenv("ARTEX_PG_DSN") == "" {
t.Skip("requires isolated PostgreSQL")
}
@@ -284,41 +285,63 @@ func TestRemoteArchiveStageRecoveryAfterCommit(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} {
- t.Run(state, func(t *testing.T) {
- committed := state != pgdb.ArchiveQueued
- checksum := ""
- if committed {
- checksum = strings.Repeat("a", 64)
- }
- if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, "s3://test/tasks/package.tar.zst", checksum); err != nil {
- t.Fatal(err)
- }
- file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt")
- if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
- t.Fatal(err)
- }
- if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil {
- t.Fatal(err)
- }
- stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID)
- if err != nil {
- t.Fatal(err)
- }
- if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
- t.Fatal(err)
- }
- if _, err := os.Stat(stage.root); !os.IsNotExist(err) {
- t.Fatalf("staging cleanup incomplete: %v", err)
- }
- if committed {
- if _, err := os.Stat(file); !os.IsNotExist(err) {
- t.Fatalf("resurrected committed workspace: %v", err)
+ for _, backend := range []string{"local", "s3"} {
+ for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} {
+ t.Run(backend+"/"+state, func(t *testing.T) {
+ committed := state != pgdb.ArchiveQueued
+ checksum := ""
+ if committed {
+ checksum = strings.Repeat("a", 64)
}
- } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" {
- t.Fatalf("uncommitted archive lost source: %v", err)
- }
- })
+ packagePath := taskArchivePath(m.dir, job.ID, task.ID)
+ if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(packagePath, []byte("verified package"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ location := packagePath
+ if backend == "s3" {
+ location = "s3://test/tasks/package.tar.zst"
+ }
+ if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, location, checksum); err != nil {
+ t.Fatal(err)
+ }
+ file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt")
+ if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(stage.root); !os.IsNotExist(err) {
+ t.Fatalf("staging cleanup incomplete: %v", err)
+ }
+ if backend == "local" && committed {
+ if data, err := os.ReadFile(packagePath); err != nil || string(data) != "verified package" {
+ t.Fatal("committed local package lost", err)
+ }
+ } else if !committed {
+ if _, err := os.Stat(packagePath); !os.IsNotExist(err) {
+ t.Fatal("uncommitted temporary package retained", err)
+ }
+ }
+ if committed {
+ if _, err := os.Stat(file); !os.IsNotExist(err) {
+ t.Fatalf("resurrected committed workspace: %v", err)
+ }
+ } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" {
+ t.Fatalf("uncommitted archive lost source: %v", err)
+ }
+ })
+ }
}
}
From a40781ae1fc0d65d7045e669e5e4d87465e69e77 Mon Sep 17 00:00:00 2001
From: Chubboy Ren
Date: Wed, 18 Nov 2026 02:05:17 +0000
Subject: [PATCH 3/4] refactor: split archive backend from server and UI
integration
---
README.md | 6 -
db/task_archives.go | 5 -
docs/task-archive-storage.md | 160 --------
server/server.go | 9 +-
server/task_archive_interrupt_test.go | 291 --------------
server/task_archive_package.go | 13 +-
server/task_archive_remote.go | 171 ---------
server/task_archive_remote_test.go | 419 ---------------------
server/task_archives.go | 65 +---
web/src/app/(main)/function/tasks/page.tsx | 5 +-
web/src/lib/types.ts | 1 -
11 files changed, 15 insertions(+), 1130 deletions(-)
delete mode 100644 docs/task-archive-storage.md
delete mode 100644 server/task_archive_interrupt_test.go
delete mode 100644 server/task_archive_remote.go
delete mode 100644 server/task_archive_remote_test.go
diff --git a/README.md b/README.md
index 2d91397..ad8efec 100644
--- a/README.md
+++ b/README.md
@@ -421,12 +421,6 @@ runtime supports it.
`/chat/completions` (or `/responses` for `openai-responses`), so their base **does** end in `/v1`
(for example `https://api.openai.com/v1`).
-**Task archive storage**: local disk by default. Set `ARTEX_ARCHIVE_STORAGE=s3` to
-store cold task packages in an S3-compatible bucket and release their local
-package space after verification. Existing local archives remain usable. See
-[Task archive storage](docs/task-archive-storage.md) for configuration, permissions,
-integrity checks, temporary disk requirements and recovery.
-
**Concurrency**: work agents per task, in "System Settings" (default 3).
**Untrusted-output boundary (on by default)**: tool results for the worker, planner, chat and main
diff --git a/db/task_archives.go b/db/task_archives.go
index 9f07341..bb9aae2 100644
--- a/db/task_archives.go
+++ b/db/task_archives.go
@@ -60,7 +60,6 @@ type TaskArchive struct {
Warnings json.RawMessage `json:"warnings"`
FormatVersion int `json:"format_version"`
ArchivePath string `json:"-"`
- Storage string `json:"storage"`
SHA256 string `json:"sha256,omitempty"`
OriginalSize int64 `json:"original_size"`
CompressedSize int64 `json:"compressed_size"`
@@ -162,10 +161,6 @@ func scanTaskArchive(sc interface{ Scan(...any) error }) (*TaskArchive, error) {
if err != nil {
return nil, err
}
- item.Storage = "local"
- if strings.HasPrefix(item.ArchivePath, "s3://") {
- item.Storage = "s3"
- }
if err := json.Unmarshal([]byte(sources), &item.SourceTaskIDs); err != nil {
return nil, err
}
diff --git a/docs/task-archive-storage.md b/docs/task-archive-storage.md
deleted file mode 100644
index 113921a..0000000
--- a/docs/task-archive-storage.md
+++ /dev/null
@@ -1,160 +0,0 @@
-# Task archive storage
-
-Task archives use local disk by default. The optional S3-compatible backend moves
-cold task packages to object storage while retaining Archive, Restore and Delete.
-Active workspaces and PostgreSQL remain local. This is task cold storage, not a
-full-installation backup; a recovery plan must preserve the database, data
-directory, bucket objects, and the credentials or roles needed to read them.
-
-## Configuration
-
-Use a private bucket and a separate prefix for each ARTEX installation. Configure
-the bucket's default encryption to match the selected mode below, and enforce
-that mode/key with a bucket policy. ARTEX does not provision or inspect bucket
-encryption settings.
-
-Set the following server environment and restart ARTEX:
-
-```dotenv
-ARTEX_ARCHIVE_STORAGE=s3
-ARTEX_ARCHIVE_S3_ENDPOINT=https://s3.us-east-1.amazonaws.com
-ARTEX_ARCHIVE_S3_REGION=us-east-1
-ARTEX_ARCHIVE_S3_BUCKET=your-artex-archives
-ARTEX_ARCHIVE_S3_PREFIX=artex/tasks
-ARTEX_ARCHIVE_S3_SSE=AES256
-```
-
-With an EC2 instance role, ECS task role or other SDK-supported credential source,
-no archive-specific access keys are needed.
-
-| Variable | Meaning |
-| --- | --- |
-| `ARTEX_ARCHIVE_STORAGE` | `local` (default) or `s3`. |
-| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local tests. |
-| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. |
-| `ARTEX_ARCHIVE_S3_BUCKET` | Required bucket name. |
-| `ARTEX_ARCHIVE_S3_PREFIX` | Object prefix; defaults to `artex/tasks`. |
-| `ARTEX_ARCHIVE_S3_SSE` | `AES256` (default, SSE-S3) or `aws:kms` (SSE-KMS). There is no unencrypted mode. |
-| `ARTEX_ARCHIVE_S3_KMS_KEY_ID` | Optional KMS key ARN/ID/alias; valid only with `aws:kms`. Omit to use the provider's managed KMS key. |
-| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Optional dedicated credential override; must be supplied with the secret key. |
-| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Secret for the dedicated override. |
-| `ARTEX_ARCHIVE_S3_SESSION_TOKEN` | Optional session token for dedicated temporary credentials. |
-| `ARTEX_ARCHIVE_S3_PATH_STYLE` | `true` for path-style servers; otherwise virtual-hosted addressing is used. |
-
-An invalid mode or partial credential override fails configuration instead of
-silently falling back to local storage or ambient credentials. Preserve the
-bucket, endpoint, prefix and read/delete access while remote archives remain.
-Older local archives remain usable; they are not uploaded automatically.
-
-## Encryption and permissions
-
-Every single-object upload and multipart initiation explicitly requests the
-configured server-side encryption. Upload verification also requires HEAD to
-report the requested mode, and a KMS key identifier for SSE-KMS, before accepting
-the object. Size and full SHA-256 readback remain mandatory; encryption metadata
-does not replace content verification.
-
-For SSE-KMS, set both the bucket's default encryption and ARTEX to `aws:kms`;
-set `ARTEX_ARCHIVE_S3_KMS_KEY_ID` to the intended key when using a customer-managed
-key. An explicit `AES256` request takes precedence over a bucket's KMS default,
-so do not leave ARTEX on SSE-S3 when the bucket is intended to use KMS.
-
-The principal needs object upload, read (HEAD and ranged GET), delete and
-multipart upload/abort permissions. KMS uploads/readback also need the applicable
-key-policy and IAM permissions for `kms:GenerateDataKey` and `kms:Decrypt`.
-Retain old KMS keys and decrypt permissions when changing the key or mode; restore
-does not require an older object to use the current upload policy.
-
-Compatible providers must support the requested SSE mode and its HEAD metadata,
-plus conditional ranged reads. A provider that ignores the encryption request
-or cannot confirm it is rejected; the original task is retained. Local source and
-temporary files keep their existing local-disk protection and restricted modes.
-
-See the official [PutObject encryption and permission contract](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html)
-and [HEAD encryption response rules](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html).
-
-## Credential sources
-
-Without the archive-specific override, ARTEX uses the AWS SDK default credential
-chain, including standard AWS environment variables, shared profiles, web
-identity, ECS task credentials and EC2 instance credentials through IMDS. Role
-credentials retain the SDK's automatic refresh behavior. A named shared profile
-can be selected with `AWS_PROFILE`.
-
-For an external S3-compatible provider, dedicated keys can be supplied through
-the archive-specific variables. Both key fields are required; a session token
-is supported for temporary credentials. This explicit override takes precedence
-over ambient AWS credentials. Operator-supplied temporary credentials must be
-rotated by the operator; they are not an IAM role provider.
-
-Keep secrets in a protected service environment file outside the repository and
-frontend. A systemd `EnvironmentFile=` can point to a root-owned mode-0600 file.
-See the [AWS SDK credential-chain documentation](https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-gosdk.html).
-
-## Lifecycle
-
-1. Pause or finish a task and use its **Archive** action.
-2. ARTEX stages the source files and builds a local `.tar.zst` package. Large
- packages use multipart upload, initially 8 MiB parts with two upload workers;
- the SDK increases the part size for exceptionally large seekable files.
-3. ARTEX confirms encryption metadata and reads the complete object back,
- checking size and SHA-256. ETags and uploaded checksum metadata alone are
- insufficient.
-4. **Only after verification and a successful PostgreSQL commit** does ARTEX
- remove the local package and staged source files. The committed archive
- location is an `s3://` URI; there is no schema or archive-format migration.
-5. **Restore** downloads and verifies before installing files and database rows.
- The remote package is consumed only after restoration commits.
-6. **Delete** commits removal of the archive stub, then cleans up the object.
- A durable journal retains failed cleanup work.
-
-The archive list identifies **Object storage** and **Local storage** and shows
-upload/download verification phases. Reads use bounded 8 MiB ranges pinned to the
-object's ETag, with retries for interrupted ranges rather than the entire object.
-
-## Interruption recovery
-
-Source staging and restore journals live with task archives. Remote cleanup
-journals live under `data/archives/tasks/.remote`; they contain locations, not
-credentials. Keep this directory with PostgreSQL and the installation data.
-
-| Interruption | Recovery |
-| --- | --- |
-| During staging/compression, before an upload journal exists | Roll staged source files back. Remove any temporary package only after rollback succeeds. The task remains in PostgreSQL. |
-| After the upload journal, before successful verification | Roll source files back and clean up any uncommitted object/package. Failed cleanup retains its journal for retry. |
-| After verification, before the database commit | Treat the object as uncommitted: retain the original task and clean up the orphan object. Verification alone never authorizes local deletion. |
-| After the database commit, before package/staging cleanup | Keep the committed object and finish local cleanup. The persisted location and checksum establish commit, even if Restore or Delete has since been queued. |
-| Download/extraction before restored files are installed | Remove temporary extraction data; keep the remote object and archive stub for another attempt. |
-| Restored files installed, before database restoration commits | Roll back installed files using the restore journal. Retain the remote package. |
-| Database restoration committed, before remote cleanup | Recognize the already-restored task and finish cleanup without replaying database rows. |
-| Object consumed, before the restored archive stub is removed | Skip download/replay; repeat idempotent object deletion and remove the stub. |
-| Delete journal written, before database deletion commits | Keep the object while its archive stub exists. The queued Delete operation can retry. |
-| Database deletion committed, before object deletion succeeds | The durable outbox retains the URI and retries object deletion. |
-| An old failed upload survives a retry using local storage | Remove the abandoned remote object, but retain a local package referenced by the committed archive row. |
-
-The staging recovery rule also applies to local archives. Dedicated tests cover
-both storage modes with ready, restore-queued, delete-queued and uncommitted
-states. Additional PostgreSQL/HTTP-S3 tests reconstruct interrupted operations at
-13 persisted boundaries and repeat recovery to check idempotence.
-
-Handled multipart-upload failures are aborted by the SDK. Abrupt process death
-may leave unfinished multipart uploads; configure bucket lifecycle expiration
-for those parts. Existing object versions, if versioning is enabled, are governed
-by the bucket's version-retention policy.
-
-## Disk and transfer requirements
-
-Archive creation still requires temporary local space for the compressed
-package. Restore requires space for the download and extracted files. Full
-upload readback and restores incur storage reads and applicable transfer charges.
-Failed remote cleanup does not prevent unrelated local archive jobs from being
-processed once the failed cleanup request returns.
-
-## Optional connectivity check
-
-Run `go run ./cmd/artex-archive-check` with the archive environment to upload,
-verify, download and delete a small unique object. It performs real bucket
-operations but does not touch task data, and reports cleanup failures.
-
-Automated tests use HTTP fixtures for S3 and metadata credentials, plus isolated
-PostgreSQL. They do not establish compatibility with a live cloud bucket.
diff --git a/server/server.go b/server/server.go
index 7d750d3..baf4423 100644
--- a/server/server.go
+++ b/server/server.go
@@ -23,7 +23,6 @@ import (
"time"
"github.com/Autumn-27/artex/agent"
- "github.com/Autumn-27/artex/archivestore"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/egress"
"github.com/Autumn-27/artex/intercept"
@@ -145,11 +144,9 @@ type Server struct {
// Cold task archives run through one persistent FIFO worker. The buffered wake
// channel coalesces enqueue bursts; the database remains the source of truth.
- archiveWake chan struct{}
- archiveWG sync.WaitGroup
- archiveStore *archivestore.Store
- archiveStoreErr error
- side *sideQuestionState
+ archiveWake chan struct{}
+ archiveWG sync.WaitGroup
+ side *sideQuestionState
}
// provEntry is a cached provider + its config for one LLM profile id.
diff --git a/server/task_archive_interrupt_test.go b/server/task_archive_interrupt_test.go
deleted file mode 100644
index 2f4a44e..0000000
--- a/server/task_archive_interrupt_test.go
+++ /dev/null
@@ -1,291 +0,0 @@
-package server
-
-import (
- "context"
- "fmt"
- "io"
- "net/http"
- "net/http/httptest"
- "os"
- "path/filepath"
- "strconv"
- "sync"
- "testing"
-
- "github.com/Autumn-27/artex/archivestore"
- pgdb "github.com/Autumn-27/artex/db"
-)
-
-// Recreate persisted states at operation boundaries, then run the same recovery
-// functions used at startup. PostgreSQL, files, and HTTP object storage are real
-// isolated fixtures; no production failpoint hooks are needed.
-func TestRemoteArchiveInterruptionRecovery(t *testing.T) {
- if os.Getenv("ARTEX_PG_DSN") == "" {
- t.Skip("requires isolated PostgreSQL")
- }
- phases := []string{
- "staged_before_compression", "compressed_before_journal", "journal_before_upload",
- "uploaded_before_verification", "verified_before_database",
- "database_before_local_cleanup", "package_cleaned_before_stage_cleanup",
- "restore_download_before_install", "restore_files_before_database", "restore_database_before_object_delete", "restore_object_deleted_before_stub",
- "delete_journal_before_database", "delete_database_before_object",
- }
- for _, phase := range phases {
- t.Run(phase, func(t *testing.T) {
- m, err := NewManager(t.TempDir(), "")
- if err != nil {
- t.Fatal(err)
- }
- defer m.Close()
- store, objects := newRecoveryObjectStore(t)
- s := &Server{ctx: context.Background(), m: m, archiveStore: store}
- task, err := m.CreateTask("interruption "+phase, "preserve source", nil, 0, 0)
- if err != nil {
- t.Fatal(err)
- }
- id, _ := strconv.ParseInt(task.ID, 10, 64)
- defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }()
- if err := m.pg.SetPaused(id, true); err != nil {
- t.Fatal(err)
- }
- job, err := m.pg.QueueTaskArchive(id)
- if err != nil {
- t.Fatal(err)
- }
- if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Archiving); err != nil {
- t.Fatal(err)
- }
- source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt")
- if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil {
- t.Fatal(err)
- }
- if err := os.WriteFile(source, []byte("original task data"), 0600); err != nil {
- t.Fatal(err)
- }
- snapshot, err := m.pg.SnapshotTaskArchive(id)
- if err != nil {
- t.Fatal(err)
- }
- stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID)
- if err != nil {
- t.Fatal(err)
- }
- packagePath := taskArchivePath(m.dir, job.ID, task.ID)
- var original, compressed int64
- var checksum, uri, journal string
- if phase != "staged_before_compression" {
- original, compressed, checksum, err = writeTaskArchivePackage(packagePath, stage.payload, snapshot)
- if err != nil {
- t.Fatal(err)
- }
- uri = store.URI(id, job.ID, checksum)
- }
- if phase != "staged_before_compression" && phase != "compressed_before_journal" {
- journal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, LocalPath: packagePath, Operation: "upload"})
- if err != nil {
- t.Fatal(err)
- }
- }
- committed := phase != "staged_before_compression" && phase != "compressed_before_journal" &&
- phase != "journal_before_upload" && phase != "uploaded_before_verification" && phase != "verified_before_database"
- if phase == "uploaded_before_verification" {
- data, err := os.ReadFile(packagePath)
- if err != nil {
- t.Fatal(err)
- }
- objects.put(uri, data)
- } else if phase == "verified_before_database" || committed {
- if err := store.Upload(s.ctx, uri, packagePath, checksum); err != nil {
- t.Fatal(err)
- }
- }
- if committed {
- if err := m.pg.CompleteTaskArchive(job.ID, snapshot, uri, checksum, original, compressed); err != nil {
- t.Fatal(err)
- }
- }
- afterArchive := phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" ||
- phase == "restore_object_deleted_before_stub" || phase == "delete_journal_before_database" || phase == "delete_database_before_object"
- if phase == "package_cleaned_before_stage_cleanup" || afterArchive {
- if err := os.Remove(packagePath); err != nil {
- t.Fatal(err)
- }
- if err := os.Remove(journal); err != nil {
- t.Fatal(err)
- }
- }
- if afterArchive {
- if err := stage.commit(); err != nil {
- t.Fatal(err)
- }
- }
- restored := false
- remoteDeleted := false
- if phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" {
- if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Restoring); err != nil {
- t.Fatal(err)
- }
- parent := filepath.Join(taskArchiveRoot(m.dir), ".restore")
- if err := os.MkdirAll(parent, 0700); err != nil {
- t.Fatal(err)
- }
- extracted, err := os.MkdirTemp(parent, "interrupted-")
- if err != nil {
- t.Fatal(err)
- }
- downloaded := filepath.Join(extracted, "download.tar.zst")
- if err := store.Download(s.ctx, uri, downloaded, checksum, compressed); err != nil {
- t.Fatal(err)
- }
- if err := extractTaskArchivePackage(downloaded, checksum, extracted); err != nil {
- t.Fatal(err)
- }
- if phase != "restore_download_before_install" {
- if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID); err != nil {
- t.Fatal(err)
- }
- }
- if phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" {
- if _, err := m.pg.RestoreTaskArchive(job.ID, snapshot, 0); err != nil {
- t.Fatal(err)
- }
- restored = true
- }
- if phase == "restore_object_deleted_before_stub" {
- if err := store.Delete(s.ctx, uri); err != nil {
- t.Fatal(err)
- }
- remoteDeleted = true
- }
- }
- deleting := phase == "delete_journal_before_database" || phase == "delete_database_before_object"
- if deleting {
- if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Deleting); err != nil {
- t.Fatal(err)
- }
- if _, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, Operation: "delete"}); err != nil {
- t.Fatal(err)
- }
- if phase == "delete_database_before_object" {
- if err := m.pg.DeleteTaskArchiveStub(job.ID); err != nil {
- t.Fatal(err)
- }
- remoteDeleted = true
- }
- }
- if err := m.pg.RecoverTaskArchiveJobs(); err != nil {
- t.Fatal(err)
- }
- if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
- t.Fatal(err)
- }
- if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil {
- t.Fatal(err)
- }
- if err := s.recoverRemoteArchiveJournals(); err != nil {
- t.Fatal(err)
- }
- wantSource := !committed || restored
- data, fileErr := os.ReadFile(source)
- if wantSource && (fileErr != nil || string(data) != "original task data") {
- t.Fatalf("source data lost: %v", fileErr)
- }
- if !wantSource && !os.IsNotExist(fileErr) {
- t.Fatalf("archived source resurrected: %v", fileErr)
- }
- if _, err := os.Stat(packagePath); !os.IsNotExist(err) {
- t.Fatalf("temporary package remains: %v", err)
- }
- if got, want := objects.has(uri), committed && !remoteDeleted; got != want {
- t.Fatalf("remote object exists=%v, want=%v", got, want)
- }
- if committed && !deleting {
- actual, err := m.pg.IsTaskArchiveRestored(job.ID)
- if err != nil || actual != restored {
- t.Fatalf("restored=%v want=%v err=%v", actual, restored, err)
- }
- }
- // A second startup must be harmless at every boundary.
- if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
- t.Fatal(err)
- }
- if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil {
- t.Fatal(err)
- }
- if err := s.recoverRemoteArchiveJournals(); err != nil {
- t.Fatal(err)
- }
- })
- }
-}
-
-type recoveryObjects struct {
- mu sync.Mutex
- data map[string][]byte
-}
-
-func (o *recoveryObjects) put(uri string, data []byte) {
- o.mu.Lock()
- defer o.mu.Unlock()
- // Test URIs always use the fixture bucket with path-style addressing.
- o.data["/test/"+uri[len("s3://test/"):]] = append([]byte(nil), data...)
-}
-func (o *recoveryObjects) has(uri string) bool {
- o.mu.Lock()
- defer o.mu.Unlock()
- if uri == "" {
- return false
- }
- _, ok := o.data["/test/"+uri[len("s3://test/"):]]
- return ok
-}
-func newRecoveryObjectStore(t *testing.T) (*archivestore.Store, *recoveryObjects) {
- t.Helper()
- objects := &recoveryObjects{data: map[string][]byte{}}
- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- objects.mu.Lock()
- defer objects.mu.Unlock()
- data, exists := objects.data[r.URL.Path]
- w.Header().Set("ETag", `"recovery-etag"`)
- switch r.Method {
- case "PUT":
- if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" {
- t.Error("missing SSE")
- }
- objects.data[r.URL.Path], _ = io.ReadAll(r.Body)
- case "HEAD":
- if !exists {
- w.WriteHeader(404)
- return
- }
- w.Header().Set("Content-Length", strconv.Itoa(len(data)))
- w.Header().Set("X-Amz-Server-Side-Encryption", "AES256")
- case "GET":
- if !exists {
- w.WriteHeader(404)
- return
- }
- var start, end int
- if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end >= len(data) || r.Header.Get("If-Match") != `"recovery-etag"` {
- w.WriteHeader(416)
- return
- }
- w.Header().Set("Content-Length", strconv.Itoa(end-start+1))
- w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
- w.WriteHeader(206)
- _, _ = w.Write(data[start : end+1])
- case "DELETE":
- delete(objects.data, r.URL.Path)
- w.WriteHeader(204)
- default:
- t.Errorf("unexpected object request %s", r.Method)
- w.WriteHeader(400)
- }
- }))
- t.Cleanup(srv.Close)
- store, err := archivestore.New(archivestore.Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
- if err != nil {
- t.Fatal(err)
- }
- return store, objects
-}
diff --git a/server/task_archive_package.go b/server/task_archive_package.go
index e365ea3..f23c4c8 100644
--- a/server/task_archive_package.go
+++ b/server/task_archive_package.go
@@ -428,7 +428,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error {
errs = append(errs, getErr)
continue
}
- if archive != nil && archive.ArchivePath != "" && archive.SHA256 != "" {
+ if archive != nil && archive.State == pgdb.ArchiveReady {
if err := os.RemoveAll(root); err != nil {
errs = append(errs, err)
}
@@ -437,16 +437,9 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error {
stage := &taskArchiveFileStage{root: root, payload: filepath.Join(root, "payload"), journal: journal}
if err := stage.rollback(); err != nil {
errs = append(errs, err)
- continue
}
- // The process can exit after compression but before an upload journal or
- // database location exists. Remove that temporary package only after the
- // source-file rollback succeeds, for both local and remote backends.
- packagePath := taskArchivePath(dataDir, journal.ArchiveID, journal.TaskID)
- if err := validateArchivePath(dataDir, packagePath); err != nil {
- errs = append(errs, err)
- } else if err := removeLocalArchivePackage(packagePath); err != nil {
- errs = append(errs, err)
+ if archive != nil && archive.ArchivePath != "" {
+ _ = os.Remove(archive.ArchivePath)
}
}
return errors.Join(errs...)
diff --git a/server/task_archive_remote.go b/server/task_archive_remote.go
deleted file mode 100644
index 87e230f..0000000
--- a/server/task_archive_remote.go
+++ /dev/null
@@ -1,171 +0,0 @@
-package server
-
-import (
- "encoding/json"
- "errors"
- "fmt"
- "log"
- "os"
- "path/filepath"
- "strings"
-
- pgdb "github.com/Autumn-27/artex/db"
-)
-
-// An outbox bridges the local PostgreSQL transaction and object storage. Never
-// delete the only package before metadata permits it. Unfinished cleanup is
-// retried after a restart (and before subsequent jobs).
-type remoteArchiveJournal struct {
- ArchiveID int64 `json:"archive_id"`
- URI string `json:"uri"`
- LocalPath string `json:"local_path,omitempty"`
- Operation string `json:"operation"`
-}
-
-func (s *Server) validateStoredArchive(uri string) error {
- if strings.HasPrefix(uri, "s3://") {
- return s.archiveStore.Validate(uri)
- }
- return validateArchivePath(s.m.dir, uri)
-}
-
-func removeLocalArchivePackage(filename string) error {
- if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
- return err
- }
- return nil
-}
-
-func (s *Server) removeStoredArchive(uri string) error {
- if err := s.validateStoredArchive(uri); err != nil {
- return err
- }
- if strings.HasPrefix(uri, "s3://") {
- return s.archiveStore.Delete(s.ctx, uri)
- }
- return removeLocalArchivePackage(uri)
-}
-
-func (s *Server) writeRemoteArchiveJournal(j remoteArchiveJournal) (string, error) {
- dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote")
- if err := os.MkdirAll(dir, archiveDirMode); err != nil {
- return "", err
- }
- f, err := os.CreateTemp(dir, ".journal-")
- if err != nil {
- return "", err
- }
- tmp := f.Name()
- defer os.Remove(tmp)
- encodeErr := json.NewEncoder(f).Encode(j)
- syncErr := f.Sync()
- if err := errors.Join(encodeErr, syncErr, f.Close()); err != nil {
- return "", err
- }
- name := tmp + ".json"
- if err := os.Rename(tmp, name); err != nil {
- return "", err
- }
- // Persist the directory entry before an upload or database deletion starts.
- d, err := os.Open(dir)
- if err != nil {
- return "", err
- }
- if err := errors.Join(d.Sync(), d.Close()); err != nil {
- return "", err
- }
- // The .remote directory itself may have just been created.
- parent, err := os.Open(filepath.Dir(dir))
- if err != nil {
- return "", err
- }
- return name, errors.Join(parent.Sync(), parent.Close())
-}
-
-func (s *Server) recoverRemoteArchiveJournals() error {
- dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote")
- entries, err := os.ReadDir(dir)
- if os.IsNotExist(err) {
- return nil
- }
- if err != nil {
- return err
- }
- var errs []error
- for _, entry := range entries {
- if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
- continue
- }
- name := filepath.Join(dir, entry.Name())
- if err := s.recoverRemoteArchiveJournal(name); err != nil {
- errs = append(errs, err)
- }
- }
- return errors.Join(errs...)
-}
-
-func (s *Server) recoverRemoteArchiveJournal(name string) error {
- raw, err := os.ReadFile(name)
- if err != nil {
- return err
- }
- var j remoteArchiveJournal
- if err := json.Unmarshal(raw, &j); err != nil {
- return err
- }
- if j.ArchiveID <= 0 {
- return errors.New("invalid remote archive journal ID")
- }
- if err := s.archiveStore.Validate(j.URI); err != nil {
- return err
- }
- item, err := s.m.pg.GetTaskArchive(j.ArchiveID)
- if err != nil {
- return err
- }
- switch j.Operation {
- case "upload":
- if item == nil || item.ArchivePath != j.URI {
- if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil {
- return err
- }
- }
- if err := validateArchivePath(s.m.dir, j.LocalPath); err != nil {
- return err
- }
- // An old failed upload may outlive a retry after switching back to
- // local storage. The retry reuses this pathname: keep its committed
- // package instead of treating it as the old upload's temporary copy.
- if item == nil || item.ArchivePath != j.LocalPath {
- if err := removeLocalArchivePackage(j.LocalPath); err != nil {
- return err
- }
- }
- case "delete":
- if item == nil {
- if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil {
- return err
- }
- }
- default:
- return fmt.Errorf("unknown remote archive operation %q", j.Operation)
- }
- return os.Remove(name)
-}
-
-func (s *Server) deleteRemoteTaskArchive(job *pgdb.TaskArchive) error {
- journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, Operation: "delete"})
- if err != nil {
- return err
- }
- if err := s.m.pg.DeleteTaskArchiveStub(job.ID); err != nil {
- // A connection failure can leave the transaction's commit ambiguous.
- // Recovery consults PostgreSQL before deciding whether to delete.
- return err
- }
- if err := s.recoverRemoteArchiveJournal(journal); err != nil {
- // The task is gone; the durable outbox retries the remaining object cleanup.
- log.Printf("[task-archive] archive %d deleted; remote cleanup pending: %v", job.ID, err)
- }
- return nil
-}
diff --git a/server/task_archive_remote_test.go b/server/task_archive_remote_test.go
deleted file mode 100644
index ac2a44d..0000000
--- a/server/task_archive_remote_test.go
+++ /dev/null
@@ -1,419 +0,0 @@
-package server
-
-import (
- "bytes"
- "context"
- "errors"
- "fmt"
- "io"
- "net/http"
- "net/http/httptest"
- "os"
- "path/filepath"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
- "testing"
-
- "github.com/Autumn-27/artex/archivestore"
- pgdb "github.com/Autumn-27/artex/db"
-)
-
-func TestRemoteArchiveLifecycle(t *testing.T) {
- if os.Getenv("ARTEX_PG_DSN") == "" {
- t.Skip("requires isolated PostgreSQL")
- }
- t.Setenv("ARTEX_ARCHIVE_STORAGE", "local")
- m, err := NewManager(t.TempDir(), "")
- if err != nil {
- t.Fatal(err)
- }
- defer m.Close()
- ctx, cancel := context.WithCancel(context.Background())
- cancel()
- s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
- s.archiveWG.Wait()
- s.ctx = context.Background()
- var mu sync.Mutex
- objects := map[string][]byte{}
- corrupt, failDelete := false, false
- remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- mu.Lock()
- defer mu.Unlock()
- switch r.Method {
- case "PUT":
- objects[r.URL.Path], _ = io.ReadAll(r.Body)
- w.Header().Set("ETag", `"test"`)
- case "HEAD":
- w.Header().Set("X-Amz-Server-Side-Encryption", "AES256")
- w.Header().Set("ETag", `"test"`)
- w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path])))
- case "GET":
- data := objects[r.URL.Path]
- if corrupt && len(data) > 0 {
- data = bytes.Clone(data)
- data[0] ^= 0xff
- }
- var start, end int64
- if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) || r.Header.Get("If-Match") != `"test"` {
- w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
- return
- }
- w.Header().Set("ETag", `"test"`)
- w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10))
- w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
- w.WriteHeader(http.StatusPartialContent)
- _, _ = w.Write(data[start : end+1])
- case "DELETE":
- if failDelete {
- w.WriteHeader(403)
- return
- }
- delete(objects, r.URL.Path)
- w.WriteHeader(204)
- default:
- w.WriteHeader(400)
- }
- }))
- defer remote.Close()
- s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
- if err != nil {
- t.Fatal(err)
- }
- task, err := m.CreateTask("S3 round trip", "preserve data", nil, 0, 0)
- if err != nil {
- t.Fatal(err)
- }
- id, _ := strconv.ParseInt(task.ID, 10, 64)
- defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }()
- if err := m.pg.SetPaused(id, true); err != nil {
- t.Fatal(err)
- }
- file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "important.txt")
- if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
- t.Fatal(err)
- }
- if err := os.WriteFile(file, []byte("keep this content"), 0600); err != nil {
- t.Fatal(err)
- }
- pgTask, err := m.pg.GetTask(id)
- if err != nil {
- t.Fatal(err)
- }
- if _, err := m.pg.Exploration(pgTask.ExplorationID).AppendActivity(pgdb.Activity{Worker: "system", Kind: "system", Summary: "archive proof"}); err != nil {
- t.Fatal(err)
- }
- job, err := m.pg.QueueTaskArchive(id)
- if err != nil {
- t.Fatal(err)
- }
- // A configured-but-invalid backend must fail before moving any source data.
- s.archiveStoreErr = errors.New("invalid fixture configuration")
- if err := s.runOneTaskArchiveJob(); !errors.Is(err, s.archiveStoreErr) {
- t.Fatalf("configuration error was not returned: %v", err)
- }
- if got, err := os.ReadFile(file); err != nil || string(got) != "keep this content" {
- t.Fatalf("bad configuration lost source: %v", err)
- }
- if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil {
- t.Fatal("bad configuration lost task", err)
- }
- s.archiveStoreErr = nil
- if _, err := m.pg.QueueTaskArchive(id); err != nil {
- t.Fatal(err)
- }
- // A corrupt readback must roll back filesystem staging and leave PG intact.
- mu.Lock()
- corrupt = true
- mu.Unlock()
- if err := s.runOneTaskArchiveJob(); err == nil {
- t.Fatal("expected integrity failure")
- }
- if got, _ := os.ReadFile(file); string(got) != "keep this content" {
- t.Fatal("upload failure lost source")
- }
- if task, err := m.pg.GetTask(id); err != nil || task == nil {
- t.Fatal("upload failure lost task")
- }
- mu.Lock()
- corrupt = false
- mu.Unlock()
- if err := s.recoverRemoteArchiveJournals(); err != nil {
- t.Fatal(err)
- }
- if _, err := m.pg.QueueTaskArchive(id); err != nil {
- t.Fatal(err)
- }
- if err := s.runOneTaskArchiveJob(); err != nil {
- t.Fatal(err)
- }
- job, err = m.pg.GetTaskArchive(job.ID)
- if err != nil {
- t.Fatal(err)
- }
- if job.State != pgdb.ArchiveReady || job.Storage != "s3" || !strings.HasPrefix(job.ArchivePath, "s3://") {
- t.Fatalf("unexpected archive %+v", job)
- }
- if _, err := os.Stat(file); !os.IsNotExist(err) {
- t.Fatal("workspace not released")
- }
- if _, err := os.Stat(taskArchivePath(m.dir, job.ID, task.ID)); !os.IsNotExist(err) {
- t.Fatal("local package retained")
- }
- // Simulate the crash window after commit: cleanup keeps the remote object.
- journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, LocalPath: taskArchivePath(m.dir, job.ID, task.ID), Operation: "upload"})
- if err != nil {
- t.Fatal(err)
- }
- if err := s.recoverRemoteArchiveJournal(journal); err != nil {
- t.Fatal(err)
- }
- // A corrupted download must leave the archive and its remote package intact.
- mu.Lock()
- corrupt = true
- mu.Unlock()
- if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
- t.Fatal(err)
- }
- if err := s.runOneTaskArchiveJob(); err == nil {
- t.Fatal("expected corrupted restore to fail")
- }
- if _, err := os.Stat(file); !os.IsNotExist(err) {
- t.Fatal("corrupted archive installed workspace files")
- }
- if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || restored {
- t.Fatalf("corrupted archive restored database: %v %v", restored, err)
- }
- mu.Lock()
- if len(objects) == 0 {
- t.Error("corrupted restore deleted the only remote package")
- }
- corrupt = false
- mu.Unlock()
- // Restoration commits files and PG first. Failed remote deletion is retryable.
- mu.Lock()
- failDelete = true
- mu.Unlock()
- if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
- t.Fatal(err)
- }
- if err := s.runOneTaskArchiveJob(); err == nil {
- t.Fatal("expected remote delete failure")
- }
- if got, _ := os.ReadFile(file); string(got) != "keep this content" {
- t.Fatal("restored content differs")
- }
- if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || !restored {
- t.Fatalf("PG restore %v %v", restored, err)
- }
- mu.Lock()
- failDelete = false
- mu.Unlock()
- if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
- t.Fatal(err)
- }
- if err := s.runOneTaskArchiveJob(); err != nil {
- t.Fatal(err)
- }
- if item, err := m.pg.GetTaskArchive(job.ID); err != nil || item != nil {
- t.Fatal("restore did not consume stub")
- }
- var count int
- if err := m.pg.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='archive proof'`, pgTask.ExplorationID).Scan(&count); err != nil || count != 1 {
- t.Fatalf("activity restore count=%d err=%v", count, err)
- }
- // Re-archive, then crash/retry the deletion outbox after PG removed its stub.
- job, err = m.pg.QueueTaskArchive(id)
- if err != nil {
- t.Fatal(err)
- }
- if err := s.runOneTaskArchiveJob(); err != nil {
- t.Fatal(err)
- }
- mu.Lock()
- failDelete = true
- mu.Unlock()
- if _, err := m.pg.QueueTaskArchiveDelete(job.ID); err != nil {
- t.Fatal(err)
- }
- if err := s.runOneTaskArchiveJob(); err != nil {
- t.Fatal(err)
- }
- mu.Lock()
- failDelete = false
- mu.Unlock()
- if err := s.recoverRemoteArchiveJournals(); err != nil {
- t.Fatal(err)
- }
- mu.Lock()
- remaining := len(objects)
- mu.Unlock()
- if remaining != 0 {
- t.Fatalf("left %d objects", remaining)
- }
-}
-
-// Once PostgreSQL has committed an archive location and checksum, a subsequent
-// queued operation must not make startup recovery resurrect the old workspace.
-func TestTaskArchiveStageRecoveryAfterCommit(t *testing.T) {
- if os.Getenv("ARTEX_PG_DSN") == "" {
- t.Skip("requires isolated PostgreSQL")
- }
- m, err := NewManager(t.TempDir(), "")
- if err != nil {
- t.Fatal(err)
- }
- defer m.Close()
- task, err := m.CreateTask("remote stage recovery", "test", nil, 0, 0)
- if err != nil {
- t.Fatal(err)
- }
- taskID, _ := strconv.ParseInt(task.ID, 10, 64)
- defer func() {
- _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID)
- _ = m.pg.DeleteTask(taskID)
- }()
- if err := m.pg.SetPaused(taskID, true); err != nil {
- t.Fatal(err)
- }
- job, err := m.pg.QueueTaskArchive(taskID)
- if err != nil {
- t.Fatal(err)
- }
- pgTask, err := m.pg.GetTask(taskID)
- if err != nil {
- t.Fatal(err)
- }
- for _, backend := range []string{"local", "s3"} {
- for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} {
- t.Run(backend+"/"+state, func(t *testing.T) {
- committed := state != pgdb.ArchiveQueued
- checksum := ""
- if committed {
- checksum = strings.Repeat("a", 64)
- }
- packagePath := taskArchivePath(m.dir, job.ID, task.ID)
- if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil {
- t.Fatal(err)
- }
- if err := os.WriteFile(packagePath, []byte("verified package"), 0600); err != nil {
- t.Fatal(err)
- }
- location := packagePath
- if backend == "s3" {
- location = "s3://test/tasks/package.tar.zst"
- }
- if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, location, checksum); err != nil {
- t.Fatal(err)
- }
- file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt")
- if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
- t.Fatal(err)
- }
- if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil {
- t.Fatal(err)
- }
- stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID)
- if err != nil {
- t.Fatal(err)
- }
- if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
- t.Fatal(err)
- }
- if _, err := os.Stat(stage.root); !os.IsNotExist(err) {
- t.Fatalf("staging cleanup incomplete: %v", err)
- }
- if backend == "local" && committed {
- if data, err := os.ReadFile(packagePath); err != nil || string(data) != "verified package" {
- t.Fatal("committed local package lost", err)
- }
- } else if !committed {
- if _, err := os.Stat(packagePath); !os.IsNotExist(err) {
- t.Fatal("uncommitted temporary package retained", err)
- }
- }
- if committed {
- if _, err := os.Stat(file); !os.IsNotExist(err) {
- t.Fatalf("resurrected committed workspace: %v", err)
- }
- } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" {
- t.Fatalf("uncommitted archive lost source: %v", err)
- }
- })
- }
- }
-}
-
-// Failed remote cleanup can survive switching to local storage and retrying the
-// same archive ID. Its journal must never remove the newly committed local copy.
-func TestRemoteArchiveJournalPreservesLocalRetry(t *testing.T) {
- if os.Getenv("ARTEX_PG_DSN") == "" {
- t.Skip("requires isolated PostgreSQL")
- }
- m, err := NewManager(t.TempDir(), "")
- if err != nil {
- t.Fatal(err)
- }
- defer m.Close()
- task, err := m.CreateTask("local retry", "test", nil, 0, 0)
- if err != nil {
- t.Fatal(err)
- }
- taskID, _ := strconv.ParseInt(task.ID, 10, 64)
- defer func() {
- _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID)
- _ = m.pg.DeleteTask(taskID)
- }()
- if err := m.pg.SetPaused(taskID, true); err != nil {
- t.Fatal(err)
- }
- job, err := m.pg.QueueTaskArchive(taskID)
- if err != nil {
- t.Fatal(err)
- }
- filename := taskArchivePath(m.dir, job.ID, task.ID)
- if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil {
- t.Fatal(err)
- }
- if err := os.WriteFile(filename, []byte("committed local archive"), 0600); err != nil {
- t.Fatal(err)
- }
- if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, pgdb.ArchiveReady, filename, strings.Repeat("a", 64)); err != nil {
- t.Fatal(err)
- }
- var deletes atomic.Int32
- remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.Method != http.MethodDelete {
- w.WriteHeader(http.StatusMethodNotAllowed)
- return
- }
- deletes.Add(1)
- w.WriteHeader(http.StatusNoContent)
- }))
- defer remote.Close()
- store, err := archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
- if err != nil {
- t.Fatal(err)
- }
- s := &Server{ctx: context.Background(), m: m, archiveStore: store}
- journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{
- ArchiveID: job.ID, URI: store.URI(taskID, job.ID, strings.Repeat("b", 64)),
- LocalPath: filename, Operation: "upload",
- })
- if err != nil {
- t.Fatal(err)
- }
- if err := s.recoverRemoteArchiveJournal(journal); err != nil {
- t.Fatal(err)
- }
- if data, err := os.ReadFile(filename); err != nil || string(data) != "committed local archive" {
- t.Fatalf("old upload cleanup removed local retry: %v", err)
- }
- if deletes.Load() != 1 {
- t.Fatal("uncommitted remote object was not removed")
- }
- if _, err := os.Stat(journal); !os.IsNotExist(err) {
- t.Fatalf("finished journal remains: %v", err)
- }
-}
diff --git a/server/task_archives.go b/server/task_archives.go
index 9a7af52..836c2d8 100644
--- a/server/task_archives.go
+++ b/server/task_archives.go
@@ -14,7 +14,6 @@ import (
"time"
"github.com/Autumn-27/artex/agent"
- "github.com/Autumn-27/artex/archivestore"
pgdb "github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/traffic"
)
@@ -38,12 +37,6 @@ func (s *Server) startTaskArchiveWorker() {
if s.m == nil || s.m.pg == nil {
return
}
- s.archiveStore, s.archiveStoreErr = archivestore.FromEnv()
- if s.archiveStoreErr != nil {
- log.Printf("[task-archive] storage configuration: %v", s.archiveStoreErr)
- } else if s.archiveStore != nil {
- log.Printf("[task-archive] new archives use S3 object storage with full SHA-256 readback verification")
- }
if err := s.m.pg.RecoverTaskArchiveJobs(); err != nil {
log.Printf("[task-archive] recover jobs: %v", err)
}
@@ -95,14 +88,6 @@ func (s *Server) notifyTaskArchiveWorker() {
}
func (s *Server) runOneTaskArchiveJob() error {
- if s.ctx.Err() != nil {
- return s.ctx.Err()
- }
- // Retry deferred remote cleanup before claiming more work. Failed cleanup
- // keeps its durable journal and does not block unrelated local archives.
- if err := s.recoverRemoteArchiveJournals(); err != nil {
- log.Printf("[task-archive] deferred object cleanup: %v", err)
- }
job, err := s.m.pg.ClaimTaskArchiveJob(s.ctx)
if err != nil || job == nil {
return err
@@ -130,9 +115,6 @@ func (s *Server) runOneTaskArchiveJob() error {
}
func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
- if s.archiveStoreErr != nil {
- return s.archiveStoreErr
- }
taskID := strconv.FormatInt(job.TaskID, 10)
if !s.beginTaskDelete(taskID) {
return errors.New("task is already undergoing another archive or delete operation")
@@ -217,21 +199,6 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
_ = os.Remove(archivePath)
}
}()
- storedPath := archivePath
- var remoteJournal string
- if s.archiveStore != nil {
- storedPath = s.archiveStore.URI(job.TaskID, job.ID, checksum)
- remoteJournal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{
- ArchiveID: job.ID, URI: storedPath, LocalPath: archivePath, Operation: "upload",
- })
- if err != nil {
- return err
- }
- _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "upload_and_verify", 65)
- if err := s.archiveStore.Upload(s.ctx, storedPath, archivePath, checksum); err != nil {
- return err
- }
- }
var trafficStage *traffic.HostDeleteStage
if s.m.traffic != nil && len(snapshot.ExclusiveHosts) > 0 {
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_traffic", 68)
@@ -246,18 +213,11 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
}()
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_database", 78)
- if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, storedPath, checksum, originalSize, compressedSize); err != nil {
+ if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, archivePath, checksum, originalSize, compressedSize); err != nil {
return err
}
committed = true
- removePackage = s.archiveStore != nil
- if remoteJournal != "" {
- // The journal also covers a crash after database commit but before local
- // package removal. Keep it if cleanup fails, so the worker can retry.
- if err := removeLocalArchivePackage(archivePath); err == nil {
- _ = os.Remove(remoteJournal)
- }
- }
+ removePackage = false
if trafficStage != nil {
if err := trafficStage.Commit(); err != nil {
warning := "archive completed, but cleaning the exclusive traffic hot store failed (the archive package can still restore it): " + err.Error()
@@ -279,7 +239,7 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
}
func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
- if err := s.validateStoredArchive(job.ArchivePath); err != nil {
+ if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil {
return err
}
// A prior attempt may have committed PostgreSQL and files, then failed only
@@ -287,7 +247,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
if restored, err := s.m.pg.IsTaskArchiveRestored(job.ID); err != nil {
return err
} else if restored {
- if err := s.removeStoredArchive(job.ArchivePath); err != nil {
+ if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) {
return err
}
if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil {
@@ -306,15 +266,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
return err
}
defer func() { _ = os.RemoveAll(extracted) }()
- packagePath := job.ArchivePath
- if strings.HasPrefix(packagePath, "s3://") {
- _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "download_and_verify", 12)
- packagePath = filepath.Join(extracted, "download.tar.zst")
- if err := s.archiveStore.Download(s.ctx, job.ArchivePath, packagePath, job.SHA256, job.CompressedSize); err != nil {
- return err
- }
- }
- if err := extractTaskArchivePackage(packagePath, job.SHA256, extracted); err != nil {
+ if err := extractTaskArchivePackage(job.ArchivePath, job.SHA256, extracted); err != nil {
return err
}
manifestRaw, err := os.ReadFile(filepath.Join(extracted, "manifest.json"))
@@ -404,7 +356,7 @@ func (s *Server) restoreTaskArchivePayload(job *pgdb.TaskArchive, snapshot *pgdb
return err
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "consume_package", 95)
- if err := s.removeStoredArchive(job.ArchivePath); err != nil {
+ if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) {
return err
}
if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil {
@@ -441,12 +393,9 @@ func (s *Server) loadRestoredTask(taskID int64) {
}
func (s *Server) deleteTaskArchive(job *pgdb.TaskArchive) error {
- if err := s.validateStoredArchive(job.ArchivePath); err != nil {
+ if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil {
return err
}
- if strings.HasPrefix(job.ArchivePath, "s3://") {
- return s.deleteRemoteTaskArchive(job)
- }
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "stage_package_delete", 35)
staged, moved, err := stageTaskArchivePackageDelete(job.ArchivePath, job.ID)
if err != nil {
diff --git a/web/src/app/(main)/function/tasks/page.tsx b/web/src/app/(main)/function/tasks/page.tsx
index c2f9cc3..6b8ba8e 100644
--- a/web/src/app/(main)/function/tasks/page.tsx
+++ b/web/src/app/(main)/function/tasks/page.tsx
@@ -1494,7 +1494,7 @@ function ArchiveConfirmDialog({
{count === 1 ? "Archive task" : `Archive ${count} tasks`}
- Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into the configured archive storage. Once archived, a task can be restored from "Archived". Restoring requires free space on this server.
+ Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into local cold storage. Once archived, a task can be restored from "Archived".
@@ -1934,7 +1934,6 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {
title={`Before compression: ${formatArchiveBytes(archive.original_size)}`}
>
{archiveCompressionLabel(archive)}
- {archive.storage === "s3" ? "Object storage" : "Local storage"}
{archiveDataTotal(archive).toLocaleString()}
@@ -1953,7 +1952,7 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {
))}
- {archive.phase === "upload_and_verify" ? "Uploading and verifying" : archive.phase === "download_and_verify" ? "Downloading and verifying" : archive.phase} · {archive.progress}%
+ {archive.phase} · {archive.progress}%
diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts
index f31b18c..cd53b64 100644
--- a/web/src/lib/types.ts
+++ b/web/src/lib/types.ts
@@ -101,7 +101,6 @@ export interface TaskArchiveTokenStats {
}
export interface TaskArchive {
- storage?: "local" | "s3";
id: number;
task_id: number;
state: TaskArchiveState;
From 3755e9550433095f3763ccd216f6251542b13ccd Mon Sep 17 00:00:00 2001
From: Chubboy Ren
Date: Thu, 19 Nov 2026 03:33:51 +0000
Subject: [PATCH 4/4] fix: enforce archive encryption and clean failed S3
uploads
---
.github/workflows/ci.yml | 2 +-
archivestore/config_test.go | 7 ++
archivestore/s3.go | 89 ++++++++++----
archivestore/s3_test.go | 2 +
archivestore/safety_test.go | 231 ++++++++++++++++++++++++++++++++++++
5 files changed, 310 insertions(+), 21 deletions(-)
create mode 100644 archivestore/safety_test.go
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f949165..2fac967 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -71,7 +71,7 @@ jobs:
ARTEX_PG_DSN: postgres://artex:testpass@localhost:5432/artextest_race?sslmode=disable
run: >-
- go test -race -p 1 ./agent ./cmd/artex ./config ./db ./evidence ./guard
+ go test -race -p 1 ./agent ./archivestore ./cmd/artex ./config ./db ./evidence ./guard
./intercept ./llmpool ./llmrec ./mcphttp ./notify ./server ./sidequestion
./traffic ./traffic/rewrite
diff --git a/archivestore/config_test.go b/archivestore/config_test.go
index 73afc73..1fe54a5 100644
--- a/archivestore/config_test.go
+++ b/archivestore/config_test.go
@@ -76,6 +76,13 @@ func TestArchiveEncryptionConfiguration(t *testing.T) {
t.Fatal("KMS key accepted with SSE-S3")
}
c.SSE = "aws:kms"
+ for _, key := range []string{"alias/archive", "bare-key-id", "arn:aws:kms:us-east-1:123456789012:alias/archive"} {
+ c.KMSKeyID = key
+ if _, err := New(c); err == nil {
+ t.Fatalf("accepted noncanonical KMS key %q", key)
+ }
+ }
+ c.KMSKeyID = "arn:aws:kms:us-east-1:123456789012:key/test-key"
if _, err := New(c); err != nil {
t.Fatal(err)
}
diff --git a/archivestore/s3.go b/archivestore/s3.go
index 27c50af..5761a83 100644
--- a/archivestore/s3.go
+++ b/archivestore/s3.go
@@ -17,6 +17,9 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/aws/arn"
+ "github.com/aws/aws-sdk-go-v2/aws/retry"
+ awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
@@ -93,24 +96,37 @@ func New(c Config) (*Store, error) {
if c.KMSKeyID != "" && c.SSE != "aws:kms" {
return nil, errors.New("archive S3 KMS key requires SSE=aws:kms")
}
+ if c.KMSKeyID != "" {
+ key, err := arn.Parse(c.KMSKeyID)
+ if err != nil || key.Service != "kms" || key.Region == "" || key.AccountID == "" ||
+ !strings.HasPrefix(key.Resource, "key/") || len(key.Resource) <= len("key/") {
+ return nil, errors.New("archive S3 KMS key must be a full key ARN, not an alias or bare key ID")
+ }
+ }
if c.Region == "" {
c.Region = "us-east-1"
}
- transport := http.DefaultTransport.(*http.Transport).Clone()
- transport.ResponseHeaderTimeout = 60 * time.Second
- httpClient := &http.Client{Transport: transport}
- cfg := aws.Config{Region: c.Region, HTTPClient: httpClient}
+ // Load shared AWS settings (including custom CAs) for both credential modes.
+ httpClient := awshttp.NewBuildableClient().WithTransportOptions(func(t *http.Transport) {
+ t.ResponseHeaderTimeout = 60 * time.Second
+ })
+ options := []func(*awsconfig.LoadOptions) error{
+ awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient),
+ }
if explicitCredentials {
- cfg.Credentials = credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken)
- } else {
- // Keep the SDK's refreshable default chain: standard AWS environment/shared
- // profiles, web identity, ECS task roles and EC2 instance roles (IMDSv2).
- var err error
- cfg, err = awsconfig.LoadDefaultConfig(context.Background(),
- awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient))
- if err != nil {
- return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err)
- }
+ options = append(options, awsconfig.WithCredentialsProvider(
+ credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken)))
+ }
+ cfg, err := awsconfig.LoadDefaultConfig(context.Background(), options...)
+ if err != nil {
+ return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err)
+ }
+ // The SDK's default client still follows 307/308. Use its CA-configured
+ // transport, but never redirect a signed archive request to another origin.
+ configuredClient := cfg.HTTPClient.(*awshttp.BuildableClient)
+ cfg.HTTPClient = &http.Client{
+ Transport: configuredClient.GetTransport(), Timeout: configuredClient.GetTimeout(),
+ CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(c.Endpoint)
@@ -166,6 +182,7 @@ func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) erro
uploader := manager.NewUploader(s.client, func(u *manager.Uploader) {
u.PartSize = 8 << 20
u.Concurrency = 2
+ u.LeavePartsOnError = true // Abort below with a fresh, bounded cleanup context.
u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
})
input := &s3.PutObjectInput{
@@ -176,11 +193,36 @@ func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) erro
if s.kmsKeyID != "" {
input.SSEKMSKeyId = aws.String(s.kmsKeyID)
}
- _, err = uploader.Upload(ctx, input)
+ result, err := uploader.Upload(ctx, input)
if err != nil {
- return fmt.Errorf("upload archive: %w", err)
+ uploadErr := fmt.Errorf("upload archive: %w", err)
+ var multipart manager.MultiUploadFailure
+ if errors.As(err, &multipart) {
+ cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+ defer cancel()
+ _, cleanupErr := s.client.AbortMultipartUpload(cleanupCtx, &s3.AbortMultipartUploadInput{
+ Bucket: aws.String(s.bucket), Key: aws.String(key), UploadId: aws.String(multipart.UploadID()),
+ })
+ if cleanupErr != nil {
+ return errors.Join(uploadErr, fmt.Errorf("abort archive multipart upload: %w", cleanupErr))
+ }
+ }
+ return uploadErr
}
- return s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true)
+ if err := s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true); err != nil {
+ cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+ defer cancel()
+ // Versioned buckets require deleting the version just uploaded; an
+ // unversioned DELETE would only hide it behind a delete marker.
+ _, cleanupErr := s.client.DeleteObject(cleanupCtx, &s3.DeleteObjectInput{
+ Bucket: aws.String(s.bucket), Key: aws.String(key), VersionId: result.VersionID,
+ })
+ if cleanupErr != nil {
+ return errors.Join(err, fmt.Errorf("delete unverified archive: %w", cleanupErr))
+ }
+ return err
+ }
+ return nil
}
func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64, requireEncryption bool) error {
@@ -195,7 +237,8 @@ func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, che
return fmt.Errorf("inspect archive: %w", err)
}
if requireEncryption && (object.ServerSideEncryption != s.encryption ||
- (s.encryption == types.ServerSideEncryptionAwsKms && aws.ToString(object.SSEKMSKeyId) == "")) {
+ (s.encryption == types.ServerSideEncryptionAwsKms && (aws.ToString(object.SSEKMSKeyId) == "" ||
+ (s.kmsKeyID != "" && aws.ToString(object.SSEKMSKeyId) != s.kmsKeyID)))) {
return errors.New("remote archive did not confirm the requested server-side encryption; source data retained")
}
if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" {
@@ -265,11 +308,17 @@ func (s *Store) readArchiveChunkOnce(ctx context.Context, key, etag string, star
object, err := s.client.GetObject(requestCtx, &s3.GetObjectInput{
Bucket: aws.String(s.bucket), Key: aws.String(key), IfMatch: aws.String(etag),
Range: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)),
+ }, func(o *s3.Options) {
+ // This loop owns both HTTP and interrupted-body retries: at most five
+ // wire requests per chunk, rather than five times the SDK's attempts.
+ o.RetryMaxAttempts = 1
})
if err != nil {
var networkError net.Error
- retry := errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded)
- return nil, retry, fmt.Errorf("read archive range: %w", err)
+ var responseError *awshttp.ResponseError
+ throttled := errors.As(err, &responseError) && responseError.HTTPStatusCode() == http.StatusTooManyRequests
+ shouldRetry := throttled || retry.NewStandard().IsErrorRetryable(err) || errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded)
+ return nil, shouldRetry, fmt.Errorf("read archive range: %w", err)
}
defer object.Body.Close()
want := end - start + 1
diff --git a/archivestore/s3_test.go b/archivestore/s3_test.go
index dc618c1..84049e3 100644
--- a/archivestore/s3_test.go
+++ b/archivestore/s3_test.go
@@ -340,6 +340,8 @@ func TestS3UploadRejectsUnconfirmedEncryption(t *testing.T) {
switch r.Method {
case "PUT":
_, _ = io.Copy(io.Discard, r.Body)
+ case "DELETE":
+ w.WriteHeader(204)
case "HEAD":
w.Header().Set("ETag", `"test-etag"`)
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
diff --git a/archivestore/safety_test.go b/archivestore/safety_test.go
new file mode 100644
index 0000000..9afe47b
--- /dev/null
+++ b/archivestore/safety_test.go
@@ -0,0 +1,231 @@
+package archivestore
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/pem"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "testing"
+)
+
+func archiveSource(t *testing.T, data []byte) (string, string) {
+ t.Helper()
+ filename := filepath.Join(t.TempDir(), "source")
+ if err := os.WriteFile(filename, data, 0600); err != nil {
+ t.Fatal(err)
+ }
+ sum := sha256.Sum256(data)
+ return filename, hex.EncodeToString(sum[:])
+}
+
+func TestS3VerificationFailureCleansUploadedVersion(t *testing.T) {
+ for _, failure := range []string{"wrong-kms", "checksum", "cancelled", "delete-denied"} {
+ t.Run(failure, func(t *testing.T) {
+ data := []byte("private archive")
+ source, checksum := archiveSource(t, data)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ var present, deleted atomic.Bool
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.Method {
+ case "PUT":
+ _, _ = io.Copy(io.Discard, r.Body)
+ present.Store(true)
+ w.Header().Set("X-Amz-Version-Id", "uploaded-version")
+ case "HEAD":
+ w.Header().Set("ETag", `"test-etag"`)
+ w.Header().Set("Content-Length", strconv.Itoa(len(data)))
+ w.Header().Set("X-Amz-Server-Side-Encryption", "aws:kms")
+ kms := "arn:aws:kms:us-east-1:123456789012:key/expected"
+ if failure == "wrong-kms" {
+ kms = "arn:aws:kms:us-east-1:123456789012:key/different"
+ }
+ w.Header().Set("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id", kms)
+ if failure == "cancelled" {
+ cancel()
+ }
+ case "GET":
+ corrupt := bytes.Clone(data)
+ corrupt[0] ^= 0xff
+ serveArchiveRange(t, w, r, corrupt)
+ case "DELETE":
+ if r.URL.Query().Get("versionId") != "uploaded-version" {
+ t.Error("cleanup did not target uploaded version")
+ }
+ deleted.Store(true)
+ if failure == "delete-denied" {
+ w.WriteHeader(403)
+ return
+ }
+ present.Store(false)
+ w.WriteHeader(204)
+ default:
+ t.Errorf("unexpected request %s", r.Method)
+ w.WriteHeader(400)
+ }
+ }))
+ defer srv.Close()
+ store, err := New(Config{Endpoint: srv.URL, Bucket: "test", AccessKey: "test", SecretKey: "test", PathStyle: true,
+ SSE: "aws:kms", KMSKeyID: "arn:aws:kms:us-east-1:123456789012:key/expected"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = store.Upload(ctx, store.URI(1, 2, checksum), source, checksum)
+ if err == nil {
+ t.Fatal("accepted failed verification")
+ }
+ if failure == "wrong-kms" && !strings.Contains(err.Error(), "encryption") {
+ t.Fatal(err)
+ }
+ if failure == "cancelled" && !errors.Is(err, context.Canceled) {
+ t.Fatal(err)
+ }
+ if failure == "delete-denied" && (!strings.Contains(err.Error(), "SHA-256") || !strings.Contains(err.Error(), "delete unverified archive")) {
+ t.Fatalf("verification and cleanup errors must both survive: %v", err)
+ }
+ if !deleted.Load() || present.Load() != (failure == "delete-denied") {
+ t.Fatal("unverified object cleanup was not attempted/completed")
+ }
+ if got, readErr := os.ReadFile(source); readErr != nil || !bytes.Equal(got, data) {
+ t.Fatal("source changed", readErr)
+ }
+ })
+ }
+}
+
+func TestS3CancelledMultipartAbortsWithFreshContext(t *testing.T) {
+ for _, denyAbort := range []bool{false, true} {
+ t.Run(strconv.FormatBool(denyAbort), func(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ var aborted atomic.Bool
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ q := r.URL.Query()
+ switch {
+ case r.Method == "POST" && q.Has("uploads"):
+ fmt.Fprint(w, `cancelled-upload`)
+ case r.Method == "PUT" && q.Has("partNumber"):
+ cancel()
+ _, _ = io.Copy(io.Discard, r.Body)
+ w.WriteHeader(403)
+ case r.Method == "DELETE" && q.Get("uploadId") == "cancelled-upload":
+ aborted.Store(true)
+ if denyAbort {
+ w.WriteHeader(403)
+ } else {
+ w.WriteHeader(204)
+ }
+ default:
+ t.Errorf("unexpected request %s", r.Method)
+ w.WriteHeader(400)
+ }
+ }))
+ defer srv.Close()
+ data := bytes.Repeat([]byte("x"), 9<<20)
+ source, sum := archiveSource(t, data)
+ store := testRangeStore(t, srv.URL)
+ err := store.Upload(ctx, store.URI(1, 2, sum), source, sum)
+ if !errors.Is(err, context.Canceled) || !aborted.Load() {
+ t.Fatalf("cancelled upload not aborted: %v", err)
+ }
+ if denyAbort && !strings.Contains(err.Error(), "abort archive multipart upload") {
+ t.Fatalf("cleanup failure lost: %v", err)
+ }
+ if got, readErr := os.ReadFile(source); readErr != nil || !bytes.Equal(got, data) {
+ t.Fatal("source changed", readErr)
+ }
+ })
+ }
+}
+
+func TestS3SignedRequestsNeverFollowRedirects(t *testing.T) {
+ var reached atomic.Int32
+ target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reached.Add(1); w.WriteHeader(204) }))
+ defer target.Close()
+ for _, status := range []int{301, 302, 303, 307, 308} {
+ t.Run(strconv.Itoa(status), func(t *testing.T) {
+ origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") == "" || r.Header.Get("X-Amz-Security-Token") != "fixture-session" {
+ t.Error("fixture request must be signed with session credentials")
+ }
+ w.Header().Set("Location", target.URL+"/redirected")
+ w.WriteHeader(status)
+ }))
+ defer origin.Close()
+ store, err := New(Config{Endpoint: origin.URL, Bucket: "test", AccessKey: "test", SecretKey: "test", SessionToken: "fixture-session", PathStyle: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := store.Delete(context.Background(), store.URI(1, 2, "test")); err == nil {
+ t.Fatal("redirect accepted")
+ }
+ if reached.Load() != 0 {
+ t.Fatal("signed request followed redirect")
+ }
+ })
+ }
+}
+
+func TestS3CustomCABundleForBothCredentialModes(t *testing.T) {
+ srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) }))
+ defer srv.Close()
+ for _, explicit := range []bool{false, true} {
+ t.Run(strconv.FormatBool(explicit), func(t *testing.T) {
+ isolateAWSCredentials(t)
+ ca := filepath.Join(t.TempDir(), "ca.pem")
+ if err := os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}), 0600); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("AWS_CA_BUNDLE", ca)
+ c := Config{Endpoint: srv.URL, Bucket: "test", PathStyle: true}
+ if explicit {
+ c.AccessKey, c.SecretKey = "fixture", "fixture"
+ } else {
+ t.Setenv("AWS_ACCESS_KEY_ID", "fixture")
+ t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture")
+ }
+ store, err := New(c)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := store.Delete(context.Background(), store.URI(1, 2, "test")); err != nil {
+ t.Fatalf("configured CA not used: %v", err)
+ }
+ })
+ }
+}
+
+func TestS3RangeRetriesHaveOneWireAttemptEach(t *testing.T) {
+ for _, status := range []int{403, 429, 503} {
+ t.Run(strconv.Itoa(status), func(t *testing.T) {
+ var gets atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gets.Add(1)
+ w.WriteHeader(status)
+ }))
+ defer srv.Close()
+ store := testRangeStore(t, srv.URL)
+ if _, err := store.readArchiveChunk(context.Background(), "key", `"etag"`, 0, 9, 10); err == nil {
+ t.Fatal("failed request accepted")
+ }
+ want := int32(archiveReadAttempts)
+ if status == 403 {
+ want = 1
+ }
+ if got := gets.Load(); got != want {
+ t.Fatalf("wire requests=%d want=%d", got, want)
+ }
+ })
+ }
+}