From da16b8d81a6946726da123a3bf69b9aae5375a5a Mon Sep 17 00:00:00 2001 From: Chubboy Ren Date: Tue, 17 Nov 2026 11:56:15 +0000 Subject: [PATCH 1/4] feat: archive cold tasks to S3-compatible object storage --- README.md | 6 + archivestore/config_test.go | 62 ++++ archivestore/s3.go | 280 +++++++++++++++ archivestore/s3_test.go | 311 ++++++++++++++++ cmd/artex-archive-check/main.go | 64 ++++ db/task_archives.go | 5 + docs/task-archive-storage.md | 104 ++++++ go.mod | 13 + go.sum | 38 ++ server/server.go | 9 +- server/task_archive_package.go | 4 +- server/task_archive_remote.go | 171 +++++++++ server/task_archive_remote_test.go | 396 +++++++++++++++++++++ server/task_archives.go | 65 +++- web/src/app/(main)/function/tasks/page.tsx | 5 +- web/src/lib/types.ts | 1 + 16 files changed, 1520 insertions(+), 14 deletions(-) create mode 100644 archivestore/config_test.go create mode 100644 archivestore/s3.go create mode 100644 archivestore/s3_test.go create mode 100644 cmd/artex-archive-check/main.go create mode 100644 docs/task-archive-storage.md create mode 100644 server/task_archive_remote.go create mode 100644 server/task_archive_remote_test.go diff --git a/README.md b/README.md index d15b112..7cc166c 100644 --- a/README.md +++ b/README.md @@ -418,6 +418,12 @@ runtime supports it. `/chat/completions` (or `/responses` for `openai-responses`), so their base **does** end in `/v1` (for example `https://api.openai.com/v1`). +**Task archive storage**: local disk by default. Set `ARTEX_ARCHIVE_STORAGE=s3` to +store cold task packages in an S3-compatible bucket and release their local +package space after verification. Existing local archives remain usable. See +[Task archive storage](docs/task-archive-storage.md) for configuration, permissions, +integrity checks, temporary disk requirements and recovery. + **Concurrency**: work agents per task, in "System Settings" (default 3). **Untrusted-output boundary (on by default)**: tool results for the worker, planner, chat and main diff --git a/archivestore/config_test.go b/archivestore/config_test.go new file mode 100644 index 0000000..cc12532 --- /dev/null +++ b/archivestore/config_test.go @@ -0,0 +1,62 @@ +package archivestore + +import ( + "strings" + "testing" +) + +func TestArchiveStorageEnvironment(t *testing.T) { + for _, key := range []string{ + "ARTEX_ARCHIVE_STORAGE", "ARTEX_ARCHIVE_S3_ENDPOINT", "ARTEX_ARCHIVE_S3_REGION", + "ARTEX_ARCHIVE_S3_BUCKET", "ARTEX_ARCHIVE_S3_PREFIX", "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", + "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_PATH_STYLE", + } { + t.Setenv(key, "") + } + for _, backend := range []string{"", "local"} { + t.Setenv("ARTEX_ARCHIVE_STORAGE", backend) + if store, err := FromEnv(); err != nil || store != nil { + t.Fatalf("local default: %v %v", store, err) + } + } + for _, backend := range []string{"unknown", "s3"} { + t.Setenv("ARTEX_ARCHIVE_STORAGE", backend) + if _, err := FromEnv(); err == nil { + t.Fatalf("%q silently fell back to local", backend) + } + } + t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", "http://127.0.0.1:9000") + t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "test-bucket") + t.Setenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "test-key") + t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "test-secret") + t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true") + store, err := FromEnv() + if err != nil { + t.Fatal(err) + } + if uri := store.URI(1, 2, "checksum"); uri != "s3://test-bucket/artex/tasks/task-1-2-checksum.tar.zst" { + t.Fatal("unexpected default object location", uri) + } + t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "") + if _, err := FromEnv(); err == nil { + t.Fatal("missing credentials accepted") + } +} + +func TestArchiveEndpointValidation(t *testing.T) { + config := Config{Bucket: "test", AccessKey: "test-key", SecretKey: "test-secret"} + for _, endpoint := range []string{"https://objects.example.test", "http://127.0.0.1:9000", "http://127.0.0.2:9000", "http://[::1]:9000", "http://localhost:9000"} { + config.Endpoint = endpoint + if _, err := New(config); err != nil { + t.Errorf("rejected %s: %v", endpoint, err) + } + } + for _, endpoint := range []string{"http://objects.example.test", "https://user:test-secret@objects.example.test", "https://objects.example.test/bucket", "https://objects.example.test?token=test-secret", "https://objects.example.test#fragment"} { + config.Endpoint = endpoint + if _, err := New(config); err == nil { + t.Error("accepted endpoint with non-loopback HTTP, credentials, path, query or fragment") + } else if strings.Contains(err.Error(), "test-secret") { + t.Fatal("configuration error disclosed a secret") + } + } +} diff --git a/archivestore/s3.go b/archivestore/s3.go new file mode 100644 index 0000000..edc1513 --- /dev/null +++ b/archivestore/s3.go @@ -0,0 +1,280 @@ +// Package archivestore stores immutable cold-task packages in S3-compatible storage. +package archivestore + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type Config struct { + Endpoint, Region, Bucket, Prefix, AccessKey, SecretKey string + PathStyle bool +} + +type Store struct { + client *s3.Client + bucket, prefix string +} + +// FromEnv returns nil for the default local backend. A configured but invalid +// backend is an error, never a silent fallback to the machine's local disk. +func FromEnv() (*Store, error) { + switch os.Getenv("ARTEX_ARCHIVE_STORAGE") { + case "", "local": + return nil, nil + case "s3": + return New(Config{ + Endpoint: os.Getenv("ARTEX_ARCHIVE_S3_ENDPOINT"), Region: os.Getenv("ARTEX_ARCHIVE_S3_REGION"), + Bucket: os.Getenv("ARTEX_ARCHIVE_S3_BUCKET"), Prefix: os.Getenv("ARTEX_ARCHIVE_S3_PREFIX"), + AccessKey: os.Getenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID"), SecretKey: os.Getenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY"), + PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true", + }) + default: + return nil, errors.New("ARTEX_ARCHIVE_STORAGE must be local or s3") + } +} + +func New(c Config) (*Store, error) { + u, err := url.Parse(c.Endpoint) + loopback := false + if err == nil { + loopback = u.Hostname() == "localhost" + if ip := net.ParseIP(u.Hostname()); ip != nil { + loopback = ip.IsLoopback() + } + } + if err != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") || + (u.Scheme != "https" && !(u.Scheme == "http" && loopback)) { + return nil, errors.New("archive S3 endpoint must be an HTTPS origin (HTTP allowed only on loopback)") + } + c.Prefix = strings.Trim(c.Prefix, "/") + if c.Prefix == "" { + c.Prefix = "artex/tasks" + } + if path.Clean(c.Prefix) != c.Prefix || strings.Contains(c.Prefix, "..") || strings.ContainsAny(c.Prefix, "\\?#") { + return nil, errors.New("invalid archive S3 prefix") + } + if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") || c.AccessKey == "" || c.SecretKey == "" { + return nil, errors.New("archive S3 bucket and dedicated access credentials are required") + } + if c.Region == "" { + c.Region = "us-east-1" + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.ResponseHeaderTimeout = 60 * time.Second + client := s3.New(s3.Options{ + Region: c.Region, BaseEndpoint: aws.String(c.Endpoint), UsePathStyle: c.PathStyle, + Credentials: credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, ""), + HTTPClient: &http.Client{Transport: transport}, + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + }) + return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/"}, nil +} + +func (s *Store) URI(taskID, archiveID int64, checksum string) string { + return "s3://" + s.bucket + "/" + s.prefix + fmt.Sprintf("task-%d-%d-%s.tar.zst", taskID, archiveID, checksum) +} + +func (s *Store) key(uri string) (string, error) { + if s == nil { + return "", errors.New("S3 archive storage is not configured") + } + u, err := url.Parse(uri) + if err != nil || u.Scheme != "s3" || u.Host != s.bucket || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return "", errors.New("archive object is outside the configured bucket") + } + key := strings.TrimPrefix(u.Path, "/") + if !strings.HasPrefix(key, s.prefix) || len(key) <= len(s.prefix) || path.Clean(key) != key || strings.Contains(key, "\\") { + return "", errors.New("archive object is outside the configured prefix") + } + return key, nil +} + +func (s *Store) Validate(uri string) error { _, err := s.key(uri); return err } + +// Upload verifies the entire uploaded object by reading it back, including +// multipart uploads whose ETag is not a checksum. Callers retain source data +// until verification succeeds and PostgreSQL commits the remote location. +func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) error { + key, err := s.key(uri) + if err != nil { + return err + } + if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size { + return errors.New("invalid archive SHA-256") + } + f, err := os.Open(filename) + if err != nil { + return err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return err + } + uploader := manager.NewUploader(s.client, func(u *manager.Uploader) { + u.PartSize = 8 << 20 + u.Concurrency = 2 + u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + }) + _, err = uploader.Upload(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), Key: aws.String(key), Body: f, + ContentType: aws.String("application/zstd"), Metadata: map[string]string{"sha256": checksum}, + }) + if err != nil { + return fmt.Errorf("upload archive: %w", err) + } + return s.readVerified(ctx, key, io.Discard, checksum, info.Size()) +} + +func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64) error { + if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size { + return errors.New("invalid archive SHA-256") + } + if expectedSize < 0 { + return errors.New("invalid archive size") + } + object, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}) + if err != nil { + return fmt.Errorf("inspect archive: %w", err) + } + if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" { + return errors.New("remote archive size mismatch or missing ETag; source data retained") + } + // Commit only complete, validated chunks to the output and hash. A broken + // connection can then retry its chunk without duplicating bytes or restarting + // a multi-gigabyte read. IfMatch pins every request to the same object. + h := sha256.New() + for offset := int64(0); offset < expectedSize; { + end := offset + min(archiveReadChunkSize, expectedSize-offset) - 1 + data, err := s.readArchiveChunk(ctx, key, aws.ToString(object.ETag), offset, end, expectedSize) + if err != nil { + return err + } + n, err := out.Write(data) + if err != nil { + return err + } + if n != len(data) { + return io.ErrShortWrite + } + _, _ = h.Write(data) + offset += int64(len(data)) + } + if !strings.EqualFold(hex.EncodeToString(h.Sum(nil)), checksum) { + return errors.New("remote archive size or SHA-256 mismatch; source data retained") + } + return nil +} + +const ( + archiveReadChunkSize = int64(8 << 20) + archiveReadAttempts = 5 +) + +func (s *Store) readArchiveChunk(ctx context.Context, key, etag string, start, end, total int64) ([]byte, error) { + var lastErr error + for attempt := 0; attempt < archiveReadAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return nil, err + } + if attempt > 0 { + timer := time.NewTimer((250 * time.Millisecond) << (attempt - 1)) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + data, retry, err := s.readArchiveChunkOnce(ctx, key, etag, start, end, total) + if err == nil { + return data, nil + } + if !retry { + return nil, err + } + lastErr = err + } + return nil, fmt.Errorf("read archive bytes %d-%d after %d attempts: %w", start, end, archiveReadAttempts, lastErr) +} + +func (s *Store) readArchiveChunkOnce(ctx context.Context, key, etag string, start, end, total int64) ([]byte, bool, error) { + requestCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + object, err := s.client.GetObject(requestCtx, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), Key: aws.String(key), IfMatch: aws.String(etag), + Range: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)), + }) + if err != nil { + var networkError net.Error + retry := errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded) + return nil, retry, fmt.Errorf("read archive range: %w", err) + } + defer object.Body.Close() + want := end - start + 1 + if aws.ToString(object.ETag) != etag || aws.ToString(object.ContentRange) != fmt.Sprintf("bytes %d-%d/%d", start, end, total) || object.ContentLength == nil || *object.ContentLength != want { + return nil, false, errors.New("remote archive changed or returned an invalid byte range; source data retained") + } + data, err := io.ReadAll(io.LimitReader(object.Body, want+1)) + if err != nil { + return nil, true, err + } + if int64(len(data)) < want { + return nil, true, io.ErrUnexpectedEOF + } + if int64(len(data)) != want { + return nil, false, errors.New("remote archive range exceeds its declared size") + } + return data, false, nil +} + +func (s *Store) Download(ctx context.Context, uri, filename, checksum string, size int64) (err error) { + key, err := s.key(uri) + if err != nil { + return err + } + if size <= 0 { + return errors.New("invalid archive size") + } + f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + defer func() { + err = errors.Join(err, f.Close()) + if err != nil { + _ = os.Remove(filename) + } + }() + if err = s.readVerified(ctx, key, f, checksum, size); err != nil { + return err + } + return f.Sync() +} + +func (s *Store) Delete(ctx context.Context, uri string) error { + key, err := s.key(uri) + if err != nil { + return err + } + _, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}) + return err +} diff --git a/archivestore/s3_test.go b/archivestore/s3_test.go new file mode 100644 index 0000000..316e770 --- /dev/null +++ b/archivestore/s3_test.go @@ -0,0 +1,311 @@ +package archivestore + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "sync" + "testing" +) + +func TestS3RoundTripAndIntegrity(t *testing.T) { + for _, size := range []int{1024, 10 << 20} { + t.Run(strconv.Itoa(size), func(t *testing.T) { + var mu sync.Mutex + var object []byte + parts := map[int][]byte{} + corrupt, multipart := false, false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + q := r.URL.Query() + w.Header().Set("ETag", `"test-etag"`) + switch { + case r.Method == "POST" && q.Has("uploads"): + multipart = true + fmt.Fprint(w, `test-upload`) + case r.Method == "PUT" && q.Has("partNumber"): + n, _ := strconv.Atoi(q.Get("partNumber")) + parts[n], _ = io.ReadAll(r.Body) + case r.Method == "POST" && q.Has("uploadId"): + for i := 1; i <= len(parts); i++ { + object = append(object, parts[i]...) + } + fmt.Fprint(w, `test-etag`) + case r.Method == "PUT": + object, _ = io.ReadAll(r.Body) + case r.Method == "HEAD": + w.Header().Set("Content-Length", strconv.Itoa(len(object))) + case r.Method == "GET": + data := object + if corrupt { + data = bytes.Clone(object) + data[0] ^= 0xff + } + serveArchiveRange(t, w, r, data) + case r.Method == "DELETE": + object = nil + w.WriteHeader(204) + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL) + w.WriteHeader(400) + } + })) + defer srv.Close() + store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + data := bytes.Repeat([]byte{0x42}, size) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + root := t.TempDir() + source := filepath.Join(root, "source") + if err := os.WriteFile(source, data, 0600); err != nil { + t.Fatal(err) + } + uri := store.URI(1, 2, checksum) + ctx := context.Background() + if err := store.Upload(ctx, uri, source, checksum); err != nil { + t.Fatal(err) + } + if multipart != (size > 8<<20) { + t.Fatalf("multipart=%v size=%d", multipart, size) + } + dest := filepath.Join(root, "download") + if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil { + t.Fatal(err) + } + got, _ := os.ReadFile(dest) + if !bytes.Equal(got, data) { + t.Fatal("download differs") + } + mu.Lock() + corrupt = true + mu.Unlock() + bad := filepath.Join(root, "bad") + if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil { + t.Fatal("accepted corrupt object") + } + if _, err := os.Stat(bad); !os.IsNotExist(err) { + t.Fatal("left corrupt download") + } + if err := store.Upload(ctx, uri, source, checksum); err == nil { + t.Fatal("upload accepted failed readback") + } + if err := store.Delete(ctx, uri); err != nil { + t.Fatal(err) + } + }) + } +} + +func serveArchiveRange(t *testing.T, w http.ResponseWriter, r *http.Request, data []byte) { + t.Helper() + var start, end int64 + if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) { + t.Errorf("invalid range %q", r.Header.Get("Range")) + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return + } + if r.Header.Get("If-Match") != `"test-etag"` { + t.Errorf("missing object version condition: %q", r.Header.Get("If-Match")) + w.WriteHeader(http.StatusPreconditionFailed) + return + } + w.Header().Set("ETag", `"test-etag"`) + w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[start : end+1]) +} + +func TestS3ReadbackRetriesInterruptedRange(t *testing.T) { + data := append(bytes.Repeat([]byte{0x42}, int(archiveReadChunkSize)), bytes.Repeat([]byte{0x73}, 1024)...) + var mu sync.Mutex + var ranges []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + w.Header().Set("ETag", `"test-etag"`) + if r.Method == "HEAD" { + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + return + } + ranges = append(ranges, r.Header.Get("Range")) + if len(ranges) == 2 { + w.Header().Set("Content-Length", strconv.FormatInt(int64(len(data))-archiveReadChunkSize, 10)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", archiveReadChunkSize, len(data)-1, len(data))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[archiveReadChunkSize : archiveReadChunkSize+128]) // Close during a later chunk. + return + } + serveArchiveRange(t, w, r, data) + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + dest := filepath.Join(t.TempDir(), "archive") + if err := store.Download(context.Background(), store.URI(1, 2, checksum), dest, checksum, int64(len(data))); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(dest) + if err != nil || !bytes.Equal(got, data) { + t.Fatalf("retried download differs: %v", err) + } + mu.Lock() + defer mu.Unlock() + if len(ranges) != 3 || ranges[1] != ranges[2] || ranges[0] != fmt.Sprintf("bytes=0-%d", archiveReadChunkSize-1) || ranges[2] != fmt.Sprintf("bytes=%d-%d", archiveReadChunkSize, len(data)-1) { + t.Fatalf("retry did not stay within the interrupted range: %v", ranges) + } +} + +func testRangeStore(t *testing.T, endpoint string) *Store { + t.Helper() + s, err := New(Config{Endpoint: endpoint, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + return s +} + +func TestS3ReadbackRejectsInvalidRangesAndExhaustedRetries(t *testing.T) { + for _, mode := range []string{"ignored-range", "changed-etag", "changed-object", "truncated"} { + t.Run(mode, func(t *testing.T) { + data := bytes.Repeat([]byte{0x42}, 1024) + var mu sync.Mutex + gets := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + w.Header().Set("ETag", `"test-etag"`) + if r.Method == "HEAD" { + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + return + } + gets++ + if mode == "changed-object" { + w.WriteHeader(http.StatusPreconditionFailed) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + if mode == "ignored-range" { + _, _ = w.Write(data) + return + } + if mode == "changed-etag" { + w.Header().Set("ETag", `"different-object"`) + } + w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", len(data)-1, len(data))) + w.WriteHeader(http.StatusPartialContent) + if mode == "truncated" { + _, _ = w.Write(data[:128]) + } else { + _, _ = w.Write(data) + } + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + dest := filepath.Join(t.TempDir(), "archive") + if err := store.Download(context.Background(), store.URI(1, 2, checksum), dest, checksum, int64(len(data))); err == nil { + t.Fatal("accepted invalid or incomplete readback") + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatal("left a failed download on disk") + } + mu.Lock() + defer mu.Unlock() + want := 1 + if mode == "truncated" { + want = archiveReadAttempts + } + if gets != want { + t.Fatalf("GET count %d, want %d", gets, want) + } + }) + } +} + +type failedArchiveWriter struct{ err error } + +func (w failedArchiveWriter) Write([]byte) (int, error) { return 0, w.err } + +func TestS3ReadbackPropagatesOutputFailure(t *testing.T) { + data := []byte("test archive") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"test-etag"`) + if r.Method == "HEAD" { + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + return + } + serveArchiveRange(t, w, r, data) + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + h := sha256.Sum256(data) + outputError := errors.New("output disk full") + err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data))) + if !errors.Is(err, outputError) { + t.Fatalf("lost output error: %v", err) + } +} + +func TestS3ReadbackStopsOnCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + data := bytes.Repeat([]byte{0x42}, 1024) + gets := make(chan struct{}, archiveReadAttempts+1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"test-etag"`) + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + if r.Method == "HEAD" { + return + } + gets <- struct{}{} + w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", len(data)-1, len(data))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[:128]) + w.(http.Flusher).Flush() + cancel() + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + dest := filepath.Join(t.TempDir(), "archive") + err := store.Download(ctx, store.URI(1, 2, checksum), dest, checksum, int64(len(data))) + if !errors.Is(err, context.Canceled) || len(gets) != 1 { + t.Fatalf("cancellation was not honored: error=%v GETs=%d", err, len(gets)) + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatal("left a cancelled download on disk") + } +} + +func TestS3RejectsForeignLocations(t *testing.T) { + s, err := New(Config{Endpoint: "https://sfo3.digitaloceanspaces.com", Bucket: "allowed", Prefix: "archives/tasks", AccessKey: "test", SecretKey: "test"}) + if err != nil { + t.Fatal(err) + } + for _, uri := range []string{"s3://other/archives/tasks/a", "s3://allowed/archives/other/a", "s3://allowed/archives/tasks/../a", "s3://allowed/archives/tasks/%2e%2e/a", "s3://allowed/archives/tasks/a?secret=x", "s3://user@allowed/archives/tasks/a", "file:///etc/passwd"} { + if err := s.Validate(uri); err == nil { + t.Errorf("accepted %s", uri) + } + } + var absent *Store + if absent.Validate("s3://allowed/archives/tasks/a") == nil { + t.Fatal("missing config accepted") + } +} diff --git a/cmd/artex-archive-check/main.go b/cmd/artex-archive-check/main.go new file mode 100644 index 0000000..af0efd9 --- /dev/null +++ b/cmd/artex-archive-check/main.go @@ -0,0 +1,64 @@ +// artex-archive-check tests the configured object store without touching tasks. +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "log" + "os" + "path/filepath" + "time" + + "github.com/Autumn-27/artex/archivestore" +) + +func main() { + if err := check(); err != nil { + log.Fatal(err) + } +} + +func check() error { + s, err := archivestore.FromEnv() + if err != nil { + return err + } + if s == nil { + return fmt.Errorf("S3 archive storage is not configured") + } + dir, err := os.MkdirTemp("", "artex-archive-check-") + if err != nil { + return err + } + defer os.RemoveAll(dir) + data := []byte("ARTEX object storage connectivity check " + time.Now().UTC().Format(time.RFC3339Nano)) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + uri := s.URI(0, time.Now().UnixNano(), checksum) + file := filepath.Join(dir, "check") + if err := os.WriteFile(file, data, 0600); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + defer func() { + cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := s.Delete(cleanup, uri); err != nil { + log.Printf("test object cleanup failed for %s: %v", uri, err) + } + }() + if err := s.Upload(ctx, uri, file, checksum); err != nil { + return err + } + if err := s.Download(ctx, uri, filepath.Join(dir, "download"), checksum, int64(len(data))); err != nil { + return err + } + if err := s.Delete(ctx, uri); err != nil { + return err + } + fmt.Println("S3 upload, full SHA-256 verification, download and delete passed") + return nil +} diff --git a/db/task_archives.go b/db/task_archives.go index f312000..676f1d2 100644 --- a/db/task_archives.go +++ b/db/task_archives.go @@ -60,6 +60,7 @@ type TaskArchive struct { Warnings json.RawMessage `json:"warnings"` FormatVersion int `json:"format_version"` ArchivePath string `json:"-"` + Storage string `json:"storage"` SHA256 string `json:"sha256,omitempty"` OriginalSize int64 `json:"original_size"` CompressedSize int64 `json:"compressed_size"` @@ -161,6 +162,10 @@ func scanTaskArchive(sc interface{ Scan(...any) error }) (*TaskArchive, error) { if err != nil { return nil, err } + item.Storage = "local" + if strings.HasPrefix(item.ArchivePath, "s3://") { + item.Storage = "s3" + } if err := json.Unmarshal([]byte(sources), &item.SourceTaskIDs); err != nil { return nil, err } diff --git a/docs/task-archive-storage.md b/docs/task-archive-storage.md new file mode 100644 index 0000000..18bb3b4 --- /dev/null +++ b/docs/task-archive-storage.md @@ -0,0 +1,104 @@ +# Task archive storage + +Task archives use local disk by default. The optional S3-compatible backend moves +cold task packages to object storage after verifying their contents, releasing +local package space while retaining ARTEX's Archive, Restore and Delete actions. + +Active workspaces and the live PostgreSQL database stay local. This feature is +task-level cold storage; use the existing backup workflow for a full installation +backup. A complete recovery plan must also preserve the bucket objects and +access to them. + +## Configuration + +Create a private bucket and a dedicated credential scoped to the archive prefix. +The backend needs object upload, read (including HEAD and byte ranges), delete, +and multipart upload/abort permissions. Use a separate prefix for each deployment. + +Set these environment variables for the ARTEX server, then restart it: + +```dotenv +ARTEX_ARCHIVE_STORAGE=s3 +ARTEX_ARCHIVE_S3_ENDPOINT=https://s3.us-east-1.amazonaws.com +ARTEX_ARCHIVE_S3_REGION=us-east-1 +ARTEX_ARCHIVE_S3_BUCKET=your-artex-archives +ARTEX_ARCHIVE_S3_PREFIX=artex/tasks +ARTEX_ARCHIVE_S3_ACCESS_KEY_ID=REPLACE +ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY=REPLACE +``` + +Store credentials in the service's protected environment file, outside the +repository and frontend. For systemd, a root-owned environment file with mode +0600 can be referenced through `EnvironmentFile=`. + +| Variable | Meaning | +| --- | --- | +| `ARTEX_ARCHIVE_STORAGE` | `local` (default) or `s3`. | +| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local testing. | +| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. Match the provider's requirements. | +| `ARTEX_ARCHIVE_S3_BUCKET` | Required bucket name. | +| `ARTEX_ARCHIVE_S3_PREFIX` | Object prefix; defaults to `artex/tasks`. | +| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Dedicated access key. | +| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Dedicated secret key. | +| `ARTEX_ARCHIVE_S3_PATH_STYLE` | Set to `true` for compatible path-style servers; otherwise virtual-hosted addressing is used. | + +For DigitalOcean Spaces, use its regional origin, such as +`https://sfo3.digitaloceanspaces.com`, and the provider's signing region. For a +local path-style test server, use an endpoint such as `http://127.0.0.1:9000` and +set `ARTEX_ARCHIVE_S3_PATH_STYLE=true`. + +Invalid S3 configuration fails new archive jobs instead of silently falling back +to local disk. Preserve the configured bucket, endpoint, prefix and read/delete +access while remote archives remain. The same installation can still restore or +delete older local archives. + +## Lifecycle and integrity + +1. Pause or finish a task and use its existing **Archive** action. +2. ARTEX builds its `.tar.zst` package locally and uploads it. Large packages use + multipart upload, starting with 8 MiB parts and two upload workers (the SDK + increases the part size for exceptionally large seekable files). +3. ARTEX reads the entire object back and checks its size and SHA-256. Multipart + ETags and uploaded checksum metadata alone are not treated as proof of content. +4. Only after verification does PostgreSQL commit the `s3://` location and + compact the archived task. ARTEX then removes local source files and the + temporary package. The archive list shows **Object storage** or **Local storage**. +5. **Restore** downloads and verifies the package before restoring task files and + database records. The remote package is consumed after restoration commits. + Failed remote cleanup can be retried without replaying restored records. +6. **Delete** removes the archive stub and queues durable cleanup of its remote + object. Cleanup is retried after restart and before subsequent archive jobs. + +Reads use bounded 8 MiB ranges pinned to the object's ETag. Interrupted ranges +are retried without duplicating output or restarting a multi-gigabyte download. +An object change, checksum mismatch or invalid range fails verification. Failed +downloads are removed, and upload/verification failures retain the original task. + +There is no database schema or archive format migration. Existing local archive +packages remain compatible. Files outside ARTEX's task archive database are not +automatically imported or uploaded. + +## Disk, transfer and recovery + +Archive creation still needs space for the temporary compressed package. +Restoration needs space for the downloaded package and extracted files. Full +upload readback and later restores incur object-storage reads and any applicable +transfer charges. Configure expiration of abandoned multipart uploads for abrupt +process termination. + +Recovery journals under `data/archives/tasks/.remote` contain object locations, +not credentials. Preserve them with PostgreSQL and the installation's data +directory. The worker uses these journals to clean up uploads that never +committed, keep committed objects, and finish deferred deletions. Failed remote +cleanup does not block unrelated local archive jobs. + +## Optional connectivity check + +Run `go run ./cmd/artex-archive-check` with the same archive environment as the +server to upload, verify, download and delete a small uniquely named test object. +This command performs real bucket operations but does not touch task data. It +reports leftover test objects if cleanup fails. + +Automated tests use an HTTP S3 fixture and isolated PostgreSQL databases. They +cover multipart transfers, corruption, interrupted range retries, object changes, +cancellation, upload rollback, corrupted restores and persistent cleanup. diff --git a/go.mod b/go.mod index ba11f2c..7eb3dcf 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,10 @@ go 1.26.10 require ( github.com/Autumn-27/norma v0.4.10 + github.com/aws/aws-sdk-go-v2 v1.47.5 + github.com/aws/aws-sdk-go-v2/credentials v1.20.13 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 @@ -22,6 +26,15 @@ require ( require ( github.com/andybalholm/brotli v1.2.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect + github.com/aws/smithy-go v1.28.7 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/creack/pty v1.1.24 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/go.sum b/go.sum index 15b752d..bb0b1af 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,44 @@ github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eT github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4= +github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 h1:MW0Iql1SOTgQPj8mCxrU/eH/K/AIDWi08ItOsXaWRLU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25/go.mod h1:+ucVYmHYB3JqVueY0E1RHahdXAnKQbIHqybSbvOUbOM= +github.com/aws/aws-sdk-go-v2/config v1.33.13 h1:u52x6ISQOx172tpwM0VRxhJzEsTJYpMcEYUGM2A1wrw= +github.com/aws/aws-sdk-go-v2/config v1.33.13/go.mod h1:/ixtz6XPHb42zekjQ0LqD9XjTpTysNWhIGwufY1OzI4= +github.com/aws/aws-sdk-go-v2/credentials v1.20.13 h1:j4z41OtspmQ3ftQycaDunJ26b69a9SHfaSVUPqgRv8k= +github.com/aws/aws-sdk-go-v2/credentials v1.20.13/go.mod h1:SgPA1oS3b+dlMxZb618jDmGNscpXSCaxz15igS52OlE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19 h1:+JWfo9Mkg5HWwDwnpRPu7WNiiyEmiQRTaNSFsZ/1KIY= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19/go.mod h1:l4/tI+TfymypMqq2oRCkhMAB7Gi7nlt5JaoUUz+cUbI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 h1:GACgECI422Xt5n4Pojr3O0aocPv8lfH4PJe2Aon5Wv8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9/go.mod h1:lPfnRkHpn1qWZJLbVEw10rIqSuCee1pNQTyGkUjAYh8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 h1:RS1Wt+K0r4X0ci9Vf4JYTUfHzsQihYA2LgGJeEneKpk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8/go.mod h1:adMnTy5kyoUW86jhXnGSVc/mvMQI4GZd1qYWMeaS8M8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 h1:to7+Vr3rLkpYRwVKJ5QRriI0eE306vrVN/6TnVHgxvY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6/go.mod h1:Jc89ASa2k7HMAskj3bSnt8bf+k+p2IApzkoXnEjkgUw= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.1 h1:WZLC+k/CZx11kZGQDqu9mpA8yhiXGX2I//jhuJE1FDo= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.1/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.1 h1:4VB/dnDo7pmXSLQMszSoBzWndENuc7mzwTpD6lQn77A= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.1/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 h1:w8yui2ZqdHmGwVBDGvazoIvKw/DnLENBLkYnN8q8iqo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw= +github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE= +github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= diff --git a/server/server.go b/server/server.go index ac78a25..0d4134b 100644 --- a/server/server.go +++ b/server/server.go @@ -22,6 +22,7 @@ import ( "time" "github.com/Autumn-27/artex/agent" + "github.com/Autumn-27/artex/archivestore" "github.com/Autumn-27/artex/db" "github.com/Autumn-27/artex/egress" "github.com/Autumn-27/artex/intercept" @@ -127,9 +128,11 @@ type Server struct { // Cold task archives run through one persistent FIFO worker. The buffered wake // channel coalesces enqueue bursts; the database remains the source of truth. - archiveWake chan struct{} - archiveWG sync.WaitGroup - side *sideQuestionState + archiveWake chan struct{} + archiveWG sync.WaitGroup + archiveStore *archivestore.Store + archiveStoreErr error + side *sideQuestionState } // provEntry is a cached provider + its config for one LLM profile id. diff --git a/server/task_archive_package.go b/server/task_archive_package.go index f23c4c8..e9a451d 100644 --- a/server/task_archive_package.go +++ b/server/task_archive_package.go @@ -428,7 +428,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { errs = append(errs, getErr) continue } - if archive != nil && archive.State == pgdb.ArchiveReady { + if archive != nil && archive.ArchivePath != "" && archive.SHA256 != "" { if err := os.RemoveAll(root); err != nil { errs = append(errs, err) } @@ -438,7 +438,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { if err := stage.rollback(); err != nil { errs = append(errs, err) } - if archive != nil && archive.ArchivePath != "" { + if archive != nil && archive.ArchivePath != "" && !strings.HasPrefix(archive.ArchivePath, "s3://") { _ = os.Remove(archive.ArchivePath) } } diff --git a/server/task_archive_remote.go b/server/task_archive_remote.go new file mode 100644 index 0000000..87e230f --- /dev/null +++ b/server/task_archive_remote.go @@ -0,0 +1,171 @@ +package server + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + pgdb "github.com/Autumn-27/artex/db" +) + +// An outbox bridges the local PostgreSQL transaction and object storage. Never +// delete the only package before metadata permits it. Unfinished cleanup is +// retried after a restart (and before subsequent jobs). +type remoteArchiveJournal struct { + ArchiveID int64 `json:"archive_id"` + URI string `json:"uri"` + LocalPath string `json:"local_path,omitempty"` + Operation string `json:"operation"` +} + +func (s *Server) validateStoredArchive(uri string) error { + if strings.HasPrefix(uri, "s3://") { + return s.archiveStore.Validate(uri) + } + return validateArchivePath(s.m.dir, uri) +} + +func removeLocalArchivePackage(filename string) error { + if err := os.Remove(filename); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +func (s *Server) removeStoredArchive(uri string) error { + if err := s.validateStoredArchive(uri); err != nil { + return err + } + if strings.HasPrefix(uri, "s3://") { + return s.archiveStore.Delete(s.ctx, uri) + } + return removeLocalArchivePackage(uri) +} + +func (s *Server) writeRemoteArchiveJournal(j remoteArchiveJournal) (string, error) { + dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote") + if err := os.MkdirAll(dir, archiveDirMode); err != nil { + return "", err + } + f, err := os.CreateTemp(dir, ".journal-") + if err != nil { + return "", err + } + tmp := f.Name() + defer os.Remove(tmp) + encodeErr := json.NewEncoder(f).Encode(j) + syncErr := f.Sync() + if err := errors.Join(encodeErr, syncErr, f.Close()); err != nil { + return "", err + } + name := tmp + ".json" + if err := os.Rename(tmp, name); err != nil { + return "", err + } + // Persist the directory entry before an upload or database deletion starts. + d, err := os.Open(dir) + if err != nil { + return "", err + } + if err := errors.Join(d.Sync(), d.Close()); err != nil { + return "", err + } + // The .remote directory itself may have just been created. + parent, err := os.Open(filepath.Dir(dir)) + if err != nil { + return "", err + } + return name, errors.Join(parent.Sync(), parent.Close()) +} + +func (s *Server) recoverRemoteArchiveJournals() error { + dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote") + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + var errs []error + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + name := filepath.Join(dir, entry.Name()) + if err := s.recoverRemoteArchiveJournal(name); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (s *Server) recoverRemoteArchiveJournal(name string) error { + raw, err := os.ReadFile(name) + if err != nil { + return err + } + var j remoteArchiveJournal + if err := json.Unmarshal(raw, &j); err != nil { + return err + } + if j.ArchiveID <= 0 { + return errors.New("invalid remote archive journal ID") + } + if err := s.archiveStore.Validate(j.URI); err != nil { + return err + } + item, err := s.m.pg.GetTaskArchive(j.ArchiveID) + if err != nil { + return err + } + switch j.Operation { + case "upload": + if item == nil || item.ArchivePath != j.URI { + if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil { + return err + } + } + if err := validateArchivePath(s.m.dir, j.LocalPath); err != nil { + return err + } + // An old failed upload may outlive a retry after switching back to + // local storage. The retry reuses this pathname: keep its committed + // package instead of treating it as the old upload's temporary copy. + if item == nil || item.ArchivePath != j.LocalPath { + if err := removeLocalArchivePackage(j.LocalPath); err != nil { + return err + } + } + case "delete": + if item == nil { + if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil { + return err + } + } + default: + return fmt.Errorf("unknown remote archive operation %q", j.Operation) + } + return os.Remove(name) +} + +func (s *Server) deleteRemoteTaskArchive(job *pgdb.TaskArchive) error { + journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, Operation: "delete"}) + if err != nil { + return err + } + if err := s.m.pg.DeleteTaskArchiveStub(job.ID); err != nil { + // A connection failure can leave the transaction's commit ambiguous. + // Recovery consults PostgreSQL before deciding whether to delete. + return err + } + if err := s.recoverRemoteArchiveJournal(journal); err != nil { + // The task is gone; the durable outbox retries the remaining object cleanup. + log.Printf("[task-archive] archive %d deleted; remote cleanup pending: %v", job.ID, err) + } + return nil +} diff --git a/server/task_archive_remote_test.go b/server/task_archive_remote_test.go new file mode 100644 index 0000000..34e0848 --- /dev/null +++ b/server/task_archive_remote_test.go @@ -0,0 +1,396 @@ +package server + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/Autumn-27/artex/archivestore" + pgdb "github.com/Autumn-27/artex/db" +) + +func TestRemoteArchiveLifecycle(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + t.Setenv("ARTEX_ARCHIVE_STORAGE", "local") + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir()) + s.archiveWG.Wait() + s.ctx = context.Background() + var mu sync.Mutex + objects := map[string][]byte{} + corrupt, failDelete := false, false + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + switch r.Method { + case "PUT": + objects[r.URL.Path], _ = io.ReadAll(r.Body) + w.Header().Set("ETag", `"test"`) + case "HEAD": + w.Header().Set("ETag", `"test"`) + w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path]))) + case "GET": + data := objects[r.URL.Path] + if corrupt && len(data) > 0 { + data = bytes.Clone(data) + data[0] ^= 0xff + } + var start, end int64 + if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) || r.Header.Get("If-Match") != `"test"` { + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return + } + w.Header().Set("ETag", `"test"`) + w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[start : end+1]) + case "DELETE": + if failDelete { + w.WriteHeader(403) + return + } + delete(objects, r.URL.Path) + w.WriteHeader(204) + default: + w.WriteHeader(400) + } + })) + defer remote.Close() + s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + task, err := m.CreateTask("S3 round trip", "preserve data", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + id, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }() + if err := m.pg.SetPaused(id, true); err != nil { + t.Fatal(err) + } + file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "important.txt") + if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte("keep this content"), 0600); err != nil { + t.Fatal(err) + } + pgTask, err := m.pg.GetTask(id) + if err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exploration(pgTask.ExplorationID).AppendActivity(pgdb.Activity{Worker: "system", Kind: "system", Summary: "archive proof"}); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + // A configured-but-invalid backend must fail before moving any source data. + s.archiveStoreErr = errors.New("invalid fixture configuration") + if err := s.runOneTaskArchiveJob(); !errors.Is(err, s.archiveStoreErr) { + t.Fatalf("configuration error was not returned: %v", err) + } + if got, err := os.ReadFile(file); err != nil || string(got) != "keep this content" { + t.Fatalf("bad configuration lost source: %v", err) + } + if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil { + t.Fatal("bad configuration lost task", err) + } + s.archiveStoreErr = nil + if _, err := m.pg.QueueTaskArchive(id); err != nil { + t.Fatal(err) + } + // A corrupt readback must roll back filesystem staging and leave PG intact. + mu.Lock() + corrupt = true + mu.Unlock() + if err := s.runOneTaskArchiveJob(); err == nil { + t.Fatal("expected integrity failure") + } + if got, _ := os.ReadFile(file); string(got) != "keep this content" { + t.Fatal("upload failure lost source") + } + if task, err := m.pg.GetTask(id); err != nil || task == nil { + t.Fatal("upload failure lost task") + } + mu.Lock() + corrupt = false + mu.Unlock() + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + if _, err := m.pg.QueueTaskArchive(id); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + job, err = m.pg.GetTaskArchive(job.ID) + if err != nil { + t.Fatal(err) + } + if job.State != pgdb.ArchiveReady || job.Storage != "s3" || !strings.HasPrefix(job.ArchivePath, "s3://") { + t.Fatalf("unexpected archive %+v", job) + } + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Fatal("workspace not released") + } + if _, err := os.Stat(taskArchivePath(m.dir, job.ID, task.ID)); !os.IsNotExist(err) { + t.Fatal("local package retained") + } + // Simulate the crash window after commit: cleanup keeps the remote object. + journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, LocalPath: taskArchivePath(m.dir, job.ID, task.ID), Operation: "upload"}) + if err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournal(journal); err != nil { + t.Fatal(err) + } + // A corrupted download must leave the archive and its remote package intact. + mu.Lock() + corrupt = true + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err == nil { + t.Fatal("expected corrupted restore to fail") + } + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Fatal("corrupted archive installed workspace files") + } + if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || restored { + t.Fatalf("corrupted archive restored database: %v %v", restored, err) + } + mu.Lock() + if len(objects) == 0 { + t.Error("corrupted restore deleted the only remote package") + } + corrupt = false + mu.Unlock() + // Restoration commits files and PG first. Failed remote deletion is retryable. + mu.Lock() + failDelete = true + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err == nil { + t.Fatal("expected remote delete failure") + } + if got, _ := os.ReadFile(file); string(got) != "keep this content" { + t.Fatal("restored content differs") + } + if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || !restored { + t.Fatalf("PG restore %v %v", restored, err) + } + mu.Lock() + failDelete = false + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + if item, err := m.pg.GetTaskArchive(job.ID); err != nil || item != nil { + t.Fatal("restore did not consume stub") + } + var count int + if err := m.pg.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='archive proof'`, pgTask.ExplorationID).Scan(&count); err != nil || count != 1 { + t.Fatalf("activity restore count=%d err=%v", count, err) + } + // Re-archive, then crash/retry the deletion outbox after PG removed its stub. + job, err = m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + mu.Lock() + failDelete = true + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveDelete(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + mu.Lock() + failDelete = false + mu.Unlock() + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + mu.Lock() + remaining := len(objects) + mu.Unlock() + if remaining != 0 { + t.Fatalf("left %d objects", remaining) + } +} + +// Once PostgreSQL has committed an archive location and checksum, a subsequent +// queued operation must not make startup recovery resurrect the old workspace. +func TestRemoteArchiveStageRecoveryAfterCommit(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + task, err := m.CreateTask("remote stage recovery", "test", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + taskID, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { + _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID) + _ = m.pg.DeleteTask(taskID) + }() + if err := m.pg.SetPaused(taskID, true); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(taskID) + if err != nil { + t.Fatal(err) + } + pgTask, err := m.pg.GetTask(taskID) + if err != nil { + t.Fatal(err) + } + for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} { + t.Run(state, func(t *testing.T) { + committed := state != pgdb.ArchiveQueued + checksum := "" + if committed { + checksum = strings.Repeat("a", 64) + } + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, "s3://test/tasks/package.tar.zst", checksum); err != nil { + t.Fatal(err) + } + file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt") + if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil { + t.Fatal(err) + } + stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID) + if err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stage.root); !os.IsNotExist(err) { + t.Fatalf("staging cleanup incomplete: %v", err) + } + if committed { + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Fatalf("resurrected committed workspace: %v", err) + } + } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" { + t.Fatalf("uncommitted archive lost source: %v", err) + } + }) + } +} + +// Failed remote cleanup can survive switching to local storage and retrying the +// same archive ID. Its journal must never remove the newly committed local copy. +func TestRemoteArchiveJournalPreservesLocalRetry(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + task, err := m.CreateTask("local retry", "test", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + taskID, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { + _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID) + _ = m.pg.DeleteTask(taskID) + }() + if err := m.pg.SetPaused(taskID, true); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(taskID) + if err != nil { + t.Fatal(err) + } + filename := taskArchivePath(m.dir, job.ID, task.ID) + if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filename, []byte("committed local archive"), 0600); err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, pgdb.ArchiveReady, filename, strings.Repeat("a", 64)); err != nil { + t.Fatal(err) + } + var deletes atomic.Int32 + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + deletes.Add(1) + w.WriteHeader(http.StatusNoContent) + })) + defer remote.Close() + store, err := archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + s := &Server{ctx: context.Background(), m: m, archiveStore: store} + journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ + ArchiveID: job.ID, URI: store.URI(taskID, job.ID, strings.Repeat("b", 64)), + LocalPath: filename, Operation: "upload", + }) + if err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournal(journal); err != nil { + t.Fatal(err) + } + if data, err := os.ReadFile(filename); err != nil || string(data) != "committed local archive" { + t.Fatalf("old upload cleanup removed local retry: %v", err) + } + if deletes.Load() != 1 { + t.Fatal("uncommitted remote object was not removed") + } + if _, err := os.Stat(journal); !os.IsNotExist(err) { + t.Fatalf("finished journal remains: %v", err) + } +} diff --git a/server/task_archives.go b/server/task_archives.go index b73dfb2..3ac6341 100644 --- a/server/task_archives.go +++ b/server/task_archives.go @@ -14,6 +14,7 @@ import ( "time" "github.com/Autumn-27/artex/agent" + "github.com/Autumn-27/artex/archivestore" pgdb "github.com/Autumn-27/artex/db" "github.com/Autumn-27/artex/traffic" ) @@ -37,6 +38,12 @@ func (s *Server) startTaskArchiveWorker() { if s.m == nil || s.m.pg == nil { return } + s.archiveStore, s.archiveStoreErr = archivestore.FromEnv() + if s.archiveStoreErr != nil { + log.Printf("[task-archive] storage configuration: %v", s.archiveStoreErr) + } else if s.archiveStore != nil { + log.Printf("[task-archive] new archives use S3 object storage with full SHA-256 readback verification") + } if err := s.m.pg.RecoverTaskArchiveJobs(); err != nil { log.Printf("[task-archive] recover jobs: %v", err) } @@ -88,6 +95,14 @@ func (s *Server) notifyTaskArchiveWorker() { } func (s *Server) runOneTaskArchiveJob() error { + if s.ctx.Err() != nil { + return s.ctx.Err() + } + // Retry deferred remote cleanup before claiming more work. Failed cleanup + // keeps its durable journal and does not block unrelated local archives. + if err := s.recoverRemoteArchiveJournals(); err != nil { + log.Printf("[task-archive] deferred object cleanup: %v", err) + } job, err := s.m.pg.ClaimTaskArchiveJob(s.ctx) if err != nil || job == nil { return err @@ -115,6 +130,9 @@ func (s *Server) runOneTaskArchiveJob() error { } func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { + if s.archiveStoreErr != nil { + return s.archiveStoreErr + } taskID := strconv.FormatInt(job.TaskID, 10) if !s.beginTaskDelete(taskID) { return errors.New("task is already undergoing another archive or delete operation") @@ -199,6 +217,21 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { _ = os.Remove(archivePath) } }() + storedPath := archivePath + var remoteJournal string + if s.archiveStore != nil { + storedPath = s.archiveStore.URI(job.TaskID, job.ID, checksum) + remoteJournal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ + ArchiveID: job.ID, URI: storedPath, LocalPath: archivePath, Operation: "upload", + }) + if err != nil { + return err + } + _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "upload_and_verify", 65) + if err := s.archiveStore.Upload(s.ctx, storedPath, archivePath, checksum); err != nil { + return err + } + } var trafficStage *traffic.HostDeleteStage if s.m.traffic != nil && len(snapshot.ExclusiveHosts) > 0 { _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_traffic", 68) @@ -213,11 +246,18 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { }() } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_database", 78) - if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, archivePath, checksum, originalSize, compressedSize); err != nil { + if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, storedPath, checksum, originalSize, compressedSize); err != nil { return err } committed = true - removePackage = false + removePackage = s.archiveStore != nil + if remoteJournal != "" { + // The journal also covers a crash after database commit but before local + // package removal. Keep it if cleanup fails, so the worker can retry. + if err := removeLocalArchivePackage(archivePath); err == nil { + _ = os.Remove(remoteJournal) + } + } if trafficStage != nil { if err := trafficStage.Commit(); err != nil { warning := "archive completed, but cleaning the exclusive traffic hot store failed (the archive package can still restore it): " + err.Error() @@ -239,7 +279,7 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { } func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { - if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil { + if err := s.validateStoredArchive(job.ArchivePath); err != nil { return err } // A prior attempt may have committed PostgreSQL and files, then failed only @@ -247,7 +287,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { if restored, err := s.m.pg.IsTaskArchiveRestored(job.ID); err != nil { return err } else if restored { - if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) { + if err := s.removeStoredArchive(job.ArchivePath); err != nil { return err } if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil { @@ -266,7 +306,15 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { return err } defer func() { _ = os.RemoveAll(extracted) }() - if err := extractTaskArchivePackage(job.ArchivePath, job.SHA256, extracted); err != nil { + packagePath := job.ArchivePath + if strings.HasPrefix(packagePath, "s3://") { + _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "download_and_verify", 12) + packagePath = filepath.Join(extracted, "download.tar.zst") + if err := s.archiveStore.Download(s.ctx, job.ArchivePath, packagePath, job.SHA256, job.CompressedSize); err != nil { + return err + } + } + if err := extractTaskArchivePackage(packagePath, job.SHA256, extracted); err != nil { return err } manifestRaw, err := os.ReadFile(filepath.Join(extracted, "manifest.json")) @@ -356,7 +404,7 @@ func (s *Server) restoreTaskArchivePayload(job *pgdb.TaskArchive, snapshot *pgdb return err } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "consume_package", 95) - if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) { + if err := s.removeStoredArchive(job.ArchivePath); err != nil { return err } if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil { @@ -393,9 +441,12 @@ func (s *Server) loadRestoredTask(taskID int64) { } func (s *Server) deleteTaskArchive(job *pgdb.TaskArchive) error { - if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil { + if err := s.validateStoredArchive(job.ArchivePath); err != nil { return err } + if strings.HasPrefix(job.ArchivePath, "s3://") { + return s.deleteRemoteTaskArchive(job) + } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "stage_package_delete", 35) staged, moved, err := stageTaskArchivePackageDelete(job.ArchivePath, job.ID) if err != nil { diff --git a/web/src/app/(main)/function/tasks/page.tsx b/web/src/app/(main)/function/tasks/page.tsx index 6b8ba8e..c2f9cc3 100644 --- a/web/src/app/(main)/function/tasks/page.tsx +++ b/web/src/app/(main)/function/tasks/page.tsx @@ -1494,7 +1494,7 @@ function ArchiveConfirmDialog({ {count === 1 ? "Archive task" : `Archive ${count} tasks`} - Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into local cold storage. Once archived, a task can be restored from "Archived". + Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into the configured archive storage. Once archived, a task can be restored from "Archived". Restoring requires free space on this server. @@ -1934,6 +1934,7 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) { title={`Before compression: ${formatArchiveBytes(archive.original_size)}`} > {archiveCompressionLabel(archive)} +
{archive.storage === "s3" ? "Object storage" : "Local storage"}
{archiveDataTotal(archive).toLocaleString()} @@ -1952,7 +1953,7 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {

))} - {archive.phase} · {archive.progress}% + {archive.phase === "upload_and_verify" ? "Uploading and verifying" : archive.phase === "download_and_verify" ? "Downloading and verifying" : archive.phase} · {archive.progress}%
diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index d098f9e..4a0b479 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -101,6 +101,7 @@ export interface TaskArchiveTokenStats { } export interface TaskArchive { + storage?: "local" | "s3"; id: number; task_id: number; state: TaskArchiveState; From da7cc62d4eee283586fb3c0202467ec4e44b2fce Mon Sep 17 00:00:00 2001 From: Chubboy Ren Date: Wed, 18 Nov 2026 02:05:17 +0000 Subject: [PATCH 2/4] feat: encrypt S3 archives and test credentials and interruption recovery --- archivestore/config_test.go | 20 ++ archivestore/credentials_test.go | 132 ++++++++++++ archivestore/s3.go | 73 +++++-- archivestore/s3_test.go | 279 ++++++++++++++++-------- docs/task-archive-storage.md | 194 +++++++++++------ go.mod | 6 + server/task_archive_interrupt_test.go | 291 ++++++++++++++++++++++++++ server/task_archive_package.go | 11 +- server/task_archive_remote_test.go | 93 ++++---- 9 files changed, 892 insertions(+), 207 deletions(-) create mode 100644 archivestore/credentials_test.go create mode 100644 server/task_archive_interrupt_test.go diff --git a/archivestore/config_test.go b/archivestore/config_test.go index cc12532..73afc73 100644 --- a/archivestore/config_test.go +++ b/archivestore/config_test.go @@ -10,6 +10,7 @@ func TestArchiveStorageEnvironment(t *testing.T) { "ARTEX_ARCHIVE_STORAGE", "ARTEX_ARCHIVE_S3_ENDPOINT", "ARTEX_ARCHIVE_S3_REGION", "ARTEX_ARCHIVE_S3_BUCKET", "ARTEX_ARCHIVE_S3_PREFIX", "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_PATH_STYLE", + "ARTEX_ARCHIVE_S3_SESSION_TOKEN", "ARTEX_ARCHIVE_S3_SSE", "ARTEX_ARCHIVE_S3_KMS_KEY_ID", } { t.Setenv(key, "") } @@ -60,3 +61,22 @@ func TestArchiveEndpointValidation(t *testing.T) { } } } + +func TestArchiveEncryptionConfiguration(t *testing.T) { + c := Config{Endpoint: "https://objects.example.test", Bucket: "test", AccessKey: "test", SecretKey: "test"} + for _, mode := range []string{"none", "invalid"} { + c.SSE = mode + if _, err := New(c); err == nil { + t.Fatal("unencrypted/invalid mode accepted") + } + } + c.SSE = "AES256" + c.KMSKeyID = "alias/archive" + if _, err := New(c); err == nil { + t.Fatal("KMS key accepted with SSE-S3") + } + c.SSE = "aws:kms" + if _, err := New(c); err != nil { + t.Fatal(err) + } +} diff --git a/archivestore/credentials_test.go b/archivestore/credentials_test.go new file mode 100644 index 0000000..6c667ed --- /dev/null +++ b/archivestore/credentials_test.go @@ -0,0 +1,132 @@ +package archivestore + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func isolateAWSCredentials(t *testing.T) { + t.Helper() + for _, key := range []string{ + "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AWS_PROFILE", "AWS_DEFAULT_PROFILE", + "AWS_ROLE_ARN", "AWS_ROLE_SESSION_NAME", "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_CREDENTIALS_FULL_URI", + "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", + "AWS_EC2_METADATA_SERVICE_ENDPOINT", "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE", + } { + t.Setenv(key, "") + } + dir := t.TempDir() + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(dir, "credentials")) + t.Setenv("AWS_CONFIG_FILE", filepath.Join(dir, "config")) + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") +} + +func TestArchiveCredentialChain(t *testing.T) { + config := Config{Endpoint: "https://objects.example.test", Bucket: "test"} + for _, mode := range []string{"explicit-session", "environment", "shared-profile", "ecs", "imds"} { + t.Run(mode, func(t *testing.T) { + isolateAWSCredentials(t) + c := config + switch mode { + case "explicit-session": + t.Setenv("AWS_ACCESS_KEY_ID", "other-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "other-secret") + c.AccessKey, c.SecretKey, c.SessionToken = "fixture-key", "fixture-secret", "fixture-session" + case "environment": + t.Setenv("AWS_ACCESS_KEY_ID", "fixture-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture-secret") + t.Setenv("AWS_SESSION_TOKEN", "fixture-session") + case "shared-profile": + t.Setenv("AWS_PROFILE", "archive") + data := `[archive] +aws_access_key_id=fixture-key +aws_secret_access_key=fixture-secret +aws_session_token=fixture-session +` + if err := os.WriteFile(os.Getenv("AWS_SHARED_CREDENTIALS_FILE"), []byte(data), 0600); err != nil { + t.Fatal(err) + } + case "ecs", "imds": + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if mode == "imds" { + switch r.URL.Path { + case "/latest/api/token": + if r.Method != "PUT" { + t.Error("IMDS did not use token request") + } + w.Header().Set("X-aws-ec2-metadata-token-ttl-seconds", "21600") + fmt.Fprint(w, "fixture-imds-token") + return + case "/latest/meta-data/iam/security-credentials/": + if r.Header.Get("X-aws-ec2-metadata-token") != "fixture-imds-token" { + t.Error("missing IMDSv2 token") + } + fmt.Fprint(w, "archive-role") + return + case "/latest/meta-data/iam/security-credentials/archive-role": + if r.Header.Get("X-aws-ec2-metadata-token") != "fixture-imds-token" { + t.Error("missing IMDSv2 token") + } + default: + t.Errorf("unexpected IMDS request %s", r.URL.Path) + w.WriteHeader(404) + return + } + } else if r.Header.Get("Authorization") != "fixture-container-token" { + t.Error("missing ECS authorization token") + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "Code": "Success", "AccessKeyId": "fixture-key", "SecretAccessKey": "fixture-secret", + "Token": "fixture-session", "Expiration": time.Now().Add(time.Hour).UTC().Format(time.RFC3339), + }) + })) + defer srv.Close() + if mode == "imds" { + t.Setenv("AWS_EC2_METADATA_DISABLED", "false") + t.Setenv("AWS_EC2_METADATA_V1_DISABLED", "true") + t.Setenv("AWS_EC2_METADATA_SERVICE_ENDPOINT", srv.URL) + } else { + t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", srv.URL+"/credentials") + t.Setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN", "fixture-container-token") + } + } + store, err := New(c) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + credentials, err := store.client.Options().Credentials.Retrieve(ctx) + if err != nil { + t.Fatal(err) + } + if credentials.AccessKeyID != "fixture-key" || credentials.SecretAccessKey != "fixture-secret" || credentials.SessionToken != "fixture-session" { + t.Fatal("wrong credential provider or session token") + } + if (mode == "imds" || mode == "ecs") && !credentials.CanExpire { + t.Fatal("role credentials must remain refreshable") + } + }) + } +} + +func TestArchiveIncompleteCredentialOverride(t *testing.T) { + isolateAWSCredentials(t) + t.Setenv("AWS_ACCESS_KEY_ID", "valid-ambient-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "valid-ambient-secret") + for _, c := range []Config{{AccessKey: "private-value"}, {SecretKey: "private-value"}, {SessionToken: "private-value"}} { + c.Endpoint, c.Bucket = "https://objects.example.test", "test" + if _, err := New(c); err == nil || strings.Contains(err.Error(), "private-value") { + t.Fatalf("partial override must fail without disclosing credentials: %v", err) + } + } +} diff --git a/archivestore/s3.go b/archivestore/s3.go index edc1513..27c50af 100644 --- a/archivestore/s3.go +++ b/archivestore/s3.go @@ -17,19 +17,24 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" ) type Config struct { Endpoint, Region, Bucket, Prefix, AccessKey, SecretKey string PathStyle bool + SessionToken, SSE, KMSKeyID string } type Store struct { client *s3.Client bucket, prefix string + encryption types.ServerSideEncryption + kmsKeyID string } // FromEnv returns nil for the default local backend. A configured but invalid @@ -43,7 +48,9 @@ func FromEnv() (*Store, error) { Endpoint: os.Getenv("ARTEX_ARCHIVE_S3_ENDPOINT"), Region: os.Getenv("ARTEX_ARCHIVE_S3_REGION"), Bucket: os.Getenv("ARTEX_ARCHIVE_S3_BUCKET"), Prefix: os.Getenv("ARTEX_ARCHIVE_S3_PREFIX"), AccessKey: os.Getenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID"), SecretKey: os.Getenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY"), - PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true", + PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true", + SessionToken: os.Getenv("ARTEX_ARCHIVE_S3_SESSION_TOKEN"), + SSE: os.Getenv("ARTEX_ARCHIVE_S3_SSE"), KMSKeyID: os.Getenv("ARTEX_ARCHIVE_S3_KMS_KEY_ID"), }) default: return nil, errors.New("ARTEX_ARCHIVE_STORAGE must be local or s3") @@ -70,22 +77,49 @@ func New(c Config) (*Store, error) { if path.Clean(c.Prefix) != c.Prefix || strings.Contains(c.Prefix, "..") || strings.ContainsAny(c.Prefix, "\\?#") { return nil, errors.New("invalid archive S3 prefix") } - if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") || c.AccessKey == "" || c.SecretKey == "" { - return nil, errors.New("archive S3 bucket and dedicated access credentials are required") + if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") { + return nil, errors.New("a valid archive S3 bucket is required") + } + explicitCredentials := c.AccessKey != "" || c.SecretKey != "" || c.SessionToken != "" + if explicitCredentials && (c.AccessKey == "" || c.SecretKey == "") { + return nil, errors.New("archive S3 access key and secret key must both be set when overriding AWS credentials") + } + if c.SSE == "" { + c.SSE = "AES256" + } + if c.SSE != "AES256" && c.SSE != "aws:kms" { + return nil, errors.New("archive S3 SSE must be AES256 or aws:kms; encryption cannot be disabled") + } + if c.KMSKeyID != "" && c.SSE != "aws:kms" { + return nil, errors.New("archive S3 KMS key requires SSE=aws:kms") } if c.Region == "" { c.Region = "us-east-1" } transport := http.DefaultTransport.(*http.Transport).Clone() transport.ResponseHeaderTimeout = 60 * time.Second - client := s3.New(s3.Options{ - Region: c.Region, BaseEndpoint: aws.String(c.Endpoint), UsePathStyle: c.PathStyle, - Credentials: credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, ""), - HTTPClient: &http.Client{Transport: transport}, - RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, - ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + httpClient := &http.Client{Transport: transport} + cfg := aws.Config{Region: c.Region, HTTPClient: httpClient} + if explicitCredentials { + cfg.Credentials = credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken) + } else { + // Keep the SDK's refreshable default chain: standard AWS environment/shared + // profiles, web identity, ECS task roles and EC2 instance roles (IMDSv2). + var err error + cfg, err = awsconfig.LoadDefaultConfig(context.Background(), + awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient)) + if err != nil { + return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err) + } + } + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(c.Endpoint) + o.UsePathStyle = c.PathStyle + o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired }) - return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/"}, nil + return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/", + encryption: types.ServerSideEncryption(c.SSE), kmsKeyID: c.KMSKeyID}, nil } func (s *Store) URI(taskID, archiveID int64, checksum string) string { @@ -134,17 +168,22 @@ func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) erro u.Concurrency = 2 u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired }) - _, err = uploader.Upload(ctx, &s3.PutObjectInput{ + input := &s3.PutObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(key), Body: f, ContentType: aws.String("application/zstd"), Metadata: map[string]string{"sha256": checksum}, - }) + ServerSideEncryption: s.encryption, + } + if s.kmsKeyID != "" { + input.SSEKMSKeyId = aws.String(s.kmsKeyID) + } + _, err = uploader.Upload(ctx, input) if err != nil { return fmt.Errorf("upload archive: %w", err) } - return s.readVerified(ctx, key, io.Discard, checksum, info.Size()) + return s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true) } -func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64) error { +func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64, requireEncryption bool) error { if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size { return errors.New("invalid archive SHA-256") } @@ -155,6 +194,10 @@ func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, che if err != nil { return fmt.Errorf("inspect archive: %w", err) } + if requireEncryption && (object.ServerSideEncryption != s.encryption || + (s.encryption == types.ServerSideEncryptionAwsKms && aws.ToString(object.SSEKMSKeyId) == "")) { + return errors.New("remote archive did not confirm the requested server-side encryption; source data retained") + } if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" { return errors.New("remote archive size mismatch or missing ETag; source data retained") } @@ -264,7 +307,7 @@ func (s *Store) Download(ctx context.Context, uri, filename, checksum string, si _ = os.Remove(filename) } }() - if err = s.readVerified(ctx, key, f, checksum, size); err != nil { + if err = s.readVerified(ctx, key, f, checksum, size, false); err != nil { return err } return f.Sync() diff --git a/archivestore/s3_test.go b/archivestore/s3_test.go index 316e770..dc618c1 100644 --- a/archivestore/s3_test.go +++ b/archivestore/s3_test.go @@ -13,99 +13,121 @@ import ( "os" "path/filepath" "strconv" + "strings" "sync" + "sync/atomic" "testing" ) func TestS3RoundTripAndIntegrity(t *testing.T) { - for _, size := range []int{1024, 10 << 20} { - t.Run(strconv.Itoa(size), func(t *testing.T) { - var mu sync.Mutex - var object []byte - parts := map[int][]byte{} - corrupt, multipart := false, false - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - mu.Lock() - defer mu.Unlock() - q := r.URL.Query() - w.Header().Set("ETag", `"test-etag"`) - switch { - case r.Method == "POST" && q.Has("uploads"): - multipart = true - fmt.Fprint(w, `test-upload`) - case r.Method == "PUT" && q.Has("partNumber"): - n, _ := strconv.Atoi(q.Get("partNumber")) - parts[n], _ = io.ReadAll(r.Body) - case r.Method == "POST" && q.Has("uploadId"): - for i := 1; i <= len(parts); i++ { - object = append(object, parts[i]...) + for _, sse := range []string{"AES256", "aws:kms"} { + kms := "" + if sse == "aws:kms" { + kms = "arn:aws:kms:us-east-1:123456789012:key/test-key" + } + for _, size := range []int{1024, 10 << 20} { + t.Run(sse+"/"+strconv.Itoa(size), func(t *testing.T) { + var mu sync.Mutex + var object []byte + parts := map[int][]byte{} + corrupt, multipart := false, false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + q := r.URL.Query() + if (r.Method == "PUT" && !q.Has("partNumber")) || (r.Method == "POST" && q.Has("uploads")) { + if r.Header.Get("X-Amz-Server-Side-Encryption") != sse || r.Header.Get("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id") != kms { + t.Error("upload did not explicitly request the configured encryption") + } } - fmt.Fprint(w, `test-etag`) - case r.Method == "PUT": - object, _ = io.ReadAll(r.Body) - case r.Method == "HEAD": - w.Header().Set("Content-Length", strconv.Itoa(len(object))) - case r.Method == "GET": - data := object - if corrupt { - data = bytes.Clone(object) - data[0] ^= 0xff + if r.Method == "HEAD" { + if r.Header.Get("X-Amz-Server-Side-Encryption") != "" { + t.Error("encryption write header leaked into HEAD") + } + w.Header().Set("X-Amz-Server-Side-Encryption", sse) + if kms != "" { + w.Header().Set("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id", kms) + } } - serveArchiveRange(t, w, r, data) - case r.Method == "DELETE": - object = nil - w.WriteHeader(204) - default: - t.Errorf("unexpected request %s %s", r.Method, r.URL) - w.WriteHeader(400) + w.Header().Set("ETag", `"test-etag"`) + switch { + case r.Method == "POST" && q.Has("uploads"): + multipart = true + fmt.Fprint(w, `test-upload`) + case r.Method == "PUT" && q.Has("partNumber"): + n, _ := strconv.Atoi(q.Get("partNumber")) + parts[n], _ = io.ReadAll(r.Body) + case r.Method == "POST" && q.Has("uploadId"): + for i := 1; i <= len(parts); i++ { + object = append(object, parts[i]...) + } + fmt.Fprint(w, `test-etag`) + case r.Method == "PUT": + object, _ = io.ReadAll(r.Body) + case r.Method == "HEAD": + w.Header().Set("Content-Length", strconv.Itoa(len(object))) + case r.Method == "GET": + data := object + if corrupt { + data = bytes.Clone(object) + data[0] ^= 0xff + } + serveArchiveRange(t, w, r, data) + case r.Method == "DELETE": + object = nil + w.WriteHeader(204) + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL) + w.WriteHeader(400) + } + })) + defer srv.Close() + store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true, SSE: sse, KMSKeyID: kms}) + if err != nil { + t.Fatal(err) } - })) - defer srv.Close() - store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) - if err != nil { - t.Fatal(err) - } - data := bytes.Repeat([]byte{0x42}, size) - h := sha256.Sum256(data) - checksum := hex.EncodeToString(h[:]) - root := t.TempDir() - source := filepath.Join(root, "source") - if err := os.WriteFile(source, data, 0600); err != nil { - t.Fatal(err) - } - uri := store.URI(1, 2, checksum) - ctx := context.Background() - if err := store.Upload(ctx, uri, source, checksum); err != nil { - t.Fatal(err) - } - if multipart != (size > 8<<20) { - t.Fatalf("multipart=%v size=%d", multipart, size) - } - dest := filepath.Join(root, "download") - if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil { - t.Fatal(err) - } - got, _ := os.ReadFile(dest) - if !bytes.Equal(got, data) { - t.Fatal("download differs") - } - mu.Lock() - corrupt = true - mu.Unlock() - bad := filepath.Join(root, "bad") - if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil { - t.Fatal("accepted corrupt object") - } - if _, err := os.Stat(bad); !os.IsNotExist(err) { - t.Fatal("left corrupt download") - } - if err := store.Upload(ctx, uri, source, checksum); err == nil { - t.Fatal("upload accepted failed readback") - } - if err := store.Delete(ctx, uri); err != nil { - t.Fatal(err) - } - }) + data := bytes.Repeat([]byte{0x42}, size) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + root := t.TempDir() + source := filepath.Join(root, "source") + if err := os.WriteFile(source, data, 0600); err != nil { + t.Fatal(err) + } + uri := store.URI(1, 2, checksum) + ctx := context.Background() + if err := store.Upload(ctx, uri, source, checksum); err != nil { + t.Fatal(err) + } + if multipart != (size > 8<<20) { + t.Fatalf("multipart=%v size=%d", multipart, size) + } + dest := filepath.Join(root, "download") + if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil { + t.Fatal(err) + } + got, _ := os.ReadFile(dest) + if !bytes.Equal(got, data) { + t.Fatal("download differs") + } + mu.Lock() + corrupt = true + mu.Unlock() + bad := filepath.Join(root, "bad") + if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil { + t.Fatal("accepted corrupt object") + } + if _, err := os.Stat(bad); !os.IsNotExist(err) { + t.Fatal("left corrupt download") + } + if err := store.Upload(ctx, uri, source, checksum); err == nil { + t.Fatal("upload accepted failed readback") + } + if err := store.Delete(ctx, uri); err != nil { + t.Fatal(err) + } + }) + } } } @@ -256,7 +278,7 @@ func TestS3ReadbackPropagatesOutputFailure(t *testing.T) { store := testRangeStore(t, srv.URL) h := sha256.Sum256(data) outputError := errors.New("output disk full") - err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data))) + err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data)), false) if !errors.Is(err, outputError) { t.Fatalf("lost output error: %v", err) } @@ -309,3 +331,88 @@ func TestS3RejectsForeignLocations(t *testing.T) { t.Fatal("missing config accepted") } } + +func TestS3UploadRejectsUnconfirmedEncryption(t *testing.T) { + for _, mode := range []string{"missing", "different", "kms-without-key"} { + t.Run(mode, func(t *testing.T) { + data := []byte("private archive") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case "PUT": + _, _ = io.Copy(io.Discard, r.Body) + case "HEAD": + w.Header().Set("ETag", `"test-etag"`) + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + if mode != "missing" { + w.Header().Set("X-Amz-Server-Side-Encryption", "aws:kms") + } + default: + t.Errorf("content must not be accepted before encryption confirmation: %s", r.Method) + w.WriteHeader(400) + } + })) + defer srv.Close() + c := Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true} + if mode == "kms-without-key" { + c.SSE = "aws:kms" + } + store, err := New(c) + if err != nil { + t.Fatal(err) + } + filename := filepath.Join(t.TempDir(), "source") + if err := os.WriteFile(filename, data, 0600); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(data) + checksum := hex.EncodeToString(sum[:]) + if err := store.Upload(context.Background(), store.URI(1, 2, checksum), filename, checksum); err == nil || !strings.Contains(err.Error(), "encryption") { + t.Fatalf("unconfirmed encryption accepted: %v", err) + } + if got, err := os.ReadFile(filename); err != nil || !bytes.Equal(got, data) { + t.Fatal("source was not retained", err) + } + }) + } +} + +func TestS3MultipartFailureAbortsUpload(t *testing.T) { + var aborted atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + switch { + case r.Method == "POST" && q.Has("uploads"): + if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" { + t.Error("multipart encryption missing") + } + fmt.Fprint(w, `failed-upload`) + case r.Method == "PUT" && q.Has("partNumber"): + _, _ = io.Copy(io.Discard, r.Body) + w.WriteHeader(403) + case r.Method == "DELETE" && q.Get("uploadId") == "failed-upload": + aborted.Store(true) + w.WriteHeader(204) + default: + t.Errorf("unexpected failed-upload request: %s", r.Method) + w.WriteHeader(400) + } + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + data := bytes.Repeat([]byte("x"), 9<<20) + checksum := sha256.Sum256(data) + sum := hex.EncodeToString(checksum[:]) + filename := filepath.Join(t.TempDir(), "source") + if err := os.WriteFile(filename, data, 0600); err != nil { + t.Fatal(err) + } + if err := store.Upload(context.Background(), store.URI(1, 2, sum), filename, sum); err == nil { + t.Fatal("failed part accepted") + } + if !aborted.Load() { + t.Fatal("failed multipart upload was not aborted") + } + if info, err := os.Stat(filename); err != nil || info.Size() != int64(len(data)) { + t.Fatal("source was not retained", err) + } +} diff --git a/docs/task-archive-storage.md b/docs/task-archive-storage.md index 18bb3b4..113921a 100644 --- a/docs/task-archive-storage.md +++ b/docs/task-archive-storage.md @@ -1,21 +1,19 @@ # Task archive storage Task archives use local disk by default. The optional S3-compatible backend moves -cold task packages to object storage after verifying their contents, releasing -local package space while retaining ARTEX's Archive, Restore and Delete actions. - -Active workspaces and the live PostgreSQL database stay local. This feature is -task-level cold storage; use the existing backup workflow for a full installation -backup. A complete recovery plan must also preserve the bucket objects and -access to them. +cold task packages to object storage while retaining Archive, Restore and Delete. +Active workspaces and PostgreSQL remain local. This is task cold storage, not a +full-installation backup; a recovery plan must preserve the database, data +directory, bucket objects, and the credentials or roles needed to read them. ## Configuration -Create a private bucket and a dedicated credential scoped to the archive prefix. -The backend needs object upload, read (including HEAD and byte ranges), delete, -and multipart upload/abort permissions. Use a separate prefix for each deployment. +Use a private bucket and a separate prefix for each ARTEX installation. Configure +the bucket's default encryption to match the selected mode below, and enforce +that mode/key with a bucket policy. ARTEX does not provision or inspect bucket +encryption settings. -Set these environment variables for the ARTEX server, then restart it: +Set the following server environment and restart ARTEX: ```dotenv ARTEX_ARCHIVE_STORAGE=s3 @@ -23,82 +21,140 @@ ARTEX_ARCHIVE_S3_ENDPOINT=https://s3.us-east-1.amazonaws.com ARTEX_ARCHIVE_S3_REGION=us-east-1 ARTEX_ARCHIVE_S3_BUCKET=your-artex-archives ARTEX_ARCHIVE_S3_PREFIX=artex/tasks -ARTEX_ARCHIVE_S3_ACCESS_KEY_ID=REPLACE -ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY=REPLACE +ARTEX_ARCHIVE_S3_SSE=AES256 ``` -Store credentials in the service's protected environment file, outside the -repository and frontend. For systemd, a root-owned environment file with mode -0600 can be referenced through `EnvironmentFile=`. +With an EC2 instance role, ECS task role or other SDK-supported credential source, +no archive-specific access keys are needed. | Variable | Meaning | | --- | --- | | `ARTEX_ARCHIVE_STORAGE` | `local` (default) or `s3`. | -| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local testing. | -| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. Match the provider's requirements. | +| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local tests. | +| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. | | `ARTEX_ARCHIVE_S3_BUCKET` | Required bucket name. | | `ARTEX_ARCHIVE_S3_PREFIX` | Object prefix; defaults to `artex/tasks`. | -| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Dedicated access key. | -| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Dedicated secret key. | -| `ARTEX_ARCHIVE_S3_PATH_STYLE` | Set to `true` for compatible path-style servers; otherwise virtual-hosted addressing is used. | +| `ARTEX_ARCHIVE_S3_SSE` | `AES256` (default, SSE-S3) or `aws:kms` (SSE-KMS). There is no unencrypted mode. | +| `ARTEX_ARCHIVE_S3_KMS_KEY_ID` | Optional KMS key ARN/ID/alias; valid only with `aws:kms`. Omit to use the provider's managed KMS key. | +| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Optional dedicated credential override; must be supplied with the secret key. | +| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Secret for the dedicated override. | +| `ARTEX_ARCHIVE_S3_SESSION_TOKEN` | Optional session token for dedicated temporary credentials. | +| `ARTEX_ARCHIVE_S3_PATH_STYLE` | `true` for path-style servers; otherwise virtual-hosted addressing is used. | -For DigitalOcean Spaces, use its regional origin, such as -`https://sfo3.digitaloceanspaces.com`, and the provider's signing region. For a -local path-style test server, use an endpoint such as `http://127.0.0.1:9000` and -set `ARTEX_ARCHIVE_S3_PATH_STYLE=true`. +An invalid mode or partial credential override fails configuration instead of +silently falling back to local storage or ambient credentials. Preserve the +bucket, endpoint, prefix and read/delete access while remote archives remain. +Older local archives remain usable; they are not uploaded automatically. -Invalid S3 configuration fails new archive jobs instead of silently falling back -to local disk. Preserve the configured bucket, endpoint, prefix and read/delete -access while remote archives remain. The same installation can still restore or -delete older local archives. +## Encryption and permissions -## Lifecycle and integrity +Every single-object upload and multipart initiation explicitly requests the +configured server-side encryption. Upload verification also requires HEAD to +report the requested mode, and a KMS key identifier for SSE-KMS, before accepting +the object. Size and full SHA-256 readback remain mandatory; encryption metadata +does not replace content verification. -1. Pause or finish a task and use its existing **Archive** action. -2. ARTEX builds its `.tar.zst` package locally and uploads it. Large packages use - multipart upload, starting with 8 MiB parts and two upload workers (the SDK - increases the part size for exceptionally large seekable files). -3. ARTEX reads the entire object back and checks its size and SHA-256. Multipart - ETags and uploaded checksum metadata alone are not treated as proof of content. -4. Only after verification does PostgreSQL commit the `s3://` location and - compact the archived task. ARTEX then removes local source files and the - temporary package. The archive list shows **Object storage** or **Local storage**. -5. **Restore** downloads and verifies the package before restoring task files and - database records. The remote package is consumed after restoration commits. - Failed remote cleanup can be retried without replaying restored records. -6. **Delete** removes the archive stub and queues durable cleanup of its remote - object. Cleanup is retried after restart and before subsequent archive jobs. +For SSE-KMS, set both the bucket's default encryption and ARTEX to `aws:kms`; +set `ARTEX_ARCHIVE_S3_KMS_KEY_ID` to the intended key when using a customer-managed +key. An explicit `AES256` request takes precedence over a bucket's KMS default, +so do not leave ARTEX on SSE-S3 when the bucket is intended to use KMS. -Reads use bounded 8 MiB ranges pinned to the object's ETag. Interrupted ranges -are retried without duplicating output or restarting a multi-gigabyte download. -An object change, checksum mismatch or invalid range fails verification. Failed -downloads are removed, and upload/verification failures retain the original task. +The principal needs object upload, read (HEAD and ranged GET), delete and +multipart upload/abort permissions. KMS uploads/readback also need the applicable +key-policy and IAM permissions for `kms:GenerateDataKey` and `kms:Decrypt`. +Retain old KMS keys and decrypt permissions when changing the key or mode; restore +does not require an older object to use the current upload policy. -There is no database schema or archive format migration. Existing local archive -packages remain compatible. Files outside ARTEX's task archive database are not -automatically imported or uploaded. +Compatible providers must support the requested SSE mode and its HEAD metadata, +plus conditional ranged reads. A provider that ignores the encryption request +or cannot confirm it is rejected; the original task is retained. Local source and +temporary files keep their existing local-disk protection and restricted modes. -## Disk, transfer and recovery +See the official [PutObject encryption and permission contract](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html) +and [HEAD encryption response rules](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html). -Archive creation still needs space for the temporary compressed package. -Restoration needs space for the downloaded package and extracted files. Full -upload readback and later restores incur object-storage reads and any applicable -transfer charges. Configure expiration of abandoned multipart uploads for abrupt -process termination. +## Credential sources -Recovery journals under `data/archives/tasks/.remote` contain object locations, -not credentials. Preserve them with PostgreSQL and the installation's data -directory. The worker uses these journals to clean up uploads that never -committed, keep committed objects, and finish deferred deletions. Failed remote -cleanup does not block unrelated local archive jobs. +Without the archive-specific override, ARTEX uses the AWS SDK default credential +chain, including standard AWS environment variables, shared profiles, web +identity, ECS task credentials and EC2 instance credentials through IMDS. Role +credentials retain the SDK's automatic refresh behavior. A named shared profile +can be selected with `AWS_PROFILE`. + +For an external S3-compatible provider, dedicated keys can be supplied through +the archive-specific variables. Both key fields are required; a session token +is supported for temporary credentials. This explicit override takes precedence +over ambient AWS credentials. Operator-supplied temporary credentials must be +rotated by the operator; they are not an IAM role provider. + +Keep secrets in a protected service environment file outside the repository and +frontend. A systemd `EnvironmentFile=` can point to a root-owned mode-0600 file. +See the [AWS SDK credential-chain documentation](https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-gosdk.html). + +## Lifecycle + +1. Pause or finish a task and use its **Archive** action. +2. ARTEX stages the source files and builds a local `.tar.zst` package. Large + packages use multipart upload, initially 8 MiB parts with two upload workers; + the SDK increases the part size for exceptionally large seekable files. +3. ARTEX confirms encryption metadata and reads the complete object back, + checking size and SHA-256. ETags and uploaded checksum metadata alone are + insufficient. +4. **Only after verification and a successful PostgreSQL commit** does ARTEX + remove the local package and staged source files. The committed archive + location is an `s3://` URI; there is no schema or archive-format migration. +5. **Restore** downloads and verifies before installing files and database rows. + The remote package is consumed only after restoration commits. +6. **Delete** commits removal of the archive stub, then cleans up the object. + A durable journal retains failed cleanup work. + +The archive list identifies **Object storage** and **Local storage** and shows +upload/download verification phases. Reads use bounded 8 MiB ranges pinned to the +object's ETag, with retries for interrupted ranges rather than the entire object. + +## Interruption recovery + +Source staging and restore journals live with task archives. Remote cleanup +journals live under `data/archives/tasks/.remote`; they contain locations, not +credentials. Keep this directory with PostgreSQL and the installation data. + +| Interruption | Recovery | +| --- | --- | +| During staging/compression, before an upload journal exists | Roll staged source files back. Remove any temporary package only after rollback succeeds. The task remains in PostgreSQL. | +| After the upload journal, before successful verification | Roll source files back and clean up any uncommitted object/package. Failed cleanup retains its journal for retry. | +| After verification, before the database commit | Treat the object as uncommitted: retain the original task and clean up the orphan object. Verification alone never authorizes local deletion. | +| After the database commit, before package/staging cleanup | Keep the committed object and finish local cleanup. The persisted location and checksum establish commit, even if Restore or Delete has since been queued. | +| Download/extraction before restored files are installed | Remove temporary extraction data; keep the remote object and archive stub for another attempt. | +| Restored files installed, before database restoration commits | Roll back installed files using the restore journal. Retain the remote package. | +| Database restoration committed, before remote cleanup | Recognize the already-restored task and finish cleanup without replaying database rows. | +| Object consumed, before the restored archive stub is removed | Skip download/replay; repeat idempotent object deletion and remove the stub. | +| Delete journal written, before database deletion commits | Keep the object while its archive stub exists. The queued Delete operation can retry. | +| Database deletion committed, before object deletion succeeds | The durable outbox retains the URI and retries object deletion. | +| An old failed upload survives a retry using local storage | Remove the abandoned remote object, but retain a local package referenced by the committed archive row. | + +The staging recovery rule also applies to local archives. Dedicated tests cover +both storage modes with ready, restore-queued, delete-queued and uncommitted +states. Additional PostgreSQL/HTTP-S3 tests reconstruct interrupted operations at +13 persisted boundaries and repeat recovery to check idempotence. + +Handled multipart-upload failures are aborted by the SDK. Abrupt process death +may leave unfinished multipart uploads; configure bucket lifecycle expiration +for those parts. Existing object versions, if versioning is enabled, are governed +by the bucket's version-retention policy. + +## Disk and transfer requirements + +Archive creation still requires temporary local space for the compressed +package. Restore requires space for the download and extracted files. Full +upload readback and restores incur storage reads and applicable transfer charges. +Failed remote cleanup does not prevent unrelated local archive jobs from being +processed once the failed cleanup request returns. ## Optional connectivity check -Run `go run ./cmd/artex-archive-check` with the same archive environment as the -server to upload, verify, download and delete a small uniquely named test object. -This command performs real bucket operations but does not touch task data. It -reports leftover test objects if cleanup fails. +Run `go run ./cmd/artex-archive-check` with the archive environment to upload, +verify, download and delete a small unique object. It performs real bucket +operations but does not touch task data, and reports cleanup failures. -Automated tests use an HTTP S3 fixture and isolated PostgreSQL databases. They -cover multipart transfers, corruption, interrupted range retries, object changes, -cancellation, upload rollback, corrupted restores and persistent cleanup. +Automated tests use HTTP fixtures for S3 and metadata credentials, plus isolated +PostgreSQL. They do not establish compatibility with a live cloud bucket. diff --git a/go.mod b/go.mod index 1ce0982..353f837 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.26.10 require ( github.com/Autumn-27/norma v0.4.11 github.com/aws/aws-sdk-go-v2 v1.47.5 + github.com/aws/aws-sdk-go-v2/config v1.33.13 github.com/aws/aws-sdk-go-v2/credentials v1.20.13 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19 github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 @@ -27,6 +28,7 @@ require ( github.com/andybalholm/brotli v1.2.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect @@ -34,6 +36,10 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.39.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect github.com/aws/smithy-go v1.28.7 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/creack/pty v1.1.24 // indirect diff --git a/server/task_archive_interrupt_test.go b/server/task_archive_interrupt_test.go new file mode 100644 index 0000000..2f4a44e --- /dev/null +++ b/server/task_archive_interrupt_test.go @@ -0,0 +1,291 @@ +package server + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "sync" + "testing" + + "github.com/Autumn-27/artex/archivestore" + pgdb "github.com/Autumn-27/artex/db" +) + +// Recreate persisted states at operation boundaries, then run the same recovery +// functions used at startup. PostgreSQL, files, and HTTP object storage are real +// isolated fixtures; no production failpoint hooks are needed. +func TestRemoteArchiveInterruptionRecovery(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + phases := []string{ + "staged_before_compression", "compressed_before_journal", "journal_before_upload", + "uploaded_before_verification", "verified_before_database", + "database_before_local_cleanup", "package_cleaned_before_stage_cleanup", + "restore_download_before_install", "restore_files_before_database", "restore_database_before_object_delete", "restore_object_deleted_before_stub", + "delete_journal_before_database", "delete_database_before_object", + } + for _, phase := range phases { + t.Run(phase, func(t *testing.T) { + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + store, objects := newRecoveryObjectStore(t) + s := &Server{ctx: context.Background(), m: m, archiveStore: store} + task, err := m.CreateTask("interruption "+phase, "preserve source", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + id, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }() + if err := m.pg.SetPaused(id, true); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Archiving); err != nil { + t.Fatal(err) + } + source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt") + if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source, []byte("original task data"), 0600); err != nil { + t.Fatal(err) + } + snapshot, err := m.pg.SnapshotTaskArchive(id) + if err != nil { + t.Fatal(err) + } + stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID) + if err != nil { + t.Fatal(err) + } + packagePath := taskArchivePath(m.dir, job.ID, task.ID) + var original, compressed int64 + var checksum, uri, journal string + if phase != "staged_before_compression" { + original, compressed, checksum, err = writeTaskArchivePackage(packagePath, stage.payload, snapshot) + if err != nil { + t.Fatal(err) + } + uri = store.URI(id, job.ID, checksum) + } + if phase != "staged_before_compression" && phase != "compressed_before_journal" { + journal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, LocalPath: packagePath, Operation: "upload"}) + if err != nil { + t.Fatal(err) + } + } + committed := phase != "staged_before_compression" && phase != "compressed_before_journal" && + phase != "journal_before_upload" && phase != "uploaded_before_verification" && phase != "verified_before_database" + if phase == "uploaded_before_verification" { + data, err := os.ReadFile(packagePath) + if err != nil { + t.Fatal(err) + } + objects.put(uri, data) + } else if phase == "verified_before_database" || committed { + if err := store.Upload(s.ctx, uri, packagePath, checksum); err != nil { + t.Fatal(err) + } + } + if committed { + if err := m.pg.CompleteTaskArchive(job.ID, snapshot, uri, checksum, original, compressed); err != nil { + t.Fatal(err) + } + } + afterArchive := phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || + phase == "restore_object_deleted_before_stub" || phase == "delete_journal_before_database" || phase == "delete_database_before_object" + if phase == "package_cleaned_before_stage_cleanup" || afterArchive { + if err := os.Remove(packagePath); err != nil { + t.Fatal(err) + } + if err := os.Remove(journal); err != nil { + t.Fatal(err) + } + } + if afterArchive { + if err := stage.commit(); err != nil { + t.Fatal(err) + } + } + restored := false + remoteDeleted := false + if phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" { + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Restoring); err != nil { + t.Fatal(err) + } + parent := filepath.Join(taskArchiveRoot(m.dir), ".restore") + if err := os.MkdirAll(parent, 0700); err != nil { + t.Fatal(err) + } + extracted, err := os.MkdirTemp(parent, "interrupted-") + if err != nil { + t.Fatal(err) + } + downloaded := filepath.Join(extracted, "download.tar.zst") + if err := store.Download(s.ctx, uri, downloaded, checksum, compressed); err != nil { + t.Fatal(err) + } + if err := extractTaskArchivePackage(downloaded, checksum, extracted); err != nil { + t.Fatal(err) + } + if phase != "restore_download_before_install" { + if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID); err != nil { + t.Fatal(err) + } + } + if phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" { + if _, err := m.pg.RestoreTaskArchive(job.ID, snapshot, 0); err != nil { + t.Fatal(err) + } + restored = true + } + if phase == "restore_object_deleted_before_stub" { + if err := store.Delete(s.ctx, uri); err != nil { + t.Fatal(err) + } + remoteDeleted = true + } + } + deleting := phase == "delete_journal_before_database" || phase == "delete_database_before_object" + if deleting { + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Deleting); err != nil { + t.Fatal(err) + } + if _, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, Operation: "delete"}); err != nil { + t.Fatal(err) + } + if phase == "delete_database_before_object" { + if err := m.pg.DeleteTaskArchiveStub(job.ID); err != nil { + t.Fatal(err) + } + remoteDeleted = true + } + } + if err := m.pg.RecoverTaskArchiveJobs(); err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + wantSource := !committed || restored + data, fileErr := os.ReadFile(source) + if wantSource && (fileErr != nil || string(data) != "original task data") { + t.Fatalf("source data lost: %v", fileErr) + } + if !wantSource && !os.IsNotExist(fileErr) { + t.Fatalf("archived source resurrected: %v", fileErr) + } + if _, err := os.Stat(packagePath); !os.IsNotExist(err) { + t.Fatalf("temporary package remains: %v", err) + } + if got, want := objects.has(uri), committed && !remoteDeleted; got != want { + t.Fatalf("remote object exists=%v, want=%v", got, want) + } + if committed && !deleting { + actual, err := m.pg.IsTaskArchiveRestored(job.ID) + if err != nil || actual != restored { + t.Fatalf("restored=%v want=%v err=%v", actual, restored, err) + } + } + // A second startup must be harmless at every boundary. + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + }) + } +} + +type recoveryObjects struct { + mu sync.Mutex + data map[string][]byte +} + +func (o *recoveryObjects) put(uri string, data []byte) { + o.mu.Lock() + defer o.mu.Unlock() + // Test URIs always use the fixture bucket with path-style addressing. + o.data["/test/"+uri[len("s3://test/"):]] = append([]byte(nil), data...) +} +func (o *recoveryObjects) has(uri string) bool { + o.mu.Lock() + defer o.mu.Unlock() + if uri == "" { + return false + } + _, ok := o.data["/test/"+uri[len("s3://test/"):]] + return ok +} +func newRecoveryObjectStore(t *testing.T) (*archivestore.Store, *recoveryObjects) { + t.Helper() + objects := &recoveryObjects{data: map[string][]byte{}} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + objects.mu.Lock() + defer objects.mu.Unlock() + data, exists := objects.data[r.URL.Path] + w.Header().Set("ETag", `"recovery-etag"`) + switch r.Method { + case "PUT": + if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" { + t.Error("missing SSE") + } + objects.data[r.URL.Path], _ = io.ReadAll(r.Body) + case "HEAD": + if !exists { + w.WriteHeader(404) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + w.Header().Set("X-Amz-Server-Side-Encryption", "AES256") + case "GET": + if !exists { + w.WriteHeader(404) + return + } + var start, end int + if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end >= len(data) || r.Header.Get("If-Match") != `"recovery-etag"` { + w.WriteHeader(416) + return + } + w.Header().Set("Content-Length", strconv.Itoa(end-start+1)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + w.WriteHeader(206) + _, _ = w.Write(data[start : end+1]) + case "DELETE": + delete(objects.data, r.URL.Path) + w.WriteHeader(204) + default: + t.Errorf("unexpected object request %s", r.Method) + w.WriteHeader(400) + } + })) + t.Cleanup(srv.Close) + store, err := archivestore.New(archivestore.Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + return store, objects +} diff --git a/server/task_archive_package.go b/server/task_archive_package.go index e9a451d..e365ea3 100644 --- a/server/task_archive_package.go +++ b/server/task_archive_package.go @@ -437,9 +437,16 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { stage := &taskArchiveFileStage{root: root, payload: filepath.Join(root, "payload"), journal: journal} if err := stage.rollback(); err != nil { errs = append(errs, err) + continue } - if archive != nil && archive.ArchivePath != "" && !strings.HasPrefix(archive.ArchivePath, "s3://") { - _ = os.Remove(archive.ArchivePath) + // The process can exit after compression but before an upload journal or + // database location exists. Remove that temporary package only after the + // source-file rollback succeeds, for both local and remote backends. + packagePath := taskArchivePath(dataDir, journal.ArchiveID, journal.TaskID) + if err := validateArchivePath(dataDir, packagePath); err != nil { + errs = append(errs, err) + } else if err := removeLocalArchivePackage(packagePath); err != nil { + errs = append(errs, err) } } return errors.Join(errs...) diff --git a/server/task_archive_remote_test.go b/server/task_archive_remote_test.go index 34e0848..ac2a44d 100644 --- a/server/task_archive_remote_test.go +++ b/server/task_archive_remote_test.go @@ -46,6 +46,7 @@ func TestRemoteArchiveLifecycle(t *testing.T) { objects[r.URL.Path], _ = io.ReadAll(r.Body) w.Header().Set("ETag", `"test"`) case "HEAD": + w.Header().Set("X-Amz-Server-Side-Encryption", "AES256") w.Header().Set("ETag", `"test"`) w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path]))) case "GET": @@ -255,7 +256,7 @@ func TestRemoteArchiveLifecycle(t *testing.T) { // Once PostgreSQL has committed an archive location and checksum, a subsequent // queued operation must not make startup recovery resurrect the old workspace. -func TestRemoteArchiveStageRecoveryAfterCommit(t *testing.T) { +func TestTaskArchiveStageRecoveryAfterCommit(t *testing.T) { if os.Getenv("ARTEX_PG_DSN") == "" { t.Skip("requires isolated PostgreSQL") } @@ -284,41 +285,63 @@ func TestRemoteArchiveStageRecoveryAfterCommit(t *testing.T) { if err != nil { t.Fatal(err) } - for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} { - t.Run(state, func(t *testing.T) { - committed := state != pgdb.ArchiveQueued - checksum := "" - if committed { - checksum = strings.Repeat("a", 64) - } - if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, "s3://test/tasks/package.tar.zst", checksum); err != nil { - t.Fatal(err) - } - file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt") - if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil { - t.Fatal(err) - } - stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID) - if err != nil { - t.Fatal(err) - } - if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(stage.root); !os.IsNotExist(err) { - t.Fatalf("staging cleanup incomplete: %v", err) - } - if committed { - if _, err := os.Stat(file); !os.IsNotExist(err) { - t.Fatalf("resurrected committed workspace: %v", err) + for _, backend := range []string{"local", "s3"} { + for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} { + t.Run(backend+"/"+state, func(t *testing.T) { + committed := state != pgdb.ArchiveQueued + checksum := "" + if committed { + checksum = strings.Repeat("a", 64) } - } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" { - t.Fatalf("uncommitted archive lost source: %v", err) - } - }) + packagePath := taskArchivePath(m.dir, job.ID, task.ID) + if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(packagePath, []byte("verified package"), 0600); err != nil { + t.Fatal(err) + } + location := packagePath + if backend == "s3" { + location = "s3://test/tasks/package.tar.zst" + } + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, location, checksum); err != nil { + t.Fatal(err) + } + file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt") + if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil { + t.Fatal(err) + } + stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID) + if err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stage.root); !os.IsNotExist(err) { + t.Fatalf("staging cleanup incomplete: %v", err) + } + if backend == "local" && committed { + if data, err := os.ReadFile(packagePath); err != nil || string(data) != "verified package" { + t.Fatal("committed local package lost", err) + } + } else if !committed { + if _, err := os.Stat(packagePath); !os.IsNotExist(err) { + t.Fatal("uncommitted temporary package retained", err) + } + } + if committed { + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Fatalf("resurrected committed workspace: %v", err) + } + } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" { + t.Fatalf("uncommitted archive lost source: %v", err) + } + }) + } } } From a40781ae1fc0d65d7045e669e5e4d87465e69e77 Mon Sep 17 00:00:00 2001 From: Chubboy Ren Date: Wed, 18 Nov 2026 02:05:17 +0000 Subject: [PATCH 3/4] refactor: split archive backend from server and UI integration --- README.md | 6 - db/task_archives.go | 5 - docs/task-archive-storage.md | 160 -------- server/server.go | 9 +- server/task_archive_interrupt_test.go | 291 -------------- server/task_archive_package.go | 13 +- server/task_archive_remote.go | 171 --------- server/task_archive_remote_test.go | 419 --------------------- server/task_archives.go | 65 +--- web/src/app/(main)/function/tasks/page.tsx | 5 +- web/src/lib/types.ts | 1 - 11 files changed, 15 insertions(+), 1130 deletions(-) delete mode 100644 docs/task-archive-storage.md delete mode 100644 server/task_archive_interrupt_test.go delete mode 100644 server/task_archive_remote.go delete mode 100644 server/task_archive_remote_test.go diff --git a/README.md b/README.md index 2d91397..ad8efec 100644 --- a/README.md +++ b/README.md @@ -421,12 +421,6 @@ runtime supports it. `/chat/completions` (or `/responses` for `openai-responses`), so their base **does** end in `/v1` (for example `https://api.openai.com/v1`). -**Task archive storage**: local disk by default. Set `ARTEX_ARCHIVE_STORAGE=s3` to -store cold task packages in an S3-compatible bucket and release their local -package space after verification. Existing local archives remain usable. See -[Task archive storage](docs/task-archive-storage.md) for configuration, permissions, -integrity checks, temporary disk requirements and recovery. - **Concurrency**: work agents per task, in "System Settings" (default 3). **Untrusted-output boundary (on by default)**: tool results for the worker, planner, chat and main diff --git a/db/task_archives.go b/db/task_archives.go index 9f07341..bb9aae2 100644 --- a/db/task_archives.go +++ b/db/task_archives.go @@ -60,7 +60,6 @@ type TaskArchive struct { Warnings json.RawMessage `json:"warnings"` FormatVersion int `json:"format_version"` ArchivePath string `json:"-"` - Storage string `json:"storage"` SHA256 string `json:"sha256,omitempty"` OriginalSize int64 `json:"original_size"` CompressedSize int64 `json:"compressed_size"` @@ -162,10 +161,6 @@ func scanTaskArchive(sc interface{ Scan(...any) error }) (*TaskArchive, error) { if err != nil { return nil, err } - item.Storage = "local" - if strings.HasPrefix(item.ArchivePath, "s3://") { - item.Storage = "s3" - } if err := json.Unmarshal([]byte(sources), &item.SourceTaskIDs); err != nil { return nil, err } diff --git a/docs/task-archive-storage.md b/docs/task-archive-storage.md deleted file mode 100644 index 113921a..0000000 --- a/docs/task-archive-storage.md +++ /dev/null @@ -1,160 +0,0 @@ -# Task archive storage - -Task archives use local disk by default. The optional S3-compatible backend moves -cold task packages to object storage while retaining Archive, Restore and Delete. -Active workspaces and PostgreSQL remain local. This is task cold storage, not a -full-installation backup; a recovery plan must preserve the database, data -directory, bucket objects, and the credentials or roles needed to read them. - -## Configuration - -Use a private bucket and a separate prefix for each ARTEX installation. Configure -the bucket's default encryption to match the selected mode below, and enforce -that mode/key with a bucket policy. ARTEX does not provision or inspect bucket -encryption settings. - -Set the following server environment and restart ARTEX: - -```dotenv -ARTEX_ARCHIVE_STORAGE=s3 -ARTEX_ARCHIVE_S3_ENDPOINT=https://s3.us-east-1.amazonaws.com -ARTEX_ARCHIVE_S3_REGION=us-east-1 -ARTEX_ARCHIVE_S3_BUCKET=your-artex-archives -ARTEX_ARCHIVE_S3_PREFIX=artex/tasks -ARTEX_ARCHIVE_S3_SSE=AES256 -``` - -With an EC2 instance role, ECS task role or other SDK-supported credential source, -no archive-specific access keys are needed. - -| Variable | Meaning | -| --- | --- | -| `ARTEX_ARCHIVE_STORAGE` | `local` (default) or `s3`. | -| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local tests. | -| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. | -| `ARTEX_ARCHIVE_S3_BUCKET` | Required bucket name. | -| `ARTEX_ARCHIVE_S3_PREFIX` | Object prefix; defaults to `artex/tasks`. | -| `ARTEX_ARCHIVE_S3_SSE` | `AES256` (default, SSE-S3) or `aws:kms` (SSE-KMS). There is no unencrypted mode. | -| `ARTEX_ARCHIVE_S3_KMS_KEY_ID` | Optional KMS key ARN/ID/alias; valid only with `aws:kms`. Omit to use the provider's managed KMS key. | -| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Optional dedicated credential override; must be supplied with the secret key. | -| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Secret for the dedicated override. | -| `ARTEX_ARCHIVE_S3_SESSION_TOKEN` | Optional session token for dedicated temporary credentials. | -| `ARTEX_ARCHIVE_S3_PATH_STYLE` | `true` for path-style servers; otherwise virtual-hosted addressing is used. | - -An invalid mode or partial credential override fails configuration instead of -silently falling back to local storage or ambient credentials. Preserve the -bucket, endpoint, prefix and read/delete access while remote archives remain. -Older local archives remain usable; they are not uploaded automatically. - -## Encryption and permissions - -Every single-object upload and multipart initiation explicitly requests the -configured server-side encryption. Upload verification also requires HEAD to -report the requested mode, and a KMS key identifier for SSE-KMS, before accepting -the object. Size and full SHA-256 readback remain mandatory; encryption metadata -does not replace content verification. - -For SSE-KMS, set both the bucket's default encryption and ARTEX to `aws:kms`; -set `ARTEX_ARCHIVE_S3_KMS_KEY_ID` to the intended key when using a customer-managed -key. An explicit `AES256` request takes precedence over a bucket's KMS default, -so do not leave ARTEX on SSE-S3 when the bucket is intended to use KMS. - -The principal needs object upload, read (HEAD and ranged GET), delete and -multipart upload/abort permissions. KMS uploads/readback also need the applicable -key-policy and IAM permissions for `kms:GenerateDataKey` and `kms:Decrypt`. -Retain old KMS keys and decrypt permissions when changing the key or mode; restore -does not require an older object to use the current upload policy. - -Compatible providers must support the requested SSE mode and its HEAD metadata, -plus conditional ranged reads. A provider that ignores the encryption request -or cannot confirm it is rejected; the original task is retained. Local source and -temporary files keep their existing local-disk protection and restricted modes. - -See the official [PutObject encryption and permission contract](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html) -and [HEAD encryption response rules](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html). - -## Credential sources - -Without the archive-specific override, ARTEX uses the AWS SDK default credential -chain, including standard AWS environment variables, shared profiles, web -identity, ECS task credentials and EC2 instance credentials through IMDS. Role -credentials retain the SDK's automatic refresh behavior. A named shared profile -can be selected with `AWS_PROFILE`. - -For an external S3-compatible provider, dedicated keys can be supplied through -the archive-specific variables. Both key fields are required; a session token -is supported for temporary credentials. This explicit override takes precedence -over ambient AWS credentials. Operator-supplied temporary credentials must be -rotated by the operator; they are not an IAM role provider. - -Keep secrets in a protected service environment file outside the repository and -frontend. A systemd `EnvironmentFile=` can point to a root-owned mode-0600 file. -See the [AWS SDK credential-chain documentation](https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-gosdk.html). - -## Lifecycle - -1. Pause or finish a task and use its **Archive** action. -2. ARTEX stages the source files and builds a local `.tar.zst` package. Large - packages use multipart upload, initially 8 MiB parts with two upload workers; - the SDK increases the part size for exceptionally large seekable files. -3. ARTEX confirms encryption metadata and reads the complete object back, - checking size and SHA-256. ETags and uploaded checksum metadata alone are - insufficient. -4. **Only after verification and a successful PostgreSQL commit** does ARTEX - remove the local package and staged source files. The committed archive - location is an `s3://` URI; there is no schema or archive-format migration. -5. **Restore** downloads and verifies before installing files and database rows. - The remote package is consumed only after restoration commits. -6. **Delete** commits removal of the archive stub, then cleans up the object. - A durable journal retains failed cleanup work. - -The archive list identifies **Object storage** and **Local storage** and shows -upload/download verification phases. Reads use bounded 8 MiB ranges pinned to the -object's ETag, with retries for interrupted ranges rather than the entire object. - -## Interruption recovery - -Source staging and restore journals live with task archives. Remote cleanup -journals live under `data/archives/tasks/.remote`; they contain locations, not -credentials. Keep this directory with PostgreSQL and the installation data. - -| Interruption | Recovery | -| --- | --- | -| During staging/compression, before an upload journal exists | Roll staged source files back. Remove any temporary package only after rollback succeeds. The task remains in PostgreSQL. | -| After the upload journal, before successful verification | Roll source files back and clean up any uncommitted object/package. Failed cleanup retains its journal for retry. | -| After verification, before the database commit | Treat the object as uncommitted: retain the original task and clean up the orphan object. Verification alone never authorizes local deletion. | -| After the database commit, before package/staging cleanup | Keep the committed object and finish local cleanup. The persisted location and checksum establish commit, even if Restore or Delete has since been queued. | -| Download/extraction before restored files are installed | Remove temporary extraction data; keep the remote object and archive stub for another attempt. | -| Restored files installed, before database restoration commits | Roll back installed files using the restore journal. Retain the remote package. | -| Database restoration committed, before remote cleanup | Recognize the already-restored task and finish cleanup without replaying database rows. | -| Object consumed, before the restored archive stub is removed | Skip download/replay; repeat idempotent object deletion and remove the stub. | -| Delete journal written, before database deletion commits | Keep the object while its archive stub exists. The queued Delete operation can retry. | -| Database deletion committed, before object deletion succeeds | The durable outbox retains the URI and retries object deletion. | -| An old failed upload survives a retry using local storage | Remove the abandoned remote object, but retain a local package referenced by the committed archive row. | - -The staging recovery rule also applies to local archives. Dedicated tests cover -both storage modes with ready, restore-queued, delete-queued and uncommitted -states. Additional PostgreSQL/HTTP-S3 tests reconstruct interrupted operations at -13 persisted boundaries and repeat recovery to check idempotence. - -Handled multipart-upload failures are aborted by the SDK. Abrupt process death -may leave unfinished multipart uploads; configure bucket lifecycle expiration -for those parts. Existing object versions, if versioning is enabled, are governed -by the bucket's version-retention policy. - -## Disk and transfer requirements - -Archive creation still requires temporary local space for the compressed -package. Restore requires space for the download and extracted files. Full -upload readback and restores incur storage reads and applicable transfer charges. -Failed remote cleanup does not prevent unrelated local archive jobs from being -processed once the failed cleanup request returns. - -## Optional connectivity check - -Run `go run ./cmd/artex-archive-check` with the archive environment to upload, -verify, download and delete a small unique object. It performs real bucket -operations but does not touch task data, and reports cleanup failures. - -Automated tests use HTTP fixtures for S3 and metadata credentials, plus isolated -PostgreSQL. They do not establish compatibility with a live cloud bucket. diff --git a/server/server.go b/server/server.go index 7d750d3..baf4423 100644 --- a/server/server.go +++ b/server/server.go @@ -23,7 +23,6 @@ import ( "time" "github.com/Autumn-27/artex/agent" - "github.com/Autumn-27/artex/archivestore" "github.com/Autumn-27/artex/db" "github.com/Autumn-27/artex/egress" "github.com/Autumn-27/artex/intercept" @@ -145,11 +144,9 @@ type Server struct { // Cold task archives run through one persistent FIFO worker. The buffered wake // channel coalesces enqueue bursts; the database remains the source of truth. - archiveWake chan struct{} - archiveWG sync.WaitGroup - archiveStore *archivestore.Store - archiveStoreErr error - side *sideQuestionState + archiveWake chan struct{} + archiveWG sync.WaitGroup + side *sideQuestionState } // provEntry is a cached provider + its config for one LLM profile id. diff --git a/server/task_archive_interrupt_test.go b/server/task_archive_interrupt_test.go deleted file mode 100644 index 2f4a44e..0000000 --- a/server/task_archive_interrupt_test.go +++ /dev/null @@ -1,291 +0,0 @@ -package server - -import ( - "context" - "fmt" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strconv" - "sync" - "testing" - - "github.com/Autumn-27/artex/archivestore" - pgdb "github.com/Autumn-27/artex/db" -) - -// Recreate persisted states at operation boundaries, then run the same recovery -// functions used at startup. PostgreSQL, files, and HTTP object storage are real -// isolated fixtures; no production failpoint hooks are needed. -func TestRemoteArchiveInterruptionRecovery(t *testing.T) { - if os.Getenv("ARTEX_PG_DSN") == "" { - t.Skip("requires isolated PostgreSQL") - } - phases := []string{ - "staged_before_compression", "compressed_before_journal", "journal_before_upload", - "uploaded_before_verification", "verified_before_database", - "database_before_local_cleanup", "package_cleaned_before_stage_cleanup", - "restore_download_before_install", "restore_files_before_database", "restore_database_before_object_delete", "restore_object_deleted_before_stub", - "delete_journal_before_database", "delete_database_before_object", - } - for _, phase := range phases { - t.Run(phase, func(t *testing.T) { - m, err := NewManager(t.TempDir(), "") - if err != nil { - t.Fatal(err) - } - defer m.Close() - store, objects := newRecoveryObjectStore(t) - s := &Server{ctx: context.Background(), m: m, archiveStore: store} - task, err := m.CreateTask("interruption "+phase, "preserve source", nil, 0, 0) - if err != nil { - t.Fatal(err) - } - id, _ := strconv.ParseInt(task.ID, 10, 64) - defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }() - if err := m.pg.SetPaused(id, true); err != nil { - t.Fatal(err) - } - job, err := m.pg.QueueTaskArchive(id) - if err != nil { - t.Fatal(err) - } - if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Archiving); err != nil { - t.Fatal(err) - } - source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt") - if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(source, []byte("original task data"), 0600); err != nil { - t.Fatal(err) - } - snapshot, err := m.pg.SnapshotTaskArchive(id) - if err != nil { - t.Fatal(err) - } - stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID) - if err != nil { - t.Fatal(err) - } - packagePath := taskArchivePath(m.dir, job.ID, task.ID) - var original, compressed int64 - var checksum, uri, journal string - if phase != "staged_before_compression" { - original, compressed, checksum, err = writeTaskArchivePackage(packagePath, stage.payload, snapshot) - if err != nil { - t.Fatal(err) - } - uri = store.URI(id, job.ID, checksum) - } - if phase != "staged_before_compression" && phase != "compressed_before_journal" { - journal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, LocalPath: packagePath, Operation: "upload"}) - if err != nil { - t.Fatal(err) - } - } - committed := phase != "staged_before_compression" && phase != "compressed_before_journal" && - phase != "journal_before_upload" && phase != "uploaded_before_verification" && phase != "verified_before_database" - if phase == "uploaded_before_verification" { - data, err := os.ReadFile(packagePath) - if err != nil { - t.Fatal(err) - } - objects.put(uri, data) - } else if phase == "verified_before_database" || committed { - if err := store.Upload(s.ctx, uri, packagePath, checksum); err != nil { - t.Fatal(err) - } - } - if committed { - if err := m.pg.CompleteTaskArchive(job.ID, snapshot, uri, checksum, original, compressed); err != nil { - t.Fatal(err) - } - } - afterArchive := phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || - phase == "restore_object_deleted_before_stub" || phase == "delete_journal_before_database" || phase == "delete_database_before_object" - if phase == "package_cleaned_before_stage_cleanup" || afterArchive { - if err := os.Remove(packagePath); err != nil { - t.Fatal(err) - } - if err := os.Remove(journal); err != nil { - t.Fatal(err) - } - } - if afterArchive { - if err := stage.commit(); err != nil { - t.Fatal(err) - } - } - restored := false - remoteDeleted := false - if phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" { - if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Restoring); err != nil { - t.Fatal(err) - } - parent := filepath.Join(taskArchiveRoot(m.dir), ".restore") - if err := os.MkdirAll(parent, 0700); err != nil { - t.Fatal(err) - } - extracted, err := os.MkdirTemp(parent, "interrupted-") - if err != nil { - t.Fatal(err) - } - downloaded := filepath.Join(extracted, "download.tar.zst") - if err := store.Download(s.ctx, uri, downloaded, checksum, compressed); err != nil { - t.Fatal(err) - } - if err := extractTaskArchivePackage(downloaded, checksum, extracted); err != nil { - t.Fatal(err) - } - if phase != "restore_download_before_install" { - if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID); err != nil { - t.Fatal(err) - } - } - if phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" { - if _, err := m.pg.RestoreTaskArchive(job.ID, snapshot, 0); err != nil { - t.Fatal(err) - } - restored = true - } - if phase == "restore_object_deleted_before_stub" { - if err := store.Delete(s.ctx, uri); err != nil { - t.Fatal(err) - } - remoteDeleted = true - } - } - deleting := phase == "delete_journal_before_database" || phase == "delete_database_before_object" - if deleting { - if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Deleting); err != nil { - t.Fatal(err) - } - if _, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, Operation: "delete"}); err != nil { - t.Fatal(err) - } - if phase == "delete_database_before_object" { - if err := m.pg.DeleteTaskArchiveStub(job.ID); err != nil { - t.Fatal(err) - } - remoteDeleted = true - } - } - if err := m.pg.RecoverTaskArchiveJobs(); err != nil { - t.Fatal(err) - } - if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { - t.Fatal(err) - } - if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil { - t.Fatal(err) - } - if err := s.recoverRemoteArchiveJournals(); err != nil { - t.Fatal(err) - } - wantSource := !committed || restored - data, fileErr := os.ReadFile(source) - if wantSource && (fileErr != nil || string(data) != "original task data") { - t.Fatalf("source data lost: %v", fileErr) - } - if !wantSource && !os.IsNotExist(fileErr) { - t.Fatalf("archived source resurrected: %v", fileErr) - } - if _, err := os.Stat(packagePath); !os.IsNotExist(err) { - t.Fatalf("temporary package remains: %v", err) - } - if got, want := objects.has(uri), committed && !remoteDeleted; got != want { - t.Fatalf("remote object exists=%v, want=%v", got, want) - } - if committed && !deleting { - actual, err := m.pg.IsTaskArchiveRestored(job.ID) - if err != nil || actual != restored { - t.Fatalf("restored=%v want=%v err=%v", actual, restored, err) - } - } - // A second startup must be harmless at every boundary. - if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { - t.Fatal(err) - } - if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil { - t.Fatal(err) - } - if err := s.recoverRemoteArchiveJournals(); err != nil { - t.Fatal(err) - } - }) - } -} - -type recoveryObjects struct { - mu sync.Mutex - data map[string][]byte -} - -func (o *recoveryObjects) put(uri string, data []byte) { - o.mu.Lock() - defer o.mu.Unlock() - // Test URIs always use the fixture bucket with path-style addressing. - o.data["/test/"+uri[len("s3://test/"):]] = append([]byte(nil), data...) -} -func (o *recoveryObjects) has(uri string) bool { - o.mu.Lock() - defer o.mu.Unlock() - if uri == "" { - return false - } - _, ok := o.data["/test/"+uri[len("s3://test/"):]] - return ok -} -func newRecoveryObjectStore(t *testing.T) (*archivestore.Store, *recoveryObjects) { - t.Helper() - objects := &recoveryObjects{data: map[string][]byte{}} - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - objects.mu.Lock() - defer objects.mu.Unlock() - data, exists := objects.data[r.URL.Path] - w.Header().Set("ETag", `"recovery-etag"`) - switch r.Method { - case "PUT": - if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" { - t.Error("missing SSE") - } - objects.data[r.URL.Path], _ = io.ReadAll(r.Body) - case "HEAD": - if !exists { - w.WriteHeader(404) - return - } - w.Header().Set("Content-Length", strconv.Itoa(len(data))) - w.Header().Set("X-Amz-Server-Side-Encryption", "AES256") - case "GET": - if !exists { - w.WriteHeader(404) - return - } - var start, end int - if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end >= len(data) || r.Header.Get("If-Match") != `"recovery-etag"` { - w.WriteHeader(416) - return - } - w.Header().Set("Content-Length", strconv.Itoa(end-start+1)) - w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) - w.WriteHeader(206) - _, _ = w.Write(data[start : end+1]) - case "DELETE": - delete(objects.data, r.URL.Path) - w.WriteHeader(204) - default: - t.Errorf("unexpected object request %s", r.Method) - w.WriteHeader(400) - } - })) - t.Cleanup(srv.Close) - store, err := archivestore.New(archivestore.Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) - if err != nil { - t.Fatal(err) - } - return store, objects -} diff --git a/server/task_archive_package.go b/server/task_archive_package.go index e365ea3..f23c4c8 100644 --- a/server/task_archive_package.go +++ b/server/task_archive_package.go @@ -428,7 +428,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { errs = append(errs, getErr) continue } - if archive != nil && archive.ArchivePath != "" && archive.SHA256 != "" { + if archive != nil && archive.State == pgdb.ArchiveReady { if err := os.RemoveAll(root); err != nil { errs = append(errs, err) } @@ -437,16 +437,9 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { stage := &taskArchiveFileStage{root: root, payload: filepath.Join(root, "payload"), journal: journal} if err := stage.rollback(); err != nil { errs = append(errs, err) - continue } - // The process can exit after compression but before an upload journal or - // database location exists. Remove that temporary package only after the - // source-file rollback succeeds, for both local and remote backends. - packagePath := taskArchivePath(dataDir, journal.ArchiveID, journal.TaskID) - if err := validateArchivePath(dataDir, packagePath); err != nil { - errs = append(errs, err) - } else if err := removeLocalArchivePackage(packagePath); err != nil { - errs = append(errs, err) + if archive != nil && archive.ArchivePath != "" { + _ = os.Remove(archive.ArchivePath) } } return errors.Join(errs...) diff --git a/server/task_archive_remote.go b/server/task_archive_remote.go deleted file mode 100644 index 87e230f..0000000 --- a/server/task_archive_remote.go +++ /dev/null @@ -1,171 +0,0 @@ -package server - -import ( - "encoding/json" - "errors" - "fmt" - "log" - "os" - "path/filepath" - "strings" - - pgdb "github.com/Autumn-27/artex/db" -) - -// An outbox bridges the local PostgreSQL transaction and object storage. Never -// delete the only package before metadata permits it. Unfinished cleanup is -// retried after a restart (and before subsequent jobs). -type remoteArchiveJournal struct { - ArchiveID int64 `json:"archive_id"` - URI string `json:"uri"` - LocalPath string `json:"local_path,omitempty"` - Operation string `json:"operation"` -} - -func (s *Server) validateStoredArchive(uri string) error { - if strings.HasPrefix(uri, "s3://") { - return s.archiveStore.Validate(uri) - } - return validateArchivePath(s.m.dir, uri) -} - -func removeLocalArchivePackage(filename string) error { - if err := os.Remove(filename); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -func (s *Server) removeStoredArchive(uri string) error { - if err := s.validateStoredArchive(uri); err != nil { - return err - } - if strings.HasPrefix(uri, "s3://") { - return s.archiveStore.Delete(s.ctx, uri) - } - return removeLocalArchivePackage(uri) -} - -func (s *Server) writeRemoteArchiveJournal(j remoteArchiveJournal) (string, error) { - dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote") - if err := os.MkdirAll(dir, archiveDirMode); err != nil { - return "", err - } - f, err := os.CreateTemp(dir, ".journal-") - if err != nil { - return "", err - } - tmp := f.Name() - defer os.Remove(tmp) - encodeErr := json.NewEncoder(f).Encode(j) - syncErr := f.Sync() - if err := errors.Join(encodeErr, syncErr, f.Close()); err != nil { - return "", err - } - name := tmp + ".json" - if err := os.Rename(tmp, name); err != nil { - return "", err - } - // Persist the directory entry before an upload or database deletion starts. - d, err := os.Open(dir) - if err != nil { - return "", err - } - if err := errors.Join(d.Sync(), d.Close()); err != nil { - return "", err - } - // The .remote directory itself may have just been created. - parent, err := os.Open(filepath.Dir(dir)) - if err != nil { - return "", err - } - return name, errors.Join(parent.Sync(), parent.Close()) -} - -func (s *Server) recoverRemoteArchiveJournals() error { - dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote") - entries, err := os.ReadDir(dir) - if os.IsNotExist(err) { - return nil - } - if err != nil { - return err - } - var errs []error - for _, entry := range entries { - if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { - continue - } - name := filepath.Join(dir, entry.Name()) - if err := s.recoverRemoteArchiveJournal(name); err != nil { - errs = append(errs, err) - } - } - return errors.Join(errs...) -} - -func (s *Server) recoverRemoteArchiveJournal(name string) error { - raw, err := os.ReadFile(name) - if err != nil { - return err - } - var j remoteArchiveJournal - if err := json.Unmarshal(raw, &j); err != nil { - return err - } - if j.ArchiveID <= 0 { - return errors.New("invalid remote archive journal ID") - } - if err := s.archiveStore.Validate(j.URI); err != nil { - return err - } - item, err := s.m.pg.GetTaskArchive(j.ArchiveID) - if err != nil { - return err - } - switch j.Operation { - case "upload": - if item == nil || item.ArchivePath != j.URI { - if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil { - return err - } - } - if err := validateArchivePath(s.m.dir, j.LocalPath); err != nil { - return err - } - // An old failed upload may outlive a retry after switching back to - // local storage. The retry reuses this pathname: keep its committed - // package instead of treating it as the old upload's temporary copy. - if item == nil || item.ArchivePath != j.LocalPath { - if err := removeLocalArchivePackage(j.LocalPath); err != nil { - return err - } - } - case "delete": - if item == nil { - if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil { - return err - } - } - default: - return fmt.Errorf("unknown remote archive operation %q", j.Operation) - } - return os.Remove(name) -} - -func (s *Server) deleteRemoteTaskArchive(job *pgdb.TaskArchive) error { - journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, Operation: "delete"}) - if err != nil { - return err - } - if err := s.m.pg.DeleteTaskArchiveStub(job.ID); err != nil { - // A connection failure can leave the transaction's commit ambiguous. - // Recovery consults PostgreSQL before deciding whether to delete. - return err - } - if err := s.recoverRemoteArchiveJournal(journal); err != nil { - // The task is gone; the durable outbox retries the remaining object cleanup. - log.Printf("[task-archive] archive %d deleted; remote cleanup pending: %v", job.ID, err) - } - return nil -} diff --git a/server/task_archive_remote_test.go b/server/task_archive_remote_test.go deleted file mode 100644 index ac2a44d..0000000 --- a/server/task_archive_remote_test.go +++ /dev/null @@ -1,419 +0,0 @@ -package server - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strconv" - "strings" - "sync" - "sync/atomic" - "testing" - - "github.com/Autumn-27/artex/archivestore" - pgdb "github.com/Autumn-27/artex/db" -) - -func TestRemoteArchiveLifecycle(t *testing.T) { - if os.Getenv("ARTEX_PG_DSN") == "" { - t.Skip("requires isolated PostgreSQL") - } - t.Setenv("ARTEX_ARCHIVE_STORAGE", "local") - m, err := NewManager(t.TempDir(), "") - if err != nil { - t.Fatal(err) - } - defer m.Close() - ctx, cancel := context.WithCancel(context.Background()) - cancel() - s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir()) - s.archiveWG.Wait() - s.ctx = context.Background() - var mu sync.Mutex - objects := map[string][]byte{} - corrupt, failDelete := false, false - remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - mu.Lock() - defer mu.Unlock() - switch r.Method { - case "PUT": - objects[r.URL.Path], _ = io.ReadAll(r.Body) - w.Header().Set("ETag", `"test"`) - case "HEAD": - w.Header().Set("X-Amz-Server-Side-Encryption", "AES256") - w.Header().Set("ETag", `"test"`) - w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path]))) - case "GET": - data := objects[r.URL.Path] - if corrupt && len(data) > 0 { - data = bytes.Clone(data) - data[0] ^= 0xff - } - var start, end int64 - if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) || r.Header.Get("If-Match") != `"test"` { - w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) - return - } - w.Header().Set("ETag", `"test"`) - w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10)) - w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) - w.WriteHeader(http.StatusPartialContent) - _, _ = w.Write(data[start : end+1]) - case "DELETE": - if failDelete { - w.WriteHeader(403) - return - } - delete(objects, r.URL.Path) - w.WriteHeader(204) - default: - w.WriteHeader(400) - } - })) - defer remote.Close() - s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) - if err != nil { - t.Fatal(err) - } - task, err := m.CreateTask("S3 round trip", "preserve data", nil, 0, 0) - if err != nil { - t.Fatal(err) - } - id, _ := strconv.ParseInt(task.ID, 10, 64) - defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }() - if err := m.pg.SetPaused(id, true); err != nil { - t.Fatal(err) - } - file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "important.txt") - if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(file, []byte("keep this content"), 0600); err != nil { - t.Fatal(err) - } - pgTask, err := m.pg.GetTask(id) - if err != nil { - t.Fatal(err) - } - if _, err := m.pg.Exploration(pgTask.ExplorationID).AppendActivity(pgdb.Activity{Worker: "system", Kind: "system", Summary: "archive proof"}); err != nil { - t.Fatal(err) - } - job, err := m.pg.QueueTaskArchive(id) - if err != nil { - t.Fatal(err) - } - // A configured-but-invalid backend must fail before moving any source data. - s.archiveStoreErr = errors.New("invalid fixture configuration") - if err := s.runOneTaskArchiveJob(); !errors.Is(err, s.archiveStoreErr) { - t.Fatalf("configuration error was not returned: %v", err) - } - if got, err := os.ReadFile(file); err != nil || string(got) != "keep this content" { - t.Fatalf("bad configuration lost source: %v", err) - } - if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil { - t.Fatal("bad configuration lost task", err) - } - s.archiveStoreErr = nil - if _, err := m.pg.QueueTaskArchive(id); err != nil { - t.Fatal(err) - } - // A corrupt readback must roll back filesystem staging and leave PG intact. - mu.Lock() - corrupt = true - mu.Unlock() - if err := s.runOneTaskArchiveJob(); err == nil { - t.Fatal("expected integrity failure") - } - if got, _ := os.ReadFile(file); string(got) != "keep this content" { - t.Fatal("upload failure lost source") - } - if task, err := m.pg.GetTask(id); err != nil || task == nil { - t.Fatal("upload failure lost task") - } - mu.Lock() - corrupt = false - mu.Unlock() - if err := s.recoverRemoteArchiveJournals(); err != nil { - t.Fatal(err) - } - if _, err := m.pg.QueueTaskArchive(id); err != nil { - t.Fatal(err) - } - if err := s.runOneTaskArchiveJob(); err != nil { - t.Fatal(err) - } - job, err = m.pg.GetTaskArchive(job.ID) - if err != nil { - t.Fatal(err) - } - if job.State != pgdb.ArchiveReady || job.Storage != "s3" || !strings.HasPrefix(job.ArchivePath, "s3://") { - t.Fatalf("unexpected archive %+v", job) - } - if _, err := os.Stat(file); !os.IsNotExist(err) { - t.Fatal("workspace not released") - } - if _, err := os.Stat(taskArchivePath(m.dir, job.ID, task.ID)); !os.IsNotExist(err) { - t.Fatal("local package retained") - } - // Simulate the crash window after commit: cleanup keeps the remote object. - journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, LocalPath: taskArchivePath(m.dir, job.ID, task.ID), Operation: "upload"}) - if err != nil { - t.Fatal(err) - } - if err := s.recoverRemoteArchiveJournal(journal); err != nil { - t.Fatal(err) - } - // A corrupted download must leave the archive and its remote package intact. - mu.Lock() - corrupt = true - mu.Unlock() - if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { - t.Fatal(err) - } - if err := s.runOneTaskArchiveJob(); err == nil { - t.Fatal("expected corrupted restore to fail") - } - if _, err := os.Stat(file); !os.IsNotExist(err) { - t.Fatal("corrupted archive installed workspace files") - } - if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || restored { - t.Fatalf("corrupted archive restored database: %v %v", restored, err) - } - mu.Lock() - if len(objects) == 0 { - t.Error("corrupted restore deleted the only remote package") - } - corrupt = false - mu.Unlock() - // Restoration commits files and PG first. Failed remote deletion is retryable. - mu.Lock() - failDelete = true - mu.Unlock() - if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { - t.Fatal(err) - } - if err := s.runOneTaskArchiveJob(); err == nil { - t.Fatal("expected remote delete failure") - } - if got, _ := os.ReadFile(file); string(got) != "keep this content" { - t.Fatal("restored content differs") - } - if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || !restored { - t.Fatalf("PG restore %v %v", restored, err) - } - mu.Lock() - failDelete = false - mu.Unlock() - if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { - t.Fatal(err) - } - if err := s.runOneTaskArchiveJob(); err != nil { - t.Fatal(err) - } - if item, err := m.pg.GetTaskArchive(job.ID); err != nil || item != nil { - t.Fatal("restore did not consume stub") - } - var count int - if err := m.pg.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='archive proof'`, pgTask.ExplorationID).Scan(&count); err != nil || count != 1 { - t.Fatalf("activity restore count=%d err=%v", count, err) - } - // Re-archive, then crash/retry the deletion outbox after PG removed its stub. - job, err = m.pg.QueueTaskArchive(id) - if err != nil { - t.Fatal(err) - } - if err := s.runOneTaskArchiveJob(); err != nil { - t.Fatal(err) - } - mu.Lock() - failDelete = true - mu.Unlock() - if _, err := m.pg.QueueTaskArchiveDelete(job.ID); err != nil { - t.Fatal(err) - } - if err := s.runOneTaskArchiveJob(); err != nil { - t.Fatal(err) - } - mu.Lock() - failDelete = false - mu.Unlock() - if err := s.recoverRemoteArchiveJournals(); err != nil { - t.Fatal(err) - } - mu.Lock() - remaining := len(objects) - mu.Unlock() - if remaining != 0 { - t.Fatalf("left %d objects", remaining) - } -} - -// Once PostgreSQL has committed an archive location and checksum, a subsequent -// queued operation must not make startup recovery resurrect the old workspace. -func TestTaskArchiveStageRecoveryAfterCommit(t *testing.T) { - if os.Getenv("ARTEX_PG_DSN") == "" { - t.Skip("requires isolated PostgreSQL") - } - m, err := NewManager(t.TempDir(), "") - if err != nil { - t.Fatal(err) - } - defer m.Close() - task, err := m.CreateTask("remote stage recovery", "test", nil, 0, 0) - if err != nil { - t.Fatal(err) - } - taskID, _ := strconv.ParseInt(task.ID, 10, 64) - defer func() { - _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID) - _ = m.pg.DeleteTask(taskID) - }() - if err := m.pg.SetPaused(taskID, true); err != nil { - t.Fatal(err) - } - job, err := m.pg.QueueTaskArchive(taskID) - if err != nil { - t.Fatal(err) - } - pgTask, err := m.pg.GetTask(taskID) - if err != nil { - t.Fatal(err) - } - for _, backend := range []string{"local", "s3"} { - for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} { - t.Run(backend+"/"+state, func(t *testing.T) { - committed := state != pgdb.ArchiveQueued - checksum := "" - if committed { - checksum = strings.Repeat("a", 64) - } - packagePath := taskArchivePath(m.dir, job.ID, task.ID) - if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(packagePath, []byte("verified package"), 0600); err != nil { - t.Fatal(err) - } - location := packagePath - if backend == "s3" { - location = "s3://test/tasks/package.tar.zst" - } - if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, location, checksum); err != nil { - t.Fatal(err) - } - file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt") - if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil { - t.Fatal(err) - } - stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID) - if err != nil { - t.Fatal(err) - } - if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(stage.root); !os.IsNotExist(err) { - t.Fatalf("staging cleanup incomplete: %v", err) - } - if backend == "local" && committed { - if data, err := os.ReadFile(packagePath); err != nil || string(data) != "verified package" { - t.Fatal("committed local package lost", err) - } - } else if !committed { - if _, err := os.Stat(packagePath); !os.IsNotExist(err) { - t.Fatal("uncommitted temporary package retained", err) - } - } - if committed { - if _, err := os.Stat(file); !os.IsNotExist(err) { - t.Fatalf("resurrected committed workspace: %v", err) - } - } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" { - t.Fatalf("uncommitted archive lost source: %v", err) - } - }) - } - } -} - -// Failed remote cleanup can survive switching to local storage and retrying the -// same archive ID. Its journal must never remove the newly committed local copy. -func TestRemoteArchiveJournalPreservesLocalRetry(t *testing.T) { - if os.Getenv("ARTEX_PG_DSN") == "" { - t.Skip("requires isolated PostgreSQL") - } - m, err := NewManager(t.TempDir(), "") - if err != nil { - t.Fatal(err) - } - defer m.Close() - task, err := m.CreateTask("local retry", "test", nil, 0, 0) - if err != nil { - t.Fatal(err) - } - taskID, _ := strconv.ParseInt(task.ID, 10, 64) - defer func() { - _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID) - _ = m.pg.DeleteTask(taskID) - }() - if err := m.pg.SetPaused(taskID, true); err != nil { - t.Fatal(err) - } - job, err := m.pg.QueueTaskArchive(taskID) - if err != nil { - t.Fatal(err) - } - filename := taskArchivePath(m.dir, job.ID, task.ID) - if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filename, []byte("committed local archive"), 0600); err != nil { - t.Fatal(err) - } - if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, pgdb.ArchiveReady, filename, strings.Repeat("a", 64)); err != nil { - t.Fatal(err) - } - var deletes atomic.Int32 - remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodDelete { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - deletes.Add(1) - w.WriteHeader(http.StatusNoContent) - })) - defer remote.Close() - store, err := archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) - if err != nil { - t.Fatal(err) - } - s := &Server{ctx: context.Background(), m: m, archiveStore: store} - journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ - ArchiveID: job.ID, URI: store.URI(taskID, job.ID, strings.Repeat("b", 64)), - LocalPath: filename, Operation: "upload", - }) - if err != nil { - t.Fatal(err) - } - if err := s.recoverRemoteArchiveJournal(journal); err != nil { - t.Fatal(err) - } - if data, err := os.ReadFile(filename); err != nil || string(data) != "committed local archive" { - t.Fatalf("old upload cleanup removed local retry: %v", err) - } - if deletes.Load() != 1 { - t.Fatal("uncommitted remote object was not removed") - } - if _, err := os.Stat(journal); !os.IsNotExist(err) { - t.Fatalf("finished journal remains: %v", err) - } -} diff --git a/server/task_archives.go b/server/task_archives.go index 9a7af52..836c2d8 100644 --- a/server/task_archives.go +++ b/server/task_archives.go @@ -14,7 +14,6 @@ import ( "time" "github.com/Autumn-27/artex/agent" - "github.com/Autumn-27/artex/archivestore" pgdb "github.com/Autumn-27/artex/db" "github.com/Autumn-27/artex/traffic" ) @@ -38,12 +37,6 @@ func (s *Server) startTaskArchiveWorker() { if s.m == nil || s.m.pg == nil { return } - s.archiveStore, s.archiveStoreErr = archivestore.FromEnv() - if s.archiveStoreErr != nil { - log.Printf("[task-archive] storage configuration: %v", s.archiveStoreErr) - } else if s.archiveStore != nil { - log.Printf("[task-archive] new archives use S3 object storage with full SHA-256 readback verification") - } if err := s.m.pg.RecoverTaskArchiveJobs(); err != nil { log.Printf("[task-archive] recover jobs: %v", err) } @@ -95,14 +88,6 @@ func (s *Server) notifyTaskArchiveWorker() { } func (s *Server) runOneTaskArchiveJob() error { - if s.ctx.Err() != nil { - return s.ctx.Err() - } - // Retry deferred remote cleanup before claiming more work. Failed cleanup - // keeps its durable journal and does not block unrelated local archives. - if err := s.recoverRemoteArchiveJournals(); err != nil { - log.Printf("[task-archive] deferred object cleanup: %v", err) - } job, err := s.m.pg.ClaimTaskArchiveJob(s.ctx) if err != nil || job == nil { return err @@ -130,9 +115,6 @@ func (s *Server) runOneTaskArchiveJob() error { } func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { - if s.archiveStoreErr != nil { - return s.archiveStoreErr - } taskID := strconv.FormatInt(job.TaskID, 10) if !s.beginTaskDelete(taskID) { return errors.New("task is already undergoing another archive or delete operation") @@ -217,21 +199,6 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { _ = os.Remove(archivePath) } }() - storedPath := archivePath - var remoteJournal string - if s.archiveStore != nil { - storedPath = s.archiveStore.URI(job.TaskID, job.ID, checksum) - remoteJournal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ - ArchiveID: job.ID, URI: storedPath, LocalPath: archivePath, Operation: "upload", - }) - if err != nil { - return err - } - _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "upload_and_verify", 65) - if err := s.archiveStore.Upload(s.ctx, storedPath, archivePath, checksum); err != nil { - return err - } - } var trafficStage *traffic.HostDeleteStage if s.m.traffic != nil && len(snapshot.ExclusiveHosts) > 0 { _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_traffic", 68) @@ -246,18 +213,11 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { }() } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_database", 78) - if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, storedPath, checksum, originalSize, compressedSize); err != nil { + if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, archivePath, checksum, originalSize, compressedSize); err != nil { return err } committed = true - removePackage = s.archiveStore != nil - if remoteJournal != "" { - // The journal also covers a crash after database commit but before local - // package removal. Keep it if cleanup fails, so the worker can retry. - if err := removeLocalArchivePackage(archivePath); err == nil { - _ = os.Remove(remoteJournal) - } - } + removePackage = false if trafficStage != nil { if err := trafficStage.Commit(); err != nil { warning := "archive completed, but cleaning the exclusive traffic hot store failed (the archive package can still restore it): " + err.Error() @@ -279,7 +239,7 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { } func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { - if err := s.validateStoredArchive(job.ArchivePath); err != nil { + if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil { return err } // A prior attempt may have committed PostgreSQL and files, then failed only @@ -287,7 +247,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { if restored, err := s.m.pg.IsTaskArchiveRestored(job.ID); err != nil { return err } else if restored { - if err := s.removeStoredArchive(job.ArchivePath); err != nil { + if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) { return err } if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil { @@ -306,15 +266,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { return err } defer func() { _ = os.RemoveAll(extracted) }() - packagePath := job.ArchivePath - if strings.HasPrefix(packagePath, "s3://") { - _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "download_and_verify", 12) - packagePath = filepath.Join(extracted, "download.tar.zst") - if err := s.archiveStore.Download(s.ctx, job.ArchivePath, packagePath, job.SHA256, job.CompressedSize); err != nil { - return err - } - } - if err := extractTaskArchivePackage(packagePath, job.SHA256, extracted); err != nil { + if err := extractTaskArchivePackage(job.ArchivePath, job.SHA256, extracted); err != nil { return err } manifestRaw, err := os.ReadFile(filepath.Join(extracted, "manifest.json")) @@ -404,7 +356,7 @@ func (s *Server) restoreTaskArchivePayload(job *pgdb.TaskArchive, snapshot *pgdb return err } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "consume_package", 95) - if err := s.removeStoredArchive(job.ArchivePath); err != nil { + if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) { return err } if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil { @@ -441,12 +393,9 @@ func (s *Server) loadRestoredTask(taskID int64) { } func (s *Server) deleteTaskArchive(job *pgdb.TaskArchive) error { - if err := s.validateStoredArchive(job.ArchivePath); err != nil { + if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil { return err } - if strings.HasPrefix(job.ArchivePath, "s3://") { - return s.deleteRemoteTaskArchive(job) - } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "stage_package_delete", 35) staged, moved, err := stageTaskArchivePackageDelete(job.ArchivePath, job.ID) if err != nil { diff --git a/web/src/app/(main)/function/tasks/page.tsx b/web/src/app/(main)/function/tasks/page.tsx index c2f9cc3..6b8ba8e 100644 --- a/web/src/app/(main)/function/tasks/page.tsx +++ b/web/src/app/(main)/function/tasks/page.tsx @@ -1494,7 +1494,7 @@ function ArchiveConfirmDialog({ {count === 1 ? "Archive task" : `Archive ${count} tasks`} - Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into the configured archive storage. Once archived, a task can be restored from "Archived". Restoring requires free space on this server. + Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into local cold storage. Once archived, a task can be restored from "Archived". @@ -1934,7 +1934,6 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) { title={`Before compression: ${formatArchiveBytes(archive.original_size)}`} > {archiveCompressionLabel(archive)} -
{archive.storage === "s3" ? "Object storage" : "Local storage"}
{archiveDataTotal(archive).toLocaleString()} @@ -1953,7 +1952,7 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {

))} - {archive.phase === "upload_and_verify" ? "Uploading and verifying" : archive.phase === "download_and_verify" ? "Downloading and verifying" : archive.phase} · {archive.progress}% + {archive.phase} · {archive.progress}%
diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index f31b18c..cd53b64 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -101,7 +101,6 @@ export interface TaskArchiveTokenStats { } export interface TaskArchive { - storage?: "local" | "s3"; id: number; task_id: number; state: TaskArchiveState; From 3755e9550433095f3763ccd216f6251542b13ccd Mon Sep 17 00:00:00 2001 From: Chubboy Ren Date: Thu, 19 Nov 2026 03:33:51 +0000 Subject: [PATCH 4/4] fix: enforce archive encryption and clean failed S3 uploads --- .github/workflows/ci.yml | 2 +- archivestore/config_test.go | 7 ++ archivestore/s3.go | 89 ++++++++++---- archivestore/s3_test.go | 2 + archivestore/safety_test.go | 231 ++++++++++++++++++++++++++++++++++++ 5 files changed, 310 insertions(+), 21 deletions(-) create mode 100644 archivestore/safety_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f949165..2fac967 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: ARTEX_PG_DSN: postgres://artex:testpass@localhost:5432/artextest_race?sslmode=disable run: >- - go test -race -p 1 ./agent ./cmd/artex ./config ./db ./evidence ./guard + go test -race -p 1 ./agent ./archivestore ./cmd/artex ./config ./db ./evidence ./guard ./intercept ./llmpool ./llmrec ./mcphttp ./notify ./server ./sidequestion ./traffic ./traffic/rewrite diff --git a/archivestore/config_test.go b/archivestore/config_test.go index 73afc73..1fe54a5 100644 --- a/archivestore/config_test.go +++ b/archivestore/config_test.go @@ -76,6 +76,13 @@ func TestArchiveEncryptionConfiguration(t *testing.T) { t.Fatal("KMS key accepted with SSE-S3") } c.SSE = "aws:kms" + for _, key := range []string{"alias/archive", "bare-key-id", "arn:aws:kms:us-east-1:123456789012:alias/archive"} { + c.KMSKeyID = key + if _, err := New(c); err == nil { + t.Fatalf("accepted noncanonical KMS key %q", key) + } + } + c.KMSKeyID = "arn:aws:kms:us-east-1:123456789012:key/test-key" if _, err := New(c); err != nil { t.Fatal(err) } diff --git a/archivestore/s3.go b/archivestore/s3.go index 27c50af..5761a83 100644 --- a/archivestore/s3.go +++ b/archivestore/s3.go @@ -17,6 +17,9 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/aws/aws-sdk-go-v2/aws/retry" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" @@ -93,24 +96,37 @@ func New(c Config) (*Store, error) { if c.KMSKeyID != "" && c.SSE != "aws:kms" { return nil, errors.New("archive S3 KMS key requires SSE=aws:kms") } + if c.KMSKeyID != "" { + key, err := arn.Parse(c.KMSKeyID) + if err != nil || key.Service != "kms" || key.Region == "" || key.AccountID == "" || + !strings.HasPrefix(key.Resource, "key/") || len(key.Resource) <= len("key/") { + return nil, errors.New("archive S3 KMS key must be a full key ARN, not an alias or bare key ID") + } + } if c.Region == "" { c.Region = "us-east-1" } - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.ResponseHeaderTimeout = 60 * time.Second - httpClient := &http.Client{Transport: transport} - cfg := aws.Config{Region: c.Region, HTTPClient: httpClient} + // Load shared AWS settings (including custom CAs) for both credential modes. + httpClient := awshttp.NewBuildableClient().WithTransportOptions(func(t *http.Transport) { + t.ResponseHeaderTimeout = 60 * time.Second + }) + options := []func(*awsconfig.LoadOptions) error{ + awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient), + } if explicitCredentials { - cfg.Credentials = credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken) - } else { - // Keep the SDK's refreshable default chain: standard AWS environment/shared - // profiles, web identity, ECS task roles and EC2 instance roles (IMDSv2). - var err error - cfg, err = awsconfig.LoadDefaultConfig(context.Background(), - awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient)) - if err != nil { - return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err) - } + options = append(options, awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken))) + } + cfg, err := awsconfig.LoadDefaultConfig(context.Background(), options...) + if err != nil { + return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err) + } + // The SDK's default client still follows 307/308. Use its CA-configured + // transport, but never redirect a signed archive request to another origin. + configuredClient := cfg.HTTPClient.(*awshttp.BuildableClient) + cfg.HTTPClient = &http.Client{ + Transport: configuredClient.GetTransport(), Timeout: configuredClient.GetTimeout(), + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, } client := s3.NewFromConfig(cfg, func(o *s3.Options) { o.BaseEndpoint = aws.String(c.Endpoint) @@ -166,6 +182,7 @@ func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) erro uploader := manager.NewUploader(s.client, func(u *manager.Uploader) { u.PartSize = 8 << 20 u.Concurrency = 2 + u.LeavePartsOnError = true // Abort below with a fresh, bounded cleanup context. u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired }) input := &s3.PutObjectInput{ @@ -176,11 +193,36 @@ func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) erro if s.kmsKeyID != "" { input.SSEKMSKeyId = aws.String(s.kmsKeyID) } - _, err = uploader.Upload(ctx, input) + result, err := uploader.Upload(ctx, input) if err != nil { - return fmt.Errorf("upload archive: %w", err) + uploadErr := fmt.Errorf("upload archive: %w", err) + var multipart manager.MultiUploadFailure + if errors.As(err, &multipart) { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + _, cleanupErr := s.client.AbortMultipartUpload(cleanupCtx, &s3.AbortMultipartUploadInput{ + Bucket: aws.String(s.bucket), Key: aws.String(key), UploadId: aws.String(multipart.UploadID()), + }) + if cleanupErr != nil { + return errors.Join(uploadErr, fmt.Errorf("abort archive multipart upload: %w", cleanupErr)) + } + } + return uploadErr } - return s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true) + if err := s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true); err != nil { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + // Versioned buckets require deleting the version just uploaded; an + // unversioned DELETE would only hide it behind a delete marker. + _, cleanupErr := s.client.DeleteObject(cleanupCtx, &s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), Key: aws.String(key), VersionId: result.VersionID, + }) + if cleanupErr != nil { + return errors.Join(err, fmt.Errorf("delete unverified archive: %w", cleanupErr)) + } + return err + } + return nil } func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64, requireEncryption bool) error { @@ -195,7 +237,8 @@ func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, che return fmt.Errorf("inspect archive: %w", err) } if requireEncryption && (object.ServerSideEncryption != s.encryption || - (s.encryption == types.ServerSideEncryptionAwsKms && aws.ToString(object.SSEKMSKeyId) == "")) { + (s.encryption == types.ServerSideEncryptionAwsKms && (aws.ToString(object.SSEKMSKeyId) == "" || + (s.kmsKeyID != "" && aws.ToString(object.SSEKMSKeyId) != s.kmsKeyID)))) { return errors.New("remote archive did not confirm the requested server-side encryption; source data retained") } if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" { @@ -265,11 +308,17 @@ func (s *Store) readArchiveChunkOnce(ctx context.Context, key, etag string, star object, err := s.client.GetObject(requestCtx, &s3.GetObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(key), IfMatch: aws.String(etag), Range: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)), + }, func(o *s3.Options) { + // This loop owns both HTTP and interrupted-body retries: at most five + // wire requests per chunk, rather than five times the SDK's attempts. + o.RetryMaxAttempts = 1 }) if err != nil { var networkError net.Error - retry := errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded) - return nil, retry, fmt.Errorf("read archive range: %w", err) + var responseError *awshttp.ResponseError + throttled := errors.As(err, &responseError) && responseError.HTTPStatusCode() == http.StatusTooManyRequests + shouldRetry := throttled || retry.NewStandard().IsErrorRetryable(err) || errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded) + return nil, shouldRetry, fmt.Errorf("read archive range: %w", err) } defer object.Body.Close() want := end - start + 1 diff --git a/archivestore/s3_test.go b/archivestore/s3_test.go index dc618c1..84049e3 100644 --- a/archivestore/s3_test.go +++ b/archivestore/s3_test.go @@ -340,6 +340,8 @@ func TestS3UploadRejectsUnconfirmedEncryption(t *testing.T) { switch r.Method { case "PUT": _, _ = io.Copy(io.Discard, r.Body) + case "DELETE": + w.WriteHeader(204) case "HEAD": w.Header().Set("ETag", `"test-etag"`) w.Header().Set("Content-Length", strconv.Itoa(len(data))) diff --git a/archivestore/safety_test.go b/archivestore/safety_test.go new file mode 100644 index 0000000..9afe47b --- /dev/null +++ b/archivestore/safety_test.go @@ -0,0 +1,231 @@ +package archivestore + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" +) + +func archiveSource(t *testing.T, data []byte) (string, string) { + t.Helper() + filename := filepath.Join(t.TempDir(), "source") + if err := os.WriteFile(filename, data, 0600); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(data) + return filename, hex.EncodeToString(sum[:]) +} + +func TestS3VerificationFailureCleansUploadedVersion(t *testing.T) { + for _, failure := range []string{"wrong-kms", "checksum", "cancelled", "delete-denied"} { + t.Run(failure, func(t *testing.T) { + data := []byte("private archive") + source, checksum := archiveSource(t, data) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var present, deleted atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case "PUT": + _, _ = io.Copy(io.Discard, r.Body) + present.Store(true) + w.Header().Set("X-Amz-Version-Id", "uploaded-version") + case "HEAD": + w.Header().Set("ETag", `"test-etag"`) + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + w.Header().Set("X-Amz-Server-Side-Encryption", "aws:kms") + kms := "arn:aws:kms:us-east-1:123456789012:key/expected" + if failure == "wrong-kms" { + kms = "arn:aws:kms:us-east-1:123456789012:key/different" + } + w.Header().Set("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id", kms) + if failure == "cancelled" { + cancel() + } + case "GET": + corrupt := bytes.Clone(data) + corrupt[0] ^= 0xff + serveArchiveRange(t, w, r, corrupt) + case "DELETE": + if r.URL.Query().Get("versionId") != "uploaded-version" { + t.Error("cleanup did not target uploaded version") + } + deleted.Store(true) + if failure == "delete-denied" { + w.WriteHeader(403) + return + } + present.Store(false) + w.WriteHeader(204) + default: + t.Errorf("unexpected request %s", r.Method) + w.WriteHeader(400) + } + })) + defer srv.Close() + store, err := New(Config{Endpoint: srv.URL, Bucket: "test", AccessKey: "test", SecretKey: "test", PathStyle: true, + SSE: "aws:kms", KMSKeyID: "arn:aws:kms:us-east-1:123456789012:key/expected"}) + if err != nil { + t.Fatal(err) + } + err = store.Upload(ctx, store.URI(1, 2, checksum), source, checksum) + if err == nil { + t.Fatal("accepted failed verification") + } + if failure == "wrong-kms" && !strings.Contains(err.Error(), "encryption") { + t.Fatal(err) + } + if failure == "cancelled" && !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if failure == "delete-denied" && (!strings.Contains(err.Error(), "SHA-256") || !strings.Contains(err.Error(), "delete unverified archive")) { + t.Fatalf("verification and cleanup errors must both survive: %v", err) + } + if !deleted.Load() || present.Load() != (failure == "delete-denied") { + t.Fatal("unverified object cleanup was not attempted/completed") + } + if got, readErr := os.ReadFile(source); readErr != nil || !bytes.Equal(got, data) { + t.Fatal("source changed", readErr) + } + }) + } +} + +func TestS3CancelledMultipartAbortsWithFreshContext(t *testing.T) { + for _, denyAbort := range []bool{false, true} { + t.Run(strconv.FormatBool(denyAbort), func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var aborted atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + switch { + case r.Method == "POST" && q.Has("uploads"): + fmt.Fprint(w, `cancelled-upload`) + case r.Method == "PUT" && q.Has("partNumber"): + cancel() + _, _ = io.Copy(io.Discard, r.Body) + w.WriteHeader(403) + case r.Method == "DELETE" && q.Get("uploadId") == "cancelled-upload": + aborted.Store(true) + if denyAbort { + w.WriteHeader(403) + } else { + w.WriteHeader(204) + } + default: + t.Errorf("unexpected request %s", r.Method) + w.WriteHeader(400) + } + })) + defer srv.Close() + data := bytes.Repeat([]byte("x"), 9<<20) + source, sum := archiveSource(t, data) + store := testRangeStore(t, srv.URL) + err := store.Upload(ctx, store.URI(1, 2, sum), source, sum) + if !errors.Is(err, context.Canceled) || !aborted.Load() { + t.Fatalf("cancelled upload not aborted: %v", err) + } + if denyAbort && !strings.Contains(err.Error(), "abort archive multipart upload") { + t.Fatalf("cleanup failure lost: %v", err) + } + if got, readErr := os.ReadFile(source); readErr != nil || !bytes.Equal(got, data) { + t.Fatal("source changed", readErr) + } + }) + } +} + +func TestS3SignedRequestsNeverFollowRedirects(t *testing.T) { + var reached atomic.Int32 + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reached.Add(1); w.WriteHeader(204) })) + defer target.Close() + for _, status := range []int{301, 302, 303, 307, 308} { + t.Run(strconv.Itoa(status), func(t *testing.T) { + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") == "" || r.Header.Get("X-Amz-Security-Token") != "fixture-session" { + t.Error("fixture request must be signed with session credentials") + } + w.Header().Set("Location", target.URL+"/redirected") + w.WriteHeader(status) + })) + defer origin.Close() + store, err := New(Config{Endpoint: origin.URL, Bucket: "test", AccessKey: "test", SecretKey: "test", SessionToken: "fixture-session", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + if err := store.Delete(context.Background(), store.URI(1, 2, "test")); err == nil { + t.Fatal("redirect accepted") + } + if reached.Load() != 0 { + t.Fatal("signed request followed redirect") + } + }) + } +} + +func TestS3CustomCABundleForBothCredentialModes(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) })) + defer srv.Close() + for _, explicit := range []bool{false, true} { + t.Run(strconv.FormatBool(explicit), func(t *testing.T) { + isolateAWSCredentials(t) + ca := filepath.Join(t.TempDir(), "ca.pem") + if err := os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("AWS_CA_BUNDLE", ca) + c := Config{Endpoint: srv.URL, Bucket: "test", PathStyle: true} + if explicit { + c.AccessKey, c.SecretKey = "fixture", "fixture" + } else { + t.Setenv("AWS_ACCESS_KEY_ID", "fixture") + t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture") + } + store, err := New(c) + if err != nil { + t.Fatal(err) + } + if err := store.Delete(context.Background(), store.URI(1, 2, "test")); err != nil { + t.Fatalf("configured CA not used: %v", err) + } + }) + } +} + +func TestS3RangeRetriesHaveOneWireAttemptEach(t *testing.T) { + for _, status := range []int{403, 429, 503} { + t.Run(strconv.Itoa(status), func(t *testing.T) { + var gets atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gets.Add(1) + w.WriteHeader(status) + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + if _, err := store.readArchiveChunk(context.Background(), "key", `"etag"`, 0, 9, 10); err == nil { + t.Fatal("failed request accepted") + } + want := int32(archiveReadAttempts) + if status == 403 { + want = 1 + } + if got := gets.Load(); got != want { + t.Fatalf("wire requests=%d want=%d", got, want) + } + }) + } +}