From da16b8d81a6946726da123a3bf69b9aae5375a5a Mon Sep 17 00:00:00 2001 From: Chubboy Ren Date: Tue, 17 Nov 2026 11:56:15 +0000 Subject: [PATCH] feat: archive cold tasks to S3-compatible object storage --- README.md | 6 + archivestore/config_test.go | 62 ++++ archivestore/s3.go | 280 +++++++++++++++ archivestore/s3_test.go | 311 ++++++++++++++++ cmd/artex-archive-check/main.go | 64 ++++ db/task_archives.go | 5 + docs/task-archive-storage.md | 104 ++++++ go.mod | 13 + go.sum | 38 ++ server/server.go | 9 +- server/task_archive_package.go | 4 +- server/task_archive_remote.go | 171 +++++++++ server/task_archive_remote_test.go | 396 +++++++++++++++++++++ server/task_archives.go | 65 +++- web/src/app/(main)/function/tasks/page.tsx | 5 +- web/src/lib/types.ts | 1 + 16 files changed, 1520 insertions(+), 14 deletions(-) create mode 100644 archivestore/config_test.go create mode 100644 archivestore/s3.go create mode 100644 archivestore/s3_test.go create mode 100644 cmd/artex-archive-check/main.go create mode 100644 docs/task-archive-storage.md create mode 100644 server/task_archive_remote.go create mode 100644 server/task_archive_remote_test.go diff --git a/README.md b/README.md index d15b112..7cc166c 100644 --- a/README.md +++ b/README.md @@ -418,6 +418,12 @@ runtime supports it. `/chat/completions` (or `/responses` for `openai-responses`), so their base **does** end in `/v1` (for example `https://api.openai.com/v1`). +**Task archive storage**: local disk by default. Set `ARTEX_ARCHIVE_STORAGE=s3` to +store cold task packages in an S3-compatible bucket and release their local +package space after verification. Existing local archives remain usable. See +[Task archive storage](docs/task-archive-storage.md) for configuration, permissions, +integrity checks, temporary disk requirements and recovery. + **Concurrency**: work agents per task, in "System Settings" (default 3). **Untrusted-output boundary (on by default)**: tool results for the worker, planner, chat and main diff --git a/archivestore/config_test.go b/archivestore/config_test.go new file mode 100644 index 0000000..cc12532 --- /dev/null +++ b/archivestore/config_test.go @@ -0,0 +1,62 @@ +package archivestore + +import ( + "strings" + "testing" +) + +func TestArchiveStorageEnvironment(t *testing.T) { + for _, key := range []string{ + "ARTEX_ARCHIVE_STORAGE", "ARTEX_ARCHIVE_S3_ENDPOINT", "ARTEX_ARCHIVE_S3_REGION", + "ARTEX_ARCHIVE_S3_BUCKET", "ARTEX_ARCHIVE_S3_PREFIX", "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", + "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_PATH_STYLE", + } { + t.Setenv(key, "") + } + for _, backend := range []string{"", "local"} { + t.Setenv("ARTEX_ARCHIVE_STORAGE", backend) + if store, err := FromEnv(); err != nil || store != nil { + t.Fatalf("local default: %v %v", store, err) + } + } + for _, backend := range []string{"unknown", "s3"} { + t.Setenv("ARTEX_ARCHIVE_STORAGE", backend) + if _, err := FromEnv(); err == nil { + t.Fatalf("%q silently fell back to local", backend) + } + } + t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", "http://127.0.0.1:9000") + t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "test-bucket") + t.Setenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "test-key") + t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "test-secret") + t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true") + store, err := FromEnv() + if err != nil { + t.Fatal(err) + } + if uri := store.URI(1, 2, "checksum"); uri != "s3://test-bucket/artex/tasks/task-1-2-checksum.tar.zst" { + t.Fatal("unexpected default object location", uri) + } + t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "") + if _, err := FromEnv(); err == nil { + t.Fatal("missing credentials accepted") + } +} + +func TestArchiveEndpointValidation(t *testing.T) { + config := Config{Bucket: "test", AccessKey: "test-key", SecretKey: "test-secret"} + for _, endpoint := range []string{"https://objects.example.test", "http://127.0.0.1:9000", "http://127.0.0.2:9000", "http://[::1]:9000", "http://localhost:9000"} { + config.Endpoint = endpoint + if _, err := New(config); err != nil { + t.Errorf("rejected %s: %v", endpoint, err) + } + } + for _, endpoint := range []string{"http://objects.example.test", "https://user:test-secret@objects.example.test", "https://objects.example.test/bucket", "https://objects.example.test?token=test-secret", "https://objects.example.test#fragment"} { + config.Endpoint = endpoint + if _, err := New(config); err == nil { + t.Error("accepted endpoint with non-loopback HTTP, credentials, path, query or fragment") + } else if strings.Contains(err.Error(), "test-secret") { + t.Fatal("configuration error disclosed a secret") + } + } +} diff --git a/archivestore/s3.go b/archivestore/s3.go new file mode 100644 index 0000000..edc1513 --- /dev/null +++ b/archivestore/s3.go @@ -0,0 +1,280 @@ +// Package archivestore stores immutable cold-task packages in S3-compatible storage. +package archivestore + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type Config struct { + Endpoint, Region, Bucket, Prefix, AccessKey, SecretKey string + PathStyle bool +} + +type Store struct { + client *s3.Client + bucket, prefix string +} + +// FromEnv returns nil for the default local backend. A configured but invalid +// backend is an error, never a silent fallback to the machine's local disk. +func FromEnv() (*Store, error) { + switch os.Getenv("ARTEX_ARCHIVE_STORAGE") { + case "", "local": + return nil, nil + case "s3": + return New(Config{ + Endpoint: os.Getenv("ARTEX_ARCHIVE_S3_ENDPOINT"), Region: os.Getenv("ARTEX_ARCHIVE_S3_REGION"), + Bucket: os.Getenv("ARTEX_ARCHIVE_S3_BUCKET"), Prefix: os.Getenv("ARTEX_ARCHIVE_S3_PREFIX"), + AccessKey: os.Getenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID"), SecretKey: os.Getenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY"), + PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true", + }) + default: + return nil, errors.New("ARTEX_ARCHIVE_STORAGE must be local or s3") + } +} + +func New(c Config) (*Store, error) { + u, err := url.Parse(c.Endpoint) + loopback := false + if err == nil { + loopback = u.Hostname() == "localhost" + if ip := net.ParseIP(u.Hostname()); ip != nil { + loopback = ip.IsLoopback() + } + } + if err != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") || + (u.Scheme != "https" && !(u.Scheme == "http" && loopback)) { + return nil, errors.New("archive S3 endpoint must be an HTTPS origin (HTTP allowed only on loopback)") + } + c.Prefix = strings.Trim(c.Prefix, "/") + if c.Prefix == "" { + c.Prefix = "artex/tasks" + } + if path.Clean(c.Prefix) != c.Prefix || strings.Contains(c.Prefix, "..") || strings.ContainsAny(c.Prefix, "\\?#") { + return nil, errors.New("invalid archive S3 prefix") + } + if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") || c.AccessKey == "" || c.SecretKey == "" { + return nil, errors.New("archive S3 bucket and dedicated access credentials are required") + } + if c.Region == "" { + c.Region = "us-east-1" + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.ResponseHeaderTimeout = 60 * time.Second + client := s3.New(s3.Options{ + Region: c.Region, BaseEndpoint: aws.String(c.Endpoint), UsePathStyle: c.PathStyle, + Credentials: credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, ""), + HTTPClient: &http.Client{Transport: transport}, + RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, + ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + }) + return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/"}, nil +} + +func (s *Store) URI(taskID, archiveID int64, checksum string) string { + return "s3://" + s.bucket + "/" + s.prefix + fmt.Sprintf("task-%d-%d-%s.tar.zst", taskID, archiveID, checksum) +} + +func (s *Store) key(uri string) (string, error) { + if s == nil { + return "", errors.New("S3 archive storage is not configured") + } + u, err := url.Parse(uri) + if err != nil || u.Scheme != "s3" || u.Host != s.bucket || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return "", errors.New("archive object is outside the configured bucket") + } + key := strings.TrimPrefix(u.Path, "/") + if !strings.HasPrefix(key, s.prefix) || len(key) <= len(s.prefix) || path.Clean(key) != key || strings.Contains(key, "\\") { + return "", errors.New("archive object is outside the configured prefix") + } + return key, nil +} + +func (s *Store) Validate(uri string) error { _, err := s.key(uri); return err } + +// Upload verifies the entire uploaded object by reading it back, including +// multipart uploads whose ETag is not a checksum. Callers retain source data +// until verification succeeds and PostgreSQL commits the remote location. +func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) error { + key, err := s.key(uri) + if err != nil { + return err + } + if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size { + return errors.New("invalid archive SHA-256") + } + f, err := os.Open(filename) + if err != nil { + return err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return err + } + uploader := manager.NewUploader(s.client, func(u *manager.Uploader) { + u.PartSize = 8 << 20 + u.Concurrency = 2 + u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + }) + _, err = uploader.Upload(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), Key: aws.String(key), Body: f, + ContentType: aws.String("application/zstd"), Metadata: map[string]string{"sha256": checksum}, + }) + if err != nil { + return fmt.Errorf("upload archive: %w", err) + } + return s.readVerified(ctx, key, io.Discard, checksum, info.Size()) +} + +func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64) error { + if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size { + return errors.New("invalid archive SHA-256") + } + if expectedSize < 0 { + return errors.New("invalid archive size") + } + object, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}) + if err != nil { + return fmt.Errorf("inspect archive: %w", err) + } + if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" { + return errors.New("remote archive size mismatch or missing ETag; source data retained") + } + // Commit only complete, validated chunks to the output and hash. A broken + // connection can then retry its chunk without duplicating bytes or restarting + // a multi-gigabyte read. IfMatch pins every request to the same object. + h := sha256.New() + for offset := int64(0); offset < expectedSize; { + end := offset + min(archiveReadChunkSize, expectedSize-offset) - 1 + data, err := s.readArchiveChunk(ctx, key, aws.ToString(object.ETag), offset, end, expectedSize) + if err != nil { + return err + } + n, err := out.Write(data) + if err != nil { + return err + } + if n != len(data) { + return io.ErrShortWrite + } + _, _ = h.Write(data) + offset += int64(len(data)) + } + if !strings.EqualFold(hex.EncodeToString(h.Sum(nil)), checksum) { + return errors.New("remote archive size or SHA-256 mismatch; source data retained") + } + return nil +} + +const ( + archiveReadChunkSize = int64(8 << 20) + archiveReadAttempts = 5 +) + +func (s *Store) readArchiveChunk(ctx context.Context, key, etag string, start, end, total int64) ([]byte, error) { + var lastErr error + for attempt := 0; attempt < archiveReadAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return nil, err + } + if attempt > 0 { + timer := time.NewTimer((250 * time.Millisecond) << (attempt - 1)) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + data, retry, err := s.readArchiveChunkOnce(ctx, key, etag, start, end, total) + if err == nil { + return data, nil + } + if !retry { + return nil, err + } + lastErr = err + } + return nil, fmt.Errorf("read archive bytes %d-%d after %d attempts: %w", start, end, archiveReadAttempts, lastErr) +} + +func (s *Store) readArchiveChunkOnce(ctx context.Context, key, etag string, start, end, total int64) ([]byte, bool, error) { + requestCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + object, err := s.client.GetObject(requestCtx, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), Key: aws.String(key), IfMatch: aws.String(etag), + Range: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)), + }) + if err != nil { + var networkError net.Error + retry := errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded) + return nil, retry, fmt.Errorf("read archive range: %w", err) + } + defer object.Body.Close() + want := end - start + 1 + if aws.ToString(object.ETag) != etag || aws.ToString(object.ContentRange) != fmt.Sprintf("bytes %d-%d/%d", start, end, total) || object.ContentLength == nil || *object.ContentLength != want { + return nil, false, errors.New("remote archive changed or returned an invalid byte range; source data retained") + } + data, err := io.ReadAll(io.LimitReader(object.Body, want+1)) + if err != nil { + return nil, true, err + } + if int64(len(data)) < want { + return nil, true, io.ErrUnexpectedEOF + } + if int64(len(data)) != want { + return nil, false, errors.New("remote archive range exceeds its declared size") + } + return data, false, nil +} + +func (s *Store) Download(ctx context.Context, uri, filename, checksum string, size int64) (err error) { + key, err := s.key(uri) + if err != nil { + return err + } + if size <= 0 { + return errors.New("invalid archive size") + } + f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + defer func() { + err = errors.Join(err, f.Close()) + if err != nil { + _ = os.Remove(filename) + } + }() + if err = s.readVerified(ctx, key, f, checksum, size); err != nil { + return err + } + return f.Sync() +} + +func (s *Store) Delete(ctx context.Context, uri string) error { + key, err := s.key(uri) + if err != nil { + return err + } + _, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}) + return err +} diff --git a/archivestore/s3_test.go b/archivestore/s3_test.go new file mode 100644 index 0000000..316e770 --- /dev/null +++ b/archivestore/s3_test.go @@ -0,0 +1,311 @@ +package archivestore + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "sync" + "testing" +) + +func TestS3RoundTripAndIntegrity(t *testing.T) { + for _, size := range []int{1024, 10 << 20} { + t.Run(strconv.Itoa(size), func(t *testing.T) { + var mu sync.Mutex + var object []byte + parts := map[int][]byte{} + corrupt, multipart := false, false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + q := r.URL.Query() + w.Header().Set("ETag", `"test-etag"`) + switch { + case r.Method == "POST" && q.Has("uploads"): + multipart = true + fmt.Fprint(w, `test-upload`) + case r.Method == "PUT" && q.Has("partNumber"): + n, _ := strconv.Atoi(q.Get("partNumber")) + parts[n], _ = io.ReadAll(r.Body) + case r.Method == "POST" && q.Has("uploadId"): + for i := 1; i <= len(parts); i++ { + object = append(object, parts[i]...) + } + fmt.Fprint(w, `test-etag`) + case r.Method == "PUT": + object, _ = io.ReadAll(r.Body) + case r.Method == "HEAD": + w.Header().Set("Content-Length", strconv.Itoa(len(object))) + case r.Method == "GET": + data := object + if corrupt { + data = bytes.Clone(object) + data[0] ^= 0xff + } + serveArchiveRange(t, w, r, data) + case r.Method == "DELETE": + object = nil + w.WriteHeader(204) + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL) + w.WriteHeader(400) + } + })) + defer srv.Close() + store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + data := bytes.Repeat([]byte{0x42}, size) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + root := t.TempDir() + source := filepath.Join(root, "source") + if err := os.WriteFile(source, data, 0600); err != nil { + t.Fatal(err) + } + uri := store.URI(1, 2, checksum) + ctx := context.Background() + if err := store.Upload(ctx, uri, source, checksum); err != nil { + t.Fatal(err) + } + if multipart != (size > 8<<20) { + t.Fatalf("multipart=%v size=%d", multipart, size) + } + dest := filepath.Join(root, "download") + if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil { + t.Fatal(err) + } + got, _ := os.ReadFile(dest) + if !bytes.Equal(got, data) { + t.Fatal("download differs") + } + mu.Lock() + corrupt = true + mu.Unlock() + bad := filepath.Join(root, "bad") + if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil { + t.Fatal("accepted corrupt object") + } + if _, err := os.Stat(bad); !os.IsNotExist(err) { + t.Fatal("left corrupt download") + } + if err := store.Upload(ctx, uri, source, checksum); err == nil { + t.Fatal("upload accepted failed readback") + } + if err := store.Delete(ctx, uri); err != nil { + t.Fatal(err) + } + }) + } +} + +func serveArchiveRange(t *testing.T, w http.ResponseWriter, r *http.Request, data []byte) { + t.Helper() + var start, end int64 + if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) { + t.Errorf("invalid range %q", r.Header.Get("Range")) + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return + } + if r.Header.Get("If-Match") != `"test-etag"` { + t.Errorf("missing object version condition: %q", r.Header.Get("If-Match")) + w.WriteHeader(http.StatusPreconditionFailed) + return + } + w.Header().Set("ETag", `"test-etag"`) + w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[start : end+1]) +} + +func TestS3ReadbackRetriesInterruptedRange(t *testing.T) { + data := append(bytes.Repeat([]byte{0x42}, int(archiveReadChunkSize)), bytes.Repeat([]byte{0x73}, 1024)...) + var mu sync.Mutex + var ranges []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + w.Header().Set("ETag", `"test-etag"`) + if r.Method == "HEAD" { + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + return + } + ranges = append(ranges, r.Header.Get("Range")) + if len(ranges) == 2 { + w.Header().Set("Content-Length", strconv.FormatInt(int64(len(data))-archiveReadChunkSize, 10)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", archiveReadChunkSize, len(data)-1, len(data))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[archiveReadChunkSize : archiveReadChunkSize+128]) // Close during a later chunk. + return + } + serveArchiveRange(t, w, r, data) + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + dest := filepath.Join(t.TempDir(), "archive") + if err := store.Download(context.Background(), store.URI(1, 2, checksum), dest, checksum, int64(len(data))); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(dest) + if err != nil || !bytes.Equal(got, data) { + t.Fatalf("retried download differs: %v", err) + } + mu.Lock() + defer mu.Unlock() + if len(ranges) != 3 || ranges[1] != ranges[2] || ranges[0] != fmt.Sprintf("bytes=0-%d", archiveReadChunkSize-1) || ranges[2] != fmt.Sprintf("bytes=%d-%d", archiveReadChunkSize, len(data)-1) { + t.Fatalf("retry did not stay within the interrupted range: %v", ranges) + } +} + +func testRangeStore(t *testing.T, endpoint string) *Store { + t.Helper() + s, err := New(Config{Endpoint: endpoint, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + return s +} + +func TestS3ReadbackRejectsInvalidRangesAndExhaustedRetries(t *testing.T) { + for _, mode := range []string{"ignored-range", "changed-etag", "changed-object", "truncated"} { + t.Run(mode, func(t *testing.T) { + data := bytes.Repeat([]byte{0x42}, 1024) + var mu sync.Mutex + gets := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + w.Header().Set("ETag", `"test-etag"`) + if r.Method == "HEAD" { + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + return + } + gets++ + if mode == "changed-object" { + w.WriteHeader(http.StatusPreconditionFailed) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + if mode == "ignored-range" { + _, _ = w.Write(data) + return + } + if mode == "changed-etag" { + w.Header().Set("ETag", `"different-object"`) + } + w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", len(data)-1, len(data))) + w.WriteHeader(http.StatusPartialContent) + if mode == "truncated" { + _, _ = w.Write(data[:128]) + } else { + _, _ = w.Write(data) + } + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + dest := filepath.Join(t.TempDir(), "archive") + if err := store.Download(context.Background(), store.URI(1, 2, checksum), dest, checksum, int64(len(data))); err == nil { + t.Fatal("accepted invalid or incomplete readback") + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatal("left a failed download on disk") + } + mu.Lock() + defer mu.Unlock() + want := 1 + if mode == "truncated" { + want = archiveReadAttempts + } + if gets != want { + t.Fatalf("GET count %d, want %d", gets, want) + } + }) + } +} + +type failedArchiveWriter struct{ err error } + +func (w failedArchiveWriter) Write([]byte) (int, error) { return 0, w.err } + +func TestS3ReadbackPropagatesOutputFailure(t *testing.T) { + data := []byte("test archive") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"test-etag"`) + if r.Method == "HEAD" { + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + return + } + serveArchiveRange(t, w, r, data) + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + h := sha256.Sum256(data) + outputError := errors.New("output disk full") + err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data))) + if !errors.Is(err, outputError) { + t.Fatalf("lost output error: %v", err) + } +} + +func TestS3ReadbackStopsOnCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + data := bytes.Repeat([]byte{0x42}, 1024) + gets := make(chan struct{}, archiveReadAttempts+1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"test-etag"`) + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + if r.Method == "HEAD" { + return + } + gets <- struct{}{} + w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", len(data)-1, len(data))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[:128]) + w.(http.Flusher).Flush() + cancel() + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + dest := filepath.Join(t.TempDir(), "archive") + err := store.Download(ctx, store.URI(1, 2, checksum), dest, checksum, int64(len(data))) + if !errors.Is(err, context.Canceled) || len(gets) != 1 { + t.Fatalf("cancellation was not honored: error=%v GETs=%d", err, len(gets)) + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatal("left a cancelled download on disk") + } +} + +func TestS3RejectsForeignLocations(t *testing.T) { + s, err := New(Config{Endpoint: "https://sfo3.digitaloceanspaces.com", Bucket: "allowed", Prefix: "archives/tasks", AccessKey: "test", SecretKey: "test"}) + if err != nil { + t.Fatal(err) + } + for _, uri := range []string{"s3://other/archives/tasks/a", "s3://allowed/archives/other/a", "s3://allowed/archives/tasks/../a", "s3://allowed/archives/tasks/%2e%2e/a", "s3://allowed/archives/tasks/a?secret=x", "s3://user@allowed/archives/tasks/a", "file:///etc/passwd"} { + if err := s.Validate(uri); err == nil { + t.Errorf("accepted %s", uri) + } + } + var absent *Store + if absent.Validate("s3://allowed/archives/tasks/a") == nil { + t.Fatal("missing config accepted") + } +} diff --git a/cmd/artex-archive-check/main.go b/cmd/artex-archive-check/main.go new file mode 100644 index 0000000..af0efd9 --- /dev/null +++ b/cmd/artex-archive-check/main.go @@ -0,0 +1,64 @@ +// artex-archive-check tests the configured object store without touching tasks. +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "log" + "os" + "path/filepath" + "time" + + "github.com/Autumn-27/artex/archivestore" +) + +func main() { + if err := check(); err != nil { + log.Fatal(err) + } +} + +func check() error { + s, err := archivestore.FromEnv() + if err != nil { + return err + } + if s == nil { + return fmt.Errorf("S3 archive storage is not configured") + } + dir, err := os.MkdirTemp("", "artex-archive-check-") + if err != nil { + return err + } + defer os.RemoveAll(dir) + data := []byte("ARTEX object storage connectivity check " + time.Now().UTC().Format(time.RFC3339Nano)) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + uri := s.URI(0, time.Now().UnixNano(), checksum) + file := filepath.Join(dir, "check") + if err := os.WriteFile(file, data, 0600); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + defer func() { + cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := s.Delete(cleanup, uri); err != nil { + log.Printf("test object cleanup failed for %s: %v", uri, err) + } + }() + if err := s.Upload(ctx, uri, file, checksum); err != nil { + return err + } + if err := s.Download(ctx, uri, filepath.Join(dir, "download"), checksum, int64(len(data))); err != nil { + return err + } + if err := s.Delete(ctx, uri); err != nil { + return err + } + fmt.Println("S3 upload, full SHA-256 verification, download and delete passed") + return nil +} diff --git a/db/task_archives.go b/db/task_archives.go index f312000..676f1d2 100644 --- a/db/task_archives.go +++ b/db/task_archives.go @@ -60,6 +60,7 @@ type TaskArchive struct { Warnings json.RawMessage `json:"warnings"` FormatVersion int `json:"format_version"` ArchivePath string `json:"-"` + Storage string `json:"storage"` SHA256 string `json:"sha256,omitempty"` OriginalSize int64 `json:"original_size"` CompressedSize int64 `json:"compressed_size"` @@ -161,6 +162,10 @@ func scanTaskArchive(sc interface{ Scan(...any) error }) (*TaskArchive, error) { if err != nil { return nil, err } + item.Storage = "local" + if strings.HasPrefix(item.ArchivePath, "s3://") { + item.Storage = "s3" + } if err := json.Unmarshal([]byte(sources), &item.SourceTaskIDs); err != nil { return nil, err } diff --git a/docs/task-archive-storage.md b/docs/task-archive-storage.md new file mode 100644 index 0000000..18bb3b4 --- /dev/null +++ b/docs/task-archive-storage.md @@ -0,0 +1,104 @@ +# Task archive storage + +Task archives use local disk by default. The optional S3-compatible backend moves +cold task packages to object storage after verifying their contents, releasing +local package space while retaining ARTEX's Archive, Restore and Delete actions. + +Active workspaces and the live PostgreSQL database stay local. This feature is +task-level cold storage; use the existing backup workflow for a full installation +backup. A complete recovery plan must also preserve the bucket objects and +access to them. + +## Configuration + +Create a private bucket and a dedicated credential scoped to the archive prefix. +The backend needs object upload, read (including HEAD and byte ranges), delete, +and multipart upload/abort permissions. Use a separate prefix for each deployment. + +Set these environment variables for the ARTEX server, then restart it: + +```dotenv +ARTEX_ARCHIVE_STORAGE=s3 +ARTEX_ARCHIVE_S3_ENDPOINT=https://s3.us-east-1.amazonaws.com +ARTEX_ARCHIVE_S3_REGION=us-east-1 +ARTEX_ARCHIVE_S3_BUCKET=your-artex-archives +ARTEX_ARCHIVE_S3_PREFIX=artex/tasks +ARTEX_ARCHIVE_S3_ACCESS_KEY_ID=REPLACE +ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY=REPLACE +``` + +Store credentials in the service's protected environment file, outside the +repository and frontend. For systemd, a root-owned environment file with mode +0600 can be referenced through `EnvironmentFile=`. + +| Variable | Meaning | +| --- | --- | +| `ARTEX_ARCHIVE_STORAGE` | `local` (default) or `s3`. | +| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local testing. | +| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. Match the provider's requirements. | +| `ARTEX_ARCHIVE_S3_BUCKET` | Required bucket name. | +| `ARTEX_ARCHIVE_S3_PREFIX` | Object prefix; defaults to `artex/tasks`. | +| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Dedicated access key. | +| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Dedicated secret key. | +| `ARTEX_ARCHIVE_S3_PATH_STYLE` | Set to `true` for compatible path-style servers; otherwise virtual-hosted addressing is used. | + +For DigitalOcean Spaces, use its regional origin, such as +`https://sfo3.digitaloceanspaces.com`, and the provider's signing region. For a +local path-style test server, use an endpoint such as `http://127.0.0.1:9000` and +set `ARTEX_ARCHIVE_S3_PATH_STYLE=true`. + +Invalid S3 configuration fails new archive jobs instead of silently falling back +to local disk. Preserve the configured bucket, endpoint, prefix and read/delete +access while remote archives remain. The same installation can still restore or +delete older local archives. + +## Lifecycle and integrity + +1. Pause or finish a task and use its existing **Archive** action. +2. ARTEX builds its `.tar.zst` package locally and uploads it. Large packages use + multipart upload, starting with 8 MiB parts and two upload workers (the SDK + increases the part size for exceptionally large seekable files). +3. ARTEX reads the entire object back and checks its size and SHA-256. Multipart + ETags and uploaded checksum metadata alone are not treated as proof of content. +4. Only after verification does PostgreSQL commit the `s3://` location and + compact the archived task. ARTEX then removes local source files and the + temporary package. The archive list shows **Object storage** or **Local storage**. +5. **Restore** downloads and verifies the package before restoring task files and + database records. The remote package is consumed after restoration commits. + Failed remote cleanup can be retried without replaying restored records. +6. **Delete** removes the archive stub and queues durable cleanup of its remote + object. Cleanup is retried after restart and before subsequent archive jobs. + +Reads use bounded 8 MiB ranges pinned to the object's ETag. Interrupted ranges +are retried without duplicating output or restarting a multi-gigabyte download. +An object change, checksum mismatch or invalid range fails verification. Failed +downloads are removed, and upload/verification failures retain the original task. + +There is no database schema or archive format migration. Existing local archive +packages remain compatible. Files outside ARTEX's task archive database are not +automatically imported or uploaded. + +## Disk, transfer and recovery + +Archive creation still needs space for the temporary compressed package. +Restoration needs space for the downloaded package and extracted files. Full +upload readback and later restores incur object-storage reads and any applicable +transfer charges. Configure expiration of abandoned multipart uploads for abrupt +process termination. + +Recovery journals under `data/archives/tasks/.remote` contain object locations, +not credentials. Preserve them with PostgreSQL and the installation's data +directory. The worker uses these journals to clean up uploads that never +committed, keep committed objects, and finish deferred deletions. Failed remote +cleanup does not block unrelated local archive jobs. + +## Optional connectivity check + +Run `go run ./cmd/artex-archive-check` with the same archive environment as the +server to upload, verify, download and delete a small uniquely named test object. +This command performs real bucket operations but does not touch task data. It +reports leftover test objects if cleanup fails. + +Automated tests use an HTTP S3 fixture and isolated PostgreSQL databases. They +cover multipart transfers, corruption, interrupted range retries, object changes, +cancellation, upload rollback, corrupted restores and persistent cleanup. diff --git a/go.mod b/go.mod index ba11f2c..7eb3dcf 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,10 @@ go 1.26.10 require ( github.com/Autumn-27/norma v0.4.10 + github.com/aws/aws-sdk-go-v2 v1.47.5 + github.com/aws/aws-sdk-go-v2/credentials v1.20.13 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 @@ -22,6 +26,15 @@ require ( require ( github.com/andybalholm/brotli v1.2.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect + github.com/aws/smithy-go v1.28.7 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/creack/pty v1.1.24 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/go.sum b/go.sum index 15b752d..bb0b1af 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,44 @@ github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eT github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4= +github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 h1:MW0Iql1SOTgQPj8mCxrU/eH/K/AIDWi08ItOsXaWRLU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25/go.mod h1:+ucVYmHYB3JqVueY0E1RHahdXAnKQbIHqybSbvOUbOM= +github.com/aws/aws-sdk-go-v2/config v1.33.13 h1:u52x6ISQOx172tpwM0VRxhJzEsTJYpMcEYUGM2A1wrw= +github.com/aws/aws-sdk-go-v2/config v1.33.13/go.mod h1:/ixtz6XPHb42zekjQ0LqD9XjTpTysNWhIGwufY1OzI4= +github.com/aws/aws-sdk-go-v2/credentials v1.20.13 h1:j4z41OtspmQ3ftQycaDunJ26b69a9SHfaSVUPqgRv8k= +github.com/aws/aws-sdk-go-v2/credentials v1.20.13/go.mod h1:SgPA1oS3b+dlMxZb618jDmGNscpXSCaxz15igS52OlE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19 h1:+JWfo9Mkg5HWwDwnpRPu7WNiiyEmiQRTaNSFsZ/1KIY= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19/go.mod h1:l4/tI+TfymypMqq2oRCkhMAB7Gi7nlt5JaoUUz+cUbI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 h1:GACgECI422Xt5n4Pojr3O0aocPv8lfH4PJe2Aon5Wv8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9/go.mod h1:lPfnRkHpn1qWZJLbVEw10rIqSuCee1pNQTyGkUjAYh8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 h1:RS1Wt+K0r4X0ci9Vf4JYTUfHzsQihYA2LgGJeEneKpk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8/go.mod h1:adMnTy5kyoUW86jhXnGSVc/mvMQI4GZd1qYWMeaS8M8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 h1:to7+Vr3rLkpYRwVKJ5QRriI0eE306vrVN/6TnVHgxvY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6/go.mod h1:Jc89ASa2k7HMAskj3bSnt8bf+k+p2IApzkoXnEjkgUw= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.1 h1:WZLC+k/CZx11kZGQDqu9mpA8yhiXGX2I//jhuJE1FDo= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.1/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.1 h1:4VB/dnDo7pmXSLQMszSoBzWndENuc7mzwTpD6lQn77A= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.1/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 h1:w8yui2ZqdHmGwVBDGvazoIvKw/DnLENBLkYnN8q8iqo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw= +github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE= +github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= diff --git a/server/server.go b/server/server.go index ac78a25..0d4134b 100644 --- a/server/server.go +++ b/server/server.go @@ -22,6 +22,7 @@ import ( "time" "github.com/Autumn-27/artex/agent" + "github.com/Autumn-27/artex/archivestore" "github.com/Autumn-27/artex/db" "github.com/Autumn-27/artex/egress" "github.com/Autumn-27/artex/intercept" @@ -127,9 +128,11 @@ type Server struct { // Cold task archives run through one persistent FIFO worker. The buffered wake // channel coalesces enqueue bursts; the database remains the source of truth. - archiveWake chan struct{} - archiveWG sync.WaitGroup - side *sideQuestionState + archiveWake chan struct{} + archiveWG sync.WaitGroup + archiveStore *archivestore.Store + archiveStoreErr error + side *sideQuestionState } // provEntry is a cached provider + its config for one LLM profile id. diff --git a/server/task_archive_package.go b/server/task_archive_package.go index f23c4c8..e9a451d 100644 --- a/server/task_archive_package.go +++ b/server/task_archive_package.go @@ -428,7 +428,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { errs = append(errs, getErr) continue } - if archive != nil && archive.State == pgdb.ArchiveReady { + if archive != nil && archive.ArchivePath != "" && archive.SHA256 != "" { if err := os.RemoveAll(root); err != nil { errs = append(errs, err) } @@ -438,7 +438,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { if err := stage.rollback(); err != nil { errs = append(errs, err) } - if archive != nil && archive.ArchivePath != "" { + if archive != nil && archive.ArchivePath != "" && !strings.HasPrefix(archive.ArchivePath, "s3://") { _ = os.Remove(archive.ArchivePath) } } diff --git a/server/task_archive_remote.go b/server/task_archive_remote.go new file mode 100644 index 0000000..87e230f --- /dev/null +++ b/server/task_archive_remote.go @@ -0,0 +1,171 @@ +package server + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + pgdb "github.com/Autumn-27/artex/db" +) + +// An outbox bridges the local PostgreSQL transaction and object storage. Never +// delete the only package before metadata permits it. Unfinished cleanup is +// retried after a restart (and before subsequent jobs). +type remoteArchiveJournal struct { + ArchiveID int64 `json:"archive_id"` + URI string `json:"uri"` + LocalPath string `json:"local_path,omitempty"` + Operation string `json:"operation"` +} + +func (s *Server) validateStoredArchive(uri string) error { + if strings.HasPrefix(uri, "s3://") { + return s.archiveStore.Validate(uri) + } + return validateArchivePath(s.m.dir, uri) +} + +func removeLocalArchivePackage(filename string) error { + if err := os.Remove(filename); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +func (s *Server) removeStoredArchive(uri string) error { + if err := s.validateStoredArchive(uri); err != nil { + return err + } + if strings.HasPrefix(uri, "s3://") { + return s.archiveStore.Delete(s.ctx, uri) + } + return removeLocalArchivePackage(uri) +} + +func (s *Server) writeRemoteArchiveJournal(j remoteArchiveJournal) (string, error) { + dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote") + if err := os.MkdirAll(dir, archiveDirMode); err != nil { + return "", err + } + f, err := os.CreateTemp(dir, ".journal-") + if err != nil { + return "", err + } + tmp := f.Name() + defer os.Remove(tmp) + encodeErr := json.NewEncoder(f).Encode(j) + syncErr := f.Sync() + if err := errors.Join(encodeErr, syncErr, f.Close()); err != nil { + return "", err + } + name := tmp + ".json" + if err := os.Rename(tmp, name); err != nil { + return "", err + } + // Persist the directory entry before an upload or database deletion starts. + d, err := os.Open(dir) + if err != nil { + return "", err + } + if err := errors.Join(d.Sync(), d.Close()); err != nil { + return "", err + } + // The .remote directory itself may have just been created. + parent, err := os.Open(filepath.Dir(dir)) + if err != nil { + return "", err + } + return name, errors.Join(parent.Sync(), parent.Close()) +} + +func (s *Server) recoverRemoteArchiveJournals() error { + dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote") + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + var errs []error + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + name := filepath.Join(dir, entry.Name()) + if err := s.recoverRemoteArchiveJournal(name); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (s *Server) recoverRemoteArchiveJournal(name string) error { + raw, err := os.ReadFile(name) + if err != nil { + return err + } + var j remoteArchiveJournal + if err := json.Unmarshal(raw, &j); err != nil { + return err + } + if j.ArchiveID <= 0 { + return errors.New("invalid remote archive journal ID") + } + if err := s.archiveStore.Validate(j.URI); err != nil { + return err + } + item, err := s.m.pg.GetTaskArchive(j.ArchiveID) + if err != nil { + return err + } + switch j.Operation { + case "upload": + if item == nil || item.ArchivePath != j.URI { + if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil { + return err + } + } + if err := validateArchivePath(s.m.dir, j.LocalPath); err != nil { + return err + } + // An old failed upload may outlive a retry after switching back to + // local storage. The retry reuses this pathname: keep its committed + // package instead of treating it as the old upload's temporary copy. + if item == nil || item.ArchivePath != j.LocalPath { + if err := removeLocalArchivePackage(j.LocalPath); err != nil { + return err + } + } + case "delete": + if item == nil { + if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil { + return err + } + } + default: + return fmt.Errorf("unknown remote archive operation %q", j.Operation) + } + return os.Remove(name) +} + +func (s *Server) deleteRemoteTaskArchive(job *pgdb.TaskArchive) error { + journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, Operation: "delete"}) + if err != nil { + return err + } + if err := s.m.pg.DeleteTaskArchiveStub(job.ID); err != nil { + // A connection failure can leave the transaction's commit ambiguous. + // Recovery consults PostgreSQL before deciding whether to delete. + return err + } + if err := s.recoverRemoteArchiveJournal(journal); err != nil { + // The task is gone; the durable outbox retries the remaining object cleanup. + log.Printf("[task-archive] archive %d deleted; remote cleanup pending: %v", job.ID, err) + } + return nil +} diff --git a/server/task_archive_remote_test.go b/server/task_archive_remote_test.go new file mode 100644 index 0000000..34e0848 --- /dev/null +++ b/server/task_archive_remote_test.go @@ -0,0 +1,396 @@ +package server + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/Autumn-27/artex/archivestore" + pgdb "github.com/Autumn-27/artex/db" +) + +func TestRemoteArchiveLifecycle(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + t.Setenv("ARTEX_ARCHIVE_STORAGE", "local") + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir()) + s.archiveWG.Wait() + s.ctx = context.Background() + var mu sync.Mutex + objects := map[string][]byte{} + corrupt, failDelete := false, false + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + switch r.Method { + case "PUT": + objects[r.URL.Path], _ = io.ReadAll(r.Body) + w.Header().Set("ETag", `"test"`) + case "HEAD": + w.Header().Set("ETag", `"test"`) + w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path]))) + case "GET": + data := objects[r.URL.Path] + if corrupt && len(data) > 0 { + data = bytes.Clone(data) + data[0] ^= 0xff + } + var start, end int64 + if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) || r.Header.Get("If-Match") != `"test"` { + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return + } + w.Header().Set("ETag", `"test"`) + w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(data[start : end+1]) + case "DELETE": + if failDelete { + w.WriteHeader(403) + return + } + delete(objects, r.URL.Path) + w.WriteHeader(204) + default: + w.WriteHeader(400) + } + })) + defer remote.Close() + s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + task, err := m.CreateTask("S3 round trip", "preserve data", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + id, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }() + if err := m.pg.SetPaused(id, true); err != nil { + t.Fatal(err) + } + file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "important.txt") + if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte("keep this content"), 0600); err != nil { + t.Fatal(err) + } + pgTask, err := m.pg.GetTask(id) + if err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exploration(pgTask.ExplorationID).AppendActivity(pgdb.Activity{Worker: "system", Kind: "system", Summary: "archive proof"}); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + // A configured-but-invalid backend must fail before moving any source data. + s.archiveStoreErr = errors.New("invalid fixture configuration") + if err := s.runOneTaskArchiveJob(); !errors.Is(err, s.archiveStoreErr) { + t.Fatalf("configuration error was not returned: %v", err) + } + if got, err := os.ReadFile(file); err != nil || string(got) != "keep this content" { + t.Fatalf("bad configuration lost source: %v", err) + } + if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil { + t.Fatal("bad configuration lost task", err) + } + s.archiveStoreErr = nil + if _, err := m.pg.QueueTaskArchive(id); err != nil { + t.Fatal(err) + } + // A corrupt readback must roll back filesystem staging and leave PG intact. + mu.Lock() + corrupt = true + mu.Unlock() + if err := s.runOneTaskArchiveJob(); err == nil { + t.Fatal("expected integrity failure") + } + if got, _ := os.ReadFile(file); string(got) != "keep this content" { + t.Fatal("upload failure lost source") + } + if task, err := m.pg.GetTask(id); err != nil || task == nil { + t.Fatal("upload failure lost task") + } + mu.Lock() + corrupt = false + mu.Unlock() + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + if _, err := m.pg.QueueTaskArchive(id); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + job, err = m.pg.GetTaskArchive(job.ID) + if err != nil { + t.Fatal(err) + } + if job.State != pgdb.ArchiveReady || job.Storage != "s3" || !strings.HasPrefix(job.ArchivePath, "s3://") { + t.Fatalf("unexpected archive %+v", job) + } + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Fatal("workspace not released") + } + if _, err := os.Stat(taskArchivePath(m.dir, job.ID, task.ID)); !os.IsNotExist(err) { + t.Fatal("local package retained") + } + // Simulate the crash window after commit: cleanup keeps the remote object. + journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, LocalPath: taskArchivePath(m.dir, job.ID, task.ID), Operation: "upload"}) + if err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournal(journal); err != nil { + t.Fatal(err) + } + // A corrupted download must leave the archive and its remote package intact. + mu.Lock() + corrupt = true + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err == nil { + t.Fatal("expected corrupted restore to fail") + } + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Fatal("corrupted archive installed workspace files") + } + if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || restored { + t.Fatalf("corrupted archive restored database: %v %v", restored, err) + } + mu.Lock() + if len(objects) == 0 { + t.Error("corrupted restore deleted the only remote package") + } + corrupt = false + mu.Unlock() + // Restoration commits files and PG first. Failed remote deletion is retryable. + mu.Lock() + failDelete = true + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err == nil { + t.Fatal("expected remote delete failure") + } + if got, _ := os.ReadFile(file); string(got) != "keep this content" { + t.Fatal("restored content differs") + } + if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || !restored { + t.Fatalf("PG restore %v %v", restored, err) + } + mu.Lock() + failDelete = false + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + if item, err := m.pg.GetTaskArchive(job.ID); err != nil || item != nil { + t.Fatal("restore did not consume stub") + } + var count int + if err := m.pg.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='archive proof'`, pgTask.ExplorationID).Scan(&count); err != nil || count != 1 { + t.Fatalf("activity restore count=%d err=%v", count, err) + } + // Re-archive, then crash/retry the deletion outbox after PG removed its stub. + job, err = m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + mu.Lock() + failDelete = true + mu.Unlock() + if _, err := m.pg.QueueTaskArchiveDelete(job.ID); err != nil { + t.Fatal(err) + } + if err := s.runOneTaskArchiveJob(); err != nil { + t.Fatal(err) + } + mu.Lock() + failDelete = false + mu.Unlock() + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + mu.Lock() + remaining := len(objects) + mu.Unlock() + if remaining != 0 { + t.Fatalf("left %d objects", remaining) + } +} + +// Once PostgreSQL has committed an archive location and checksum, a subsequent +// queued operation must not make startup recovery resurrect the old workspace. +func TestRemoteArchiveStageRecoveryAfterCommit(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + task, err := m.CreateTask("remote stage recovery", "test", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + taskID, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { + _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID) + _ = m.pg.DeleteTask(taskID) + }() + if err := m.pg.SetPaused(taskID, true); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(taskID) + if err != nil { + t.Fatal(err) + } + pgTask, err := m.pg.GetTask(taskID) + if err != nil { + t.Fatal(err) + } + for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} { + t.Run(state, func(t *testing.T) { + committed := state != pgdb.ArchiveQueued + checksum := "" + if committed { + checksum = strings.Repeat("a", 64) + } + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, "s3://test/tasks/package.tar.zst", checksum); err != nil { + t.Fatal(err) + } + file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt") + if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil { + t.Fatal(err) + } + stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID) + if err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stage.root); !os.IsNotExist(err) { + t.Fatalf("staging cleanup incomplete: %v", err) + } + if committed { + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Fatalf("resurrected committed workspace: %v", err) + } + } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" { + t.Fatalf("uncommitted archive lost source: %v", err) + } + }) + } +} + +// Failed remote cleanup can survive switching to local storage and retrying the +// same archive ID. Its journal must never remove the newly committed local copy. +func TestRemoteArchiveJournalPreservesLocalRetry(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + task, err := m.CreateTask("local retry", "test", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + taskID, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { + _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID) + _ = m.pg.DeleteTask(taskID) + }() + if err := m.pg.SetPaused(taskID, true); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(taskID) + if err != nil { + t.Fatal(err) + } + filename := taskArchivePath(m.dir, job.ID, task.ID) + if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filename, []byte("committed local archive"), 0600); err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, pgdb.ArchiveReady, filename, strings.Repeat("a", 64)); err != nil { + t.Fatal(err) + } + var deletes atomic.Int32 + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + deletes.Add(1) + w.WriteHeader(http.StatusNoContent) + })) + defer remote.Close() + store, err := archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + s := &Server{ctx: context.Background(), m: m, archiveStore: store} + journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ + ArchiveID: job.ID, URI: store.URI(taskID, job.ID, strings.Repeat("b", 64)), + LocalPath: filename, Operation: "upload", + }) + if err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournal(journal); err != nil { + t.Fatal(err) + } + if data, err := os.ReadFile(filename); err != nil || string(data) != "committed local archive" { + t.Fatalf("old upload cleanup removed local retry: %v", err) + } + if deletes.Load() != 1 { + t.Fatal("uncommitted remote object was not removed") + } + if _, err := os.Stat(journal); !os.IsNotExist(err) { + t.Fatalf("finished journal remains: %v", err) + } +} diff --git a/server/task_archives.go b/server/task_archives.go index b73dfb2..3ac6341 100644 --- a/server/task_archives.go +++ b/server/task_archives.go @@ -14,6 +14,7 @@ import ( "time" "github.com/Autumn-27/artex/agent" + "github.com/Autumn-27/artex/archivestore" pgdb "github.com/Autumn-27/artex/db" "github.com/Autumn-27/artex/traffic" ) @@ -37,6 +38,12 @@ func (s *Server) startTaskArchiveWorker() { if s.m == nil || s.m.pg == nil { return } + s.archiveStore, s.archiveStoreErr = archivestore.FromEnv() + if s.archiveStoreErr != nil { + log.Printf("[task-archive] storage configuration: %v", s.archiveStoreErr) + } else if s.archiveStore != nil { + log.Printf("[task-archive] new archives use S3 object storage with full SHA-256 readback verification") + } if err := s.m.pg.RecoverTaskArchiveJobs(); err != nil { log.Printf("[task-archive] recover jobs: %v", err) } @@ -88,6 +95,14 @@ func (s *Server) notifyTaskArchiveWorker() { } func (s *Server) runOneTaskArchiveJob() error { + if s.ctx.Err() != nil { + return s.ctx.Err() + } + // Retry deferred remote cleanup before claiming more work. Failed cleanup + // keeps its durable journal and does not block unrelated local archives. + if err := s.recoverRemoteArchiveJournals(); err != nil { + log.Printf("[task-archive] deferred object cleanup: %v", err) + } job, err := s.m.pg.ClaimTaskArchiveJob(s.ctx) if err != nil || job == nil { return err @@ -115,6 +130,9 @@ func (s *Server) runOneTaskArchiveJob() error { } func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { + if s.archiveStoreErr != nil { + return s.archiveStoreErr + } taskID := strconv.FormatInt(job.TaskID, 10) if !s.beginTaskDelete(taskID) { return errors.New("task is already undergoing another archive or delete operation") @@ -199,6 +217,21 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { _ = os.Remove(archivePath) } }() + storedPath := archivePath + var remoteJournal string + if s.archiveStore != nil { + storedPath = s.archiveStore.URI(job.TaskID, job.ID, checksum) + remoteJournal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ + ArchiveID: job.ID, URI: storedPath, LocalPath: archivePath, Operation: "upload", + }) + if err != nil { + return err + } + _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "upload_and_verify", 65) + if err := s.archiveStore.Upload(s.ctx, storedPath, archivePath, checksum); err != nil { + return err + } + } var trafficStage *traffic.HostDeleteStage if s.m.traffic != nil && len(snapshot.ExclusiveHosts) > 0 { _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_traffic", 68) @@ -213,11 +246,18 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { }() } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_database", 78) - if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, archivePath, checksum, originalSize, compressedSize); err != nil { + if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, storedPath, checksum, originalSize, compressedSize); err != nil { return err } committed = true - removePackage = false + removePackage = s.archiveStore != nil + if remoteJournal != "" { + // The journal also covers a crash after database commit but before local + // package removal. Keep it if cleanup fails, so the worker can retry. + if err := removeLocalArchivePackage(archivePath); err == nil { + _ = os.Remove(remoteJournal) + } + } if trafficStage != nil { if err := trafficStage.Commit(); err != nil { warning := "archive completed, but cleaning the exclusive traffic hot store failed (the archive package can still restore it): " + err.Error() @@ -239,7 +279,7 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) { } func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { - if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil { + if err := s.validateStoredArchive(job.ArchivePath); err != nil { return err } // A prior attempt may have committed PostgreSQL and files, then failed only @@ -247,7 +287,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { if restored, err := s.m.pg.IsTaskArchiveRestored(job.ID); err != nil { return err } else if restored { - if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) { + if err := s.removeStoredArchive(job.ArchivePath); err != nil { return err } if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil { @@ -266,7 +306,15 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) { return err } defer func() { _ = os.RemoveAll(extracted) }() - if err := extractTaskArchivePackage(job.ArchivePath, job.SHA256, extracted); err != nil { + packagePath := job.ArchivePath + if strings.HasPrefix(packagePath, "s3://") { + _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "download_and_verify", 12) + packagePath = filepath.Join(extracted, "download.tar.zst") + if err := s.archiveStore.Download(s.ctx, job.ArchivePath, packagePath, job.SHA256, job.CompressedSize); err != nil { + return err + } + } + if err := extractTaskArchivePackage(packagePath, job.SHA256, extracted); err != nil { return err } manifestRaw, err := os.ReadFile(filepath.Join(extracted, "manifest.json")) @@ -356,7 +404,7 @@ func (s *Server) restoreTaskArchivePayload(job *pgdb.TaskArchive, snapshot *pgdb return err } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "consume_package", 95) - if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) { + if err := s.removeStoredArchive(job.ArchivePath); err != nil { return err } if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil { @@ -393,9 +441,12 @@ func (s *Server) loadRestoredTask(taskID int64) { } func (s *Server) deleteTaskArchive(job *pgdb.TaskArchive) error { - if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil { + if err := s.validateStoredArchive(job.ArchivePath); err != nil { return err } + if strings.HasPrefix(job.ArchivePath, "s3://") { + return s.deleteRemoteTaskArchive(job) + } _ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "stage_package_delete", 35) staged, moved, err := stageTaskArchivePackageDelete(job.ArchivePath, job.ID) if err != nil { diff --git a/web/src/app/(main)/function/tasks/page.tsx b/web/src/app/(main)/function/tasks/page.tsx index 6b8ba8e..c2f9cc3 100644 --- a/web/src/app/(main)/function/tasks/page.tsx +++ b/web/src/app/(main)/function/tasks/page.tsx @@ -1494,7 +1494,7 @@ function ArchiveConfirmDialog({ {count === 1 ? "Archive task" : `Archive ${count} tasks`} - Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into local cold storage. Once archived, a task can be restored from "Archived". + Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into the configured archive storage. Once archived, a task can be restored from "Archived". Restoring requires free space on this server. @@ -1934,6 +1934,7 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) { title={`Before compression: ${formatArchiveBytes(archive.original_size)}`} > {archiveCompressionLabel(archive)} +
{archive.storage === "s3" ? "Object storage" : "Local storage"}
{archiveDataTotal(archive).toLocaleString()} @@ -1952,7 +1953,7 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {

))} - {archive.phase} · {archive.progress}% + {archive.phase === "upload_and_verify" ? "Uploading and verifying" : archive.phase === "download_and_verify" ? "Downloading and verifying" : archive.phase} · {archive.progress}%
diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index d098f9e..4a0b479 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -101,6 +101,7 @@ export interface TaskArchiveTokenStats { } export interface TaskArchive { + storage?: "local" | "s3"; id: number; task_id: number; state: TaskArchiveState;