feat: archive cold tasks to S3-compatible object storage

This commit is contained in:
Chubboy Ren
2026-11-17 11:56:15 +00:00
parent 278de8b2e0
commit da16b8d81a
16 changed files with 1520 additions and 14 deletions
+6
View File
@@ -418,6 +418,12 @@ runtime supports it.
`/chat/completions` (or `/responses` for `openai-responses`), so their base **does** end in `/v1`
(for example `https://api.openai.com/v1`).
**Task archive storage**: local disk by default. Set `ARTEX_ARCHIVE_STORAGE=s3` to
store cold task packages in an S3-compatible bucket and release their local
package space after verification. Existing local archives remain usable. See
[Task archive storage](docs/task-archive-storage.md) for configuration, permissions,
integrity checks, temporary disk requirements and recovery.
**Concurrency**: work agents per task, in "System Settings" (default 3).
**Untrusted-output boundary (on by default)**: tool results for the worker, planner, chat and main
+62
View File
@@ -0,0 +1,62 @@
package archivestore
import (
"strings"
"testing"
)
func TestArchiveStorageEnvironment(t *testing.T) {
for _, key := range []string{
"ARTEX_ARCHIVE_STORAGE", "ARTEX_ARCHIVE_S3_ENDPOINT", "ARTEX_ARCHIVE_S3_REGION",
"ARTEX_ARCHIVE_S3_BUCKET", "ARTEX_ARCHIVE_S3_PREFIX", "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID",
"ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_PATH_STYLE",
} {
t.Setenv(key, "")
}
for _, backend := range []string{"", "local"} {
t.Setenv("ARTEX_ARCHIVE_STORAGE", backend)
if store, err := FromEnv(); err != nil || store != nil {
t.Fatalf("local default: %v %v", store, err)
}
}
for _, backend := range []string{"unknown", "s3"} {
t.Setenv("ARTEX_ARCHIVE_STORAGE", backend)
if _, err := FromEnv(); err == nil {
t.Fatalf("%q silently fell back to local", backend)
}
}
t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", "http://127.0.0.1:9000")
t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "test-bucket")
t.Setenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "test-key")
t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "test-secret")
t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true")
store, err := FromEnv()
if err != nil {
t.Fatal(err)
}
if uri := store.URI(1, 2, "checksum"); uri != "s3://test-bucket/artex/tasks/task-1-2-checksum.tar.zst" {
t.Fatal("unexpected default object location", uri)
}
t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "")
if _, err := FromEnv(); err == nil {
t.Fatal("missing credentials accepted")
}
}
func TestArchiveEndpointValidation(t *testing.T) {
config := Config{Bucket: "test", AccessKey: "test-key", SecretKey: "test-secret"}
for _, endpoint := range []string{"https://objects.example.test", "http://127.0.0.1:9000", "http://127.0.0.2:9000", "http://[::1]:9000", "http://localhost:9000"} {
config.Endpoint = endpoint
if _, err := New(config); err != nil {
t.Errorf("rejected %s: %v", endpoint, err)
}
}
for _, endpoint := range []string{"http://objects.example.test", "https://user:test-secret@objects.example.test", "https://objects.example.test/bucket", "https://objects.example.test?token=test-secret", "https://objects.example.test#fragment"} {
config.Endpoint = endpoint
if _, err := New(config); err == nil {
t.Error("accepted endpoint with non-loopback HTTP, credentials, path, query or fragment")
} else if strings.Contains(err.Error(), "test-secret") {
t.Fatal("configuration error disclosed a secret")
}
}
}
+280
View File
@@ -0,0 +1,280 @@
// Package archivestore stores immutable cold-task packages in S3-compatible storage.
package archivestore
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
type Config struct {
Endpoint, Region, Bucket, Prefix, AccessKey, SecretKey string
PathStyle bool
}
type Store struct {
client *s3.Client
bucket, prefix string
}
// FromEnv returns nil for the default local backend. A configured but invalid
// backend is an error, never a silent fallback to the machine's local disk.
func FromEnv() (*Store, error) {
switch os.Getenv("ARTEX_ARCHIVE_STORAGE") {
case "", "local":
return nil, nil
case "s3":
return New(Config{
Endpoint: os.Getenv("ARTEX_ARCHIVE_S3_ENDPOINT"), Region: os.Getenv("ARTEX_ARCHIVE_S3_REGION"),
Bucket: os.Getenv("ARTEX_ARCHIVE_S3_BUCKET"), Prefix: os.Getenv("ARTEX_ARCHIVE_S3_PREFIX"),
AccessKey: os.Getenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID"), SecretKey: os.Getenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY"),
PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true",
})
default:
return nil, errors.New("ARTEX_ARCHIVE_STORAGE must be local or s3")
}
}
func New(c Config) (*Store, error) {
u, err := url.Parse(c.Endpoint)
loopback := false
if err == nil {
loopback = u.Hostname() == "localhost"
if ip := net.ParseIP(u.Hostname()); ip != nil {
loopback = ip.IsLoopback()
}
}
if err != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") ||
(u.Scheme != "https" && !(u.Scheme == "http" && loopback)) {
return nil, errors.New("archive S3 endpoint must be an HTTPS origin (HTTP allowed only on loopback)")
}
c.Prefix = strings.Trim(c.Prefix, "/")
if c.Prefix == "" {
c.Prefix = "artex/tasks"
}
if path.Clean(c.Prefix) != c.Prefix || strings.Contains(c.Prefix, "..") || strings.ContainsAny(c.Prefix, "\\?#") {
return nil, errors.New("invalid archive S3 prefix")
}
if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") || c.AccessKey == "" || c.SecretKey == "" {
return nil, errors.New("archive S3 bucket and dedicated access credentials are required")
}
if c.Region == "" {
c.Region = "us-east-1"
}
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.ResponseHeaderTimeout = 60 * time.Second
client := s3.New(s3.Options{
Region: c.Region, BaseEndpoint: aws.String(c.Endpoint), UsePathStyle: c.PathStyle,
Credentials: credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, ""),
HTTPClient: &http.Client{Transport: transport},
RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired,
})
return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/"}, nil
}
func (s *Store) URI(taskID, archiveID int64, checksum string) string {
return "s3://" + s.bucket + "/" + s.prefix + fmt.Sprintf("task-%d-%d-%s.tar.zst", taskID, archiveID, checksum)
}
func (s *Store) key(uri string) (string, error) {
if s == nil {
return "", errors.New("S3 archive storage is not configured")
}
u, err := url.Parse(uri)
if err != nil || u.Scheme != "s3" || u.Host != s.bucket || u.User != nil || u.RawQuery != "" || u.Fragment != "" {
return "", errors.New("archive object is outside the configured bucket")
}
key := strings.TrimPrefix(u.Path, "/")
if !strings.HasPrefix(key, s.prefix) || len(key) <= len(s.prefix) || path.Clean(key) != key || strings.Contains(key, "\\") {
return "", errors.New("archive object is outside the configured prefix")
}
return key, nil
}
func (s *Store) Validate(uri string) error { _, err := s.key(uri); return err }
// Upload verifies the entire uploaded object by reading it back, including
// multipart uploads whose ETag is not a checksum. Callers retain source data
// until verification succeeds and PostgreSQL commits the remote location.
func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) error {
key, err := s.key(uri)
if err != nil {
return err
}
if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size {
return errors.New("invalid archive SHA-256")
}
f, err := os.Open(filename)
if err != nil {
return err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return err
}
uploader := manager.NewUploader(s.client, func(u *manager.Uploader) {
u.PartSize = 8 << 20
u.Concurrency = 2
u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
})
_, err = uploader.Upload(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.bucket), Key: aws.String(key), Body: f,
ContentType: aws.String("application/zstd"), Metadata: map[string]string{"sha256": checksum},
})
if err != nil {
return fmt.Errorf("upload archive: %w", err)
}
return s.readVerified(ctx, key, io.Discard, checksum, info.Size())
}
func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64) error {
if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size {
return errors.New("invalid archive SHA-256")
}
if expectedSize < 0 {
return errors.New("invalid archive size")
}
object, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)})
if err != nil {
return fmt.Errorf("inspect archive: %w", err)
}
if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" {
return errors.New("remote archive size mismatch or missing ETag; source data retained")
}
// Commit only complete, validated chunks to the output and hash. A broken
// connection can then retry its chunk without duplicating bytes or restarting
// a multi-gigabyte read. IfMatch pins every request to the same object.
h := sha256.New()
for offset := int64(0); offset < expectedSize; {
end := offset + min(archiveReadChunkSize, expectedSize-offset) - 1
data, err := s.readArchiveChunk(ctx, key, aws.ToString(object.ETag), offset, end, expectedSize)
if err != nil {
return err
}
n, err := out.Write(data)
if err != nil {
return err
}
if n != len(data) {
return io.ErrShortWrite
}
_, _ = h.Write(data)
offset += int64(len(data))
}
if !strings.EqualFold(hex.EncodeToString(h.Sum(nil)), checksum) {
return errors.New("remote archive size or SHA-256 mismatch; source data retained")
}
return nil
}
const (
archiveReadChunkSize = int64(8 << 20)
archiveReadAttempts = 5
)
func (s *Store) readArchiveChunk(ctx context.Context, key, etag string, start, end, total int64) ([]byte, error) {
var lastErr error
for attempt := 0; attempt < archiveReadAttempts; attempt++ {
if err := ctx.Err(); err != nil {
return nil, err
}
if attempt > 0 {
timer := time.NewTimer((250 * time.Millisecond) << (attempt - 1))
select {
case <-ctx.Done():
timer.Stop()
return nil, ctx.Err()
case <-timer.C:
}
}
data, retry, err := s.readArchiveChunkOnce(ctx, key, etag, start, end, total)
if err == nil {
return data, nil
}
if !retry {
return nil, err
}
lastErr = err
}
return nil, fmt.Errorf("read archive bytes %d-%d after %d attempts: %w", start, end, archiveReadAttempts, lastErr)
}
func (s *Store) readArchiveChunkOnce(ctx context.Context, key, etag string, start, end, total int64) ([]byte, bool, error) {
requestCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
object, err := s.client.GetObject(requestCtx, &s3.GetObjectInput{
Bucket: aws.String(s.bucket), Key: aws.String(key), IfMatch: aws.String(etag),
Range: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)),
})
if err != nil {
var networkError net.Error
retry := errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded)
return nil, retry, fmt.Errorf("read archive range: %w", err)
}
defer object.Body.Close()
want := end - start + 1
if aws.ToString(object.ETag) != etag || aws.ToString(object.ContentRange) != fmt.Sprintf("bytes %d-%d/%d", start, end, total) || object.ContentLength == nil || *object.ContentLength != want {
return nil, false, errors.New("remote archive changed or returned an invalid byte range; source data retained")
}
data, err := io.ReadAll(io.LimitReader(object.Body, want+1))
if err != nil {
return nil, true, err
}
if int64(len(data)) < want {
return nil, true, io.ErrUnexpectedEOF
}
if int64(len(data)) != want {
return nil, false, errors.New("remote archive range exceeds its declared size")
}
return data, false, nil
}
func (s *Store) Download(ctx context.Context, uri, filename, checksum string, size int64) (err error) {
key, err := s.key(uri)
if err != nil {
return err
}
if size <= 0 {
return errors.New("invalid archive size")
}
f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
return err
}
defer func() {
err = errors.Join(err, f.Close())
if err != nil {
_ = os.Remove(filename)
}
}()
if err = s.readVerified(ctx, key, f, checksum, size); err != nil {
return err
}
return f.Sync()
}
func (s *Store) Delete(ctx context.Context, uri string) error {
key, err := s.key(uri)
if err != nil {
return err
}
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)})
return err
}
+311
View File
@@ -0,0 +1,311 @@
package archivestore
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"sync"
"testing"
)
func TestS3RoundTripAndIntegrity(t *testing.T) {
for _, size := range []int{1024, 10 << 20} {
t.Run(strconv.Itoa(size), func(t *testing.T) {
var mu sync.Mutex
var object []byte
parts := map[int][]byte{}
corrupt, multipart := false, false
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
defer mu.Unlock()
q := r.URL.Query()
w.Header().Set("ETag", `"test-etag"`)
switch {
case r.Method == "POST" && q.Has("uploads"):
multipart = true
fmt.Fprint(w, `<InitiateMultipartUploadResult><UploadId>test-upload</UploadId></InitiateMultipartUploadResult>`)
case r.Method == "PUT" && q.Has("partNumber"):
n, _ := strconv.Atoi(q.Get("partNumber"))
parts[n], _ = io.ReadAll(r.Body)
case r.Method == "POST" && q.Has("uploadId"):
for i := 1; i <= len(parts); i++ {
object = append(object, parts[i]...)
}
fmt.Fprint(w, `<CompleteMultipartUploadResult><ETag>test-etag</ETag></CompleteMultipartUploadResult>`)
case r.Method == "PUT":
object, _ = io.ReadAll(r.Body)
case r.Method == "HEAD":
w.Header().Set("Content-Length", strconv.Itoa(len(object)))
case r.Method == "GET":
data := object
if corrupt {
data = bytes.Clone(object)
data[0] ^= 0xff
}
serveArchiveRange(t, w, r, data)
case r.Method == "DELETE":
object = nil
w.WriteHeader(204)
default:
t.Errorf("unexpected request %s %s", r.Method, r.URL)
w.WriteHeader(400)
}
}))
defer srv.Close()
store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
if err != nil {
t.Fatal(err)
}
data := bytes.Repeat([]byte{0x42}, size)
h := sha256.Sum256(data)
checksum := hex.EncodeToString(h[:])
root := t.TempDir()
source := filepath.Join(root, "source")
if err := os.WriteFile(source, data, 0600); err != nil {
t.Fatal(err)
}
uri := store.URI(1, 2, checksum)
ctx := context.Background()
if err := store.Upload(ctx, uri, source, checksum); err != nil {
t.Fatal(err)
}
if multipart != (size > 8<<20) {
t.Fatalf("multipart=%v size=%d", multipart, size)
}
dest := filepath.Join(root, "download")
if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil {
t.Fatal(err)
}
got, _ := os.ReadFile(dest)
if !bytes.Equal(got, data) {
t.Fatal("download differs")
}
mu.Lock()
corrupt = true
mu.Unlock()
bad := filepath.Join(root, "bad")
if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil {
t.Fatal("accepted corrupt object")
}
if _, err := os.Stat(bad); !os.IsNotExist(err) {
t.Fatal("left corrupt download")
}
if err := store.Upload(ctx, uri, source, checksum); err == nil {
t.Fatal("upload accepted failed readback")
}
if err := store.Delete(ctx, uri); err != nil {
t.Fatal(err)
}
})
}
}
func serveArchiveRange(t *testing.T, w http.ResponseWriter, r *http.Request, data []byte) {
t.Helper()
var start, end int64
if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) {
t.Errorf("invalid range %q", r.Header.Get("Range"))
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return
}
if r.Header.Get("If-Match") != `"test-etag"` {
t.Errorf("missing object version condition: %q", r.Header.Get("If-Match"))
w.WriteHeader(http.StatusPreconditionFailed)
return
}
w.Header().Set("ETag", `"test-etag"`)
w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10))
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
w.WriteHeader(http.StatusPartialContent)
_, _ = w.Write(data[start : end+1])
}
func TestS3ReadbackRetriesInterruptedRange(t *testing.T) {
data := append(bytes.Repeat([]byte{0x42}, int(archiveReadChunkSize)), bytes.Repeat([]byte{0x73}, 1024)...)
var mu sync.Mutex
var ranges []string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
defer mu.Unlock()
w.Header().Set("ETag", `"test-etag"`)
if r.Method == "HEAD" {
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
return
}
ranges = append(ranges, r.Header.Get("Range"))
if len(ranges) == 2 {
w.Header().Set("Content-Length", strconv.FormatInt(int64(len(data))-archiveReadChunkSize, 10))
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", archiveReadChunkSize, len(data)-1, len(data)))
w.WriteHeader(http.StatusPartialContent)
_, _ = w.Write(data[archiveReadChunkSize : archiveReadChunkSize+128]) // Close during a later chunk.
return
}
serveArchiveRange(t, w, r, data)
}))
defer srv.Close()
store := testRangeStore(t, srv.URL)
h := sha256.Sum256(data)
checksum := hex.EncodeToString(h[:])
dest := filepath.Join(t.TempDir(), "archive")
if err := store.Download(context.Background(), store.URI(1, 2, checksum), dest, checksum, int64(len(data))); err != nil {
t.Fatal(err)
}
got, err := os.ReadFile(dest)
if err != nil || !bytes.Equal(got, data) {
t.Fatalf("retried download differs: %v", err)
}
mu.Lock()
defer mu.Unlock()
if len(ranges) != 3 || ranges[1] != ranges[2] || ranges[0] != fmt.Sprintf("bytes=0-%d", archiveReadChunkSize-1) || ranges[2] != fmt.Sprintf("bytes=%d-%d", archiveReadChunkSize, len(data)-1) {
t.Fatalf("retry did not stay within the interrupted range: %v", ranges)
}
}
func testRangeStore(t *testing.T, endpoint string) *Store {
t.Helper()
s, err := New(Config{Endpoint: endpoint, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
if err != nil {
t.Fatal(err)
}
return s
}
func TestS3ReadbackRejectsInvalidRangesAndExhaustedRetries(t *testing.T) {
for _, mode := range []string{"ignored-range", "changed-etag", "changed-object", "truncated"} {
t.Run(mode, func(t *testing.T) {
data := bytes.Repeat([]byte{0x42}, 1024)
var mu sync.Mutex
gets := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
defer mu.Unlock()
w.Header().Set("ETag", `"test-etag"`)
if r.Method == "HEAD" {
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
return
}
gets++
if mode == "changed-object" {
w.WriteHeader(http.StatusPreconditionFailed)
return
}
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
if mode == "ignored-range" {
_, _ = w.Write(data)
return
}
if mode == "changed-etag" {
w.Header().Set("ETag", `"different-object"`)
}
w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", len(data)-1, len(data)))
w.WriteHeader(http.StatusPartialContent)
if mode == "truncated" {
_, _ = w.Write(data[:128])
} else {
_, _ = w.Write(data)
}
}))
defer srv.Close()
store := testRangeStore(t, srv.URL)
h := sha256.Sum256(data)
checksum := hex.EncodeToString(h[:])
dest := filepath.Join(t.TempDir(), "archive")
if err := store.Download(context.Background(), store.URI(1, 2, checksum), dest, checksum, int64(len(data))); err == nil {
t.Fatal("accepted invalid or incomplete readback")
}
if _, err := os.Stat(dest); !os.IsNotExist(err) {
t.Fatal("left a failed download on disk")
}
mu.Lock()
defer mu.Unlock()
want := 1
if mode == "truncated" {
want = archiveReadAttempts
}
if gets != want {
t.Fatalf("GET count %d, want %d", gets, want)
}
})
}
}
type failedArchiveWriter struct{ err error }
func (w failedArchiveWriter) Write([]byte) (int, error) { return 0, w.err }
func TestS3ReadbackPropagatesOutputFailure(t *testing.T) {
data := []byte("test archive")
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("ETag", `"test-etag"`)
if r.Method == "HEAD" {
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
return
}
serveArchiveRange(t, w, r, data)
}))
defer srv.Close()
store := testRangeStore(t, srv.URL)
h := sha256.Sum256(data)
outputError := errors.New("output disk full")
err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data)))
if !errors.Is(err, outputError) {
t.Fatalf("lost output error: %v", err)
}
}
func TestS3ReadbackStopsOnCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
data := bytes.Repeat([]byte{0x42}, 1024)
gets := make(chan struct{}, archiveReadAttempts+1)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("ETag", `"test-etag"`)
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
if r.Method == "HEAD" {
return
}
gets <- struct{}{}
w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", len(data)-1, len(data)))
w.WriteHeader(http.StatusPartialContent)
_, _ = w.Write(data[:128])
w.(http.Flusher).Flush()
cancel()
}))
defer srv.Close()
store := testRangeStore(t, srv.URL)
h := sha256.Sum256(data)
checksum := hex.EncodeToString(h[:])
dest := filepath.Join(t.TempDir(), "archive")
err := store.Download(ctx, store.URI(1, 2, checksum), dest, checksum, int64(len(data)))
if !errors.Is(err, context.Canceled) || len(gets) != 1 {
t.Fatalf("cancellation was not honored: error=%v GETs=%d", err, len(gets))
}
if _, err := os.Stat(dest); !os.IsNotExist(err) {
t.Fatal("left a cancelled download on disk")
}
}
func TestS3RejectsForeignLocations(t *testing.T) {
s, err := New(Config{Endpoint: "https://sfo3.digitaloceanspaces.com", Bucket: "allowed", Prefix: "archives/tasks", AccessKey: "test", SecretKey: "test"})
if err != nil {
t.Fatal(err)
}
for _, uri := range []string{"s3://other/archives/tasks/a", "s3://allowed/archives/other/a", "s3://allowed/archives/tasks/../a", "s3://allowed/archives/tasks/%2e%2e/a", "s3://allowed/archives/tasks/a?secret=x", "s3://user@allowed/archives/tasks/a", "file:///etc/passwd"} {
if err := s.Validate(uri); err == nil {
t.Errorf("accepted %s", uri)
}
}
var absent *Store
if absent.Validate("s3://allowed/archives/tasks/a") == nil {
t.Fatal("missing config accepted")
}
}
+64
View File
@@ -0,0 +1,64 @@
// artex-archive-check tests the configured object store without touching tasks.
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"log"
"os"
"path/filepath"
"time"
"github.com/Autumn-27/artex/archivestore"
)
func main() {
if err := check(); err != nil {
log.Fatal(err)
}
}
func check() error {
s, err := archivestore.FromEnv()
if err != nil {
return err
}
if s == nil {
return fmt.Errorf("S3 archive storage is not configured")
}
dir, err := os.MkdirTemp("", "artex-archive-check-")
if err != nil {
return err
}
defer os.RemoveAll(dir)
data := []byte("ARTEX object storage connectivity check " + time.Now().UTC().Format(time.RFC3339Nano))
h := sha256.Sum256(data)
checksum := hex.EncodeToString(h[:])
uri := s.URI(0, time.Now().UnixNano(), checksum)
file := filepath.Join(dir, "check")
if err := os.WriteFile(file, data, 0600); err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
defer func() {
cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := s.Delete(cleanup, uri); err != nil {
log.Printf("test object cleanup failed for %s: %v", uri, err)
}
}()
if err := s.Upload(ctx, uri, file, checksum); err != nil {
return err
}
if err := s.Download(ctx, uri, filepath.Join(dir, "download"), checksum, int64(len(data))); err != nil {
return err
}
if err := s.Delete(ctx, uri); err != nil {
return err
}
fmt.Println("S3 upload, full SHA-256 verification, download and delete passed")
return nil
}
+5
View File
@@ -60,6 +60,7 @@ type TaskArchive struct {
Warnings json.RawMessage `json:"warnings"`
FormatVersion int `json:"format_version"`
ArchivePath string `json:"-"`
Storage string `json:"storage"`
SHA256 string `json:"sha256,omitempty"`
OriginalSize int64 `json:"original_size"`
CompressedSize int64 `json:"compressed_size"`
@@ -161,6 +162,10 @@ func scanTaskArchive(sc interface{ Scan(...any) error }) (*TaskArchive, error) {
if err != nil {
return nil, err
}
item.Storage = "local"
if strings.HasPrefix(item.ArchivePath, "s3://") {
item.Storage = "s3"
}
if err := json.Unmarshal([]byte(sources), &item.SourceTaskIDs); err != nil {
return nil, err
}
+104
View File
@@ -0,0 +1,104 @@
# Task archive storage
Task archives use local disk by default. The optional S3-compatible backend moves
cold task packages to object storage after verifying their contents, releasing
local package space while retaining ARTEX's Archive, Restore and Delete actions.
Active workspaces and the live PostgreSQL database stay local. This feature is
task-level cold storage; use the existing backup workflow for a full installation
backup. A complete recovery plan must also preserve the bucket objects and
access to them.
## Configuration
Create a private bucket and a dedicated credential scoped to the archive prefix.
The backend needs object upload, read (including HEAD and byte ranges), delete,
and multipart upload/abort permissions. Use a separate prefix for each deployment.
Set these environment variables for the ARTEX server, then restart it:
```dotenv
ARTEX_ARCHIVE_STORAGE=s3
ARTEX_ARCHIVE_S3_ENDPOINT=https://s3.us-east-1.amazonaws.com
ARTEX_ARCHIVE_S3_REGION=us-east-1
ARTEX_ARCHIVE_S3_BUCKET=your-artex-archives
ARTEX_ARCHIVE_S3_PREFIX=artex/tasks
ARTEX_ARCHIVE_S3_ACCESS_KEY_ID=REPLACE
ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY=REPLACE
```
Store credentials in the service's protected environment file, outside the
repository and frontend. For systemd, a root-owned environment file with mode
0600 can be referenced through `EnvironmentFile=`.
| Variable | Meaning |
| --- | --- |
| `ARTEX_ARCHIVE_STORAGE` | `local` (default) or `s3`. |
| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local testing. |
| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. Match the provider's requirements. |
| `ARTEX_ARCHIVE_S3_BUCKET` | Required bucket name. |
| `ARTEX_ARCHIVE_S3_PREFIX` | Object prefix; defaults to `artex/tasks`. |
| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Dedicated access key. |
| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Dedicated secret key. |
| `ARTEX_ARCHIVE_S3_PATH_STYLE` | Set to `true` for compatible path-style servers; otherwise virtual-hosted addressing is used. |
For DigitalOcean Spaces, use its regional origin, such as
`https://sfo3.digitaloceanspaces.com`, and the provider's signing region. For a
local path-style test server, use an endpoint such as `http://127.0.0.1:9000` and
set `ARTEX_ARCHIVE_S3_PATH_STYLE=true`.
Invalid S3 configuration fails new archive jobs instead of silently falling back
to local disk. Preserve the configured bucket, endpoint, prefix and read/delete
access while remote archives remain. The same installation can still restore or
delete older local archives.
## Lifecycle and integrity
1. Pause or finish a task and use its existing **Archive** action.
2. ARTEX builds its `.tar.zst` package locally and uploads it. Large packages use
multipart upload, starting with 8 MiB parts and two upload workers (the SDK
increases the part size for exceptionally large seekable files).
3. ARTEX reads the entire object back and checks its size and SHA-256. Multipart
ETags and uploaded checksum metadata alone are not treated as proof of content.
4. Only after verification does PostgreSQL commit the `s3://` location and
compact the archived task. ARTEX then removes local source files and the
temporary package. The archive list shows **Object storage** or **Local storage**.
5. **Restore** downloads and verifies the package before restoring task files and
database records. The remote package is consumed after restoration commits.
Failed remote cleanup can be retried without replaying restored records.
6. **Delete** removes the archive stub and queues durable cleanup of its remote
object. Cleanup is retried after restart and before subsequent archive jobs.
Reads use bounded 8 MiB ranges pinned to the object's ETag. Interrupted ranges
are retried without duplicating output or restarting a multi-gigabyte download.
An object change, checksum mismatch or invalid range fails verification. Failed
downloads are removed, and upload/verification failures retain the original task.
There is no database schema or archive format migration. Existing local archive
packages remain compatible. Files outside ARTEX's task archive database are not
automatically imported or uploaded.
## Disk, transfer and recovery
Archive creation still needs space for the temporary compressed package.
Restoration needs space for the downloaded package and extracted files. Full
upload readback and later restores incur object-storage reads and any applicable
transfer charges. Configure expiration of abandoned multipart uploads for abrupt
process termination.
Recovery journals under `data/archives/tasks/.remote` contain object locations,
not credentials. Preserve them with PostgreSQL and the installation's data
directory. The worker uses these journals to clean up uploads that never
committed, keep committed objects, and finish deferred deletions. Failed remote
cleanup does not block unrelated local archive jobs.
## Optional connectivity check
Run `go run ./cmd/artex-archive-check` with the same archive environment as the
server to upload, verify, download and delete a small uniquely named test object.
This command performs real bucket operations but does not touch task data. It
reports leftover test objects if cleanup fails.
Automated tests use an HTTP S3 fixture and isolated PostgreSQL databases. They
cover multipart transfers, corruption, interrupted range retries, object changes,
cancellation, upload rollback, corrupted restores and persistent cleanup.
+13
View File
@@ -4,6 +4,10 @@ go 1.26.10
require (
github.com/Autumn-27/norma v0.4.10
github.com/aws/aws-sdk-go-v2 v1.47.5
github.com/aws/aws-sdk-go-v2/credentials v1.20.13
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.5.3
@@ -22,6 +26,15 @@ require (
require (
github.com/andybalholm/brotli v1.2.1 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect
github.com/aws/smithy-go v1.28.7 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/creack/pty v1.1.24 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
+38
View File
@@ -4,6 +4,44 @@ github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eT
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4=
github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 h1:MW0Iql1SOTgQPj8mCxrU/eH/K/AIDWi08ItOsXaWRLU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25/go.mod h1:+ucVYmHYB3JqVueY0E1RHahdXAnKQbIHqybSbvOUbOM=
github.com/aws/aws-sdk-go-v2/config v1.33.13 h1:u52x6ISQOx172tpwM0VRxhJzEsTJYpMcEYUGM2A1wrw=
github.com/aws/aws-sdk-go-v2/config v1.33.13/go.mod h1:/ixtz6XPHb42zekjQ0LqD9XjTpTysNWhIGwufY1OzI4=
github.com/aws/aws-sdk-go-v2/credentials v1.20.13 h1:j4z41OtspmQ3ftQycaDunJ26b69a9SHfaSVUPqgRv8k=
github.com/aws/aws-sdk-go-v2/credentials v1.20.13/go.mod h1:SgPA1oS3b+dlMxZb618jDmGNscpXSCaxz15igS52OlE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19 h1:+JWfo9Mkg5HWwDwnpRPu7WNiiyEmiQRTaNSFsZ/1KIY=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19/go.mod h1:l4/tI+TfymypMqq2oRCkhMAB7Gi7nlt5JaoUUz+cUbI=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 h1:GACgECI422Xt5n4Pojr3O0aocPv8lfH4PJe2Aon5Wv8=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9/go.mod h1:lPfnRkHpn1qWZJLbVEw10rIqSuCee1pNQTyGkUjAYh8=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 h1:RS1Wt+K0r4X0ci9Vf4JYTUfHzsQihYA2LgGJeEneKpk=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8/go.mod h1:adMnTy5kyoUW86jhXnGSVc/mvMQI4GZd1qYWMeaS8M8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 h1:to7+Vr3rLkpYRwVKJ5QRriI0eE306vrVN/6TnVHgxvY=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6/go.mod h1:Jc89ASa2k7HMAskj3bSnt8bf+k+p2IApzkoXnEjkgUw=
github.com/aws/aws-sdk-go-v2/service/signin v1.12.1 h1:WZLC+k/CZx11kZGQDqu9mpA8yhiXGX2I//jhuJE1FDo=
github.com/aws/aws-sdk-go-v2/service/signin v1.12.1/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y=
github.com/aws/aws-sdk-go-v2/service/sso v1.39.1 h1:4VB/dnDo7pmXSLQMszSoBzWndENuc7mzwTpD6lQn77A=
github.com/aws/aws-sdk-go-v2/service/sso v1.39.1/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 h1:w8yui2ZqdHmGwVBDGvazoIvKw/DnLENBLkYnN8q8iqo=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw=
github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE=
github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
+6 -3
View File
@@ -22,6 +22,7 @@ import (
"time"
"github.com/Autumn-27/artex/agent"
"github.com/Autumn-27/artex/archivestore"
"github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/egress"
"github.com/Autumn-27/artex/intercept"
@@ -127,9 +128,11 @@ type Server struct {
// Cold task archives run through one persistent FIFO worker. The buffered wake
// channel coalesces enqueue bursts; the database remains the source of truth.
archiveWake chan struct{}
archiveWG sync.WaitGroup
side *sideQuestionState
archiveWake chan struct{}
archiveWG sync.WaitGroup
archiveStore *archivestore.Store
archiveStoreErr error
side *sideQuestionState
}
// provEntry is a cached provider + its config for one LLM profile id.
+2 -2
View File
@@ -428,7 +428,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error {
errs = append(errs, getErr)
continue
}
if archive != nil && archive.State == pgdb.ArchiveReady {
if archive != nil && archive.ArchivePath != "" && archive.SHA256 != "" {
if err := os.RemoveAll(root); err != nil {
errs = append(errs, err)
}
@@ -438,7 +438,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error {
if err := stage.rollback(); err != nil {
errs = append(errs, err)
}
if archive != nil && archive.ArchivePath != "" {
if archive != nil && archive.ArchivePath != "" && !strings.HasPrefix(archive.ArchivePath, "s3://") {
_ = os.Remove(archive.ArchivePath)
}
}
+171
View File
@@ -0,0 +1,171 @@
package server
import (
"encoding/json"
"errors"
"fmt"
"log"
"os"
"path/filepath"
"strings"
pgdb "github.com/Autumn-27/artex/db"
)
// An outbox bridges the local PostgreSQL transaction and object storage. Never
// delete the only package before metadata permits it. Unfinished cleanup is
// retried after a restart (and before subsequent jobs).
type remoteArchiveJournal struct {
ArchiveID int64 `json:"archive_id"`
URI string `json:"uri"`
LocalPath string `json:"local_path,omitempty"`
Operation string `json:"operation"`
}
func (s *Server) validateStoredArchive(uri string) error {
if strings.HasPrefix(uri, "s3://") {
return s.archiveStore.Validate(uri)
}
return validateArchivePath(s.m.dir, uri)
}
func removeLocalArchivePackage(filename string) error {
if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
func (s *Server) removeStoredArchive(uri string) error {
if err := s.validateStoredArchive(uri); err != nil {
return err
}
if strings.HasPrefix(uri, "s3://") {
return s.archiveStore.Delete(s.ctx, uri)
}
return removeLocalArchivePackage(uri)
}
func (s *Server) writeRemoteArchiveJournal(j remoteArchiveJournal) (string, error) {
dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote")
if err := os.MkdirAll(dir, archiveDirMode); err != nil {
return "", err
}
f, err := os.CreateTemp(dir, ".journal-")
if err != nil {
return "", err
}
tmp := f.Name()
defer os.Remove(tmp)
encodeErr := json.NewEncoder(f).Encode(j)
syncErr := f.Sync()
if err := errors.Join(encodeErr, syncErr, f.Close()); err != nil {
return "", err
}
name := tmp + ".json"
if err := os.Rename(tmp, name); err != nil {
return "", err
}
// Persist the directory entry before an upload or database deletion starts.
d, err := os.Open(dir)
if err != nil {
return "", err
}
if err := errors.Join(d.Sync(), d.Close()); err != nil {
return "", err
}
// The .remote directory itself may have just been created.
parent, err := os.Open(filepath.Dir(dir))
if err != nil {
return "", err
}
return name, errors.Join(parent.Sync(), parent.Close())
}
func (s *Server) recoverRemoteArchiveJournals() error {
dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote")
entries, err := os.ReadDir(dir)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
var errs []error
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
continue
}
name := filepath.Join(dir, entry.Name())
if err := s.recoverRemoteArchiveJournal(name); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
func (s *Server) recoverRemoteArchiveJournal(name string) error {
raw, err := os.ReadFile(name)
if err != nil {
return err
}
var j remoteArchiveJournal
if err := json.Unmarshal(raw, &j); err != nil {
return err
}
if j.ArchiveID <= 0 {
return errors.New("invalid remote archive journal ID")
}
if err := s.archiveStore.Validate(j.URI); err != nil {
return err
}
item, err := s.m.pg.GetTaskArchive(j.ArchiveID)
if err != nil {
return err
}
switch j.Operation {
case "upload":
if item == nil || item.ArchivePath != j.URI {
if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil {
return err
}
}
if err := validateArchivePath(s.m.dir, j.LocalPath); err != nil {
return err
}
// An old failed upload may outlive a retry after switching back to
// local storage. The retry reuses this pathname: keep its committed
// package instead of treating it as the old upload's temporary copy.
if item == nil || item.ArchivePath != j.LocalPath {
if err := removeLocalArchivePackage(j.LocalPath); err != nil {
return err
}
}
case "delete":
if item == nil {
if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil {
return err
}
}
default:
return fmt.Errorf("unknown remote archive operation %q", j.Operation)
}
return os.Remove(name)
}
func (s *Server) deleteRemoteTaskArchive(job *pgdb.TaskArchive) error {
journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, Operation: "delete"})
if err != nil {
return err
}
if err := s.m.pg.DeleteTaskArchiveStub(job.ID); err != nil {
// A connection failure can leave the transaction's commit ambiguous.
// Recovery consults PostgreSQL before deciding whether to delete.
return err
}
if err := s.recoverRemoteArchiveJournal(journal); err != nil {
// The task is gone; the durable outbox retries the remaining object cleanup.
log.Printf("[task-archive] archive %d deleted; remote cleanup pending: %v", job.ID, err)
}
return nil
}
+396
View File
@@ -0,0 +1,396 @@
package server
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"github.com/Autumn-27/artex/archivestore"
pgdb "github.com/Autumn-27/artex/db"
)
func TestRemoteArchiveLifecycle(t *testing.T) {
if os.Getenv("ARTEX_PG_DSN") == "" {
t.Skip("requires isolated PostgreSQL")
}
t.Setenv("ARTEX_ARCHIVE_STORAGE", "local")
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
defer m.Close()
ctx, cancel := context.WithCancel(context.Background())
cancel()
s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
s.archiveWG.Wait()
s.ctx = context.Background()
var mu sync.Mutex
objects := map[string][]byte{}
corrupt, failDelete := false, false
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
defer mu.Unlock()
switch r.Method {
case "PUT":
objects[r.URL.Path], _ = io.ReadAll(r.Body)
w.Header().Set("ETag", `"test"`)
case "HEAD":
w.Header().Set("ETag", `"test"`)
w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path])))
case "GET":
data := objects[r.URL.Path]
if corrupt && len(data) > 0 {
data = bytes.Clone(data)
data[0] ^= 0xff
}
var start, end int64
if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) || r.Header.Get("If-Match") != `"test"` {
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
return
}
w.Header().Set("ETag", `"test"`)
w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10))
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
w.WriteHeader(http.StatusPartialContent)
_, _ = w.Write(data[start : end+1])
case "DELETE":
if failDelete {
w.WriteHeader(403)
return
}
delete(objects, r.URL.Path)
w.WriteHeader(204)
default:
w.WriteHeader(400)
}
}))
defer remote.Close()
s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
if err != nil {
t.Fatal(err)
}
task, err := m.CreateTask("S3 round trip", "preserve data", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
id, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }()
if err := m.pg.SetPaused(id, true); err != nil {
t.Fatal(err)
}
file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "important.txt")
if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(file, []byte("keep this content"), 0600); err != nil {
t.Fatal(err)
}
pgTask, err := m.pg.GetTask(id)
if err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exploration(pgTask.ExplorationID).AppendActivity(pgdb.Activity{Worker: "system", Kind: "system", Summary: "archive proof"}); err != nil {
t.Fatal(err)
}
job, err := m.pg.QueueTaskArchive(id)
if err != nil {
t.Fatal(err)
}
// A configured-but-invalid backend must fail before moving any source data.
s.archiveStoreErr = errors.New("invalid fixture configuration")
if err := s.runOneTaskArchiveJob(); !errors.Is(err, s.archiveStoreErr) {
t.Fatalf("configuration error was not returned: %v", err)
}
if got, err := os.ReadFile(file); err != nil || string(got) != "keep this content" {
t.Fatalf("bad configuration lost source: %v", err)
}
if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil {
t.Fatal("bad configuration lost task", err)
}
s.archiveStoreErr = nil
if _, err := m.pg.QueueTaskArchive(id); err != nil {
t.Fatal(err)
}
// A corrupt readback must roll back filesystem staging and leave PG intact.
mu.Lock()
corrupt = true
mu.Unlock()
if err := s.runOneTaskArchiveJob(); err == nil {
t.Fatal("expected integrity failure")
}
if got, _ := os.ReadFile(file); string(got) != "keep this content" {
t.Fatal("upload failure lost source")
}
if task, err := m.pg.GetTask(id); err != nil || task == nil {
t.Fatal("upload failure lost task")
}
mu.Lock()
corrupt = false
mu.Unlock()
if err := s.recoverRemoteArchiveJournals(); err != nil {
t.Fatal(err)
}
if _, err := m.pg.QueueTaskArchive(id); err != nil {
t.Fatal(err)
}
if err := s.runOneTaskArchiveJob(); err != nil {
t.Fatal(err)
}
job, err = m.pg.GetTaskArchive(job.ID)
if err != nil {
t.Fatal(err)
}
if job.State != pgdb.ArchiveReady || job.Storage != "s3" || !strings.HasPrefix(job.ArchivePath, "s3://") {
t.Fatalf("unexpected archive %+v", job)
}
if _, err := os.Stat(file); !os.IsNotExist(err) {
t.Fatal("workspace not released")
}
if _, err := os.Stat(taskArchivePath(m.dir, job.ID, task.ID)); !os.IsNotExist(err) {
t.Fatal("local package retained")
}
// Simulate the crash window after commit: cleanup keeps the remote object.
journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, LocalPath: taskArchivePath(m.dir, job.ID, task.ID), Operation: "upload"})
if err != nil {
t.Fatal(err)
}
if err := s.recoverRemoteArchiveJournal(journal); err != nil {
t.Fatal(err)
}
// A corrupted download must leave the archive and its remote package intact.
mu.Lock()
corrupt = true
mu.Unlock()
if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
t.Fatal(err)
}
if err := s.runOneTaskArchiveJob(); err == nil {
t.Fatal("expected corrupted restore to fail")
}
if _, err := os.Stat(file); !os.IsNotExist(err) {
t.Fatal("corrupted archive installed workspace files")
}
if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || restored {
t.Fatalf("corrupted archive restored database: %v %v", restored, err)
}
mu.Lock()
if len(objects) == 0 {
t.Error("corrupted restore deleted the only remote package")
}
corrupt = false
mu.Unlock()
// Restoration commits files and PG first. Failed remote deletion is retryable.
mu.Lock()
failDelete = true
mu.Unlock()
if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
t.Fatal(err)
}
if err := s.runOneTaskArchiveJob(); err == nil {
t.Fatal("expected remote delete failure")
}
if got, _ := os.ReadFile(file); string(got) != "keep this content" {
t.Fatal("restored content differs")
}
if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || !restored {
t.Fatalf("PG restore %v %v", restored, err)
}
mu.Lock()
failDelete = false
mu.Unlock()
if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
t.Fatal(err)
}
if err := s.runOneTaskArchiveJob(); err != nil {
t.Fatal(err)
}
if item, err := m.pg.GetTaskArchive(job.ID); err != nil || item != nil {
t.Fatal("restore did not consume stub")
}
var count int
if err := m.pg.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='archive proof'`, pgTask.ExplorationID).Scan(&count); err != nil || count != 1 {
t.Fatalf("activity restore count=%d err=%v", count, err)
}
// Re-archive, then crash/retry the deletion outbox after PG removed its stub.
job, err = m.pg.QueueTaskArchive(id)
if err != nil {
t.Fatal(err)
}
if err := s.runOneTaskArchiveJob(); err != nil {
t.Fatal(err)
}
mu.Lock()
failDelete = true
mu.Unlock()
if _, err := m.pg.QueueTaskArchiveDelete(job.ID); err != nil {
t.Fatal(err)
}
if err := s.runOneTaskArchiveJob(); err != nil {
t.Fatal(err)
}
mu.Lock()
failDelete = false
mu.Unlock()
if err := s.recoverRemoteArchiveJournals(); err != nil {
t.Fatal(err)
}
mu.Lock()
remaining := len(objects)
mu.Unlock()
if remaining != 0 {
t.Fatalf("left %d objects", remaining)
}
}
// Once PostgreSQL has committed an archive location and checksum, a subsequent
// queued operation must not make startup recovery resurrect the old workspace.
func TestRemoteArchiveStageRecoveryAfterCommit(t *testing.T) {
if os.Getenv("ARTEX_PG_DSN") == "" {
t.Skip("requires isolated PostgreSQL")
}
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
defer m.Close()
task, err := m.CreateTask("remote stage recovery", "test", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() {
_, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID)
_ = m.pg.DeleteTask(taskID)
}()
if err := m.pg.SetPaused(taskID, true); err != nil {
t.Fatal(err)
}
job, err := m.pg.QueueTaskArchive(taskID)
if err != nil {
t.Fatal(err)
}
pgTask, err := m.pg.GetTask(taskID)
if err != nil {
t.Fatal(err)
}
for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} {
t.Run(state, func(t *testing.T) {
committed := state != pgdb.ArchiveQueued
checksum := ""
if committed {
checksum = strings.Repeat("a", 64)
}
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, "s3://test/tasks/package.tar.zst", checksum); err != nil {
t.Fatal(err)
}
file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt")
if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil {
t.Fatal(err)
}
stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID)
if err != nil {
t.Fatal(err)
}
if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stage.root); !os.IsNotExist(err) {
t.Fatalf("staging cleanup incomplete: %v", err)
}
if committed {
if _, err := os.Stat(file); !os.IsNotExist(err) {
t.Fatalf("resurrected committed workspace: %v", err)
}
} else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" {
t.Fatalf("uncommitted archive lost source: %v", err)
}
})
}
}
// Failed remote cleanup can survive switching to local storage and retrying the
// same archive ID. Its journal must never remove the newly committed local copy.
func TestRemoteArchiveJournalPreservesLocalRetry(t *testing.T) {
if os.Getenv("ARTEX_PG_DSN") == "" {
t.Skip("requires isolated PostgreSQL")
}
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
defer m.Close()
task, err := m.CreateTask("local retry", "test", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() {
_, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID)
_ = m.pg.DeleteTask(taskID)
}()
if err := m.pg.SetPaused(taskID, true); err != nil {
t.Fatal(err)
}
job, err := m.pg.QueueTaskArchive(taskID)
if err != nil {
t.Fatal(err)
}
filename := taskArchivePath(m.dir, job.ID, task.ID)
if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filename, []byte("committed local archive"), 0600); err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, pgdb.ArchiveReady, filename, strings.Repeat("a", 64)); err != nil {
t.Fatal(err)
}
var deletes atomic.Int32
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
deletes.Add(1)
w.WriteHeader(http.StatusNoContent)
}))
defer remote.Close()
store, err := archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
if err != nil {
t.Fatal(err)
}
s := &Server{ctx: context.Background(), m: m, archiveStore: store}
journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{
ArchiveID: job.ID, URI: store.URI(taskID, job.ID, strings.Repeat("b", 64)),
LocalPath: filename, Operation: "upload",
})
if err != nil {
t.Fatal(err)
}
if err := s.recoverRemoteArchiveJournal(journal); err != nil {
t.Fatal(err)
}
if data, err := os.ReadFile(filename); err != nil || string(data) != "committed local archive" {
t.Fatalf("old upload cleanup removed local retry: %v", err)
}
if deletes.Load() != 1 {
t.Fatal("uncommitted remote object was not removed")
}
if _, err := os.Stat(journal); !os.IsNotExist(err) {
t.Fatalf("finished journal remains: %v", err)
}
}
+58 -7
View File
@@ -14,6 +14,7 @@ import (
"time"
"github.com/Autumn-27/artex/agent"
"github.com/Autumn-27/artex/archivestore"
pgdb "github.com/Autumn-27/artex/db"
"github.com/Autumn-27/artex/traffic"
)
@@ -37,6 +38,12 @@ func (s *Server) startTaskArchiveWorker() {
if s.m == nil || s.m.pg == nil {
return
}
s.archiveStore, s.archiveStoreErr = archivestore.FromEnv()
if s.archiveStoreErr != nil {
log.Printf("[task-archive] storage configuration: %v", s.archiveStoreErr)
} else if s.archiveStore != nil {
log.Printf("[task-archive] new archives use S3 object storage with full SHA-256 readback verification")
}
if err := s.m.pg.RecoverTaskArchiveJobs(); err != nil {
log.Printf("[task-archive] recover jobs: %v", err)
}
@@ -88,6 +95,14 @@ func (s *Server) notifyTaskArchiveWorker() {
}
func (s *Server) runOneTaskArchiveJob() error {
if s.ctx.Err() != nil {
return s.ctx.Err()
}
// Retry deferred remote cleanup before claiming more work. Failed cleanup
// keeps its durable journal and does not block unrelated local archives.
if err := s.recoverRemoteArchiveJournals(); err != nil {
log.Printf("[task-archive] deferred object cleanup: %v", err)
}
job, err := s.m.pg.ClaimTaskArchiveJob(s.ctx)
if err != nil || job == nil {
return err
@@ -115,6 +130,9 @@ func (s *Server) runOneTaskArchiveJob() error {
}
func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
if s.archiveStoreErr != nil {
return s.archiveStoreErr
}
taskID := strconv.FormatInt(job.TaskID, 10)
if !s.beginTaskDelete(taskID) {
return errors.New("task is already undergoing another archive or delete operation")
@@ -199,6 +217,21 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
_ = os.Remove(archivePath)
}
}()
storedPath := archivePath
var remoteJournal string
if s.archiveStore != nil {
storedPath = s.archiveStore.URI(job.TaskID, job.ID, checksum)
remoteJournal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{
ArchiveID: job.ID, URI: storedPath, LocalPath: archivePath, Operation: "upload",
})
if err != nil {
return err
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "upload_and_verify", 65)
if err := s.archiveStore.Upload(s.ctx, storedPath, archivePath, checksum); err != nil {
return err
}
}
var trafficStage *traffic.HostDeleteStage
if s.m.traffic != nil && len(snapshot.ExclusiveHosts) > 0 {
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_traffic", 68)
@@ -213,11 +246,18 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
}()
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_database", 78)
if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, archivePath, checksum, originalSize, compressedSize); err != nil {
if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, storedPath, checksum, originalSize, compressedSize); err != nil {
return err
}
committed = true
removePackage = false
removePackage = s.archiveStore != nil
if remoteJournal != "" {
// The journal also covers a crash after database commit but before local
// package removal. Keep it if cleanup fails, so the worker can retry.
if err := removeLocalArchivePackage(archivePath); err == nil {
_ = os.Remove(remoteJournal)
}
}
if trafficStage != nil {
if err := trafficStage.Commit(); err != nil {
warning := "archive completed, but cleaning the exclusive traffic hot store failed (the archive package can still restore it): " + err.Error()
@@ -239,7 +279,7 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
}
func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil {
if err := s.validateStoredArchive(job.ArchivePath); err != nil {
return err
}
// A prior attempt may have committed PostgreSQL and files, then failed only
@@ -247,7 +287,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
if restored, err := s.m.pg.IsTaskArchiveRestored(job.ID); err != nil {
return err
} else if restored {
if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) {
if err := s.removeStoredArchive(job.ArchivePath); err != nil {
return err
}
if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil {
@@ -266,7 +306,15 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
return err
}
defer func() { _ = os.RemoveAll(extracted) }()
if err := extractTaskArchivePackage(job.ArchivePath, job.SHA256, extracted); err != nil {
packagePath := job.ArchivePath
if strings.HasPrefix(packagePath, "s3://") {
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "download_and_verify", 12)
packagePath = filepath.Join(extracted, "download.tar.zst")
if err := s.archiveStore.Download(s.ctx, job.ArchivePath, packagePath, job.SHA256, job.CompressedSize); err != nil {
return err
}
}
if err := extractTaskArchivePackage(packagePath, job.SHA256, extracted); err != nil {
return err
}
manifestRaw, err := os.ReadFile(filepath.Join(extracted, "manifest.json"))
@@ -356,7 +404,7 @@ func (s *Server) restoreTaskArchivePayload(job *pgdb.TaskArchive, snapshot *pgdb
return err
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "consume_package", 95)
if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) {
if err := s.removeStoredArchive(job.ArchivePath); err != nil {
return err
}
if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil {
@@ -393,9 +441,12 @@ func (s *Server) loadRestoredTask(taskID int64) {
}
func (s *Server) deleteTaskArchive(job *pgdb.TaskArchive) error {
if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil {
if err := s.validateStoredArchive(job.ArchivePath); err != nil {
return err
}
if strings.HasPrefix(job.ArchivePath, "s3://") {
return s.deleteRemoteTaskArchive(job)
}
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "stage_package_delete", 35)
staged, moved, err := stageTaskArchivePackageDelete(job.ArchivePath, job.ID)
if err != nil {
+3 -2
View File
@@ -1494,7 +1494,7 @@ function ArchiveConfirmDialog({
<AlertDialogHeader>
<AlertDialogTitle>{count === 1 ? "Archive task" : `Archive ${count} tasks`}</AlertDialogTitle>
<AlertDialogDescription className="[overflow-wrap:anywhere]">
Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into local cold storage. Once archived, a task can be restored from "Archived".
Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into the configured archive storage. Once archived, a task can be restored from "Archived". Restoring requires free space on this server.
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
@@ -1934,6 +1934,7 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {
title={`Before compression: ${formatArchiveBytes(archive.original_size)}`}
>
{archiveCompressionLabel(archive)}
<div className="text-muted-foreground">{archive.storage === "s3" ? "Object storage" : "Local storage"}</div>
</TableCell>
<TableCell className="text-xs tabular-nums">{archiveDataTotal(archive).toLocaleString()}</TableCell>
<TableCell>
@@ -1952,7 +1953,7 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {
</p>
))}
<span className="text-muted-foreground text-xs">
{archive.phase} · {archive.progress}%
{archive.phase === "upload_and_verify" ? "Uploading and verifying" : archive.phase === "download_and_verify" ? "Downloading and verifying" : archive.phase} · {archive.progress}%
</span>
</div>
</TableCell>
+1
View File
@@ -101,6 +101,7 @@ export interface TaskArchiveTokenStats {
}
export interface TaskArchive {
storage?: "local" | "s3";
id: number;
task_id: number;
state: TaskArchiveState;