mirror of
https://github.com/BaaRai/ARTEX.git
synced 2026-11-19 22:50:55 +08:00
feat: archive cold tasks to S3-compatible object storage
This commit is contained in:
@@ -418,6 +418,12 @@ runtime supports it.
|
||||
`/chat/completions` (or `/responses` for `openai-responses`), so their base **does** end in `/v1`
|
||||
(for example `https://api.openai.com/v1`).
|
||||
|
||||
**Task archive storage**: local disk by default. Set `ARTEX_ARCHIVE_STORAGE=s3` to
|
||||
store cold task packages in an S3-compatible bucket and release their local
|
||||
package space after verification. Existing local archives remain usable. See
|
||||
[Task archive storage](docs/task-archive-storage.md) for configuration, permissions,
|
||||
integrity checks, temporary disk requirements and recovery.
|
||||
|
||||
**Concurrency**: work agents per task, in "System Settings" (default 3).
|
||||
|
||||
**Untrusted-output boundary (on by default)**: tool results for the worker, planner, chat and main
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package archivestore
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestArchiveStorageEnvironment(t *testing.T) {
|
||||
for _, key := range []string{
|
||||
"ARTEX_ARCHIVE_STORAGE", "ARTEX_ARCHIVE_S3_ENDPOINT", "ARTEX_ARCHIVE_S3_REGION",
|
||||
"ARTEX_ARCHIVE_S3_BUCKET", "ARTEX_ARCHIVE_S3_PREFIX", "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID",
|
||||
"ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_PATH_STYLE",
|
||||
} {
|
||||
t.Setenv(key, "")
|
||||
}
|
||||
for _, backend := range []string{"", "local"} {
|
||||
t.Setenv("ARTEX_ARCHIVE_STORAGE", backend)
|
||||
if store, err := FromEnv(); err != nil || store != nil {
|
||||
t.Fatalf("local default: %v %v", store, err)
|
||||
}
|
||||
}
|
||||
for _, backend := range []string{"unknown", "s3"} {
|
||||
t.Setenv("ARTEX_ARCHIVE_STORAGE", backend)
|
||||
if _, err := FromEnv(); err == nil {
|
||||
t.Fatalf("%q silently fell back to local", backend)
|
||||
}
|
||||
}
|
||||
t.Setenv("ARTEX_ARCHIVE_S3_ENDPOINT", "http://127.0.0.1:9000")
|
||||
t.Setenv("ARTEX_ARCHIVE_S3_BUCKET", "test-bucket")
|
||||
t.Setenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "test-key")
|
||||
t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "test-secret")
|
||||
t.Setenv("ARTEX_ARCHIVE_S3_PATH_STYLE", "true")
|
||||
store, err := FromEnv()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if uri := store.URI(1, 2, "checksum"); uri != "s3://test-bucket/artex/tasks/task-1-2-checksum.tar.zst" {
|
||||
t.Fatal("unexpected default object location", uri)
|
||||
}
|
||||
t.Setenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "")
|
||||
if _, err := FromEnv(); err == nil {
|
||||
t.Fatal("missing credentials accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveEndpointValidation(t *testing.T) {
|
||||
config := Config{Bucket: "test", AccessKey: "test-key", SecretKey: "test-secret"}
|
||||
for _, endpoint := range []string{"https://objects.example.test", "http://127.0.0.1:9000", "http://127.0.0.2:9000", "http://[::1]:9000", "http://localhost:9000"} {
|
||||
config.Endpoint = endpoint
|
||||
if _, err := New(config); err != nil {
|
||||
t.Errorf("rejected %s: %v", endpoint, err)
|
||||
}
|
||||
}
|
||||
for _, endpoint := range []string{"http://objects.example.test", "https://user:test-secret@objects.example.test", "https://objects.example.test/bucket", "https://objects.example.test?token=test-secret", "https://objects.example.test#fragment"} {
|
||||
config.Endpoint = endpoint
|
||||
if _, err := New(config); err == nil {
|
||||
t.Error("accepted endpoint with non-loopback HTTP, credentials, path, query or fragment")
|
||||
} else if strings.Contains(err.Error(), "test-secret") {
|
||||
t.Fatal("configuration error disclosed a secret")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
// Package archivestore stores immutable cold-task packages in S3-compatible storage.
|
||||
package archivestore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Endpoint, Region, Bucket, Prefix, AccessKey, SecretKey string
|
||||
PathStyle bool
|
||||
}
|
||||
|
||||
type Store struct {
|
||||
client *s3.Client
|
||||
bucket, prefix string
|
||||
}
|
||||
|
||||
// FromEnv returns nil for the default local backend. A configured but invalid
|
||||
// backend is an error, never a silent fallback to the machine's local disk.
|
||||
func FromEnv() (*Store, error) {
|
||||
switch os.Getenv("ARTEX_ARCHIVE_STORAGE") {
|
||||
case "", "local":
|
||||
return nil, nil
|
||||
case "s3":
|
||||
return New(Config{
|
||||
Endpoint: os.Getenv("ARTEX_ARCHIVE_S3_ENDPOINT"), Region: os.Getenv("ARTEX_ARCHIVE_S3_REGION"),
|
||||
Bucket: os.Getenv("ARTEX_ARCHIVE_S3_BUCKET"), Prefix: os.Getenv("ARTEX_ARCHIVE_S3_PREFIX"),
|
||||
AccessKey: os.Getenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID"), SecretKey: os.Getenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY"),
|
||||
PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true",
|
||||
})
|
||||
default:
|
||||
return nil, errors.New("ARTEX_ARCHIVE_STORAGE must be local or s3")
|
||||
}
|
||||
}
|
||||
|
||||
func New(c Config) (*Store, error) {
|
||||
u, err := url.Parse(c.Endpoint)
|
||||
loopback := false
|
||||
if err == nil {
|
||||
loopback = u.Hostname() == "localhost"
|
||||
if ip := net.ParseIP(u.Hostname()); ip != nil {
|
||||
loopback = ip.IsLoopback()
|
||||
}
|
||||
}
|
||||
if err != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") ||
|
||||
(u.Scheme != "https" && !(u.Scheme == "http" && loopback)) {
|
||||
return nil, errors.New("archive S3 endpoint must be an HTTPS origin (HTTP allowed only on loopback)")
|
||||
}
|
||||
c.Prefix = strings.Trim(c.Prefix, "/")
|
||||
if c.Prefix == "" {
|
||||
c.Prefix = "artex/tasks"
|
||||
}
|
||||
if path.Clean(c.Prefix) != c.Prefix || strings.Contains(c.Prefix, "..") || strings.ContainsAny(c.Prefix, "\\?#") {
|
||||
return nil, errors.New("invalid archive S3 prefix")
|
||||
}
|
||||
if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") || c.AccessKey == "" || c.SecretKey == "" {
|
||||
return nil, errors.New("archive S3 bucket and dedicated access credentials are required")
|
||||
}
|
||||
if c.Region == "" {
|
||||
c.Region = "us-east-1"
|
||||
}
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.ResponseHeaderTimeout = 60 * time.Second
|
||||
client := s3.New(s3.Options{
|
||||
Region: c.Region, BaseEndpoint: aws.String(c.Endpoint), UsePathStyle: c.PathStyle,
|
||||
Credentials: credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, ""),
|
||||
HTTPClient: &http.Client{Transport: transport},
|
||||
RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
|
||||
ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired,
|
||||
})
|
||||
return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/"}, nil
|
||||
}
|
||||
|
||||
func (s *Store) URI(taskID, archiveID int64, checksum string) string {
|
||||
return "s3://" + s.bucket + "/" + s.prefix + fmt.Sprintf("task-%d-%d-%s.tar.zst", taskID, archiveID, checksum)
|
||||
}
|
||||
|
||||
func (s *Store) key(uri string) (string, error) {
|
||||
if s == nil {
|
||||
return "", errors.New("S3 archive storage is not configured")
|
||||
}
|
||||
u, err := url.Parse(uri)
|
||||
if err != nil || u.Scheme != "s3" || u.Host != s.bucket || u.User != nil || u.RawQuery != "" || u.Fragment != "" {
|
||||
return "", errors.New("archive object is outside the configured bucket")
|
||||
}
|
||||
key := strings.TrimPrefix(u.Path, "/")
|
||||
if !strings.HasPrefix(key, s.prefix) || len(key) <= len(s.prefix) || path.Clean(key) != key || strings.Contains(key, "\\") {
|
||||
return "", errors.New("archive object is outside the configured prefix")
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func (s *Store) Validate(uri string) error { _, err := s.key(uri); return err }
|
||||
|
||||
// Upload verifies the entire uploaded object by reading it back, including
|
||||
// multipart uploads whose ETag is not a checksum. Callers retain source data
|
||||
// until verification succeeds and PostgreSQL commits the remote location.
|
||||
func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) error {
|
||||
key, err := s.key(uri)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size {
|
||||
return errors.New("invalid archive SHA-256")
|
||||
}
|
||||
f, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
uploader := manager.NewUploader(s.client, func(u *manager.Uploader) {
|
||||
u.PartSize = 8 << 20
|
||||
u.Concurrency = 2
|
||||
u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
|
||||
})
|
||||
_, err = uploader.Upload(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucket), Key: aws.String(key), Body: f,
|
||||
ContentType: aws.String("application/zstd"), Metadata: map[string]string{"sha256": checksum},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload archive: %w", err)
|
||||
}
|
||||
return s.readVerified(ctx, key, io.Discard, checksum, info.Size())
|
||||
}
|
||||
|
||||
func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64) error {
|
||||
if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size {
|
||||
return errors.New("invalid archive SHA-256")
|
||||
}
|
||||
if expectedSize < 0 {
|
||||
return errors.New("invalid archive size")
|
||||
}
|
||||
object, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)})
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect archive: %w", err)
|
||||
}
|
||||
if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" {
|
||||
return errors.New("remote archive size mismatch or missing ETag; source data retained")
|
||||
}
|
||||
// Commit only complete, validated chunks to the output and hash. A broken
|
||||
// connection can then retry its chunk without duplicating bytes or restarting
|
||||
// a multi-gigabyte read. IfMatch pins every request to the same object.
|
||||
h := sha256.New()
|
||||
for offset := int64(0); offset < expectedSize; {
|
||||
end := offset + min(archiveReadChunkSize, expectedSize-offset) - 1
|
||||
data, err := s.readArchiveChunk(ctx, key, aws.ToString(object.ETag), offset, end, expectedSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := out.Write(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != len(data) {
|
||||
return io.ErrShortWrite
|
||||
}
|
||||
_, _ = h.Write(data)
|
||||
offset += int64(len(data))
|
||||
}
|
||||
if !strings.EqualFold(hex.EncodeToString(h.Sum(nil)), checksum) {
|
||||
return errors.New("remote archive size or SHA-256 mismatch; source data retained")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const (
|
||||
archiveReadChunkSize = int64(8 << 20)
|
||||
archiveReadAttempts = 5
|
||||
)
|
||||
|
||||
func (s *Store) readArchiveChunk(ctx context.Context, key, etag string, start, end, total int64) ([]byte, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < archiveReadAttempts; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if attempt > 0 {
|
||||
timer := time.NewTimer((250 * time.Millisecond) << (attempt - 1))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return nil, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
data, retry, err := s.readArchiveChunkOnce(ctx, key, etag, start, end, total)
|
||||
if err == nil {
|
||||
return data, nil
|
||||
}
|
||||
if !retry {
|
||||
return nil, err
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
return nil, fmt.Errorf("read archive bytes %d-%d after %d attempts: %w", start, end, archiveReadAttempts, lastErr)
|
||||
}
|
||||
|
||||
func (s *Store) readArchiveChunkOnce(ctx context.Context, key, etag string, start, end, total int64) ([]byte, bool, error) {
|
||||
requestCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
object, err := s.client.GetObject(requestCtx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket), Key: aws.String(key), IfMatch: aws.String(etag),
|
||||
Range: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)),
|
||||
})
|
||||
if err != nil {
|
||||
var networkError net.Error
|
||||
retry := errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded)
|
||||
return nil, retry, fmt.Errorf("read archive range: %w", err)
|
||||
}
|
||||
defer object.Body.Close()
|
||||
want := end - start + 1
|
||||
if aws.ToString(object.ETag) != etag || aws.ToString(object.ContentRange) != fmt.Sprintf("bytes %d-%d/%d", start, end, total) || object.ContentLength == nil || *object.ContentLength != want {
|
||||
return nil, false, errors.New("remote archive changed or returned an invalid byte range; source data retained")
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(object.Body, want+1))
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
if int64(len(data)) < want {
|
||||
return nil, true, io.ErrUnexpectedEOF
|
||||
}
|
||||
if int64(len(data)) != want {
|
||||
return nil, false, errors.New("remote archive range exceeds its declared size")
|
||||
}
|
||||
return data, false, nil
|
||||
}
|
||||
|
||||
func (s *Store) Download(ctx context.Context, uri, filename, checksum string, size int64) (err error) {
|
||||
key, err := s.key(uri)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if size <= 0 {
|
||||
return errors.New("invalid archive size")
|
||||
}
|
||||
f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
err = errors.Join(err, f.Close())
|
||||
if err != nil {
|
||||
_ = os.Remove(filename)
|
||||
}
|
||||
}()
|
||||
if err = s.readVerified(ctx, key, f, checksum, size); err != nil {
|
||||
return err
|
||||
}
|
||||
return f.Sync()
|
||||
}
|
||||
|
||||
func (s *Store) Delete(ctx context.Context, uri string) error {
|
||||
key, err := s.key(uri)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)})
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
package archivestore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestS3RoundTripAndIntegrity(t *testing.T) {
|
||||
for _, size := range []int{1024, 10 << 20} {
|
||||
t.Run(strconv.Itoa(size), func(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var object []byte
|
||||
parts := map[int][]byte{}
|
||||
corrupt, multipart := false, false
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
q := r.URL.Query()
|
||||
w.Header().Set("ETag", `"test-etag"`)
|
||||
switch {
|
||||
case r.Method == "POST" && q.Has("uploads"):
|
||||
multipart = true
|
||||
fmt.Fprint(w, `<InitiateMultipartUploadResult><UploadId>test-upload</UploadId></InitiateMultipartUploadResult>`)
|
||||
case r.Method == "PUT" && q.Has("partNumber"):
|
||||
n, _ := strconv.Atoi(q.Get("partNumber"))
|
||||
parts[n], _ = io.ReadAll(r.Body)
|
||||
case r.Method == "POST" && q.Has("uploadId"):
|
||||
for i := 1; i <= len(parts); i++ {
|
||||
object = append(object, parts[i]...)
|
||||
}
|
||||
fmt.Fprint(w, `<CompleteMultipartUploadResult><ETag>test-etag</ETag></CompleteMultipartUploadResult>`)
|
||||
case r.Method == "PUT":
|
||||
object, _ = io.ReadAll(r.Body)
|
||||
case r.Method == "HEAD":
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(object)))
|
||||
case r.Method == "GET":
|
||||
data := object
|
||||
if corrupt {
|
||||
data = bytes.Clone(object)
|
||||
data[0] ^= 0xff
|
||||
}
|
||||
serveArchiveRange(t, w, r, data)
|
||||
case r.Method == "DELETE":
|
||||
object = nil
|
||||
w.WriteHeader(204)
|
||||
default:
|
||||
t.Errorf("unexpected request %s %s", r.Method, r.URL)
|
||||
w.WriteHeader(400)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data := bytes.Repeat([]byte{0x42}, size)
|
||||
h := sha256.Sum256(data)
|
||||
checksum := hex.EncodeToString(h[:])
|
||||
root := t.TempDir()
|
||||
source := filepath.Join(root, "source")
|
||||
if err := os.WriteFile(source, data, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
uri := store.URI(1, 2, checksum)
|
||||
ctx := context.Background()
|
||||
if err := store.Upload(ctx, uri, source, checksum); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if multipart != (size > 8<<20) {
|
||||
t.Fatalf("multipart=%v size=%d", multipart, size)
|
||||
}
|
||||
dest := filepath.Join(root, "download")
|
||||
if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := os.ReadFile(dest)
|
||||
if !bytes.Equal(got, data) {
|
||||
t.Fatal("download differs")
|
||||
}
|
||||
mu.Lock()
|
||||
corrupt = true
|
||||
mu.Unlock()
|
||||
bad := filepath.Join(root, "bad")
|
||||
if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil {
|
||||
t.Fatal("accepted corrupt object")
|
||||
}
|
||||
if _, err := os.Stat(bad); !os.IsNotExist(err) {
|
||||
t.Fatal("left corrupt download")
|
||||
}
|
||||
if err := store.Upload(ctx, uri, source, checksum); err == nil {
|
||||
t.Fatal("upload accepted failed readback")
|
||||
}
|
||||
if err := store.Delete(ctx, uri); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func serveArchiveRange(t *testing.T, w http.ResponseWriter, r *http.Request, data []byte) {
|
||||
t.Helper()
|
||||
var start, end int64
|
||||
if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) {
|
||||
t.Errorf("invalid range %q", r.Header.Get("Range"))
|
||||
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return
|
||||
}
|
||||
if r.Header.Get("If-Match") != `"test-etag"` {
|
||||
t.Errorf("missing object version condition: %q", r.Header.Get("If-Match"))
|
||||
w.WriteHeader(http.StatusPreconditionFailed)
|
||||
return
|
||||
}
|
||||
w.Header().Set("ETag", `"test-etag"`)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10))
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write(data[start : end+1])
|
||||
}
|
||||
|
||||
func TestS3ReadbackRetriesInterruptedRange(t *testing.T) {
|
||||
data := append(bytes.Repeat([]byte{0x42}, int(archiveReadChunkSize)), bytes.Repeat([]byte{0x73}, 1024)...)
|
||||
var mu sync.Mutex
|
||||
var ranges []string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
w.Header().Set("ETag", `"test-etag"`)
|
||||
if r.Method == "HEAD" {
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
|
||||
return
|
||||
}
|
||||
ranges = append(ranges, r.Header.Get("Range"))
|
||||
if len(ranges) == 2 {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(int64(len(data))-archiveReadChunkSize, 10))
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", archiveReadChunkSize, len(data)-1, len(data)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write(data[archiveReadChunkSize : archiveReadChunkSize+128]) // Close during a later chunk.
|
||||
return
|
||||
}
|
||||
serveArchiveRange(t, w, r, data)
|
||||
}))
|
||||
defer srv.Close()
|
||||
store := testRangeStore(t, srv.URL)
|
||||
h := sha256.Sum256(data)
|
||||
checksum := hex.EncodeToString(h[:])
|
||||
dest := filepath.Join(t.TempDir(), "archive")
|
||||
if err := store.Download(context.Background(), store.URI(1, 2, checksum), dest, checksum, int64(len(data))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := os.ReadFile(dest)
|
||||
if err != nil || !bytes.Equal(got, data) {
|
||||
t.Fatalf("retried download differs: %v", err)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(ranges) != 3 || ranges[1] != ranges[2] || ranges[0] != fmt.Sprintf("bytes=0-%d", archiveReadChunkSize-1) || ranges[2] != fmt.Sprintf("bytes=%d-%d", archiveReadChunkSize, len(data)-1) {
|
||||
t.Fatalf("retry did not stay within the interrupted range: %v", ranges)
|
||||
}
|
||||
}
|
||||
|
||||
func testRangeStore(t *testing.T, endpoint string) *Store {
|
||||
t.Helper()
|
||||
s, err := New(Config{Endpoint: endpoint, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestS3ReadbackRejectsInvalidRangesAndExhaustedRetries(t *testing.T) {
|
||||
for _, mode := range []string{"ignored-range", "changed-etag", "changed-object", "truncated"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
data := bytes.Repeat([]byte{0x42}, 1024)
|
||||
var mu sync.Mutex
|
||||
gets := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
w.Header().Set("ETag", `"test-etag"`)
|
||||
if r.Method == "HEAD" {
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
|
||||
return
|
||||
}
|
||||
gets++
|
||||
if mode == "changed-object" {
|
||||
w.WriteHeader(http.StatusPreconditionFailed)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
|
||||
if mode == "ignored-range" {
|
||||
_, _ = w.Write(data)
|
||||
return
|
||||
}
|
||||
if mode == "changed-etag" {
|
||||
w.Header().Set("ETag", `"different-object"`)
|
||||
}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", len(data)-1, len(data)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
if mode == "truncated" {
|
||||
_, _ = w.Write(data[:128])
|
||||
} else {
|
||||
_, _ = w.Write(data)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
store := testRangeStore(t, srv.URL)
|
||||
h := sha256.Sum256(data)
|
||||
checksum := hex.EncodeToString(h[:])
|
||||
dest := filepath.Join(t.TempDir(), "archive")
|
||||
if err := store.Download(context.Background(), store.URI(1, 2, checksum), dest, checksum, int64(len(data))); err == nil {
|
||||
t.Fatal("accepted invalid or incomplete readback")
|
||||
}
|
||||
if _, err := os.Stat(dest); !os.IsNotExist(err) {
|
||||
t.Fatal("left a failed download on disk")
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
want := 1
|
||||
if mode == "truncated" {
|
||||
want = archiveReadAttempts
|
||||
}
|
||||
if gets != want {
|
||||
t.Fatalf("GET count %d, want %d", gets, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type failedArchiveWriter struct{ err error }
|
||||
|
||||
func (w failedArchiveWriter) Write([]byte) (int, error) { return 0, w.err }
|
||||
|
||||
func TestS3ReadbackPropagatesOutputFailure(t *testing.T) {
|
||||
data := []byte("test archive")
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("ETag", `"test-etag"`)
|
||||
if r.Method == "HEAD" {
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
|
||||
return
|
||||
}
|
||||
serveArchiveRange(t, w, r, data)
|
||||
}))
|
||||
defer srv.Close()
|
||||
store := testRangeStore(t, srv.URL)
|
||||
h := sha256.Sum256(data)
|
||||
outputError := errors.New("output disk full")
|
||||
err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data)))
|
||||
if !errors.Is(err, outputError) {
|
||||
t.Fatalf("lost output error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3ReadbackStopsOnCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
data := bytes.Repeat([]byte{0x42}, 1024)
|
||||
gets := make(chan struct{}, archiveReadAttempts+1)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("ETag", `"test-etag"`)
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
|
||||
if r.Method == "HEAD" {
|
||||
return
|
||||
}
|
||||
gets <- struct{}{}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", len(data)-1, len(data)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write(data[:128])
|
||||
w.(http.Flusher).Flush()
|
||||
cancel()
|
||||
}))
|
||||
defer srv.Close()
|
||||
store := testRangeStore(t, srv.URL)
|
||||
h := sha256.Sum256(data)
|
||||
checksum := hex.EncodeToString(h[:])
|
||||
dest := filepath.Join(t.TempDir(), "archive")
|
||||
err := store.Download(ctx, store.URI(1, 2, checksum), dest, checksum, int64(len(data)))
|
||||
if !errors.Is(err, context.Canceled) || len(gets) != 1 {
|
||||
t.Fatalf("cancellation was not honored: error=%v GETs=%d", err, len(gets))
|
||||
}
|
||||
if _, err := os.Stat(dest); !os.IsNotExist(err) {
|
||||
t.Fatal("left a cancelled download on disk")
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3RejectsForeignLocations(t *testing.T) {
|
||||
s, err := New(Config{Endpoint: "https://sfo3.digitaloceanspaces.com", Bucket: "allowed", Prefix: "archives/tasks", AccessKey: "test", SecretKey: "test"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, uri := range []string{"s3://other/archives/tasks/a", "s3://allowed/archives/other/a", "s3://allowed/archives/tasks/../a", "s3://allowed/archives/tasks/%2e%2e/a", "s3://allowed/archives/tasks/a?secret=x", "s3://user@allowed/archives/tasks/a", "file:///etc/passwd"} {
|
||||
if err := s.Validate(uri); err == nil {
|
||||
t.Errorf("accepted %s", uri)
|
||||
}
|
||||
}
|
||||
var absent *Store
|
||||
if absent.Validate("s3://allowed/archives/tasks/a") == nil {
|
||||
t.Fatal("missing config accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
// artex-archive-check tests the configured object store without touching tasks.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/archivestore"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := check(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func check() error {
|
||||
s, err := archivestore.FromEnv()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if s == nil {
|
||||
return fmt.Errorf("S3 archive storage is not configured")
|
||||
}
|
||||
dir, err := os.MkdirTemp("", "artex-archive-check-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
data := []byte("ARTEX object storage connectivity check " + time.Now().UTC().Format(time.RFC3339Nano))
|
||||
h := sha256.Sum256(data)
|
||||
checksum := hex.EncodeToString(h[:])
|
||||
uri := s.URI(0, time.Now().UnixNano(), checksum)
|
||||
file := filepath.Join(dir, "check")
|
||||
if err := os.WriteFile(file, data, 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
defer func() {
|
||||
cleanup, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := s.Delete(cleanup, uri); err != nil {
|
||||
log.Printf("test object cleanup failed for %s: %v", uri, err)
|
||||
}
|
||||
}()
|
||||
if err := s.Upload(ctx, uri, file, checksum); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.Download(ctx, uri, filepath.Join(dir, "download"), checksum, int64(len(data))); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.Delete(ctx, uri); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("S3 upload, full SHA-256 verification, download and delete passed")
|
||||
return nil
|
||||
}
|
||||
@@ -60,6 +60,7 @@ type TaskArchive struct {
|
||||
Warnings json.RawMessage `json:"warnings"`
|
||||
FormatVersion int `json:"format_version"`
|
||||
ArchivePath string `json:"-"`
|
||||
Storage string `json:"storage"`
|
||||
SHA256 string `json:"sha256,omitempty"`
|
||||
OriginalSize int64 `json:"original_size"`
|
||||
CompressedSize int64 `json:"compressed_size"`
|
||||
@@ -161,6 +162,10 @@ func scanTaskArchive(sc interface{ Scan(...any) error }) (*TaskArchive, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.Storage = "local"
|
||||
if strings.HasPrefix(item.ArchivePath, "s3://") {
|
||||
item.Storage = "s3"
|
||||
}
|
||||
if err := json.Unmarshal([]byte(sources), &item.SourceTaskIDs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
# Task archive storage
|
||||
|
||||
Task archives use local disk by default. The optional S3-compatible backend moves
|
||||
cold task packages to object storage after verifying their contents, releasing
|
||||
local package space while retaining ARTEX's Archive, Restore and Delete actions.
|
||||
|
||||
Active workspaces and the live PostgreSQL database stay local. This feature is
|
||||
task-level cold storage; use the existing backup workflow for a full installation
|
||||
backup. A complete recovery plan must also preserve the bucket objects and
|
||||
access to them.
|
||||
|
||||
## Configuration
|
||||
|
||||
Create a private bucket and a dedicated credential scoped to the archive prefix.
|
||||
The backend needs object upload, read (including HEAD and byte ranges), delete,
|
||||
and multipart upload/abort permissions. Use a separate prefix for each deployment.
|
||||
|
||||
Set these environment variables for the ARTEX server, then restart it:
|
||||
|
||||
```dotenv
|
||||
ARTEX_ARCHIVE_STORAGE=s3
|
||||
ARTEX_ARCHIVE_S3_ENDPOINT=https://s3.us-east-1.amazonaws.com
|
||||
ARTEX_ARCHIVE_S3_REGION=us-east-1
|
||||
ARTEX_ARCHIVE_S3_BUCKET=your-artex-archives
|
||||
ARTEX_ARCHIVE_S3_PREFIX=artex/tasks
|
||||
ARTEX_ARCHIVE_S3_ACCESS_KEY_ID=REPLACE
|
||||
ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY=REPLACE
|
||||
```
|
||||
|
||||
Store credentials in the service's protected environment file, outside the
|
||||
repository and frontend. For systemd, a root-owned environment file with mode
|
||||
0600 can be referenced through `EnvironmentFile=`.
|
||||
|
||||
| Variable | Meaning |
|
||||
| --- | --- |
|
||||
| `ARTEX_ARCHIVE_STORAGE` | `local` (default) or `s3`. |
|
||||
| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local testing. |
|
||||
| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. Match the provider's requirements. |
|
||||
| `ARTEX_ARCHIVE_S3_BUCKET` | Required bucket name. |
|
||||
| `ARTEX_ARCHIVE_S3_PREFIX` | Object prefix; defaults to `artex/tasks`. |
|
||||
| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Dedicated access key. |
|
||||
| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Dedicated secret key. |
|
||||
| `ARTEX_ARCHIVE_S3_PATH_STYLE` | Set to `true` for compatible path-style servers; otherwise virtual-hosted addressing is used. |
|
||||
|
||||
For DigitalOcean Spaces, use its regional origin, such as
|
||||
`https://sfo3.digitaloceanspaces.com`, and the provider's signing region. For a
|
||||
local path-style test server, use an endpoint such as `http://127.0.0.1:9000` and
|
||||
set `ARTEX_ARCHIVE_S3_PATH_STYLE=true`.
|
||||
|
||||
Invalid S3 configuration fails new archive jobs instead of silently falling back
|
||||
to local disk. Preserve the configured bucket, endpoint, prefix and read/delete
|
||||
access while remote archives remain. The same installation can still restore or
|
||||
delete older local archives.
|
||||
|
||||
## Lifecycle and integrity
|
||||
|
||||
1. Pause or finish a task and use its existing **Archive** action.
|
||||
2. ARTEX builds its `.tar.zst` package locally and uploads it. Large packages use
|
||||
multipart upload, starting with 8 MiB parts and two upload workers (the SDK
|
||||
increases the part size for exceptionally large seekable files).
|
||||
3. ARTEX reads the entire object back and checks its size and SHA-256. Multipart
|
||||
ETags and uploaded checksum metadata alone are not treated as proof of content.
|
||||
4. Only after verification does PostgreSQL commit the `s3://` location and
|
||||
compact the archived task. ARTEX then removes local source files and the
|
||||
temporary package. The archive list shows **Object storage** or **Local storage**.
|
||||
5. **Restore** downloads and verifies the package before restoring task files and
|
||||
database records. The remote package is consumed after restoration commits.
|
||||
Failed remote cleanup can be retried without replaying restored records.
|
||||
6. **Delete** removes the archive stub and queues durable cleanup of its remote
|
||||
object. Cleanup is retried after restart and before subsequent archive jobs.
|
||||
|
||||
Reads use bounded 8 MiB ranges pinned to the object's ETag. Interrupted ranges
|
||||
are retried without duplicating output or restarting a multi-gigabyte download.
|
||||
An object change, checksum mismatch or invalid range fails verification. Failed
|
||||
downloads are removed, and upload/verification failures retain the original task.
|
||||
|
||||
There is no database schema or archive format migration. Existing local archive
|
||||
packages remain compatible. Files outside ARTEX's task archive database are not
|
||||
automatically imported or uploaded.
|
||||
|
||||
## Disk, transfer and recovery
|
||||
|
||||
Archive creation still needs space for the temporary compressed package.
|
||||
Restoration needs space for the downloaded package and extracted files. Full
|
||||
upload readback and later restores incur object-storage reads and any applicable
|
||||
transfer charges. Configure expiration of abandoned multipart uploads for abrupt
|
||||
process termination.
|
||||
|
||||
Recovery journals under `data/archives/tasks/.remote` contain object locations,
|
||||
not credentials. Preserve them with PostgreSQL and the installation's data
|
||||
directory. The worker uses these journals to clean up uploads that never
|
||||
committed, keep committed objects, and finish deferred deletions. Failed remote
|
||||
cleanup does not block unrelated local archive jobs.
|
||||
|
||||
## Optional connectivity check
|
||||
|
||||
Run `go run ./cmd/artex-archive-check` with the same archive environment as the
|
||||
server to upload, verify, download and delete a small uniquely named test object.
|
||||
This command performs real bucket operations but does not touch task data. It
|
||||
reports leftover test objects if cleanup fails.
|
||||
|
||||
Automated tests use an HTTP S3 fixture and isolated PostgreSQL databases. They
|
||||
cover multipart transfers, corruption, interrupted range retries, object changes,
|
||||
cancellation, upload rollback, corrupted restores and persistent cleanup.
|
||||
@@ -4,6 +4,10 @@ go 1.26.10
|
||||
|
||||
require (
|
||||
github.com/Autumn-27/norma v0.4.10
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.5
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.13
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
@@ -22,6 +26,15 @@ require (
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.2.1 // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect
|
||||
github.com/aws/smithy-go v1.28.7 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/creack/pty v1.1.24 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
|
||||
@@ -4,6 +4,44 @@ github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eT
|
||||
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 h1:MW0Iql1SOTgQPj8mCxrU/eH/K/AIDWi08ItOsXaWRLU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25/go.mod h1:+ucVYmHYB3JqVueY0E1RHahdXAnKQbIHqybSbvOUbOM=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.13 h1:u52x6ISQOx172tpwM0VRxhJzEsTJYpMcEYUGM2A1wrw=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.13/go.mod h1:/ixtz6XPHb42zekjQ0LqD9XjTpTysNWhIGwufY1OzI4=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.13 h1:j4z41OtspmQ3ftQycaDunJ26b69a9SHfaSVUPqgRv8k=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.13/go.mod h1:SgPA1oS3b+dlMxZb618jDmGNscpXSCaxz15igS52OlE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19 h1:+JWfo9Mkg5HWwDwnpRPu7WNiiyEmiQRTaNSFsZ/1KIY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19/go.mod h1:l4/tI+TfymypMqq2oRCkhMAB7Gi7nlt5JaoUUz+cUbI=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 h1:GACgECI422Xt5n4Pojr3O0aocPv8lfH4PJe2Aon5Wv8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9/go.mod h1:lPfnRkHpn1qWZJLbVEw10rIqSuCee1pNQTyGkUjAYh8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 h1:RS1Wt+K0r4X0ci9Vf4JYTUfHzsQihYA2LgGJeEneKpk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8/go.mod h1:adMnTy5kyoUW86jhXnGSVc/mvMQI4GZd1qYWMeaS8M8=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 h1:to7+Vr3rLkpYRwVKJ5QRriI0eE306vrVN/6TnVHgxvY=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6/go.mod h1:Jc89ASa2k7HMAskj3bSnt8bf+k+p2IApzkoXnEjkgUw=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.12.1 h1:WZLC+k/CZx11kZGQDqu9mpA8yhiXGX2I//jhuJE1FDo=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.12.1/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.39.1 h1:4VB/dnDo7pmXSLQMszSoBzWndENuc7mzwTpD6lQn77A=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.39.1/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 h1:w8yui2ZqdHmGwVBDGvazoIvKw/DnLENBLkYnN8q8iqo=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw=
|
||||
github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE=
|
||||
github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
|
||||
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||
|
||||
+6
-3
@@ -22,6 +22,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
"github.com/Autumn-27/artex/archivestore"
|
||||
"github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/egress"
|
||||
"github.com/Autumn-27/artex/intercept"
|
||||
@@ -127,9 +128,11 @@ type Server struct {
|
||||
|
||||
// Cold task archives run through one persistent FIFO worker. The buffered wake
|
||||
// channel coalesces enqueue bursts; the database remains the source of truth.
|
||||
archiveWake chan struct{}
|
||||
archiveWG sync.WaitGroup
|
||||
side *sideQuestionState
|
||||
archiveWake chan struct{}
|
||||
archiveWG sync.WaitGroup
|
||||
archiveStore *archivestore.Store
|
||||
archiveStoreErr error
|
||||
side *sideQuestionState
|
||||
}
|
||||
|
||||
// provEntry is a cached provider + its config for one LLM profile id.
|
||||
|
||||
@@ -428,7 +428,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error {
|
||||
errs = append(errs, getErr)
|
||||
continue
|
||||
}
|
||||
if archive != nil && archive.State == pgdb.ArchiveReady {
|
||||
if archive != nil && archive.ArchivePath != "" && archive.SHA256 != "" {
|
||||
if err := os.RemoveAll(root); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
@@ -438,7 +438,7 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error {
|
||||
if err := stage.rollback(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
if archive != nil && archive.ArchivePath != "" {
|
||||
if archive != nil && archive.ArchivePath != "" && !strings.HasPrefix(archive.ArchivePath, "s3://") {
|
||||
_ = os.Remove(archive.ArchivePath)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
pgdb "github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
// An outbox bridges the local PostgreSQL transaction and object storage. Never
|
||||
// delete the only package before metadata permits it. Unfinished cleanup is
|
||||
// retried after a restart (and before subsequent jobs).
|
||||
type remoteArchiveJournal struct {
|
||||
ArchiveID int64 `json:"archive_id"`
|
||||
URI string `json:"uri"`
|
||||
LocalPath string `json:"local_path,omitempty"`
|
||||
Operation string `json:"operation"`
|
||||
}
|
||||
|
||||
func (s *Server) validateStoredArchive(uri string) error {
|
||||
if strings.HasPrefix(uri, "s3://") {
|
||||
return s.archiveStore.Validate(uri)
|
||||
}
|
||||
return validateArchivePath(s.m.dir, uri)
|
||||
}
|
||||
|
||||
func removeLocalArchivePackage(filename string) error {
|
||||
if err := os.Remove(filename); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) removeStoredArchive(uri string) error {
|
||||
if err := s.validateStoredArchive(uri); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.HasPrefix(uri, "s3://") {
|
||||
return s.archiveStore.Delete(s.ctx, uri)
|
||||
}
|
||||
return removeLocalArchivePackage(uri)
|
||||
}
|
||||
|
||||
func (s *Server) writeRemoteArchiveJournal(j remoteArchiveJournal) (string, error) {
|
||||
dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote")
|
||||
if err := os.MkdirAll(dir, archiveDirMode); err != nil {
|
||||
return "", err
|
||||
}
|
||||
f, err := os.CreateTemp(dir, ".journal-")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
tmp := f.Name()
|
||||
defer os.Remove(tmp)
|
||||
encodeErr := json.NewEncoder(f).Encode(j)
|
||||
syncErr := f.Sync()
|
||||
if err := errors.Join(encodeErr, syncErr, f.Close()); err != nil {
|
||||
return "", err
|
||||
}
|
||||
name := tmp + ".json"
|
||||
if err := os.Rename(tmp, name); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Persist the directory entry before an upload or database deletion starts.
|
||||
d, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := errors.Join(d.Sync(), d.Close()); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// The .remote directory itself may have just been created.
|
||||
parent, err := os.Open(filepath.Dir(dir))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return name, errors.Join(parent.Sync(), parent.Close())
|
||||
}
|
||||
|
||||
func (s *Server) recoverRemoteArchiveJournals() error {
|
||||
dir := filepath.Join(taskArchiveRoot(s.m.dir), ".remote")
|
||||
entries, err := os.ReadDir(dir)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
|
||||
continue
|
||||
}
|
||||
name := filepath.Join(dir, entry.Name())
|
||||
if err := s.recoverRemoteArchiveJournal(name); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func (s *Server) recoverRemoteArchiveJournal(name string) error {
|
||||
raw, err := os.ReadFile(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var j remoteArchiveJournal
|
||||
if err := json.Unmarshal(raw, &j); err != nil {
|
||||
return err
|
||||
}
|
||||
if j.ArchiveID <= 0 {
|
||||
return errors.New("invalid remote archive journal ID")
|
||||
}
|
||||
if err := s.archiveStore.Validate(j.URI); err != nil {
|
||||
return err
|
||||
}
|
||||
item, err := s.m.pg.GetTaskArchive(j.ArchiveID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch j.Operation {
|
||||
case "upload":
|
||||
if item == nil || item.ArchivePath != j.URI {
|
||||
if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := validateArchivePath(s.m.dir, j.LocalPath); err != nil {
|
||||
return err
|
||||
}
|
||||
// An old failed upload may outlive a retry after switching back to
|
||||
// local storage. The retry reuses this pathname: keep its committed
|
||||
// package instead of treating it as the old upload's temporary copy.
|
||||
if item == nil || item.ArchivePath != j.LocalPath {
|
||||
if err := removeLocalArchivePackage(j.LocalPath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case "delete":
|
||||
if item == nil {
|
||||
if err := s.archiveStore.Delete(s.ctx, j.URI); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unknown remote archive operation %q", j.Operation)
|
||||
}
|
||||
return os.Remove(name)
|
||||
}
|
||||
|
||||
func (s *Server) deleteRemoteTaskArchive(job *pgdb.TaskArchive) error {
|
||||
journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, Operation: "delete"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.m.pg.DeleteTaskArchiveStub(job.ID); err != nil {
|
||||
// A connection failure can leave the transaction's commit ambiguous.
|
||||
// Recovery consults PostgreSQL before deciding whether to delete.
|
||||
return err
|
||||
}
|
||||
if err := s.recoverRemoteArchiveJournal(journal); err != nil {
|
||||
// The task is gone; the durable outbox retries the remaining object cleanup.
|
||||
log.Printf("[task-archive] archive %d deleted; remote cleanup pending: %v", job.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,396 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/Autumn-27/artex/archivestore"
|
||||
pgdb "github.com/Autumn-27/artex/db"
|
||||
)
|
||||
|
||||
func TestRemoteArchiveLifecycle(t *testing.T) {
|
||||
if os.Getenv("ARTEX_PG_DSN") == "" {
|
||||
t.Skip("requires isolated PostgreSQL")
|
||||
}
|
||||
t.Setenv("ARTEX_ARCHIVE_STORAGE", "local")
|
||||
m, err := NewManager(t.TempDir(), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer m.Close()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
s := New(ctx, m, t.TempDir(), t.TempDir(), t.TempDir())
|
||||
s.archiveWG.Wait()
|
||||
s.ctx = context.Background()
|
||||
var mu sync.Mutex
|
||||
objects := map[string][]byte{}
|
||||
corrupt, failDelete := false, false
|
||||
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
switch r.Method {
|
||||
case "PUT":
|
||||
objects[r.URL.Path], _ = io.ReadAll(r.Body)
|
||||
w.Header().Set("ETag", `"test"`)
|
||||
case "HEAD":
|
||||
w.Header().Set("ETag", `"test"`)
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path])))
|
||||
case "GET":
|
||||
data := objects[r.URL.Path]
|
||||
if corrupt && len(data) > 0 {
|
||||
data = bytes.Clone(data)
|
||||
data[0] ^= 0xff
|
||||
}
|
||||
var start, end int64
|
||||
if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end < start || end >= int64(len(data)) || r.Header.Get("If-Match") != `"test"` {
|
||||
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return
|
||||
}
|
||||
w.Header().Set("ETag", `"test"`)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10))
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
_, _ = w.Write(data[start : end+1])
|
||||
case "DELETE":
|
||||
if failDelete {
|
||||
w.WriteHeader(403)
|
||||
return
|
||||
}
|
||||
delete(objects, r.URL.Path)
|
||||
w.WriteHeader(204)
|
||||
default:
|
||||
w.WriteHeader(400)
|
||||
}
|
||||
}))
|
||||
defer remote.Close()
|
||||
s.archiveStore, err = archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task, err := m.CreateTask("S3 round trip", "preserve data", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, _ := strconv.ParseInt(task.ID, 10, 64)
|
||||
defer func() { _, _ = m.pg.Exec(`DELETE FROM task_archives WHERE task_id=$1`, id); _ = m.pg.DeleteTask(id) }()
|
||||
if err := m.pg.SetPaused(id, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "important.txt")
|
||||
if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(file, []byte("keep this content"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pgTask, err := m.pg.GetTask(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.pg.Exploration(pgTask.ExplorationID).AppendActivity(pgdb.Activity{Worker: "system", Kind: "system", Summary: "archive proof"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job, err := m.pg.QueueTaskArchive(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A configured-but-invalid backend must fail before moving any source data.
|
||||
s.archiveStoreErr = errors.New("invalid fixture configuration")
|
||||
if err := s.runOneTaskArchiveJob(); !errors.Is(err, s.archiveStoreErr) {
|
||||
t.Fatalf("configuration error was not returned: %v", err)
|
||||
}
|
||||
if got, err := os.ReadFile(file); err != nil || string(got) != "keep this content" {
|
||||
t.Fatalf("bad configuration lost source: %v", err)
|
||||
}
|
||||
if pgTask, err := m.pg.GetTask(id); err != nil || pgTask == nil {
|
||||
t.Fatal("bad configuration lost task", err)
|
||||
}
|
||||
s.archiveStoreErr = nil
|
||||
if _, err := m.pg.QueueTaskArchive(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A corrupt readback must roll back filesystem staging and leave PG intact.
|
||||
mu.Lock()
|
||||
corrupt = true
|
||||
mu.Unlock()
|
||||
if err := s.runOneTaskArchiveJob(); err == nil {
|
||||
t.Fatal("expected integrity failure")
|
||||
}
|
||||
if got, _ := os.ReadFile(file); string(got) != "keep this content" {
|
||||
t.Fatal("upload failure lost source")
|
||||
}
|
||||
if task, err := m.pg.GetTask(id); err != nil || task == nil {
|
||||
t.Fatal("upload failure lost task")
|
||||
}
|
||||
mu.Lock()
|
||||
corrupt = false
|
||||
mu.Unlock()
|
||||
if err := s.recoverRemoteArchiveJournals(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.pg.QueueTaskArchive(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.runOneTaskArchiveJob(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job, err = m.pg.GetTaskArchive(job.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job.State != pgdb.ArchiveReady || job.Storage != "s3" || !strings.HasPrefix(job.ArchivePath, "s3://") {
|
||||
t.Fatalf("unexpected archive %+v", job)
|
||||
}
|
||||
if _, err := os.Stat(file); !os.IsNotExist(err) {
|
||||
t.Fatal("workspace not released")
|
||||
}
|
||||
if _, err := os.Stat(taskArchivePath(m.dir, job.ID, task.ID)); !os.IsNotExist(err) {
|
||||
t.Fatal("local package retained")
|
||||
}
|
||||
// Simulate the crash window after commit: cleanup keeps the remote object.
|
||||
journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: job.ArchivePath, LocalPath: taskArchivePath(m.dir, job.ID, task.ID), Operation: "upload"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.recoverRemoteArchiveJournal(journal); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A corrupted download must leave the archive and its remote package intact.
|
||||
mu.Lock()
|
||||
corrupt = true
|
||||
mu.Unlock()
|
||||
if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.runOneTaskArchiveJob(); err == nil {
|
||||
t.Fatal("expected corrupted restore to fail")
|
||||
}
|
||||
if _, err := os.Stat(file); !os.IsNotExist(err) {
|
||||
t.Fatal("corrupted archive installed workspace files")
|
||||
}
|
||||
if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || restored {
|
||||
t.Fatalf("corrupted archive restored database: %v %v", restored, err)
|
||||
}
|
||||
mu.Lock()
|
||||
if len(objects) == 0 {
|
||||
t.Error("corrupted restore deleted the only remote package")
|
||||
}
|
||||
corrupt = false
|
||||
mu.Unlock()
|
||||
// Restoration commits files and PG first. Failed remote deletion is retryable.
|
||||
mu.Lock()
|
||||
failDelete = true
|
||||
mu.Unlock()
|
||||
if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.runOneTaskArchiveJob(); err == nil {
|
||||
t.Fatal("expected remote delete failure")
|
||||
}
|
||||
if got, _ := os.ReadFile(file); string(got) != "keep this content" {
|
||||
t.Fatal("restored content differs")
|
||||
}
|
||||
if restored, err := m.pg.IsTaskArchiveRestored(job.ID); err != nil || !restored {
|
||||
t.Fatalf("PG restore %v %v", restored, err)
|
||||
}
|
||||
mu.Lock()
|
||||
failDelete = false
|
||||
mu.Unlock()
|
||||
if _, err := m.pg.QueueTaskArchiveRestore(job.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.runOneTaskArchiveJob(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if item, err := m.pg.GetTaskArchive(job.ID); err != nil || item != nil {
|
||||
t.Fatal("restore did not consume stub")
|
||||
}
|
||||
var count int
|
||||
if err := m.pg.QueryRow(`SELECT count(*) FROM activity WHERE exploration_id=$1 AND summary='archive proof'`, pgTask.ExplorationID).Scan(&count); err != nil || count != 1 {
|
||||
t.Fatalf("activity restore count=%d err=%v", count, err)
|
||||
}
|
||||
// Re-archive, then crash/retry the deletion outbox after PG removed its stub.
|
||||
job, err = m.pg.QueueTaskArchive(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.runOneTaskArchiveJob(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mu.Lock()
|
||||
failDelete = true
|
||||
mu.Unlock()
|
||||
if _, err := m.pg.QueueTaskArchiveDelete(job.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.runOneTaskArchiveJob(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mu.Lock()
|
||||
failDelete = false
|
||||
mu.Unlock()
|
||||
if err := s.recoverRemoteArchiveJournals(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mu.Lock()
|
||||
remaining := len(objects)
|
||||
mu.Unlock()
|
||||
if remaining != 0 {
|
||||
t.Fatalf("left %d objects", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
// Once PostgreSQL has committed an archive location and checksum, a subsequent
|
||||
// queued operation must not make startup recovery resurrect the old workspace.
|
||||
func TestRemoteArchiveStageRecoveryAfterCommit(t *testing.T) {
|
||||
if os.Getenv("ARTEX_PG_DSN") == "" {
|
||||
t.Skip("requires isolated PostgreSQL")
|
||||
}
|
||||
m, err := NewManager(t.TempDir(), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer m.Close()
|
||||
task, err := m.CreateTask("remote stage recovery", "test", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
|
||||
defer func() {
|
||||
_, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID)
|
||||
_ = m.pg.DeleteTask(taskID)
|
||||
}()
|
||||
if err := m.pg.SetPaused(taskID, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job, err := m.pg.QueueTaskArchive(taskID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pgTask, err := m.pg.GetTask(taskID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} {
|
||||
t.Run(state, func(t *testing.T) {
|
||||
committed := state != pgdb.ArchiveQueued
|
||||
checksum := ""
|
||||
if committed {
|
||||
checksum = strings.Repeat("a", 64)
|
||||
}
|
||||
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, "s3://test/tasks/package.tar.zst", checksum); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt")
|
||||
if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(stage.root); !os.IsNotExist(err) {
|
||||
t.Fatalf("staging cleanup incomplete: %v", err)
|
||||
}
|
||||
if committed {
|
||||
if _, err := os.Stat(file); !os.IsNotExist(err) {
|
||||
t.Fatalf("resurrected committed workspace: %v", err)
|
||||
}
|
||||
} else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" {
|
||||
t.Fatalf("uncommitted archive lost source: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Failed remote cleanup can survive switching to local storage and retrying the
|
||||
// same archive ID. Its journal must never remove the newly committed local copy.
|
||||
func TestRemoteArchiveJournalPreservesLocalRetry(t *testing.T) {
|
||||
if os.Getenv("ARTEX_PG_DSN") == "" {
|
||||
t.Skip("requires isolated PostgreSQL")
|
||||
}
|
||||
m, err := NewManager(t.TempDir(), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer m.Close()
|
||||
task, err := m.CreateTask("local retry", "test", nil, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taskID, _ := strconv.ParseInt(task.ID, 10, 64)
|
||||
defer func() {
|
||||
_, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", taskID)
|
||||
_ = m.pg.DeleteTask(taskID)
|
||||
}()
|
||||
if err := m.pg.SetPaused(taskID, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job, err := m.pg.QueueTaskArchive(taskID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
filename := taskArchivePath(m.dir, job.ID, task.ID)
|
||||
if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filename, []byte("committed local archive"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, pgdb.ArchiveReady, filename, strings.Repeat("a", 64)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var deletes atomic.Int32
|
||||
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodDelete {
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
deletes.Add(1)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer remote.Close()
|
||||
store, err := archivestore.New(archivestore.Config{Endpoint: remote.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := &Server{ctx: context.Background(), m: m, archiveStore: store}
|
||||
journal, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{
|
||||
ArchiveID: job.ID, URI: store.URI(taskID, job.ID, strings.Repeat("b", 64)),
|
||||
LocalPath: filename, Operation: "upload",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.recoverRemoteArchiveJournal(journal); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if data, err := os.ReadFile(filename); err != nil || string(data) != "committed local archive" {
|
||||
t.Fatalf("old upload cleanup removed local retry: %v", err)
|
||||
}
|
||||
if deletes.Load() != 1 {
|
||||
t.Fatal("uncommitted remote object was not removed")
|
||||
}
|
||||
if _, err := os.Stat(journal); !os.IsNotExist(err) {
|
||||
t.Fatalf("finished journal remains: %v", err)
|
||||
}
|
||||
}
|
||||
+58
-7
@@ -14,6 +14,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Autumn-27/artex/agent"
|
||||
"github.com/Autumn-27/artex/archivestore"
|
||||
pgdb "github.com/Autumn-27/artex/db"
|
||||
"github.com/Autumn-27/artex/traffic"
|
||||
)
|
||||
@@ -37,6 +38,12 @@ func (s *Server) startTaskArchiveWorker() {
|
||||
if s.m == nil || s.m.pg == nil {
|
||||
return
|
||||
}
|
||||
s.archiveStore, s.archiveStoreErr = archivestore.FromEnv()
|
||||
if s.archiveStoreErr != nil {
|
||||
log.Printf("[task-archive] storage configuration: %v", s.archiveStoreErr)
|
||||
} else if s.archiveStore != nil {
|
||||
log.Printf("[task-archive] new archives use S3 object storage with full SHA-256 readback verification")
|
||||
}
|
||||
if err := s.m.pg.RecoverTaskArchiveJobs(); err != nil {
|
||||
log.Printf("[task-archive] recover jobs: %v", err)
|
||||
}
|
||||
@@ -88,6 +95,14 @@ func (s *Server) notifyTaskArchiveWorker() {
|
||||
}
|
||||
|
||||
func (s *Server) runOneTaskArchiveJob() error {
|
||||
if s.ctx.Err() != nil {
|
||||
return s.ctx.Err()
|
||||
}
|
||||
// Retry deferred remote cleanup before claiming more work. Failed cleanup
|
||||
// keeps its durable journal and does not block unrelated local archives.
|
||||
if err := s.recoverRemoteArchiveJournals(); err != nil {
|
||||
log.Printf("[task-archive] deferred object cleanup: %v", err)
|
||||
}
|
||||
job, err := s.m.pg.ClaimTaskArchiveJob(s.ctx)
|
||||
if err != nil || job == nil {
|
||||
return err
|
||||
@@ -115,6 +130,9 @@ func (s *Server) runOneTaskArchiveJob() error {
|
||||
}
|
||||
|
||||
func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
|
||||
if s.archiveStoreErr != nil {
|
||||
return s.archiveStoreErr
|
||||
}
|
||||
taskID := strconv.FormatInt(job.TaskID, 10)
|
||||
if !s.beginTaskDelete(taskID) {
|
||||
return errors.New("task is already undergoing another archive or delete operation")
|
||||
@@ -199,6 +217,21 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
|
||||
_ = os.Remove(archivePath)
|
||||
}
|
||||
}()
|
||||
storedPath := archivePath
|
||||
var remoteJournal string
|
||||
if s.archiveStore != nil {
|
||||
storedPath = s.archiveStore.URI(job.TaskID, job.ID, checksum)
|
||||
remoteJournal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{
|
||||
ArchiveID: job.ID, URI: storedPath, LocalPath: archivePath, Operation: "upload",
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "upload_and_verify", 65)
|
||||
if err := s.archiveStore.Upload(s.ctx, storedPath, archivePath, checksum); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var trafficStage *traffic.HostDeleteStage
|
||||
if s.m.traffic != nil && len(snapshot.ExclusiveHosts) > 0 {
|
||||
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_traffic", 68)
|
||||
@@ -213,11 +246,18 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
|
||||
}()
|
||||
}
|
||||
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "compact_database", 78)
|
||||
if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, archivePath, checksum, originalSize, compressedSize); err != nil {
|
||||
if err := s.m.pg.CompleteTaskArchive(job.ID, snapshot, storedPath, checksum, originalSize, compressedSize); err != nil {
|
||||
return err
|
||||
}
|
||||
committed = true
|
||||
removePackage = false
|
||||
removePackage = s.archiveStore != nil
|
||||
if remoteJournal != "" {
|
||||
// The journal also covers a crash after database commit but before local
|
||||
// package removal. Keep it if cleanup fails, so the worker can retry.
|
||||
if err := removeLocalArchivePackage(archivePath); err == nil {
|
||||
_ = os.Remove(remoteJournal)
|
||||
}
|
||||
}
|
||||
if trafficStage != nil {
|
||||
if err := trafficStage.Commit(); err != nil {
|
||||
warning := "archive completed, but cleaning the exclusive traffic hot store failed (the archive package can still restore it): " + err.Error()
|
||||
@@ -239,7 +279,7 @@ func (s *Server) archiveTask(job *pgdb.TaskArchive) (runErr error) {
|
||||
}
|
||||
|
||||
func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
|
||||
if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil {
|
||||
if err := s.validateStoredArchive(job.ArchivePath); err != nil {
|
||||
return err
|
||||
}
|
||||
// A prior attempt may have committed PostgreSQL and files, then failed only
|
||||
@@ -247,7 +287,7 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
|
||||
if restored, err := s.m.pg.IsTaskArchiveRestored(job.ID); err != nil {
|
||||
return err
|
||||
} else if restored {
|
||||
if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) {
|
||||
if err := s.removeStoredArchive(job.ArchivePath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil {
|
||||
@@ -266,7 +306,15 @@ func (s *Server) restoreTaskArchive(job *pgdb.TaskArchive) (runErr error) {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(extracted) }()
|
||||
if err := extractTaskArchivePackage(job.ArchivePath, job.SHA256, extracted); err != nil {
|
||||
packagePath := job.ArchivePath
|
||||
if strings.HasPrefix(packagePath, "s3://") {
|
||||
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "download_and_verify", 12)
|
||||
packagePath = filepath.Join(extracted, "download.tar.zst")
|
||||
if err := s.archiveStore.Download(s.ctx, job.ArchivePath, packagePath, job.SHA256, job.CompressedSize); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := extractTaskArchivePackage(packagePath, job.SHA256, extracted); err != nil {
|
||||
return err
|
||||
}
|
||||
manifestRaw, err := os.ReadFile(filepath.Join(extracted, "manifest.json"))
|
||||
@@ -356,7 +404,7 @@ func (s *Server) restoreTaskArchivePayload(job *pgdb.TaskArchive, snapshot *pgdb
|
||||
return err
|
||||
}
|
||||
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "consume_package", 95)
|
||||
if err := os.Remove(job.ArchivePath); err != nil && !os.IsNotExist(err) {
|
||||
if err := s.removeStoredArchive(job.ArchivePath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.m.pg.CompleteTaskArchiveRestore(job.ID); err != nil {
|
||||
@@ -393,9 +441,12 @@ func (s *Server) loadRestoredTask(taskID int64) {
|
||||
}
|
||||
|
||||
func (s *Server) deleteTaskArchive(job *pgdb.TaskArchive) error {
|
||||
if err := validateArchivePath(s.m.dir, job.ArchivePath); err != nil {
|
||||
if err := s.validateStoredArchive(job.ArchivePath); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.HasPrefix(job.ArchivePath, "s3://") {
|
||||
return s.deleteRemoteTaskArchive(job)
|
||||
}
|
||||
_ = s.m.pg.UpdateTaskArchiveProgress(job.ID, "stage_package_delete", 35)
|
||||
staged, moved, err := stageTaskArchivePackageDelete(job.ArchivePath, job.ID)
|
||||
if err != nil {
|
||||
|
||||
@@ -1494,7 +1494,7 @@ function ArchiveConfirmDialog({
|
||||
<AlertDialogHeader>
|
||||
<AlertDialogTitle>{count === 1 ? "Archive task" : `Archive ${count} tasks`}</AlertDialogTitle>
|
||||
<AlertDialogDescription className="[overflow-wrap:anywhere]">
|
||||
Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into local cold storage. Once archived, a task can be restored from "Archived".
|
||||
Archiving stops task scheduling and compresses the graph, LLM history, files, and exclusive assets and traffic into the configured archive storage. Once archived, a task can be restored from "Archived". Restoring requires free space on this server.
|
||||
</AlertDialogDescription>
|
||||
</AlertDialogHeader>
|
||||
<AlertDialogFooter>
|
||||
@@ -1934,6 +1934,7 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {
|
||||
title={`Before compression: ${formatArchiveBytes(archive.original_size)}`}
|
||||
>
|
||||
{archiveCompressionLabel(archive)}
|
||||
<div className="text-muted-foreground">{archive.storage === "s3" ? "Object storage" : "Local storage"}</div>
|
||||
</TableCell>
|
||||
<TableCell className="text-xs tabular-nums">{archiveDataTotal(archive).toLocaleString()}</TableCell>
|
||||
<TableCell>
|
||||
@@ -1952,7 +1953,7 @@ function TaskArchivesPanel({ onChanged }: { onChanged: () => void }) {
|
||||
</p>
|
||||
))}
|
||||
<span className="text-muted-foreground text-xs">
|
||||
{archive.phase} · {archive.progress}%
|
||||
{archive.phase === "upload_and_verify" ? "Uploading and verifying" : archive.phase === "download_and_verify" ? "Downloading and verifying" : archive.phase} · {archive.progress}%
|
||||
</span>
|
||||
</div>
|
||||
</TableCell>
|
||||
|
||||
@@ -101,6 +101,7 @@ export interface TaskArchiveTokenStats {
|
||||
}
|
||||
|
||||
export interface TaskArchive {
|
||||
storage?: "local" | "s3";
|
||||
id: number;
|
||||
task_id: number;
|
||||
state: TaskArchiveState;
|
||||
|
||||
Reference in New Issue
Block a user