feat: encrypt S3 archives and test credentials and interruption recovery

This commit is contained in:
Chubboy Ren
2026-11-18 02:05:17 +00:00
parent 3d44e9cd0d
commit da7cc62d4e
9 changed files with 892 additions and 207 deletions
+20
View File
@@ -10,6 +10,7 @@ func TestArchiveStorageEnvironment(t *testing.T) {
"ARTEX_ARCHIVE_STORAGE", "ARTEX_ARCHIVE_S3_ENDPOINT", "ARTEX_ARCHIVE_S3_REGION",
"ARTEX_ARCHIVE_S3_BUCKET", "ARTEX_ARCHIVE_S3_PREFIX", "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID",
"ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_PATH_STYLE",
"ARTEX_ARCHIVE_S3_SESSION_TOKEN", "ARTEX_ARCHIVE_S3_SSE", "ARTEX_ARCHIVE_S3_KMS_KEY_ID",
} {
t.Setenv(key, "")
}
@@ -60,3 +61,22 @@ func TestArchiveEndpointValidation(t *testing.T) {
}
}
}
func TestArchiveEncryptionConfiguration(t *testing.T) {
c := Config{Endpoint: "https://objects.example.test", Bucket: "test", AccessKey: "test", SecretKey: "test"}
for _, mode := range []string{"none", "invalid"} {
c.SSE = mode
if _, err := New(c); err == nil {
t.Fatal("unencrypted/invalid mode accepted")
}
}
c.SSE = "AES256"
c.KMSKeyID = "alias/archive"
if _, err := New(c); err == nil {
t.Fatal("KMS key accepted with SSE-S3")
}
c.SSE = "aws:kms"
if _, err := New(c); err != nil {
t.Fatal(err)
}
}
+132
View File
@@ -0,0 +1,132 @@
package archivestore
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func isolateAWSCredentials(t *testing.T) {
t.Helper()
for _, key := range []string{
"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AWS_PROFILE", "AWS_DEFAULT_PROFILE",
"AWS_ROLE_ARN", "AWS_ROLE_SESSION_NAME", "AWS_WEB_IDENTITY_TOKEN_FILE",
"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_CREDENTIALS_FULL_URI",
"AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
"AWS_EC2_METADATA_SERVICE_ENDPOINT", "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE",
} {
t.Setenv(key, "")
}
dir := t.TempDir()
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(dir, "credentials"))
t.Setenv("AWS_CONFIG_FILE", filepath.Join(dir, "config"))
t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
}
func TestArchiveCredentialChain(t *testing.T) {
config := Config{Endpoint: "https://objects.example.test", Bucket: "test"}
for _, mode := range []string{"explicit-session", "environment", "shared-profile", "ecs", "imds"} {
t.Run(mode, func(t *testing.T) {
isolateAWSCredentials(t)
c := config
switch mode {
case "explicit-session":
t.Setenv("AWS_ACCESS_KEY_ID", "other-key")
t.Setenv("AWS_SECRET_ACCESS_KEY", "other-secret")
c.AccessKey, c.SecretKey, c.SessionToken = "fixture-key", "fixture-secret", "fixture-session"
case "environment":
t.Setenv("AWS_ACCESS_KEY_ID", "fixture-key")
t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture-secret")
t.Setenv("AWS_SESSION_TOKEN", "fixture-session")
case "shared-profile":
t.Setenv("AWS_PROFILE", "archive")
data := `[archive]
aws_access_key_id=fixture-key
aws_secret_access_key=fixture-secret
aws_session_token=fixture-session
`
if err := os.WriteFile(os.Getenv("AWS_SHARED_CREDENTIALS_FILE"), []byte(data), 0600); err != nil {
t.Fatal(err)
}
case "ecs", "imds":
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if mode == "imds" {
switch r.URL.Path {
case "/latest/api/token":
if r.Method != "PUT" {
t.Error("IMDS did not use token request")
}
w.Header().Set("X-aws-ec2-metadata-token-ttl-seconds", "21600")
fmt.Fprint(w, "fixture-imds-token")
return
case "/latest/meta-data/iam/security-credentials/":
if r.Header.Get("X-aws-ec2-metadata-token") != "fixture-imds-token" {
t.Error("missing IMDSv2 token")
}
fmt.Fprint(w, "archive-role")
return
case "/latest/meta-data/iam/security-credentials/archive-role":
if r.Header.Get("X-aws-ec2-metadata-token") != "fixture-imds-token" {
t.Error("missing IMDSv2 token")
}
default:
t.Errorf("unexpected IMDS request %s", r.URL.Path)
w.WriteHeader(404)
return
}
} else if r.Header.Get("Authorization") != "fixture-container-token" {
t.Error("missing ECS authorization token")
}
_ = json.NewEncoder(w).Encode(map[string]any{
"Code": "Success", "AccessKeyId": "fixture-key", "SecretAccessKey": "fixture-secret",
"Token": "fixture-session", "Expiration": time.Now().Add(time.Hour).UTC().Format(time.RFC3339),
})
}))
defer srv.Close()
if mode == "imds" {
t.Setenv("AWS_EC2_METADATA_DISABLED", "false")
t.Setenv("AWS_EC2_METADATA_V1_DISABLED", "true")
t.Setenv("AWS_EC2_METADATA_SERVICE_ENDPOINT", srv.URL)
} else {
t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", srv.URL+"/credentials")
t.Setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN", "fixture-container-token")
}
}
store, err := New(c)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
credentials, err := store.client.Options().Credentials.Retrieve(ctx)
if err != nil {
t.Fatal(err)
}
if credentials.AccessKeyID != "fixture-key" || credentials.SecretAccessKey != "fixture-secret" || credentials.SessionToken != "fixture-session" {
t.Fatal("wrong credential provider or session token")
}
if (mode == "imds" || mode == "ecs") && !credentials.CanExpire {
t.Fatal("role credentials must remain refreshable")
}
})
}
}
func TestArchiveIncompleteCredentialOverride(t *testing.T) {
isolateAWSCredentials(t)
t.Setenv("AWS_ACCESS_KEY_ID", "valid-ambient-key")
t.Setenv("AWS_SECRET_ACCESS_KEY", "valid-ambient-secret")
for _, c := range []Config{{AccessKey: "private-value"}, {SecretKey: "private-value"}, {SessionToken: "private-value"}} {
c.Endpoint, c.Bucket = "https://objects.example.test", "test"
if _, err := New(c); err == nil || strings.Contains(err.Error(), "private-value") {
t.Fatalf("partial override must fail without disclosing credentials: %v", err)
}
}
}
+58 -15
View File
@@ -17,19 +17,24 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
)
type Config struct {
Endpoint, Region, Bucket, Prefix, AccessKey, SecretKey string
PathStyle bool
SessionToken, SSE, KMSKeyID string
}
type Store struct {
client *s3.Client
bucket, prefix string
encryption types.ServerSideEncryption
kmsKeyID string
}
// FromEnv returns nil for the default local backend. A configured but invalid
@@ -43,7 +48,9 @@ func FromEnv() (*Store, error) {
Endpoint: os.Getenv("ARTEX_ARCHIVE_S3_ENDPOINT"), Region: os.Getenv("ARTEX_ARCHIVE_S3_REGION"),
Bucket: os.Getenv("ARTEX_ARCHIVE_S3_BUCKET"), Prefix: os.Getenv("ARTEX_ARCHIVE_S3_PREFIX"),
AccessKey: os.Getenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID"), SecretKey: os.Getenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY"),
PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true",
PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true",
SessionToken: os.Getenv("ARTEX_ARCHIVE_S3_SESSION_TOKEN"),
SSE: os.Getenv("ARTEX_ARCHIVE_S3_SSE"), KMSKeyID: os.Getenv("ARTEX_ARCHIVE_S3_KMS_KEY_ID"),
})
default:
return nil, errors.New("ARTEX_ARCHIVE_STORAGE must be local or s3")
@@ -70,22 +77,49 @@ func New(c Config) (*Store, error) {
if path.Clean(c.Prefix) != c.Prefix || strings.Contains(c.Prefix, "..") || strings.ContainsAny(c.Prefix, "\\?#") {
return nil, errors.New("invalid archive S3 prefix")
}
if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") || c.AccessKey == "" || c.SecretKey == "" {
return nil, errors.New("archive S3 bucket and dedicated access credentials are required")
if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") {
return nil, errors.New("a valid archive S3 bucket is required")
}
explicitCredentials := c.AccessKey != "" || c.SecretKey != "" || c.SessionToken != ""
if explicitCredentials && (c.AccessKey == "" || c.SecretKey == "") {
return nil, errors.New("archive S3 access key and secret key must both be set when overriding AWS credentials")
}
if c.SSE == "" {
c.SSE = "AES256"
}
if c.SSE != "AES256" && c.SSE != "aws:kms" {
return nil, errors.New("archive S3 SSE must be AES256 or aws:kms; encryption cannot be disabled")
}
if c.KMSKeyID != "" && c.SSE != "aws:kms" {
return nil, errors.New("archive S3 KMS key requires SSE=aws:kms")
}
if c.Region == "" {
c.Region = "us-east-1"
}
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.ResponseHeaderTimeout = 60 * time.Second
client := s3.New(s3.Options{
Region: c.Region, BaseEndpoint: aws.String(c.Endpoint), UsePathStyle: c.PathStyle,
Credentials: credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, ""),
HTTPClient: &http.Client{Transport: transport},
RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired,
httpClient := &http.Client{Transport: transport}
cfg := aws.Config{Region: c.Region, HTTPClient: httpClient}
if explicitCredentials {
cfg.Credentials = credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken)
} else {
// Keep the SDK's refreshable default chain: standard AWS environment/shared
// profiles, web identity, ECS task roles and EC2 instance roles (IMDSv2).
var err error
cfg, err = awsconfig.LoadDefaultConfig(context.Background(),
awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient))
if err != nil {
return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err)
}
}
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(c.Endpoint)
o.UsePathStyle = c.PathStyle
o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
})
return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/"}, nil
return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/",
encryption: types.ServerSideEncryption(c.SSE), kmsKeyID: c.KMSKeyID}, nil
}
func (s *Store) URI(taskID, archiveID int64, checksum string) string {
@@ -134,17 +168,22 @@ func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) erro
u.Concurrency = 2
u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
})
_, err = uploader.Upload(ctx, &s3.PutObjectInput{
input := &s3.PutObjectInput{
Bucket: aws.String(s.bucket), Key: aws.String(key), Body: f,
ContentType: aws.String("application/zstd"), Metadata: map[string]string{"sha256": checksum},
})
ServerSideEncryption: s.encryption,
}
if s.kmsKeyID != "" {
input.SSEKMSKeyId = aws.String(s.kmsKeyID)
}
_, err = uploader.Upload(ctx, input)
if err != nil {
return fmt.Errorf("upload archive: %w", err)
}
return s.readVerified(ctx, key, io.Discard, checksum, info.Size())
return s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true)
}
func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64) error {
func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64, requireEncryption bool) error {
if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size {
return errors.New("invalid archive SHA-256")
}
@@ -155,6 +194,10 @@ func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, che
if err != nil {
return fmt.Errorf("inspect archive: %w", err)
}
if requireEncryption && (object.ServerSideEncryption != s.encryption ||
(s.encryption == types.ServerSideEncryptionAwsKms && aws.ToString(object.SSEKMSKeyId) == "")) {
return errors.New("remote archive did not confirm the requested server-side encryption; source data retained")
}
if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" {
return errors.New("remote archive size mismatch or missing ETag; source data retained")
}
@@ -264,7 +307,7 @@ func (s *Store) Download(ctx context.Context, uri, filename, checksum string, si
_ = os.Remove(filename)
}
}()
if err = s.readVerified(ctx, key, f, checksum, size); err != nil {
if err = s.readVerified(ctx, key, f, checksum, size, false); err != nil {
return err
}
return f.Sync()
+193 -86
View File
@@ -13,99 +13,121 @@ import (
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
)
func TestS3RoundTripAndIntegrity(t *testing.T) {
for _, size := range []int{1024, 10 << 20} {
t.Run(strconv.Itoa(size), func(t *testing.T) {
var mu sync.Mutex
var object []byte
parts := map[int][]byte{}
corrupt, multipart := false, false
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
defer mu.Unlock()
q := r.URL.Query()
w.Header().Set("ETag", `"test-etag"`)
switch {
case r.Method == "POST" && q.Has("uploads"):
multipart = true
fmt.Fprint(w, `<InitiateMultipartUploadResult><UploadId>test-upload</UploadId></InitiateMultipartUploadResult>`)
case r.Method == "PUT" && q.Has("partNumber"):
n, _ := strconv.Atoi(q.Get("partNumber"))
parts[n], _ = io.ReadAll(r.Body)
case r.Method == "POST" && q.Has("uploadId"):
for i := 1; i <= len(parts); i++ {
object = append(object, parts[i]...)
for _, sse := range []string{"AES256", "aws:kms"} {
kms := ""
if sse == "aws:kms" {
kms = "arn:aws:kms:us-east-1:123456789012:key/test-key"
}
for _, size := range []int{1024, 10 << 20} {
t.Run(sse+"/"+strconv.Itoa(size), func(t *testing.T) {
var mu sync.Mutex
var object []byte
parts := map[int][]byte{}
corrupt, multipart := false, false
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
defer mu.Unlock()
q := r.URL.Query()
if (r.Method == "PUT" && !q.Has("partNumber")) || (r.Method == "POST" && q.Has("uploads")) {
if r.Header.Get("X-Amz-Server-Side-Encryption") != sse || r.Header.Get("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id") != kms {
t.Error("upload did not explicitly request the configured encryption")
}
}
fmt.Fprint(w, `<CompleteMultipartUploadResult><ETag>test-etag</ETag></CompleteMultipartUploadResult>`)
case r.Method == "PUT":
object, _ = io.ReadAll(r.Body)
case r.Method == "HEAD":
w.Header().Set("Content-Length", strconv.Itoa(len(object)))
case r.Method == "GET":
data := object
if corrupt {
data = bytes.Clone(object)
data[0] ^= 0xff
if r.Method == "HEAD" {
if r.Header.Get("X-Amz-Server-Side-Encryption") != "" {
t.Error("encryption write header leaked into HEAD")
}
w.Header().Set("X-Amz-Server-Side-Encryption", sse)
if kms != "" {
w.Header().Set("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id", kms)
}
}
serveArchiveRange(t, w, r, data)
case r.Method == "DELETE":
object = nil
w.WriteHeader(204)
default:
t.Errorf("unexpected request %s %s", r.Method, r.URL)
w.WriteHeader(400)
w.Header().Set("ETag", `"test-etag"`)
switch {
case r.Method == "POST" && q.Has("uploads"):
multipart = true
fmt.Fprint(w, `<InitiateMultipartUploadResult><UploadId>test-upload</UploadId></InitiateMultipartUploadResult>`)
case r.Method == "PUT" && q.Has("partNumber"):
n, _ := strconv.Atoi(q.Get("partNumber"))
parts[n], _ = io.ReadAll(r.Body)
case r.Method == "POST" && q.Has("uploadId"):
for i := 1; i <= len(parts); i++ {
object = append(object, parts[i]...)
}
fmt.Fprint(w, `<CompleteMultipartUploadResult><ETag>test-etag</ETag></CompleteMultipartUploadResult>`)
case r.Method == "PUT":
object, _ = io.ReadAll(r.Body)
case r.Method == "HEAD":
w.Header().Set("Content-Length", strconv.Itoa(len(object)))
case r.Method == "GET":
data := object
if corrupt {
data = bytes.Clone(object)
data[0] ^= 0xff
}
serveArchiveRange(t, w, r, data)
case r.Method == "DELETE":
object = nil
w.WriteHeader(204)
default:
t.Errorf("unexpected request %s %s", r.Method, r.URL)
w.WriteHeader(400)
}
}))
defer srv.Close()
store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true, SSE: sse, KMSKeyID: kms})
if err != nil {
t.Fatal(err)
}
}))
defer srv.Close()
store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
if err != nil {
t.Fatal(err)
}
data := bytes.Repeat([]byte{0x42}, size)
h := sha256.Sum256(data)
checksum := hex.EncodeToString(h[:])
root := t.TempDir()
source := filepath.Join(root, "source")
if err := os.WriteFile(source, data, 0600); err != nil {
t.Fatal(err)
}
uri := store.URI(1, 2, checksum)
ctx := context.Background()
if err := store.Upload(ctx, uri, source, checksum); err != nil {
t.Fatal(err)
}
if multipart != (size > 8<<20) {
t.Fatalf("multipart=%v size=%d", multipart, size)
}
dest := filepath.Join(root, "download")
if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil {
t.Fatal(err)
}
got, _ := os.ReadFile(dest)
if !bytes.Equal(got, data) {
t.Fatal("download differs")
}
mu.Lock()
corrupt = true
mu.Unlock()
bad := filepath.Join(root, "bad")
if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil {
t.Fatal("accepted corrupt object")
}
if _, err := os.Stat(bad); !os.IsNotExist(err) {
t.Fatal("left corrupt download")
}
if err := store.Upload(ctx, uri, source, checksum); err == nil {
t.Fatal("upload accepted failed readback")
}
if err := store.Delete(ctx, uri); err != nil {
t.Fatal(err)
}
})
data := bytes.Repeat([]byte{0x42}, size)
h := sha256.Sum256(data)
checksum := hex.EncodeToString(h[:])
root := t.TempDir()
source := filepath.Join(root, "source")
if err := os.WriteFile(source, data, 0600); err != nil {
t.Fatal(err)
}
uri := store.URI(1, 2, checksum)
ctx := context.Background()
if err := store.Upload(ctx, uri, source, checksum); err != nil {
t.Fatal(err)
}
if multipart != (size > 8<<20) {
t.Fatalf("multipart=%v size=%d", multipart, size)
}
dest := filepath.Join(root, "download")
if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil {
t.Fatal(err)
}
got, _ := os.ReadFile(dest)
if !bytes.Equal(got, data) {
t.Fatal("download differs")
}
mu.Lock()
corrupt = true
mu.Unlock()
bad := filepath.Join(root, "bad")
if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil {
t.Fatal("accepted corrupt object")
}
if _, err := os.Stat(bad); !os.IsNotExist(err) {
t.Fatal("left corrupt download")
}
if err := store.Upload(ctx, uri, source, checksum); err == nil {
t.Fatal("upload accepted failed readback")
}
if err := store.Delete(ctx, uri); err != nil {
t.Fatal(err)
}
})
}
}
}
@@ -256,7 +278,7 @@ func TestS3ReadbackPropagatesOutputFailure(t *testing.T) {
store := testRangeStore(t, srv.URL)
h := sha256.Sum256(data)
outputError := errors.New("output disk full")
err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data)))
err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data)), false)
if !errors.Is(err, outputError) {
t.Fatalf("lost output error: %v", err)
}
@@ -309,3 +331,88 @@ func TestS3RejectsForeignLocations(t *testing.T) {
t.Fatal("missing config accepted")
}
}
func TestS3UploadRejectsUnconfirmedEncryption(t *testing.T) {
for _, mode := range []string{"missing", "different", "kms-without-key"} {
t.Run(mode, func(t *testing.T) {
data := []byte("private archive")
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "PUT":
_, _ = io.Copy(io.Discard, r.Body)
case "HEAD":
w.Header().Set("ETag", `"test-etag"`)
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
if mode != "missing" {
w.Header().Set("X-Amz-Server-Side-Encryption", "aws:kms")
}
default:
t.Errorf("content must not be accepted before encryption confirmation: %s", r.Method)
w.WriteHeader(400)
}
}))
defer srv.Close()
c := Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}
if mode == "kms-without-key" {
c.SSE = "aws:kms"
}
store, err := New(c)
if err != nil {
t.Fatal(err)
}
filename := filepath.Join(t.TempDir(), "source")
if err := os.WriteFile(filename, data, 0600); err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(data)
checksum := hex.EncodeToString(sum[:])
if err := store.Upload(context.Background(), store.URI(1, 2, checksum), filename, checksum); err == nil || !strings.Contains(err.Error(), "encryption") {
t.Fatalf("unconfirmed encryption accepted: %v", err)
}
if got, err := os.ReadFile(filename); err != nil || !bytes.Equal(got, data) {
t.Fatal("source was not retained", err)
}
})
}
}
func TestS3MultipartFailureAbortsUpload(t *testing.T) {
var aborted atomic.Bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
switch {
case r.Method == "POST" && q.Has("uploads"):
if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" {
t.Error("multipart encryption missing")
}
fmt.Fprint(w, `<InitiateMultipartUploadResult><UploadId>failed-upload</UploadId></InitiateMultipartUploadResult>`)
case r.Method == "PUT" && q.Has("partNumber"):
_, _ = io.Copy(io.Discard, r.Body)
w.WriteHeader(403)
case r.Method == "DELETE" && q.Get("uploadId") == "failed-upload":
aborted.Store(true)
w.WriteHeader(204)
default:
t.Errorf("unexpected failed-upload request: %s", r.Method)
w.WriteHeader(400)
}
}))
defer srv.Close()
store := testRangeStore(t, srv.URL)
data := bytes.Repeat([]byte("x"), 9<<20)
checksum := sha256.Sum256(data)
sum := hex.EncodeToString(checksum[:])
filename := filepath.Join(t.TempDir(), "source")
if err := os.WriteFile(filename, data, 0600); err != nil {
t.Fatal(err)
}
if err := store.Upload(context.Background(), store.URI(1, 2, sum), filename, sum); err == nil {
t.Fatal("failed part accepted")
}
if !aborted.Load() {
t.Fatal("failed multipart upload was not aborted")
}
if info, err := os.Stat(filename); err != nil || info.Size() != int64(len(data)) {
t.Fatal("source was not retained", err)
}
}
+125 -69
View File
@@ -1,21 +1,19 @@
# Task archive storage
Task archives use local disk by default. The optional S3-compatible backend moves
cold task packages to object storage after verifying their contents, releasing
local package space while retaining ARTEX's Archive, Restore and Delete actions.
Active workspaces and the live PostgreSQL database stay local. This feature is
task-level cold storage; use the existing backup workflow for a full installation
backup. A complete recovery plan must also preserve the bucket objects and
access to them.
cold task packages to object storage while retaining Archive, Restore and Delete.
Active workspaces and PostgreSQL remain local. This is task cold storage, not a
full-installation backup; a recovery plan must preserve the database, data
directory, bucket objects, and the credentials or roles needed to read them.
## Configuration
Create a private bucket and a dedicated credential scoped to the archive prefix.
The backend needs object upload, read (including HEAD and byte ranges), delete,
and multipart upload/abort permissions. Use a separate prefix for each deployment.
Use a private bucket and a separate prefix for each ARTEX installation. Configure
the bucket's default encryption to match the selected mode below, and enforce
that mode/key with a bucket policy. ARTEX does not provision or inspect bucket
encryption settings.
Set these environment variables for the ARTEX server, then restart it:
Set the following server environment and restart ARTEX:
```dotenv
ARTEX_ARCHIVE_STORAGE=s3
@@ -23,82 +21,140 @@ ARTEX_ARCHIVE_S3_ENDPOINT=https://s3.us-east-1.amazonaws.com
ARTEX_ARCHIVE_S3_REGION=us-east-1
ARTEX_ARCHIVE_S3_BUCKET=your-artex-archives
ARTEX_ARCHIVE_S3_PREFIX=artex/tasks
ARTEX_ARCHIVE_S3_ACCESS_KEY_ID=REPLACE
ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY=REPLACE
ARTEX_ARCHIVE_S3_SSE=AES256
```
Store credentials in the service's protected environment file, outside the
repository and frontend. For systemd, a root-owned environment file with mode
0600 can be referenced through `EnvironmentFile=`.
With an EC2 instance role, ECS task role or other SDK-supported credential source,
no archive-specific access keys are needed.
| Variable | Meaning |
| --- | --- |
| `ARTEX_ARCHIVE_STORAGE` | `local` (default) or `s3`. |
| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local testing. |
| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. Match the provider's requirements. |
| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local tests. |
| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. |
| `ARTEX_ARCHIVE_S3_BUCKET` | Required bucket name. |
| `ARTEX_ARCHIVE_S3_PREFIX` | Object prefix; defaults to `artex/tasks`. |
| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Dedicated access key. |
| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Dedicated secret key. |
| `ARTEX_ARCHIVE_S3_PATH_STYLE` | Set to `true` for compatible path-style servers; otherwise virtual-hosted addressing is used. |
| `ARTEX_ARCHIVE_S3_SSE` | `AES256` (default, SSE-S3) or `aws:kms` (SSE-KMS). There is no unencrypted mode. |
| `ARTEX_ARCHIVE_S3_KMS_KEY_ID` | Optional KMS key ARN/ID/alias; valid only with `aws:kms`. Omit to use the provider's managed KMS key. |
| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Optional dedicated credential override; must be supplied with the secret key. |
| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Secret for the dedicated override. |
| `ARTEX_ARCHIVE_S3_SESSION_TOKEN` | Optional session token for dedicated temporary credentials. |
| `ARTEX_ARCHIVE_S3_PATH_STYLE` | `true` for path-style servers; otherwise virtual-hosted addressing is used. |
For DigitalOcean Spaces, use its regional origin, such as
`https://sfo3.digitaloceanspaces.com`, and the provider's signing region. For a
local path-style test server, use an endpoint such as `http://127.0.0.1:9000` and
set `ARTEX_ARCHIVE_S3_PATH_STYLE=true`.
An invalid mode or partial credential override fails configuration instead of
silently falling back to local storage or ambient credentials. Preserve the
bucket, endpoint, prefix and read/delete access while remote archives remain.
Older local archives remain usable; they are not uploaded automatically.
Invalid S3 configuration fails new archive jobs instead of silently falling back
to local disk. Preserve the configured bucket, endpoint, prefix and read/delete
access while remote archives remain. The same installation can still restore or
delete older local archives.
## Encryption and permissions
## Lifecycle and integrity
Every single-object upload and multipart initiation explicitly requests the
configured server-side encryption. Upload verification also requires HEAD to
report the requested mode, and a KMS key identifier for SSE-KMS, before accepting
the object. Size and full SHA-256 readback remain mandatory; encryption metadata
does not replace content verification.
1. Pause or finish a task and use its existing **Archive** action.
2. ARTEX builds its `.tar.zst` package locally and uploads it. Large packages use
multipart upload, starting with 8 MiB parts and two upload workers (the SDK
increases the part size for exceptionally large seekable files).
3. ARTEX reads the entire object back and checks its size and SHA-256. Multipart
ETags and uploaded checksum metadata alone are not treated as proof of content.
4. Only after verification does PostgreSQL commit the `s3://` location and
compact the archived task. ARTEX then removes local source files and the
temporary package. The archive list shows **Object storage** or **Local storage**.
5. **Restore** downloads and verifies the package before restoring task files and
database records. The remote package is consumed after restoration commits.
Failed remote cleanup can be retried without replaying restored records.
6. **Delete** removes the archive stub and queues durable cleanup of its remote
object. Cleanup is retried after restart and before subsequent archive jobs.
For SSE-KMS, set both the bucket's default encryption and ARTEX to `aws:kms`;
set `ARTEX_ARCHIVE_S3_KMS_KEY_ID` to the intended key when using a customer-managed
key. An explicit `AES256` request takes precedence over a bucket's KMS default,
so do not leave ARTEX on SSE-S3 when the bucket is intended to use KMS.
Reads use bounded 8 MiB ranges pinned to the object's ETag. Interrupted ranges
are retried without duplicating output or restarting a multi-gigabyte download.
An object change, checksum mismatch or invalid range fails verification. Failed
downloads are removed, and upload/verification failures retain the original task.
The principal needs object upload, read (HEAD and ranged GET), delete and
multipart upload/abort permissions. KMS uploads/readback also need the applicable
key-policy and IAM permissions for `kms:GenerateDataKey` and `kms:Decrypt`.
Retain old KMS keys and decrypt permissions when changing the key or mode; restore
does not require an older object to use the current upload policy.
There is no database schema or archive format migration. Existing local archive
packages remain compatible. Files outside ARTEX's task archive database are not
automatically imported or uploaded.
Compatible providers must support the requested SSE mode and its HEAD metadata,
plus conditional ranged reads. A provider that ignores the encryption request
or cannot confirm it is rejected; the original task is retained. Local source and
temporary files keep their existing local-disk protection and restricted modes.
## Disk, transfer and recovery
See the official [PutObject encryption and permission contract](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html)
and [HEAD encryption response rules](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html).
Archive creation still needs space for the temporary compressed package.
Restoration needs space for the downloaded package and extracted files. Full
upload readback and later restores incur object-storage reads and any applicable
transfer charges. Configure expiration of abandoned multipart uploads for abrupt
process termination.
## Credential sources
Recovery journals under `data/archives/tasks/.remote` contain object locations,
not credentials. Preserve them with PostgreSQL and the installation's data
directory. The worker uses these journals to clean up uploads that never
committed, keep committed objects, and finish deferred deletions. Failed remote
cleanup does not block unrelated local archive jobs.
Without the archive-specific override, ARTEX uses the AWS SDK default credential
chain, including standard AWS environment variables, shared profiles, web
identity, ECS task credentials and EC2 instance credentials through IMDS. Role
credentials retain the SDK's automatic refresh behavior. A named shared profile
can be selected with `AWS_PROFILE`.
For an external S3-compatible provider, dedicated keys can be supplied through
the archive-specific variables. Both key fields are required; a session token
is supported for temporary credentials. This explicit override takes precedence
over ambient AWS credentials. Operator-supplied temporary credentials must be
rotated by the operator; they are not an IAM role provider.
Keep secrets in a protected service environment file outside the repository and
frontend. A systemd `EnvironmentFile=` can point to a root-owned mode-0600 file.
See the [AWS SDK credential-chain documentation](https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-gosdk.html).
## Lifecycle
1. Pause or finish a task and use its **Archive** action.
2. ARTEX stages the source files and builds a local `.tar.zst` package. Large
packages use multipart upload, initially 8 MiB parts with two upload workers;
the SDK increases the part size for exceptionally large seekable files.
3. ARTEX confirms encryption metadata and reads the complete object back,
checking size and SHA-256. ETags and uploaded checksum metadata alone are
insufficient.
4. **Only after verification and a successful PostgreSQL commit** does ARTEX
remove the local package and staged source files. The committed archive
location is an `s3://` URI; there is no schema or archive-format migration.
5. **Restore** downloads and verifies before installing files and database rows.
The remote package is consumed only after restoration commits.
6. **Delete** commits removal of the archive stub, then cleans up the object.
A durable journal retains failed cleanup work.
The archive list identifies **Object storage** and **Local storage** and shows
upload/download verification phases. Reads use bounded 8 MiB ranges pinned to the
object's ETag, with retries for interrupted ranges rather than the entire object.
## Interruption recovery
Source staging and restore journals live with task archives. Remote cleanup
journals live under `data/archives/tasks/.remote`; they contain locations, not
credentials. Keep this directory with PostgreSQL and the installation data.
| Interruption | Recovery |
| --- | --- |
| During staging/compression, before an upload journal exists | Roll staged source files back. Remove any temporary package only after rollback succeeds. The task remains in PostgreSQL. |
| After the upload journal, before successful verification | Roll source files back and clean up any uncommitted object/package. Failed cleanup retains its journal for retry. |
| After verification, before the database commit | Treat the object as uncommitted: retain the original task and clean up the orphan object. Verification alone never authorizes local deletion. |
| After the database commit, before package/staging cleanup | Keep the committed object and finish local cleanup. The persisted location and checksum establish commit, even if Restore or Delete has since been queued. |
| Download/extraction before restored files are installed | Remove temporary extraction data; keep the remote object and archive stub for another attempt. |
| Restored files installed, before database restoration commits | Roll back installed files using the restore journal. Retain the remote package. |
| Database restoration committed, before remote cleanup | Recognize the already-restored task and finish cleanup without replaying database rows. |
| Object consumed, before the restored archive stub is removed | Skip download/replay; repeat idempotent object deletion and remove the stub. |
| Delete journal written, before database deletion commits | Keep the object while its archive stub exists. The queued Delete operation can retry. |
| Database deletion committed, before object deletion succeeds | The durable outbox retains the URI and retries object deletion. |
| An old failed upload survives a retry using local storage | Remove the abandoned remote object, but retain a local package referenced by the committed archive row. |
The staging recovery rule also applies to local archives. Dedicated tests cover
both storage modes with ready, restore-queued, delete-queued and uncommitted
states. Additional PostgreSQL/HTTP-S3 tests reconstruct interrupted operations at
13 persisted boundaries and repeat recovery to check idempotence.
Handled multipart-upload failures are aborted by the SDK. Abrupt process death
may leave unfinished multipart uploads; configure bucket lifecycle expiration
for those parts. Existing object versions, if versioning is enabled, are governed
by the bucket's version-retention policy.
## Disk and transfer requirements
Archive creation still requires temporary local space for the compressed
package. Restore requires space for the download and extracted files. Full
upload readback and restores incur storage reads and applicable transfer charges.
Failed remote cleanup does not prevent unrelated local archive jobs from being
processed once the failed cleanup request returns.
## Optional connectivity check
Run `go run ./cmd/artex-archive-check` with the same archive environment as the
server to upload, verify, download and delete a small uniquely named test object.
This command performs real bucket operations but does not touch task data. It
reports leftover test objects if cleanup fails.
Run `go run ./cmd/artex-archive-check` with the archive environment to upload,
verify, download and delete a small unique object. It performs real bucket
operations but does not touch task data, and reports cleanup failures.
Automated tests use an HTTP S3 fixture and isolated PostgreSQL databases. They
cover multipart transfers, corruption, interrupted range retries, object changes,
cancellation, upload rollback, corrupted restores and persistent cleanup.
Automated tests use HTTP fixtures for S3 and metadata credentials, plus isolated
PostgreSQL. They do not establish compatibility with a live cloud bucket.
+6
View File
@@ -5,6 +5,7 @@ go 1.26.10
require (
github.com/Autumn-27/norma v0.4.11
github.com/aws/aws-sdk-go-v2 v1.47.5
github.com/aws/aws-sdk-go-v2/config v1.33.13
github.com/aws/aws-sdk-go-v2/credentials v1.20.13
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6
@@ -27,6 +28,7 @@ require (
github.com/andybalholm/brotli v1.2.1 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect
@@ -34,6 +36,10 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.39.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect
github.com/aws/smithy-go v1.28.7 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/creack/pty v1.1.24 // indirect
+291
View File
@@ -0,0 +1,291 @@
package server
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"sync"
"testing"
"github.com/Autumn-27/artex/archivestore"
pgdb "github.com/Autumn-27/artex/db"
)
// Recreate persisted states at operation boundaries, then run the same recovery
// functions used at startup. PostgreSQL, files, and HTTP object storage are real
// isolated fixtures; no production failpoint hooks are needed.
func TestRemoteArchiveInterruptionRecovery(t *testing.T) {
if os.Getenv("ARTEX_PG_DSN") == "" {
t.Skip("requires isolated PostgreSQL")
}
phases := []string{
"staged_before_compression", "compressed_before_journal", "journal_before_upload",
"uploaded_before_verification", "verified_before_database",
"database_before_local_cleanup", "package_cleaned_before_stage_cleanup",
"restore_download_before_install", "restore_files_before_database", "restore_database_before_object_delete", "restore_object_deleted_before_stub",
"delete_journal_before_database", "delete_database_before_object",
}
for _, phase := range phases {
t.Run(phase, func(t *testing.T) {
m, err := NewManager(t.TempDir(), "")
if err != nil {
t.Fatal(err)
}
defer m.Close()
store, objects := newRecoveryObjectStore(t)
s := &Server{ctx: context.Background(), m: m, archiveStore: store}
task, err := m.CreateTask("interruption "+phase, "preserve source", nil, 0, 0)
if err != nil {
t.Fatal(err)
}
id, _ := strconv.ParseInt(task.ID, 10, 64)
defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }()
if err := m.pg.SetPaused(id, true); err != nil {
t.Fatal(err)
}
job, err := m.pg.QueueTaskArchive(id)
if err != nil {
t.Fatal(err)
}
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Archiving); err != nil {
t.Fatal(err)
}
source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt")
if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(source, []byte("original task data"), 0600); err != nil {
t.Fatal(err)
}
snapshot, err := m.pg.SnapshotTaskArchive(id)
if err != nil {
t.Fatal(err)
}
stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID)
if err != nil {
t.Fatal(err)
}
packagePath := taskArchivePath(m.dir, job.ID, task.ID)
var original, compressed int64
var checksum, uri, journal string
if phase != "staged_before_compression" {
original, compressed, checksum, err = writeTaskArchivePackage(packagePath, stage.payload, snapshot)
if err != nil {
t.Fatal(err)
}
uri = store.URI(id, job.ID, checksum)
}
if phase != "staged_before_compression" && phase != "compressed_before_journal" {
journal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, LocalPath: packagePath, Operation: "upload"})
if err != nil {
t.Fatal(err)
}
}
committed := phase != "staged_before_compression" && phase != "compressed_before_journal" &&
phase != "journal_before_upload" && phase != "uploaded_before_verification" && phase != "verified_before_database"
if phase == "uploaded_before_verification" {
data, err := os.ReadFile(packagePath)
if err != nil {
t.Fatal(err)
}
objects.put(uri, data)
} else if phase == "verified_before_database" || committed {
if err := store.Upload(s.ctx, uri, packagePath, checksum); err != nil {
t.Fatal(err)
}
}
if committed {
if err := m.pg.CompleteTaskArchive(job.ID, snapshot, uri, checksum, original, compressed); err != nil {
t.Fatal(err)
}
}
afterArchive := phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" ||
phase == "restore_object_deleted_before_stub" || phase == "delete_journal_before_database" || phase == "delete_database_before_object"
if phase == "package_cleaned_before_stage_cleanup" || afterArchive {
if err := os.Remove(packagePath); err != nil {
t.Fatal(err)
}
if err := os.Remove(journal); err != nil {
t.Fatal(err)
}
}
if afterArchive {
if err := stage.commit(); err != nil {
t.Fatal(err)
}
}
restored := false
remoteDeleted := false
if phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" {
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Restoring); err != nil {
t.Fatal(err)
}
parent := filepath.Join(taskArchiveRoot(m.dir), ".restore")
if err := os.MkdirAll(parent, 0700); err != nil {
t.Fatal(err)
}
extracted, err := os.MkdirTemp(parent, "interrupted-")
if err != nil {
t.Fatal(err)
}
downloaded := filepath.Join(extracted, "download.tar.zst")
if err := store.Download(s.ctx, uri, downloaded, checksum, compressed); err != nil {
t.Fatal(err)
}
if err := extractTaskArchivePackage(downloaded, checksum, extracted); err != nil {
t.Fatal(err)
}
if phase != "restore_download_before_install" {
if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID); err != nil {
t.Fatal(err)
}
}
if phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" {
if _, err := m.pg.RestoreTaskArchive(job.ID, snapshot, 0); err != nil {
t.Fatal(err)
}
restored = true
}
if phase == "restore_object_deleted_before_stub" {
if err := store.Delete(s.ctx, uri); err != nil {
t.Fatal(err)
}
remoteDeleted = true
}
}
deleting := phase == "delete_journal_before_database" || phase == "delete_database_before_object"
if deleting {
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Deleting); err != nil {
t.Fatal(err)
}
if _, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, Operation: "delete"}); err != nil {
t.Fatal(err)
}
if phase == "delete_database_before_object" {
if err := m.pg.DeleteTaskArchiveStub(job.ID); err != nil {
t.Fatal(err)
}
remoteDeleted = true
}
}
if err := m.pg.RecoverTaskArchiveJobs(); err != nil {
t.Fatal(err)
}
if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if err := s.recoverRemoteArchiveJournals(); err != nil {
t.Fatal(err)
}
wantSource := !committed || restored
data, fileErr := os.ReadFile(source)
if wantSource && (fileErr != nil || string(data) != "original task data") {
t.Fatalf("source data lost: %v", fileErr)
}
if !wantSource && !os.IsNotExist(fileErr) {
t.Fatalf("archived source resurrected: %v", fileErr)
}
if _, err := os.Stat(packagePath); !os.IsNotExist(err) {
t.Fatalf("temporary package remains: %v", err)
}
if got, want := objects.has(uri), committed && !remoteDeleted; got != want {
t.Fatalf("remote object exists=%v, want=%v", got, want)
}
if committed && !deleting {
actual, err := m.pg.IsTaskArchiveRestored(job.ID)
if err != nil || actual != restored {
t.Fatalf("restored=%v want=%v err=%v", actual, restored, err)
}
}
// A second startup must be harmless at every boundary.
if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if err := s.recoverRemoteArchiveJournals(); err != nil {
t.Fatal(err)
}
})
}
}
type recoveryObjects struct {
mu sync.Mutex
data map[string][]byte
}
func (o *recoveryObjects) put(uri string, data []byte) {
o.mu.Lock()
defer o.mu.Unlock()
// Test URIs always use the fixture bucket with path-style addressing.
o.data["/test/"+uri[len("s3://test/"):]] = append([]byte(nil), data...)
}
func (o *recoveryObjects) has(uri string) bool {
o.mu.Lock()
defer o.mu.Unlock()
if uri == "" {
return false
}
_, ok := o.data["/test/"+uri[len("s3://test/"):]]
return ok
}
func newRecoveryObjectStore(t *testing.T) (*archivestore.Store, *recoveryObjects) {
t.Helper()
objects := &recoveryObjects{data: map[string][]byte{}}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
objects.mu.Lock()
defer objects.mu.Unlock()
data, exists := objects.data[r.URL.Path]
w.Header().Set("ETag", `"recovery-etag"`)
switch r.Method {
case "PUT":
if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" {
t.Error("missing SSE")
}
objects.data[r.URL.Path], _ = io.ReadAll(r.Body)
case "HEAD":
if !exists {
w.WriteHeader(404)
return
}
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
w.Header().Set("X-Amz-Server-Side-Encryption", "AES256")
case "GET":
if !exists {
w.WriteHeader(404)
return
}
var start, end int
if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end >= len(data) || r.Header.Get("If-Match") != `"recovery-etag"` {
w.WriteHeader(416)
return
}
w.Header().Set("Content-Length", strconv.Itoa(end-start+1))
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
w.WriteHeader(206)
_, _ = w.Write(data[start : end+1])
case "DELETE":
delete(objects.data, r.URL.Path)
w.WriteHeader(204)
default:
t.Errorf("unexpected object request %s", r.Method)
w.WriteHeader(400)
}
}))
t.Cleanup(srv.Close)
store, err := archivestore.New(archivestore.Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
if err != nil {
t.Fatal(err)
}
return store, objects
}
+9 -2
View File
@@ -437,9 +437,16 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error {
stage := &taskArchiveFileStage{root: root, payload: filepath.Join(root, "payload"), journal: journal}
if err := stage.rollback(); err != nil {
errs = append(errs, err)
continue
}
if archive != nil && archive.ArchivePath != "" && !strings.HasPrefix(archive.ArchivePath, "s3://") {
_ = os.Remove(archive.ArchivePath)
// The process can exit after compression but before an upload journal or
// database location exists. Remove that temporary package only after the
// source-file rollback succeeds, for both local and remote backends.
packagePath := taskArchivePath(dataDir, journal.ArchiveID, journal.TaskID)
if err := validateArchivePath(dataDir, packagePath); err != nil {
errs = append(errs, err)
} else if err := removeLocalArchivePackage(packagePath); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
+58 -35
View File
@@ -46,6 +46,7 @@ func TestRemoteArchiveLifecycle(t *testing.T) {
objects[r.URL.Path], _ = io.ReadAll(r.Body)
w.Header().Set("ETag", `"test"`)
case "HEAD":
w.Header().Set("X-Amz-Server-Side-Encryption", "AES256")
w.Header().Set("ETag", `"test"`)
w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path])))
case "GET":
@@ -255,7 +256,7 @@ func TestRemoteArchiveLifecycle(t *testing.T) {
// Once PostgreSQL has committed an archive location and checksum, a subsequent
// queued operation must not make startup recovery resurrect the old workspace.
func TestRemoteArchiveStageRecoveryAfterCommit(t *testing.T) {
func TestTaskArchiveStageRecoveryAfterCommit(t *testing.T) {
if os.Getenv("ARTEX_PG_DSN") == "" {
t.Skip("requires isolated PostgreSQL")
}
@@ -284,41 +285,63 @@ func TestRemoteArchiveStageRecoveryAfterCommit(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} {
t.Run(state, func(t *testing.T) {
committed := state != pgdb.ArchiveQueued
checksum := ""
if committed {
checksum = strings.Repeat("a", 64)
}
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, "s3://test/tasks/package.tar.zst", checksum); err != nil {
t.Fatal(err)
}
file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt")
if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil {
t.Fatal(err)
}
stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID)
if err != nil {
t.Fatal(err)
}
if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stage.root); !os.IsNotExist(err) {
t.Fatalf("staging cleanup incomplete: %v", err)
}
if committed {
if _, err := os.Stat(file); !os.IsNotExist(err) {
t.Fatalf("resurrected committed workspace: %v", err)
for _, backend := range []string{"local", "s3"} {
for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} {
t.Run(backend+"/"+state, func(t *testing.T) {
committed := state != pgdb.ArchiveQueued
checksum := ""
if committed {
checksum = strings.Repeat("a", 64)
}
} else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" {
t.Fatalf("uncommitted archive lost source: %v", err)
}
})
packagePath := taskArchivePath(m.dir, job.ID, task.ID)
if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(packagePath, []byte("verified package"), 0600); err != nil {
t.Fatal(err)
}
location := packagePath
if backend == "s3" {
location = "s3://test/tasks/package.tar.zst"
}
if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, location, checksum); err != nil {
t.Fatal(err)
}
file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt")
if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil {
t.Fatal(err)
}
stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID)
if err != nil {
t.Fatal(err)
}
if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stage.root); !os.IsNotExist(err) {
t.Fatalf("staging cleanup incomplete: %v", err)
}
if backend == "local" && committed {
if data, err := os.ReadFile(packagePath); err != nil || string(data) != "verified package" {
t.Fatal("committed local package lost", err)
}
} else if !committed {
if _, err := os.Stat(packagePath); !os.IsNotExist(err) {
t.Fatal("uncommitted temporary package retained", err)
}
}
if committed {
if _, err := os.Stat(file); !os.IsNotExist(err) {
t.Fatalf("resurrected committed workspace: %v", err)
}
} else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" {
t.Fatalf("uncommitted archive lost source: %v", err)
}
})
}
}
}