diff --git a/archivestore/config_test.go b/archivestore/config_test.go index cc12532..73afc73 100644 --- a/archivestore/config_test.go +++ b/archivestore/config_test.go @@ -10,6 +10,7 @@ func TestArchiveStorageEnvironment(t *testing.T) { "ARTEX_ARCHIVE_STORAGE", "ARTEX_ARCHIVE_S3_ENDPOINT", "ARTEX_ARCHIVE_S3_REGION", "ARTEX_ARCHIVE_S3_BUCKET", "ARTEX_ARCHIVE_S3_PREFIX", "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID", "ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_PATH_STYLE", + "ARTEX_ARCHIVE_S3_SESSION_TOKEN", "ARTEX_ARCHIVE_S3_SSE", "ARTEX_ARCHIVE_S3_KMS_KEY_ID", } { t.Setenv(key, "") } @@ -60,3 +61,22 @@ func TestArchiveEndpointValidation(t *testing.T) { } } } + +func TestArchiveEncryptionConfiguration(t *testing.T) { + c := Config{Endpoint: "https://objects.example.test", Bucket: "test", AccessKey: "test", SecretKey: "test"} + for _, mode := range []string{"none", "invalid"} { + c.SSE = mode + if _, err := New(c); err == nil { + t.Fatal("unencrypted/invalid mode accepted") + } + } + c.SSE = "AES256" + c.KMSKeyID = "alias/archive" + if _, err := New(c); err == nil { + t.Fatal("KMS key accepted with SSE-S3") + } + c.SSE = "aws:kms" + if _, err := New(c); err != nil { + t.Fatal(err) + } +} diff --git a/archivestore/credentials_test.go b/archivestore/credentials_test.go new file mode 100644 index 0000000..6c667ed --- /dev/null +++ b/archivestore/credentials_test.go @@ -0,0 +1,132 @@ +package archivestore + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func isolateAWSCredentials(t *testing.T) { + t.Helper() + for _, key := range []string{ + "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AWS_PROFILE", "AWS_DEFAULT_PROFILE", + "AWS_ROLE_ARN", "AWS_ROLE_SESSION_NAME", "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_CREDENTIALS_FULL_URI", + "AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", + "AWS_EC2_METADATA_SERVICE_ENDPOINT", "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE", + } { + t.Setenv(key, "") + } + dir := t.TempDir() + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(dir, "credentials")) + t.Setenv("AWS_CONFIG_FILE", filepath.Join(dir, "config")) + t.Setenv("AWS_EC2_METADATA_DISABLED", "true") +} + +func TestArchiveCredentialChain(t *testing.T) { + config := Config{Endpoint: "https://objects.example.test", Bucket: "test"} + for _, mode := range []string{"explicit-session", "environment", "shared-profile", "ecs", "imds"} { + t.Run(mode, func(t *testing.T) { + isolateAWSCredentials(t) + c := config + switch mode { + case "explicit-session": + t.Setenv("AWS_ACCESS_KEY_ID", "other-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "other-secret") + c.AccessKey, c.SecretKey, c.SessionToken = "fixture-key", "fixture-secret", "fixture-session" + case "environment": + t.Setenv("AWS_ACCESS_KEY_ID", "fixture-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture-secret") + t.Setenv("AWS_SESSION_TOKEN", "fixture-session") + case "shared-profile": + t.Setenv("AWS_PROFILE", "archive") + data := `[archive] +aws_access_key_id=fixture-key +aws_secret_access_key=fixture-secret +aws_session_token=fixture-session +` + if err := os.WriteFile(os.Getenv("AWS_SHARED_CREDENTIALS_FILE"), []byte(data), 0600); err != nil { + t.Fatal(err) + } + case "ecs", "imds": + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if mode == "imds" { + switch r.URL.Path { + case "/latest/api/token": + if r.Method != "PUT" { + t.Error("IMDS did not use token request") + } + w.Header().Set("X-aws-ec2-metadata-token-ttl-seconds", "21600") + fmt.Fprint(w, "fixture-imds-token") + return + case "/latest/meta-data/iam/security-credentials/": + if r.Header.Get("X-aws-ec2-metadata-token") != "fixture-imds-token" { + t.Error("missing IMDSv2 token") + } + fmt.Fprint(w, "archive-role") + return + case "/latest/meta-data/iam/security-credentials/archive-role": + if r.Header.Get("X-aws-ec2-metadata-token") != "fixture-imds-token" { + t.Error("missing IMDSv2 token") + } + default: + t.Errorf("unexpected IMDS request %s", r.URL.Path) + w.WriteHeader(404) + return + } + } else if r.Header.Get("Authorization") != "fixture-container-token" { + t.Error("missing ECS authorization token") + } + _ = json.NewEncoder(w).Encode(map[string]any{ + "Code": "Success", "AccessKeyId": "fixture-key", "SecretAccessKey": "fixture-secret", + "Token": "fixture-session", "Expiration": time.Now().Add(time.Hour).UTC().Format(time.RFC3339), + }) + })) + defer srv.Close() + if mode == "imds" { + t.Setenv("AWS_EC2_METADATA_DISABLED", "false") + t.Setenv("AWS_EC2_METADATA_V1_DISABLED", "true") + t.Setenv("AWS_EC2_METADATA_SERVICE_ENDPOINT", srv.URL) + } else { + t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", srv.URL+"/credentials") + t.Setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN", "fixture-container-token") + } + } + store, err := New(c) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + credentials, err := store.client.Options().Credentials.Retrieve(ctx) + if err != nil { + t.Fatal(err) + } + if credentials.AccessKeyID != "fixture-key" || credentials.SecretAccessKey != "fixture-secret" || credentials.SessionToken != "fixture-session" { + t.Fatal("wrong credential provider or session token") + } + if (mode == "imds" || mode == "ecs") && !credentials.CanExpire { + t.Fatal("role credentials must remain refreshable") + } + }) + } +} + +func TestArchiveIncompleteCredentialOverride(t *testing.T) { + isolateAWSCredentials(t) + t.Setenv("AWS_ACCESS_KEY_ID", "valid-ambient-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "valid-ambient-secret") + for _, c := range []Config{{AccessKey: "private-value"}, {SecretKey: "private-value"}, {SessionToken: "private-value"}} { + c.Endpoint, c.Bucket = "https://objects.example.test", "test" + if _, err := New(c); err == nil || strings.Contains(err.Error(), "private-value") { + t.Fatalf("partial override must fail without disclosing credentials: %v", err) + } + } +} diff --git a/archivestore/s3.go b/archivestore/s3.go index edc1513..27c50af 100644 --- a/archivestore/s3.go +++ b/archivestore/s3.go @@ -17,19 +17,24 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" ) type Config struct { Endpoint, Region, Bucket, Prefix, AccessKey, SecretKey string PathStyle bool + SessionToken, SSE, KMSKeyID string } type Store struct { client *s3.Client bucket, prefix string + encryption types.ServerSideEncryption + kmsKeyID string } // FromEnv returns nil for the default local backend. A configured but invalid @@ -43,7 +48,9 @@ func FromEnv() (*Store, error) { Endpoint: os.Getenv("ARTEX_ARCHIVE_S3_ENDPOINT"), Region: os.Getenv("ARTEX_ARCHIVE_S3_REGION"), Bucket: os.Getenv("ARTEX_ARCHIVE_S3_BUCKET"), Prefix: os.Getenv("ARTEX_ARCHIVE_S3_PREFIX"), AccessKey: os.Getenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID"), SecretKey: os.Getenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY"), - PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true", + PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true", + SessionToken: os.Getenv("ARTEX_ARCHIVE_S3_SESSION_TOKEN"), + SSE: os.Getenv("ARTEX_ARCHIVE_S3_SSE"), KMSKeyID: os.Getenv("ARTEX_ARCHIVE_S3_KMS_KEY_ID"), }) default: return nil, errors.New("ARTEX_ARCHIVE_STORAGE must be local or s3") @@ -70,22 +77,49 @@ func New(c Config) (*Store, error) { if path.Clean(c.Prefix) != c.Prefix || strings.Contains(c.Prefix, "..") || strings.ContainsAny(c.Prefix, "\\?#") { return nil, errors.New("invalid archive S3 prefix") } - if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") || c.AccessKey == "" || c.SecretKey == "" { - return nil, errors.New("archive S3 bucket and dedicated access credentials are required") + if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") { + return nil, errors.New("a valid archive S3 bucket is required") + } + explicitCredentials := c.AccessKey != "" || c.SecretKey != "" || c.SessionToken != "" + if explicitCredentials && (c.AccessKey == "" || c.SecretKey == "") { + return nil, errors.New("archive S3 access key and secret key must both be set when overriding AWS credentials") + } + if c.SSE == "" { + c.SSE = "AES256" + } + if c.SSE != "AES256" && c.SSE != "aws:kms" { + return nil, errors.New("archive S3 SSE must be AES256 or aws:kms; encryption cannot be disabled") + } + if c.KMSKeyID != "" && c.SSE != "aws:kms" { + return nil, errors.New("archive S3 KMS key requires SSE=aws:kms") } if c.Region == "" { c.Region = "us-east-1" } transport := http.DefaultTransport.(*http.Transport).Clone() transport.ResponseHeaderTimeout = 60 * time.Second - client := s3.New(s3.Options{ - Region: c.Region, BaseEndpoint: aws.String(c.Endpoint), UsePathStyle: c.PathStyle, - Credentials: credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, ""), - HTTPClient: &http.Client{Transport: transport}, - RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired, - ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired, + httpClient := &http.Client{Transport: transport} + cfg := aws.Config{Region: c.Region, HTTPClient: httpClient} + if explicitCredentials { + cfg.Credentials = credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken) + } else { + // Keep the SDK's refreshable default chain: standard AWS environment/shared + // profiles, web identity, ECS task roles and EC2 instance roles (IMDSv2). + var err error + cfg, err = awsconfig.LoadDefaultConfig(context.Background(), + awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient)) + if err != nil { + return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err) + } + } + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(c.Endpoint) + o.UsePathStyle = c.PathStyle + o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired }) - return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/"}, nil + return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/", + encryption: types.ServerSideEncryption(c.SSE), kmsKeyID: c.KMSKeyID}, nil } func (s *Store) URI(taskID, archiveID int64, checksum string) string { @@ -134,17 +168,22 @@ func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) erro u.Concurrency = 2 u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired }) - _, err = uploader.Upload(ctx, &s3.PutObjectInput{ + input := &s3.PutObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(key), Body: f, ContentType: aws.String("application/zstd"), Metadata: map[string]string{"sha256": checksum}, - }) + ServerSideEncryption: s.encryption, + } + if s.kmsKeyID != "" { + input.SSEKMSKeyId = aws.String(s.kmsKeyID) + } + _, err = uploader.Upload(ctx, input) if err != nil { return fmt.Errorf("upload archive: %w", err) } - return s.readVerified(ctx, key, io.Discard, checksum, info.Size()) + return s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true) } -func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64) error { +func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64, requireEncryption bool) error { if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size { return errors.New("invalid archive SHA-256") } @@ -155,6 +194,10 @@ func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, che if err != nil { return fmt.Errorf("inspect archive: %w", err) } + if requireEncryption && (object.ServerSideEncryption != s.encryption || + (s.encryption == types.ServerSideEncryptionAwsKms && aws.ToString(object.SSEKMSKeyId) == "")) { + return errors.New("remote archive did not confirm the requested server-side encryption; source data retained") + } if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" { return errors.New("remote archive size mismatch or missing ETag; source data retained") } @@ -264,7 +307,7 @@ func (s *Store) Download(ctx context.Context, uri, filename, checksum string, si _ = os.Remove(filename) } }() - if err = s.readVerified(ctx, key, f, checksum, size); err != nil { + if err = s.readVerified(ctx, key, f, checksum, size, false); err != nil { return err } return f.Sync() diff --git a/archivestore/s3_test.go b/archivestore/s3_test.go index 316e770..dc618c1 100644 --- a/archivestore/s3_test.go +++ b/archivestore/s3_test.go @@ -13,99 +13,121 @@ import ( "os" "path/filepath" "strconv" + "strings" "sync" + "sync/atomic" "testing" ) func TestS3RoundTripAndIntegrity(t *testing.T) { - for _, size := range []int{1024, 10 << 20} { - t.Run(strconv.Itoa(size), func(t *testing.T) { - var mu sync.Mutex - var object []byte - parts := map[int][]byte{} - corrupt, multipart := false, false - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - mu.Lock() - defer mu.Unlock() - q := r.URL.Query() - w.Header().Set("ETag", `"test-etag"`) - switch { - case r.Method == "POST" && q.Has("uploads"): - multipart = true - fmt.Fprint(w, `test-upload`) - case r.Method == "PUT" && q.Has("partNumber"): - n, _ := strconv.Atoi(q.Get("partNumber")) - parts[n], _ = io.ReadAll(r.Body) - case r.Method == "POST" && q.Has("uploadId"): - for i := 1; i <= len(parts); i++ { - object = append(object, parts[i]...) + for _, sse := range []string{"AES256", "aws:kms"} { + kms := "" + if sse == "aws:kms" { + kms = "arn:aws:kms:us-east-1:123456789012:key/test-key" + } + for _, size := range []int{1024, 10 << 20} { + t.Run(sse+"/"+strconv.Itoa(size), func(t *testing.T) { + var mu sync.Mutex + var object []byte + parts := map[int][]byte{} + corrupt, multipart := false, false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + q := r.URL.Query() + if (r.Method == "PUT" && !q.Has("partNumber")) || (r.Method == "POST" && q.Has("uploads")) { + if r.Header.Get("X-Amz-Server-Side-Encryption") != sse || r.Header.Get("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id") != kms { + t.Error("upload did not explicitly request the configured encryption") + } } - fmt.Fprint(w, `test-etag`) - case r.Method == "PUT": - object, _ = io.ReadAll(r.Body) - case r.Method == "HEAD": - w.Header().Set("Content-Length", strconv.Itoa(len(object))) - case r.Method == "GET": - data := object - if corrupt { - data = bytes.Clone(object) - data[0] ^= 0xff + if r.Method == "HEAD" { + if r.Header.Get("X-Amz-Server-Side-Encryption") != "" { + t.Error("encryption write header leaked into HEAD") + } + w.Header().Set("X-Amz-Server-Side-Encryption", sse) + if kms != "" { + w.Header().Set("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id", kms) + } } - serveArchiveRange(t, w, r, data) - case r.Method == "DELETE": - object = nil - w.WriteHeader(204) - default: - t.Errorf("unexpected request %s %s", r.Method, r.URL) - w.WriteHeader(400) + w.Header().Set("ETag", `"test-etag"`) + switch { + case r.Method == "POST" && q.Has("uploads"): + multipart = true + fmt.Fprint(w, `test-upload`) + case r.Method == "PUT" && q.Has("partNumber"): + n, _ := strconv.Atoi(q.Get("partNumber")) + parts[n], _ = io.ReadAll(r.Body) + case r.Method == "POST" && q.Has("uploadId"): + for i := 1; i <= len(parts); i++ { + object = append(object, parts[i]...) + } + fmt.Fprint(w, `test-etag`) + case r.Method == "PUT": + object, _ = io.ReadAll(r.Body) + case r.Method == "HEAD": + w.Header().Set("Content-Length", strconv.Itoa(len(object))) + case r.Method == "GET": + data := object + if corrupt { + data = bytes.Clone(object) + data[0] ^= 0xff + } + serveArchiveRange(t, w, r, data) + case r.Method == "DELETE": + object = nil + w.WriteHeader(204) + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL) + w.WriteHeader(400) + } + })) + defer srv.Close() + store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true, SSE: sse, KMSKeyID: kms}) + if err != nil { + t.Fatal(err) } - })) - defer srv.Close() - store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) - if err != nil { - t.Fatal(err) - } - data := bytes.Repeat([]byte{0x42}, size) - h := sha256.Sum256(data) - checksum := hex.EncodeToString(h[:]) - root := t.TempDir() - source := filepath.Join(root, "source") - if err := os.WriteFile(source, data, 0600); err != nil { - t.Fatal(err) - } - uri := store.URI(1, 2, checksum) - ctx := context.Background() - if err := store.Upload(ctx, uri, source, checksum); err != nil { - t.Fatal(err) - } - if multipart != (size > 8<<20) { - t.Fatalf("multipart=%v size=%d", multipart, size) - } - dest := filepath.Join(root, "download") - if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil { - t.Fatal(err) - } - got, _ := os.ReadFile(dest) - if !bytes.Equal(got, data) { - t.Fatal("download differs") - } - mu.Lock() - corrupt = true - mu.Unlock() - bad := filepath.Join(root, "bad") - if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil { - t.Fatal("accepted corrupt object") - } - if _, err := os.Stat(bad); !os.IsNotExist(err) { - t.Fatal("left corrupt download") - } - if err := store.Upload(ctx, uri, source, checksum); err == nil { - t.Fatal("upload accepted failed readback") - } - if err := store.Delete(ctx, uri); err != nil { - t.Fatal(err) - } - }) + data := bytes.Repeat([]byte{0x42}, size) + h := sha256.Sum256(data) + checksum := hex.EncodeToString(h[:]) + root := t.TempDir() + source := filepath.Join(root, "source") + if err := os.WriteFile(source, data, 0600); err != nil { + t.Fatal(err) + } + uri := store.URI(1, 2, checksum) + ctx := context.Background() + if err := store.Upload(ctx, uri, source, checksum); err != nil { + t.Fatal(err) + } + if multipart != (size > 8<<20) { + t.Fatalf("multipart=%v size=%d", multipart, size) + } + dest := filepath.Join(root, "download") + if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil { + t.Fatal(err) + } + got, _ := os.ReadFile(dest) + if !bytes.Equal(got, data) { + t.Fatal("download differs") + } + mu.Lock() + corrupt = true + mu.Unlock() + bad := filepath.Join(root, "bad") + if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil { + t.Fatal("accepted corrupt object") + } + if _, err := os.Stat(bad); !os.IsNotExist(err) { + t.Fatal("left corrupt download") + } + if err := store.Upload(ctx, uri, source, checksum); err == nil { + t.Fatal("upload accepted failed readback") + } + if err := store.Delete(ctx, uri); err != nil { + t.Fatal(err) + } + }) + } } } @@ -256,7 +278,7 @@ func TestS3ReadbackPropagatesOutputFailure(t *testing.T) { store := testRangeStore(t, srv.URL) h := sha256.Sum256(data) outputError := errors.New("output disk full") - err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data))) + err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data)), false) if !errors.Is(err, outputError) { t.Fatalf("lost output error: %v", err) } @@ -309,3 +331,88 @@ func TestS3RejectsForeignLocations(t *testing.T) { t.Fatal("missing config accepted") } } + +func TestS3UploadRejectsUnconfirmedEncryption(t *testing.T) { + for _, mode := range []string{"missing", "different", "kms-without-key"} { + t.Run(mode, func(t *testing.T) { + data := []byte("private archive") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case "PUT": + _, _ = io.Copy(io.Discard, r.Body) + case "HEAD": + w.Header().Set("ETag", `"test-etag"`) + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + if mode != "missing" { + w.Header().Set("X-Amz-Server-Side-Encryption", "aws:kms") + } + default: + t.Errorf("content must not be accepted before encryption confirmation: %s", r.Method) + w.WriteHeader(400) + } + })) + defer srv.Close() + c := Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true} + if mode == "kms-without-key" { + c.SSE = "aws:kms" + } + store, err := New(c) + if err != nil { + t.Fatal(err) + } + filename := filepath.Join(t.TempDir(), "source") + if err := os.WriteFile(filename, data, 0600); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(data) + checksum := hex.EncodeToString(sum[:]) + if err := store.Upload(context.Background(), store.URI(1, 2, checksum), filename, checksum); err == nil || !strings.Contains(err.Error(), "encryption") { + t.Fatalf("unconfirmed encryption accepted: %v", err) + } + if got, err := os.ReadFile(filename); err != nil || !bytes.Equal(got, data) { + t.Fatal("source was not retained", err) + } + }) + } +} + +func TestS3MultipartFailureAbortsUpload(t *testing.T) { + var aborted atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + switch { + case r.Method == "POST" && q.Has("uploads"): + if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" { + t.Error("multipart encryption missing") + } + fmt.Fprint(w, `failed-upload`) + case r.Method == "PUT" && q.Has("partNumber"): + _, _ = io.Copy(io.Discard, r.Body) + w.WriteHeader(403) + case r.Method == "DELETE" && q.Get("uploadId") == "failed-upload": + aborted.Store(true) + w.WriteHeader(204) + default: + t.Errorf("unexpected failed-upload request: %s", r.Method) + w.WriteHeader(400) + } + })) + defer srv.Close() + store := testRangeStore(t, srv.URL) + data := bytes.Repeat([]byte("x"), 9<<20) + checksum := sha256.Sum256(data) + sum := hex.EncodeToString(checksum[:]) + filename := filepath.Join(t.TempDir(), "source") + if err := os.WriteFile(filename, data, 0600); err != nil { + t.Fatal(err) + } + if err := store.Upload(context.Background(), store.URI(1, 2, sum), filename, sum); err == nil { + t.Fatal("failed part accepted") + } + if !aborted.Load() { + t.Fatal("failed multipart upload was not aborted") + } + if info, err := os.Stat(filename); err != nil || info.Size() != int64(len(data)) { + t.Fatal("source was not retained", err) + } +} diff --git a/docs/task-archive-storage.md b/docs/task-archive-storage.md index 18bb3b4..113921a 100644 --- a/docs/task-archive-storage.md +++ b/docs/task-archive-storage.md @@ -1,21 +1,19 @@ # Task archive storage Task archives use local disk by default. The optional S3-compatible backend moves -cold task packages to object storage after verifying their contents, releasing -local package space while retaining ARTEX's Archive, Restore and Delete actions. - -Active workspaces and the live PostgreSQL database stay local. This feature is -task-level cold storage; use the existing backup workflow for a full installation -backup. A complete recovery plan must also preserve the bucket objects and -access to them. +cold task packages to object storage while retaining Archive, Restore and Delete. +Active workspaces and PostgreSQL remain local. This is task cold storage, not a +full-installation backup; a recovery plan must preserve the database, data +directory, bucket objects, and the credentials or roles needed to read them. ## Configuration -Create a private bucket and a dedicated credential scoped to the archive prefix. -The backend needs object upload, read (including HEAD and byte ranges), delete, -and multipart upload/abort permissions. Use a separate prefix for each deployment. +Use a private bucket and a separate prefix for each ARTEX installation. Configure +the bucket's default encryption to match the selected mode below, and enforce +that mode/key with a bucket policy. ARTEX does not provision or inspect bucket +encryption settings. -Set these environment variables for the ARTEX server, then restart it: +Set the following server environment and restart ARTEX: ```dotenv ARTEX_ARCHIVE_STORAGE=s3 @@ -23,82 +21,140 @@ ARTEX_ARCHIVE_S3_ENDPOINT=https://s3.us-east-1.amazonaws.com ARTEX_ARCHIVE_S3_REGION=us-east-1 ARTEX_ARCHIVE_S3_BUCKET=your-artex-archives ARTEX_ARCHIVE_S3_PREFIX=artex/tasks -ARTEX_ARCHIVE_S3_ACCESS_KEY_ID=REPLACE -ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY=REPLACE +ARTEX_ARCHIVE_S3_SSE=AES256 ``` -Store credentials in the service's protected environment file, outside the -repository and frontend. For systemd, a root-owned environment file with mode -0600 can be referenced through `EnvironmentFile=`. +With an EC2 instance role, ECS task role or other SDK-supported credential source, +no archive-specific access keys are needed. | Variable | Meaning | | --- | --- | | `ARTEX_ARCHIVE_STORAGE` | `local` (default) or `s3`. | -| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local testing. | -| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. Match the provider's requirements. | +| `ARTEX_ARCHIVE_S3_ENDPOINT` | Required HTTPS origin, without bucket/path/query. Loopback HTTP is allowed for local tests. | +| `ARTEX_ARCHIVE_S3_REGION` | Signing region; defaults to `us-east-1`. | | `ARTEX_ARCHIVE_S3_BUCKET` | Required bucket name. | | `ARTEX_ARCHIVE_S3_PREFIX` | Object prefix; defaults to `artex/tasks`. | -| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Dedicated access key. | -| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Dedicated secret key. | -| `ARTEX_ARCHIVE_S3_PATH_STYLE` | Set to `true` for compatible path-style servers; otherwise virtual-hosted addressing is used. | +| `ARTEX_ARCHIVE_S3_SSE` | `AES256` (default, SSE-S3) or `aws:kms` (SSE-KMS). There is no unencrypted mode. | +| `ARTEX_ARCHIVE_S3_KMS_KEY_ID` | Optional KMS key ARN/ID/alias; valid only with `aws:kms`. Omit to use the provider's managed KMS key. | +| `ARTEX_ARCHIVE_S3_ACCESS_KEY_ID` | Optional dedicated credential override; must be supplied with the secret key. | +| `ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY` | Secret for the dedicated override. | +| `ARTEX_ARCHIVE_S3_SESSION_TOKEN` | Optional session token for dedicated temporary credentials. | +| `ARTEX_ARCHIVE_S3_PATH_STYLE` | `true` for path-style servers; otherwise virtual-hosted addressing is used. | -For DigitalOcean Spaces, use its regional origin, such as -`https://sfo3.digitaloceanspaces.com`, and the provider's signing region. For a -local path-style test server, use an endpoint such as `http://127.0.0.1:9000` and -set `ARTEX_ARCHIVE_S3_PATH_STYLE=true`. +An invalid mode or partial credential override fails configuration instead of +silently falling back to local storage or ambient credentials. Preserve the +bucket, endpoint, prefix and read/delete access while remote archives remain. +Older local archives remain usable; they are not uploaded automatically. -Invalid S3 configuration fails new archive jobs instead of silently falling back -to local disk. Preserve the configured bucket, endpoint, prefix and read/delete -access while remote archives remain. The same installation can still restore or -delete older local archives. +## Encryption and permissions -## Lifecycle and integrity +Every single-object upload and multipart initiation explicitly requests the +configured server-side encryption. Upload verification also requires HEAD to +report the requested mode, and a KMS key identifier for SSE-KMS, before accepting +the object. Size and full SHA-256 readback remain mandatory; encryption metadata +does not replace content verification. -1. Pause or finish a task and use its existing **Archive** action. -2. ARTEX builds its `.tar.zst` package locally and uploads it. Large packages use - multipart upload, starting with 8 MiB parts and two upload workers (the SDK - increases the part size for exceptionally large seekable files). -3. ARTEX reads the entire object back and checks its size and SHA-256. Multipart - ETags and uploaded checksum metadata alone are not treated as proof of content. -4. Only after verification does PostgreSQL commit the `s3://` location and - compact the archived task. ARTEX then removes local source files and the - temporary package. The archive list shows **Object storage** or **Local storage**. -5. **Restore** downloads and verifies the package before restoring task files and - database records. The remote package is consumed after restoration commits. - Failed remote cleanup can be retried without replaying restored records. -6. **Delete** removes the archive stub and queues durable cleanup of its remote - object. Cleanup is retried after restart and before subsequent archive jobs. +For SSE-KMS, set both the bucket's default encryption and ARTEX to `aws:kms`; +set `ARTEX_ARCHIVE_S3_KMS_KEY_ID` to the intended key when using a customer-managed +key. An explicit `AES256` request takes precedence over a bucket's KMS default, +so do not leave ARTEX on SSE-S3 when the bucket is intended to use KMS. -Reads use bounded 8 MiB ranges pinned to the object's ETag. Interrupted ranges -are retried without duplicating output or restarting a multi-gigabyte download. -An object change, checksum mismatch or invalid range fails verification. Failed -downloads are removed, and upload/verification failures retain the original task. +The principal needs object upload, read (HEAD and ranged GET), delete and +multipart upload/abort permissions. KMS uploads/readback also need the applicable +key-policy and IAM permissions for `kms:GenerateDataKey` and `kms:Decrypt`. +Retain old KMS keys and decrypt permissions when changing the key or mode; restore +does not require an older object to use the current upload policy. -There is no database schema or archive format migration. Existing local archive -packages remain compatible. Files outside ARTEX's task archive database are not -automatically imported or uploaded. +Compatible providers must support the requested SSE mode and its HEAD metadata, +plus conditional ranged reads. A provider that ignores the encryption request +or cannot confirm it is rejected; the original task is retained. Local source and +temporary files keep their existing local-disk protection and restricted modes. -## Disk, transfer and recovery +See the official [PutObject encryption and permission contract](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html) +and [HEAD encryption response rules](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html). -Archive creation still needs space for the temporary compressed package. -Restoration needs space for the downloaded package and extracted files. Full -upload readback and later restores incur object-storage reads and any applicable -transfer charges. Configure expiration of abandoned multipart uploads for abrupt -process termination. +## Credential sources -Recovery journals under `data/archives/tasks/.remote` contain object locations, -not credentials. Preserve them with PostgreSQL and the installation's data -directory. The worker uses these journals to clean up uploads that never -committed, keep committed objects, and finish deferred deletions. Failed remote -cleanup does not block unrelated local archive jobs. +Without the archive-specific override, ARTEX uses the AWS SDK default credential +chain, including standard AWS environment variables, shared profiles, web +identity, ECS task credentials and EC2 instance credentials through IMDS. Role +credentials retain the SDK's automatic refresh behavior. A named shared profile +can be selected with `AWS_PROFILE`. + +For an external S3-compatible provider, dedicated keys can be supplied through +the archive-specific variables. Both key fields are required; a session token +is supported for temporary credentials. This explicit override takes precedence +over ambient AWS credentials. Operator-supplied temporary credentials must be +rotated by the operator; they are not an IAM role provider. + +Keep secrets in a protected service environment file outside the repository and +frontend. A systemd `EnvironmentFile=` can point to a root-owned mode-0600 file. +See the [AWS SDK credential-chain documentation](https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-gosdk.html). + +## Lifecycle + +1. Pause or finish a task and use its **Archive** action. +2. ARTEX stages the source files and builds a local `.tar.zst` package. Large + packages use multipart upload, initially 8 MiB parts with two upload workers; + the SDK increases the part size for exceptionally large seekable files. +3. ARTEX confirms encryption metadata and reads the complete object back, + checking size and SHA-256. ETags and uploaded checksum metadata alone are + insufficient. +4. **Only after verification and a successful PostgreSQL commit** does ARTEX + remove the local package and staged source files. The committed archive + location is an `s3://` URI; there is no schema or archive-format migration. +5. **Restore** downloads and verifies before installing files and database rows. + The remote package is consumed only after restoration commits. +6. **Delete** commits removal of the archive stub, then cleans up the object. + A durable journal retains failed cleanup work. + +The archive list identifies **Object storage** and **Local storage** and shows +upload/download verification phases. Reads use bounded 8 MiB ranges pinned to the +object's ETag, with retries for interrupted ranges rather than the entire object. + +## Interruption recovery + +Source staging and restore journals live with task archives. Remote cleanup +journals live under `data/archives/tasks/.remote`; they contain locations, not +credentials. Keep this directory with PostgreSQL and the installation data. + +| Interruption | Recovery | +| --- | --- | +| During staging/compression, before an upload journal exists | Roll staged source files back. Remove any temporary package only after rollback succeeds. The task remains in PostgreSQL. | +| After the upload journal, before successful verification | Roll source files back and clean up any uncommitted object/package. Failed cleanup retains its journal for retry. | +| After verification, before the database commit | Treat the object as uncommitted: retain the original task and clean up the orphan object. Verification alone never authorizes local deletion. | +| After the database commit, before package/staging cleanup | Keep the committed object and finish local cleanup. The persisted location and checksum establish commit, even if Restore or Delete has since been queued. | +| Download/extraction before restored files are installed | Remove temporary extraction data; keep the remote object and archive stub for another attempt. | +| Restored files installed, before database restoration commits | Roll back installed files using the restore journal. Retain the remote package. | +| Database restoration committed, before remote cleanup | Recognize the already-restored task and finish cleanup without replaying database rows. | +| Object consumed, before the restored archive stub is removed | Skip download/replay; repeat idempotent object deletion and remove the stub. | +| Delete journal written, before database deletion commits | Keep the object while its archive stub exists. The queued Delete operation can retry. | +| Database deletion committed, before object deletion succeeds | The durable outbox retains the URI and retries object deletion. | +| An old failed upload survives a retry using local storage | Remove the abandoned remote object, but retain a local package referenced by the committed archive row. | + +The staging recovery rule also applies to local archives. Dedicated tests cover +both storage modes with ready, restore-queued, delete-queued and uncommitted +states. Additional PostgreSQL/HTTP-S3 tests reconstruct interrupted operations at +13 persisted boundaries and repeat recovery to check idempotence. + +Handled multipart-upload failures are aborted by the SDK. Abrupt process death +may leave unfinished multipart uploads; configure bucket lifecycle expiration +for those parts. Existing object versions, if versioning is enabled, are governed +by the bucket's version-retention policy. + +## Disk and transfer requirements + +Archive creation still requires temporary local space for the compressed +package. Restore requires space for the download and extracted files. Full +upload readback and restores incur storage reads and applicable transfer charges. +Failed remote cleanup does not prevent unrelated local archive jobs from being +processed once the failed cleanup request returns. ## Optional connectivity check -Run `go run ./cmd/artex-archive-check` with the same archive environment as the -server to upload, verify, download and delete a small uniquely named test object. -This command performs real bucket operations but does not touch task data. It -reports leftover test objects if cleanup fails. +Run `go run ./cmd/artex-archive-check` with the archive environment to upload, +verify, download and delete a small unique object. It performs real bucket +operations but does not touch task data, and reports cleanup failures. -Automated tests use an HTTP S3 fixture and isolated PostgreSQL databases. They -cover multipart transfers, corruption, interrupted range retries, object changes, -cancellation, upload rollback, corrupted restores and persistent cleanup. +Automated tests use HTTP fixtures for S3 and metadata credentials, plus isolated +PostgreSQL. They do not establish compatibility with a live cloud bucket. diff --git a/go.mod b/go.mod index 1ce0982..353f837 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.26.10 require ( github.com/Autumn-27/norma v0.4.11 github.com/aws/aws-sdk-go-v2 v1.47.5 + github.com/aws/aws-sdk-go-v2/config v1.33.13 github.com/aws/aws-sdk-go-v2/credentials v1.20.13 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.19 github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 @@ -27,6 +28,7 @@ require ( github.com/andybalholm/brotli v1.2.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect @@ -34,6 +36,10 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.39.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect github.com/aws/smithy-go v1.28.7 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/creack/pty v1.1.24 // indirect diff --git a/server/task_archive_interrupt_test.go b/server/task_archive_interrupt_test.go new file mode 100644 index 0000000..2f4a44e --- /dev/null +++ b/server/task_archive_interrupt_test.go @@ -0,0 +1,291 @@ +package server + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "sync" + "testing" + + "github.com/Autumn-27/artex/archivestore" + pgdb "github.com/Autumn-27/artex/db" +) + +// Recreate persisted states at operation boundaries, then run the same recovery +// functions used at startup. PostgreSQL, files, and HTTP object storage are real +// isolated fixtures; no production failpoint hooks are needed. +func TestRemoteArchiveInterruptionRecovery(t *testing.T) { + if os.Getenv("ARTEX_PG_DSN") == "" { + t.Skip("requires isolated PostgreSQL") + } + phases := []string{ + "staged_before_compression", "compressed_before_journal", "journal_before_upload", + "uploaded_before_verification", "verified_before_database", + "database_before_local_cleanup", "package_cleaned_before_stage_cleanup", + "restore_download_before_install", "restore_files_before_database", "restore_database_before_object_delete", "restore_object_deleted_before_stub", + "delete_journal_before_database", "delete_database_before_object", + } + for _, phase := range phases { + t.Run(phase, func(t *testing.T) { + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + store, objects := newRecoveryObjectStore(t) + s := &Server{ctx: context.Background(), m: m, archiveStore: store} + task, err := m.CreateTask("interruption "+phase, "preserve source", nil, 0, 0) + if err != nil { + t.Fatal(err) + } + id, _ := strconv.ParseInt(task.ID, 10, 64) + defer func() { _, _ = m.pg.Exec("DELETE FROM task_archives WHERE task_id=$1", id); _ = m.pg.DeleteTask(id) }() + if err := m.pg.SetPaused(id, true); err != nil { + t.Fatal(err) + } + job, err := m.pg.QueueTaskArchive(id) + if err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Archiving); err != nil { + t.Fatal(err) + } + source := filepath.Join(m.dir, "tasks", task.ID, "workspace", "source.txt") + if err := os.MkdirAll(filepath.Dir(source), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source, []byte("original task data"), 0600); err != nil { + t.Fatal(err) + } + snapshot, err := m.pg.SnapshotTaskArchive(id) + if err != nil { + t.Fatal(err) + } + stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, snapshot.ExplorationID) + if err != nil { + t.Fatal(err) + } + packagePath := taskArchivePath(m.dir, job.ID, task.ID) + var original, compressed int64 + var checksum, uri, journal string + if phase != "staged_before_compression" { + original, compressed, checksum, err = writeTaskArchivePackage(packagePath, stage.payload, snapshot) + if err != nil { + t.Fatal(err) + } + uri = store.URI(id, job.ID, checksum) + } + if phase != "staged_before_compression" && phase != "compressed_before_journal" { + journal, err = s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, LocalPath: packagePath, Operation: "upload"}) + if err != nil { + t.Fatal(err) + } + } + committed := phase != "staged_before_compression" && phase != "compressed_before_journal" && + phase != "journal_before_upload" && phase != "uploaded_before_verification" && phase != "verified_before_database" + if phase == "uploaded_before_verification" { + data, err := os.ReadFile(packagePath) + if err != nil { + t.Fatal(err) + } + objects.put(uri, data) + } else if phase == "verified_before_database" || committed { + if err := store.Upload(s.ctx, uri, packagePath, checksum); err != nil { + t.Fatal(err) + } + } + if committed { + if err := m.pg.CompleteTaskArchive(job.ID, snapshot, uri, checksum, original, compressed); err != nil { + t.Fatal(err) + } + } + afterArchive := phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || + phase == "restore_object_deleted_before_stub" || phase == "delete_journal_before_database" || phase == "delete_database_before_object" + if phase == "package_cleaned_before_stage_cleanup" || afterArchive { + if err := os.Remove(packagePath); err != nil { + t.Fatal(err) + } + if err := os.Remove(journal); err != nil { + t.Fatal(err) + } + } + if afterArchive { + if err := stage.commit(); err != nil { + t.Fatal(err) + } + } + restored := false + remoteDeleted := false + if phase == "restore_download_before_install" || phase == "restore_files_before_database" || phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" { + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Restoring); err != nil { + t.Fatal(err) + } + parent := filepath.Join(taskArchiveRoot(m.dir), ".restore") + if err := os.MkdirAll(parent, 0700); err != nil { + t.Fatal(err) + } + extracted, err := os.MkdirTemp(parent, "interrupted-") + if err != nil { + t.Fatal(err) + } + downloaded := filepath.Join(extracted, "download.tar.zst") + if err := store.Download(s.ctx, uri, downloaded, checksum, compressed); err != nil { + t.Fatal(err) + } + if err := extractTaskArchivePackage(downloaded, checksum, extracted); err != nil { + t.Fatal(err) + } + if phase != "restore_download_before_install" { + if _, err := installTaskArchiveFiles(m.dir, extracted, task.ID, job.ID); err != nil { + t.Fatal(err) + } + } + if phase == "restore_database_before_object_delete" || phase == "restore_object_deleted_before_stub" { + if _, err := m.pg.RestoreTaskArchive(job.ID, snapshot, 0); err != nil { + t.Fatal(err) + } + restored = true + } + if phase == "restore_object_deleted_before_stub" { + if err := store.Delete(s.ctx, uri); err != nil { + t.Fatal(err) + } + remoteDeleted = true + } + } + deleting := phase == "delete_journal_before_database" || phase == "delete_database_before_object" + if deleting { + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2 WHERE id=$1", job.ID, pgdb.Deleting); err != nil { + t.Fatal(err) + } + if _, err := s.writeRemoteArchiveJournal(remoteArchiveJournal{ArchiveID: job.ID, URI: uri, Operation: "delete"}); err != nil { + t.Fatal(err) + } + if phase == "delete_database_before_object" { + if err := m.pg.DeleteTaskArchiveStub(job.ID); err != nil { + t.Fatal(err) + } + remoteDeleted = true + } + } + if err := m.pg.RecoverTaskArchiveJobs(); err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + wantSource := !committed || restored + data, fileErr := os.ReadFile(source) + if wantSource && (fileErr != nil || string(data) != "original task data") { + t.Fatalf("source data lost: %v", fileErr) + } + if !wantSource && !os.IsNotExist(fileErr) { + t.Fatalf("archived source resurrected: %v", fileErr) + } + if _, err := os.Stat(packagePath); !os.IsNotExist(err) { + t.Fatalf("temporary package remains: %v", err) + } + if got, want := objects.has(uri), committed && !remoteDeleted; got != want { + t.Fatalf("remote object exists=%v, want=%v", got, want) + } + if committed && !deleting { + actual, err := m.pg.IsTaskArchiveRestored(job.ID) + if err != nil || actual != restored { + t.Fatalf("restored=%v want=%v err=%v", actual, restored, err) + } + } + // A second startup must be harmless at every boundary. + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveRestoreStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if err := s.recoverRemoteArchiveJournals(); err != nil { + t.Fatal(err) + } + }) + } +} + +type recoveryObjects struct { + mu sync.Mutex + data map[string][]byte +} + +func (o *recoveryObjects) put(uri string, data []byte) { + o.mu.Lock() + defer o.mu.Unlock() + // Test URIs always use the fixture bucket with path-style addressing. + o.data["/test/"+uri[len("s3://test/"):]] = append([]byte(nil), data...) +} +func (o *recoveryObjects) has(uri string) bool { + o.mu.Lock() + defer o.mu.Unlock() + if uri == "" { + return false + } + _, ok := o.data["/test/"+uri[len("s3://test/"):]] + return ok +} +func newRecoveryObjectStore(t *testing.T) (*archivestore.Store, *recoveryObjects) { + t.Helper() + objects := &recoveryObjects{data: map[string][]byte{}} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + objects.mu.Lock() + defer objects.mu.Unlock() + data, exists := objects.data[r.URL.Path] + w.Header().Set("ETag", `"recovery-etag"`) + switch r.Method { + case "PUT": + if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" { + t.Error("missing SSE") + } + objects.data[r.URL.Path], _ = io.ReadAll(r.Body) + case "HEAD": + if !exists { + w.WriteHeader(404) + return + } + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + w.Header().Set("X-Amz-Server-Side-Encryption", "AES256") + case "GET": + if !exists { + w.WriteHeader(404) + return + } + var start, end int + if n, err := fmt.Sscanf(r.Header.Get("Range"), "bytes=%d-%d", &start, &end); err != nil || n != 2 || start < 0 || end >= len(data) || r.Header.Get("If-Match") != `"recovery-etag"` { + w.WriteHeader(416) + return + } + w.Header().Set("Content-Length", strconv.Itoa(end-start+1)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data))) + w.WriteHeader(206) + _, _ = w.Write(data[start : end+1]) + case "DELETE": + delete(objects.data, r.URL.Path) + w.WriteHeader(204) + default: + t.Errorf("unexpected object request %s", r.Method) + w.WriteHeader(400) + } + })) + t.Cleanup(srv.Close) + store, err := archivestore.New(archivestore.Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}) + if err != nil { + t.Fatal(err) + } + return store, objects +} diff --git a/server/task_archive_package.go b/server/task_archive_package.go index e9a451d..e365ea3 100644 --- a/server/task_archive_package.go +++ b/server/task_archive_package.go @@ -437,9 +437,16 @@ func recoverTaskArchiveStages(dataDir string, pg *pgdb.DB) error { stage := &taskArchiveFileStage{root: root, payload: filepath.Join(root, "payload"), journal: journal} if err := stage.rollback(); err != nil { errs = append(errs, err) + continue } - if archive != nil && archive.ArchivePath != "" && !strings.HasPrefix(archive.ArchivePath, "s3://") { - _ = os.Remove(archive.ArchivePath) + // The process can exit after compression but before an upload journal or + // database location exists. Remove that temporary package only after the + // source-file rollback succeeds, for both local and remote backends. + packagePath := taskArchivePath(dataDir, journal.ArchiveID, journal.TaskID) + if err := validateArchivePath(dataDir, packagePath); err != nil { + errs = append(errs, err) + } else if err := removeLocalArchivePackage(packagePath); err != nil { + errs = append(errs, err) } } return errors.Join(errs...) diff --git a/server/task_archive_remote_test.go b/server/task_archive_remote_test.go index 34e0848..ac2a44d 100644 --- a/server/task_archive_remote_test.go +++ b/server/task_archive_remote_test.go @@ -46,6 +46,7 @@ func TestRemoteArchiveLifecycle(t *testing.T) { objects[r.URL.Path], _ = io.ReadAll(r.Body) w.Header().Set("ETag", `"test"`) case "HEAD": + w.Header().Set("X-Amz-Server-Side-Encryption", "AES256") w.Header().Set("ETag", `"test"`) w.Header().Set("Content-Length", strconv.Itoa(len(objects[r.URL.Path]))) case "GET": @@ -255,7 +256,7 @@ func TestRemoteArchiveLifecycle(t *testing.T) { // Once PostgreSQL has committed an archive location and checksum, a subsequent // queued operation must not make startup recovery resurrect the old workspace. -func TestRemoteArchiveStageRecoveryAfterCommit(t *testing.T) { +func TestTaskArchiveStageRecoveryAfterCommit(t *testing.T) { if os.Getenv("ARTEX_PG_DSN") == "" { t.Skip("requires isolated PostgreSQL") } @@ -284,41 +285,63 @@ func TestRemoteArchiveStageRecoveryAfterCommit(t *testing.T) { if err != nil { t.Fatal(err) } - for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} { - t.Run(state, func(t *testing.T) { - committed := state != pgdb.ArchiveQueued - checksum := "" - if committed { - checksum = strings.Repeat("a", 64) - } - if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, "s3://test/tasks/package.tar.zst", checksum); err != nil { - t.Fatal(err) - } - file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt") - if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil { - t.Fatal(err) - } - stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID) - if err != nil { - t.Fatal(err) - } - if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(stage.root); !os.IsNotExist(err) { - t.Fatalf("staging cleanup incomplete: %v", err) - } - if committed { - if _, err := os.Stat(file); !os.IsNotExist(err) { - t.Fatalf("resurrected committed workspace: %v", err) + for _, backend := range []string{"local", "s3"} { + for _, state := range []string{pgdb.ArchiveReady, pgdb.RestoreQueued, pgdb.DeleteQueued, pgdb.ArchiveQueued} { + t.Run(backend+"/"+state, func(t *testing.T) { + committed := state != pgdb.ArchiveQueued + checksum := "" + if committed { + checksum = strings.Repeat("a", 64) } - } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" { - t.Fatalf("uncommitted archive lost source: %v", err) - } - }) + packagePath := taskArchivePath(m.dir, job.ID, task.ID) + if err := os.MkdirAll(filepath.Dir(packagePath), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(packagePath, []byte("verified package"), 0600); err != nil { + t.Fatal(err) + } + location := packagePath + if backend == "s3" { + location = "s3://test/tasks/package.tar.zst" + } + if _, err := m.pg.Exec("UPDATE task_archives SET state=$2,archive_path=$3,sha256=$4 WHERE id=$1", job.ID, state, location, checksum); err != nil { + t.Fatal(err) + } + file := filepath.Join(m.dir, "tasks", task.ID, "workspace", "remaining.txt") + if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte("retain until commit"), 0600); err != nil { + t.Fatal(err) + } + stage, err := stageTaskArchiveFiles(m.dir, job.ID, task.ID, pgTask.ExplorationID) + if err != nil { + t.Fatal(err) + } + if err := recoverTaskArchiveStages(m.dir, m.pg); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stage.root); !os.IsNotExist(err) { + t.Fatalf("staging cleanup incomplete: %v", err) + } + if backend == "local" && committed { + if data, err := os.ReadFile(packagePath); err != nil || string(data) != "verified package" { + t.Fatal("committed local package lost", err) + } + } else if !committed { + if _, err := os.Stat(packagePath); !os.IsNotExist(err) { + t.Fatal("uncommitted temporary package retained", err) + } + } + if committed { + if _, err := os.Stat(file); !os.IsNotExist(err) { + t.Fatalf("resurrected committed workspace: %v", err) + } + } else if data, err := os.ReadFile(file); err != nil || string(data) != "retain until commit" { + t.Fatalf("uncommitted archive lost source: %v", err) + } + }) + } } }