feat: encrypt S3 archives and test credentials and interruption recovery

This commit is contained in:
Chubboy Ren
2026-11-18 02:05:17 +00:00
parent 3d44e9cd0d
commit da7cc62d4e
9 changed files with 892 additions and 207 deletions
+20
View File
@@ -10,6 +10,7 @@ func TestArchiveStorageEnvironment(t *testing.T) {
"ARTEX_ARCHIVE_STORAGE", "ARTEX_ARCHIVE_S3_ENDPOINT", "ARTEX_ARCHIVE_S3_REGION",
"ARTEX_ARCHIVE_S3_BUCKET", "ARTEX_ARCHIVE_S3_PREFIX", "ARTEX_ARCHIVE_S3_ACCESS_KEY_ID",
"ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY", "ARTEX_ARCHIVE_S3_PATH_STYLE",
"ARTEX_ARCHIVE_S3_SESSION_TOKEN", "ARTEX_ARCHIVE_S3_SSE", "ARTEX_ARCHIVE_S3_KMS_KEY_ID",
} {
t.Setenv(key, "")
}
@@ -60,3 +61,22 @@ func TestArchiveEndpointValidation(t *testing.T) {
}
}
}
func TestArchiveEncryptionConfiguration(t *testing.T) {
c := Config{Endpoint: "https://objects.example.test", Bucket: "test", AccessKey: "test", SecretKey: "test"}
for _, mode := range []string{"none", "invalid"} {
c.SSE = mode
if _, err := New(c); err == nil {
t.Fatal("unencrypted/invalid mode accepted")
}
}
c.SSE = "AES256"
c.KMSKeyID = "alias/archive"
if _, err := New(c); err == nil {
t.Fatal("KMS key accepted with SSE-S3")
}
c.SSE = "aws:kms"
if _, err := New(c); err != nil {
t.Fatal(err)
}
}
+132
View File
@@ -0,0 +1,132 @@
package archivestore
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func isolateAWSCredentials(t *testing.T) {
t.Helper()
for _, key := range []string{
"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "AWS_PROFILE", "AWS_DEFAULT_PROFILE",
"AWS_ROLE_ARN", "AWS_ROLE_SESSION_NAME", "AWS_WEB_IDENTITY_TOKEN_FILE",
"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_CREDENTIALS_FULL_URI",
"AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE",
"AWS_EC2_METADATA_SERVICE_ENDPOINT", "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE",
} {
t.Setenv(key, "")
}
dir := t.TempDir()
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(dir, "credentials"))
t.Setenv("AWS_CONFIG_FILE", filepath.Join(dir, "config"))
t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
}
func TestArchiveCredentialChain(t *testing.T) {
config := Config{Endpoint: "https://objects.example.test", Bucket: "test"}
for _, mode := range []string{"explicit-session", "environment", "shared-profile", "ecs", "imds"} {
t.Run(mode, func(t *testing.T) {
isolateAWSCredentials(t)
c := config
switch mode {
case "explicit-session":
t.Setenv("AWS_ACCESS_KEY_ID", "other-key")
t.Setenv("AWS_SECRET_ACCESS_KEY", "other-secret")
c.AccessKey, c.SecretKey, c.SessionToken = "fixture-key", "fixture-secret", "fixture-session"
case "environment":
t.Setenv("AWS_ACCESS_KEY_ID", "fixture-key")
t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture-secret")
t.Setenv("AWS_SESSION_TOKEN", "fixture-session")
case "shared-profile":
t.Setenv("AWS_PROFILE", "archive")
data := `[archive]
aws_access_key_id=fixture-key
aws_secret_access_key=fixture-secret
aws_session_token=fixture-session
`
if err := os.WriteFile(os.Getenv("AWS_SHARED_CREDENTIALS_FILE"), []byte(data), 0600); err != nil {
t.Fatal(err)
}
case "ecs", "imds":
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if mode == "imds" {
switch r.URL.Path {
case "/latest/api/token":
if r.Method != "PUT" {
t.Error("IMDS did not use token request")
}
w.Header().Set("X-aws-ec2-metadata-token-ttl-seconds", "21600")
fmt.Fprint(w, "fixture-imds-token")
return
case "/latest/meta-data/iam/security-credentials/":
if r.Header.Get("X-aws-ec2-metadata-token") != "fixture-imds-token" {
t.Error("missing IMDSv2 token")
}
fmt.Fprint(w, "archive-role")
return
case "/latest/meta-data/iam/security-credentials/archive-role":
if r.Header.Get("X-aws-ec2-metadata-token") != "fixture-imds-token" {
t.Error("missing IMDSv2 token")
}
default:
t.Errorf("unexpected IMDS request %s", r.URL.Path)
w.WriteHeader(404)
return
}
} else if r.Header.Get("Authorization") != "fixture-container-token" {
t.Error("missing ECS authorization token")
}
_ = json.NewEncoder(w).Encode(map[string]any{
"Code": "Success", "AccessKeyId": "fixture-key", "SecretAccessKey": "fixture-secret",
"Token": "fixture-session", "Expiration": time.Now().Add(time.Hour).UTC().Format(time.RFC3339),
})
}))
defer srv.Close()
if mode == "imds" {
t.Setenv("AWS_EC2_METADATA_DISABLED", "false")
t.Setenv("AWS_EC2_METADATA_V1_DISABLED", "true")
t.Setenv("AWS_EC2_METADATA_SERVICE_ENDPOINT", srv.URL)
} else {
t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", srv.URL+"/credentials")
t.Setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN", "fixture-container-token")
}
}
store, err := New(c)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
credentials, err := store.client.Options().Credentials.Retrieve(ctx)
if err != nil {
t.Fatal(err)
}
if credentials.AccessKeyID != "fixture-key" || credentials.SecretAccessKey != "fixture-secret" || credentials.SessionToken != "fixture-session" {
t.Fatal("wrong credential provider or session token")
}
if (mode == "imds" || mode == "ecs") && !credentials.CanExpire {
t.Fatal("role credentials must remain refreshable")
}
})
}
}
func TestArchiveIncompleteCredentialOverride(t *testing.T) {
isolateAWSCredentials(t)
t.Setenv("AWS_ACCESS_KEY_ID", "valid-ambient-key")
t.Setenv("AWS_SECRET_ACCESS_KEY", "valid-ambient-secret")
for _, c := range []Config{{AccessKey: "private-value"}, {SecretKey: "private-value"}, {SessionToken: "private-value"}} {
c.Endpoint, c.Bucket = "https://objects.example.test", "test"
if _, err := New(c); err == nil || strings.Contains(err.Error(), "private-value") {
t.Fatalf("partial override must fail without disclosing credentials: %v", err)
}
}
}
+58 -15
View File
@@ -17,19 +17,24 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
)
type Config struct {
Endpoint, Region, Bucket, Prefix, AccessKey, SecretKey string
PathStyle bool
SessionToken, SSE, KMSKeyID string
}
type Store struct {
client *s3.Client
bucket, prefix string
encryption types.ServerSideEncryption
kmsKeyID string
}
// FromEnv returns nil for the default local backend. A configured but invalid
@@ -43,7 +48,9 @@ func FromEnv() (*Store, error) {
Endpoint: os.Getenv("ARTEX_ARCHIVE_S3_ENDPOINT"), Region: os.Getenv("ARTEX_ARCHIVE_S3_REGION"),
Bucket: os.Getenv("ARTEX_ARCHIVE_S3_BUCKET"), Prefix: os.Getenv("ARTEX_ARCHIVE_S3_PREFIX"),
AccessKey: os.Getenv("ARTEX_ARCHIVE_S3_ACCESS_KEY_ID"), SecretKey: os.Getenv("ARTEX_ARCHIVE_S3_SECRET_ACCESS_KEY"),
PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true",
PathStyle: os.Getenv("ARTEX_ARCHIVE_S3_PATH_STYLE") == "true",
SessionToken: os.Getenv("ARTEX_ARCHIVE_S3_SESSION_TOKEN"),
SSE: os.Getenv("ARTEX_ARCHIVE_S3_SSE"), KMSKeyID: os.Getenv("ARTEX_ARCHIVE_S3_KMS_KEY_ID"),
})
default:
return nil, errors.New("ARTEX_ARCHIVE_STORAGE must be local or s3")
@@ -70,22 +77,49 @@ func New(c Config) (*Store, error) {
if path.Clean(c.Prefix) != c.Prefix || strings.Contains(c.Prefix, "..") || strings.ContainsAny(c.Prefix, "\\?#") {
return nil, errors.New("invalid archive S3 prefix")
}
if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") || c.AccessKey == "" || c.SecretKey == "" {
return nil, errors.New("archive S3 bucket and dedicated access credentials are required")
if c.Bucket == "" || strings.ContainsAny(c.Bucket, "/\\:@?# ") {
return nil, errors.New("a valid archive S3 bucket is required")
}
explicitCredentials := c.AccessKey != "" || c.SecretKey != "" || c.SessionToken != ""
if explicitCredentials && (c.AccessKey == "" || c.SecretKey == "") {
return nil, errors.New("archive S3 access key and secret key must both be set when overriding AWS credentials")
}
if c.SSE == "" {
c.SSE = "AES256"
}
if c.SSE != "AES256" && c.SSE != "aws:kms" {
return nil, errors.New("archive S3 SSE must be AES256 or aws:kms; encryption cannot be disabled")
}
if c.KMSKeyID != "" && c.SSE != "aws:kms" {
return nil, errors.New("archive S3 KMS key requires SSE=aws:kms")
}
if c.Region == "" {
c.Region = "us-east-1"
}
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.ResponseHeaderTimeout = 60 * time.Second
client := s3.New(s3.Options{
Region: c.Region, BaseEndpoint: aws.String(c.Endpoint), UsePathStyle: c.PathStyle,
Credentials: credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, ""),
HTTPClient: &http.Client{Transport: transport},
RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
ResponseChecksumValidation: aws.ResponseChecksumValidationWhenRequired,
httpClient := &http.Client{Transport: transport}
cfg := aws.Config{Region: c.Region, HTTPClient: httpClient}
if explicitCredentials {
cfg.Credentials = credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken)
} else {
// Keep the SDK's refreshable default chain: standard AWS environment/shared
// profiles, web identity, ECS task roles and EC2 instance roles (IMDSv2).
var err error
cfg, err = awsconfig.LoadDefaultConfig(context.Background(),
awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient))
if err != nil {
return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err)
}
}
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(c.Endpoint)
o.UsePathStyle = c.PathStyle
o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
})
return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/"}, nil
return &Store{client: client, bucket: c.Bucket, prefix: c.Prefix + "/",
encryption: types.ServerSideEncryption(c.SSE), kmsKeyID: c.KMSKeyID}, nil
}
func (s *Store) URI(taskID, archiveID int64, checksum string) string {
@@ -134,17 +168,22 @@ func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) erro
u.Concurrency = 2
u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
})
_, err = uploader.Upload(ctx, &s3.PutObjectInput{
input := &s3.PutObjectInput{
Bucket: aws.String(s.bucket), Key: aws.String(key), Body: f,
ContentType: aws.String("application/zstd"), Metadata: map[string]string{"sha256": checksum},
})
ServerSideEncryption: s.encryption,
}
if s.kmsKeyID != "" {
input.SSEKMSKeyId = aws.String(s.kmsKeyID)
}
_, err = uploader.Upload(ctx, input)
if err != nil {
return fmt.Errorf("upload archive: %w", err)
}
return s.readVerified(ctx, key, io.Discard, checksum, info.Size())
return s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true)
}
func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64) error {
func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64, requireEncryption bool) error {
if decoded, err := hex.DecodeString(checksum); err != nil || len(decoded) != sha256.Size {
return errors.New("invalid archive SHA-256")
}
@@ -155,6 +194,10 @@ func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, che
if err != nil {
return fmt.Errorf("inspect archive: %w", err)
}
if requireEncryption && (object.ServerSideEncryption != s.encryption ||
(s.encryption == types.ServerSideEncryptionAwsKms && aws.ToString(object.SSEKMSKeyId) == "")) {
return errors.New("remote archive did not confirm the requested server-side encryption; source data retained")
}
if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" {
return errors.New("remote archive size mismatch or missing ETag; source data retained")
}
@@ -264,7 +307,7 @@ func (s *Store) Download(ctx context.Context, uri, filename, checksum string, si
_ = os.Remove(filename)
}
}()
if err = s.readVerified(ctx, key, f, checksum, size); err != nil {
if err = s.readVerified(ctx, key, f, checksum, size, false); err != nil {
return err
}
return f.Sync()
+193 -86
View File
@@ -13,99 +13,121 @@ import (
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
)
func TestS3RoundTripAndIntegrity(t *testing.T) {
for _, size := range []int{1024, 10 << 20} {
t.Run(strconv.Itoa(size), func(t *testing.T) {
var mu sync.Mutex
var object []byte
parts := map[int][]byte{}
corrupt, multipart := false, false
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
defer mu.Unlock()
q := r.URL.Query()
w.Header().Set("ETag", `"test-etag"`)
switch {
case r.Method == "POST" && q.Has("uploads"):
multipart = true
fmt.Fprint(w, `<InitiateMultipartUploadResult><UploadId>test-upload</UploadId></InitiateMultipartUploadResult>`)
case r.Method == "PUT" && q.Has("partNumber"):
n, _ := strconv.Atoi(q.Get("partNumber"))
parts[n], _ = io.ReadAll(r.Body)
case r.Method == "POST" && q.Has("uploadId"):
for i := 1; i <= len(parts); i++ {
object = append(object, parts[i]...)
for _, sse := range []string{"AES256", "aws:kms"} {
kms := ""
if sse == "aws:kms" {
kms = "arn:aws:kms:us-east-1:123456789012:key/test-key"
}
for _, size := range []int{1024, 10 << 20} {
t.Run(sse+"/"+strconv.Itoa(size), func(t *testing.T) {
var mu sync.Mutex
var object []byte
parts := map[int][]byte{}
corrupt, multipart := false, false
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
defer mu.Unlock()
q := r.URL.Query()
if (r.Method == "PUT" && !q.Has("partNumber")) || (r.Method == "POST" && q.Has("uploads")) {
if r.Header.Get("X-Amz-Server-Side-Encryption") != sse || r.Header.Get("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id") != kms {
t.Error("upload did not explicitly request the configured encryption")
}
}
fmt.Fprint(w, `<CompleteMultipartUploadResult><ETag>test-etag</ETag></CompleteMultipartUploadResult>`)
case r.Method == "PUT":
object, _ = io.ReadAll(r.Body)
case r.Method == "HEAD":
w.Header().Set("Content-Length", strconv.Itoa(len(object)))
case r.Method == "GET":
data := object
if corrupt {
data = bytes.Clone(object)
data[0] ^= 0xff
if r.Method == "HEAD" {
if r.Header.Get("X-Amz-Server-Side-Encryption") != "" {
t.Error("encryption write header leaked into HEAD")
}
w.Header().Set("X-Amz-Server-Side-Encryption", sse)
if kms != "" {
w.Header().Set("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id", kms)
}
}
serveArchiveRange(t, w, r, data)
case r.Method == "DELETE":
object = nil
w.WriteHeader(204)
default:
t.Errorf("unexpected request %s %s", r.Method, r.URL)
w.WriteHeader(400)
w.Header().Set("ETag", `"test-etag"`)
switch {
case r.Method == "POST" && q.Has("uploads"):
multipart = true
fmt.Fprint(w, `<InitiateMultipartUploadResult><UploadId>test-upload</UploadId></InitiateMultipartUploadResult>`)
case r.Method == "PUT" && q.Has("partNumber"):
n, _ := strconv.Atoi(q.Get("partNumber"))
parts[n], _ = io.ReadAll(r.Body)
case r.Method == "POST" && q.Has("uploadId"):
for i := 1; i <= len(parts); i++ {
object = append(object, parts[i]...)
}
fmt.Fprint(w, `<CompleteMultipartUploadResult><ETag>test-etag</ETag></CompleteMultipartUploadResult>`)
case r.Method == "PUT":
object, _ = io.ReadAll(r.Body)
case r.Method == "HEAD":
w.Header().Set("Content-Length", strconv.Itoa(len(object)))
case r.Method == "GET":
data := object
if corrupt {
data = bytes.Clone(object)
data[0] ^= 0xff
}
serveArchiveRange(t, w, r, data)
case r.Method == "DELETE":
object = nil
w.WriteHeader(204)
default:
t.Errorf("unexpected request %s %s", r.Method, r.URL)
w.WriteHeader(400)
}
}))
defer srv.Close()
store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true, SSE: sse, KMSKeyID: kms})
if err != nil {
t.Fatal(err)
}
}))
defer srv.Close()
store, err := New(Config{Endpoint: srv.URL, Bucket: "test-bucket", Prefix: "test/tasks", AccessKey: "test", SecretKey: "test", PathStyle: true})
if err != nil {
t.Fatal(err)
}
data := bytes.Repeat([]byte{0x42}, size)
h := sha256.Sum256(data)
checksum := hex.EncodeToString(h[:])
root := t.TempDir()
source := filepath.Join(root, "source")
if err := os.WriteFile(source, data, 0600); err != nil {
t.Fatal(err)
}
uri := store.URI(1, 2, checksum)
ctx := context.Background()
if err := store.Upload(ctx, uri, source, checksum); err != nil {
t.Fatal(err)
}
if multipart != (size > 8<<20) {
t.Fatalf("multipart=%v size=%d", multipart, size)
}
dest := filepath.Join(root, "download")
if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil {
t.Fatal(err)
}
got, _ := os.ReadFile(dest)
if !bytes.Equal(got, data) {
t.Fatal("download differs")
}
mu.Lock()
corrupt = true
mu.Unlock()
bad := filepath.Join(root, "bad")
if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil {
t.Fatal("accepted corrupt object")
}
if _, err := os.Stat(bad); !os.IsNotExist(err) {
t.Fatal("left corrupt download")
}
if err := store.Upload(ctx, uri, source, checksum); err == nil {
t.Fatal("upload accepted failed readback")
}
if err := store.Delete(ctx, uri); err != nil {
t.Fatal(err)
}
})
data := bytes.Repeat([]byte{0x42}, size)
h := sha256.Sum256(data)
checksum := hex.EncodeToString(h[:])
root := t.TempDir()
source := filepath.Join(root, "source")
if err := os.WriteFile(source, data, 0600); err != nil {
t.Fatal(err)
}
uri := store.URI(1, 2, checksum)
ctx := context.Background()
if err := store.Upload(ctx, uri, source, checksum); err != nil {
t.Fatal(err)
}
if multipart != (size > 8<<20) {
t.Fatalf("multipart=%v size=%d", multipart, size)
}
dest := filepath.Join(root, "download")
if err := store.Download(ctx, uri, dest, checksum, int64(size)); err != nil {
t.Fatal(err)
}
got, _ := os.ReadFile(dest)
if !bytes.Equal(got, data) {
t.Fatal("download differs")
}
mu.Lock()
corrupt = true
mu.Unlock()
bad := filepath.Join(root, "bad")
if err := store.Download(ctx, uri, bad, checksum, int64(size)); err == nil {
t.Fatal("accepted corrupt object")
}
if _, err := os.Stat(bad); !os.IsNotExist(err) {
t.Fatal("left corrupt download")
}
if err := store.Upload(ctx, uri, source, checksum); err == nil {
t.Fatal("upload accepted failed readback")
}
if err := store.Delete(ctx, uri); err != nil {
t.Fatal(err)
}
})
}
}
}
@@ -256,7 +278,7 @@ func TestS3ReadbackPropagatesOutputFailure(t *testing.T) {
store := testRangeStore(t, srv.URL)
h := sha256.Sum256(data)
outputError := errors.New("output disk full")
err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data)))
err := store.readVerified(context.Background(), "test/tasks/archive", failedArchiveWriter{outputError}, hex.EncodeToString(h[:]), int64(len(data)), false)
if !errors.Is(err, outputError) {
t.Fatalf("lost output error: %v", err)
}
@@ -309,3 +331,88 @@ func TestS3RejectsForeignLocations(t *testing.T) {
t.Fatal("missing config accepted")
}
}
func TestS3UploadRejectsUnconfirmedEncryption(t *testing.T) {
for _, mode := range []string{"missing", "different", "kms-without-key"} {
t.Run(mode, func(t *testing.T) {
data := []byte("private archive")
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "PUT":
_, _ = io.Copy(io.Discard, r.Body)
case "HEAD":
w.Header().Set("ETag", `"test-etag"`)
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
if mode != "missing" {
w.Header().Set("X-Amz-Server-Side-Encryption", "aws:kms")
}
default:
t.Errorf("content must not be accepted before encryption confirmation: %s", r.Method)
w.WriteHeader(400)
}
}))
defer srv.Close()
c := Config{Endpoint: srv.URL, Bucket: "test", Prefix: "tasks", AccessKey: "test", SecretKey: "test", PathStyle: true}
if mode == "kms-without-key" {
c.SSE = "aws:kms"
}
store, err := New(c)
if err != nil {
t.Fatal(err)
}
filename := filepath.Join(t.TempDir(), "source")
if err := os.WriteFile(filename, data, 0600); err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(data)
checksum := hex.EncodeToString(sum[:])
if err := store.Upload(context.Background(), store.URI(1, 2, checksum), filename, checksum); err == nil || !strings.Contains(err.Error(), "encryption") {
t.Fatalf("unconfirmed encryption accepted: %v", err)
}
if got, err := os.ReadFile(filename); err != nil || !bytes.Equal(got, data) {
t.Fatal("source was not retained", err)
}
})
}
}
func TestS3MultipartFailureAbortsUpload(t *testing.T) {
var aborted atomic.Bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
switch {
case r.Method == "POST" && q.Has("uploads"):
if r.Header.Get("X-Amz-Server-Side-Encryption") != "AES256" {
t.Error("multipart encryption missing")
}
fmt.Fprint(w, `<InitiateMultipartUploadResult><UploadId>failed-upload</UploadId></InitiateMultipartUploadResult>`)
case r.Method == "PUT" && q.Has("partNumber"):
_, _ = io.Copy(io.Discard, r.Body)
w.WriteHeader(403)
case r.Method == "DELETE" && q.Get("uploadId") == "failed-upload":
aborted.Store(true)
w.WriteHeader(204)
default:
t.Errorf("unexpected failed-upload request: %s", r.Method)
w.WriteHeader(400)
}
}))
defer srv.Close()
store := testRangeStore(t, srv.URL)
data := bytes.Repeat([]byte("x"), 9<<20)
checksum := sha256.Sum256(data)
sum := hex.EncodeToString(checksum[:])
filename := filepath.Join(t.TempDir(), "source")
if err := os.WriteFile(filename, data, 0600); err != nil {
t.Fatal(err)
}
if err := store.Upload(context.Background(), store.URI(1, 2, sum), filename, sum); err == nil {
t.Fatal("failed part accepted")
}
if !aborted.Load() {
t.Fatal("failed multipart upload was not aborted")
}
if info, err := os.Stat(filename); err != nil || info.Size() != int64(len(data)) {
t.Fatal("source was not retained", err)
}
}