fix: enforce archive encryption and clean failed S3 uploads

This commit is contained in:
Chubboy Ren
2026-11-19 03:33:51 +00:00
parent a40781ae1f
commit 3755e95504
5 changed files with 310 additions and 21 deletions
+1 -1
View File
@@ -71,7 +71,7 @@ jobs:
ARTEX_PG_DSN: postgres://artex:testpass@localhost:5432/artextest_race?sslmode=disable
run: >-
go test -race -p 1 ./agent ./cmd/artex ./config ./db ./evidence ./guard
go test -race -p 1 ./agent ./archivestore ./cmd/artex ./config ./db ./evidence ./guard
./intercept ./llmpool ./llmrec ./mcphttp ./notify ./server ./sidequestion
./traffic ./traffic/rewrite
+7
View File
@@ -76,6 +76,13 @@ func TestArchiveEncryptionConfiguration(t *testing.T) {
t.Fatal("KMS key accepted with SSE-S3")
}
c.SSE = "aws:kms"
for _, key := range []string{"alias/archive", "bare-key-id", "arn:aws:kms:us-east-1:123456789012:alias/archive"} {
c.KMSKeyID = key
if _, err := New(c); err == nil {
t.Fatalf("accepted noncanonical KMS key %q", key)
}
}
c.KMSKeyID = "arn:aws:kms:us-east-1:123456789012:key/test-key"
if _, err := New(c); err != nil {
t.Fatal(err)
}
+69 -20
View File
@@ -17,6 +17,9 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/arn"
"github.com/aws/aws-sdk-go-v2/aws/retry"
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
@@ -93,24 +96,37 @@ func New(c Config) (*Store, error) {
if c.KMSKeyID != "" && c.SSE != "aws:kms" {
return nil, errors.New("archive S3 KMS key requires SSE=aws:kms")
}
if c.KMSKeyID != "" {
key, err := arn.Parse(c.KMSKeyID)
if err != nil || key.Service != "kms" || key.Region == "" || key.AccountID == "" ||
!strings.HasPrefix(key.Resource, "key/") || len(key.Resource) <= len("key/") {
return nil, errors.New("archive S3 KMS key must be a full key ARN, not an alias or bare key ID")
}
}
if c.Region == "" {
c.Region = "us-east-1"
}
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.ResponseHeaderTimeout = 60 * time.Second
httpClient := &http.Client{Transport: transport}
cfg := aws.Config{Region: c.Region, HTTPClient: httpClient}
// Load shared AWS settings (including custom CAs) for both credential modes.
httpClient := awshttp.NewBuildableClient().WithTransportOptions(func(t *http.Transport) {
t.ResponseHeaderTimeout = 60 * time.Second
})
options := []func(*awsconfig.LoadOptions) error{
awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient),
}
if explicitCredentials {
cfg.Credentials = credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken)
} else {
// Keep the SDK's refreshable default chain: standard AWS environment/shared
// profiles, web identity, ECS task roles and EC2 instance roles (IMDSv2).
var err error
cfg, err = awsconfig.LoadDefaultConfig(context.Background(),
awsconfig.WithRegion(c.Region), awsconfig.WithHTTPClient(httpClient))
if err != nil {
return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err)
}
options = append(options, awsconfig.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, c.SessionToken)))
}
cfg, err := awsconfig.LoadDefaultConfig(context.Background(), options...)
if err != nil {
return nil, fmt.Errorf("load archive S3 AWS configuration: %w", err)
}
// The SDK's default client still follows 307/308. Use its CA-configured
// transport, but never redirect a signed archive request to another origin.
configuredClient := cfg.HTTPClient.(*awshttp.BuildableClient)
cfg.HTTPClient = &http.Client{
Transport: configuredClient.GetTransport(), Timeout: configuredClient.GetTimeout(),
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(c.Endpoint)
@@ -166,6 +182,7 @@ func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) erro
uploader := manager.NewUploader(s.client, func(u *manager.Uploader) {
u.PartSize = 8 << 20
u.Concurrency = 2
u.LeavePartsOnError = true // Abort below with a fresh, bounded cleanup context.
u.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
})
input := &s3.PutObjectInput{
@@ -176,11 +193,36 @@ func (s *Store) Upload(ctx context.Context, uri, filename, checksum string) erro
if s.kmsKeyID != "" {
input.SSEKMSKeyId = aws.String(s.kmsKeyID)
}
_, err = uploader.Upload(ctx, input)
result, err := uploader.Upload(ctx, input)
if err != nil {
return fmt.Errorf("upload archive: %w", err)
uploadErr := fmt.Errorf("upload archive: %w", err)
var multipart manager.MultiUploadFailure
if errors.As(err, &multipart) {
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
_, cleanupErr := s.client.AbortMultipartUpload(cleanupCtx, &s3.AbortMultipartUploadInput{
Bucket: aws.String(s.bucket), Key: aws.String(key), UploadId: aws.String(multipart.UploadID()),
})
if cleanupErr != nil {
return errors.Join(uploadErr, fmt.Errorf("abort archive multipart upload: %w", cleanupErr))
}
}
return uploadErr
}
return s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true)
if err := s.readVerified(ctx, key, io.Discard, checksum, info.Size(), true); err != nil {
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
// Versioned buckets require deleting the version just uploaded; an
// unversioned DELETE would only hide it behind a delete marker.
_, cleanupErr := s.client.DeleteObject(cleanupCtx, &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket), Key: aws.String(key), VersionId: result.VersionID,
})
if cleanupErr != nil {
return errors.Join(err, fmt.Errorf("delete unverified archive: %w", cleanupErr))
}
return err
}
return nil
}
func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, checksum string, expectedSize int64, requireEncryption bool) error {
@@ -195,7 +237,8 @@ func (s *Store) readVerified(ctx context.Context, key string, out io.Writer, che
return fmt.Errorf("inspect archive: %w", err)
}
if requireEncryption && (object.ServerSideEncryption != s.encryption ||
(s.encryption == types.ServerSideEncryptionAwsKms && aws.ToString(object.SSEKMSKeyId) == "")) {
(s.encryption == types.ServerSideEncryptionAwsKms && (aws.ToString(object.SSEKMSKeyId) == "" ||
(s.kmsKeyID != "" && aws.ToString(object.SSEKMSKeyId) != s.kmsKeyID)))) {
return errors.New("remote archive did not confirm the requested server-side encryption; source data retained")
}
if object.ContentLength == nil || *object.ContentLength != expectedSize || aws.ToString(object.ETag) == "" {
@@ -265,11 +308,17 @@ func (s *Store) readArchiveChunkOnce(ctx context.Context, key, etag string, star
object, err := s.client.GetObject(requestCtx, &s3.GetObjectInput{
Bucket: aws.String(s.bucket), Key: aws.String(key), IfMatch: aws.String(etag),
Range: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)),
}, func(o *s3.Options) {
// This loop owns both HTTP and interrupted-body retries: at most five
// wire requests per chunk, rather than five times the SDK's attempts.
o.RetryMaxAttempts = 1
})
if err != nil {
var networkError net.Error
retry := errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded)
return nil, retry, fmt.Errorf("read archive range: %w", err)
var responseError *awshttp.ResponseError
throttled := errors.As(err, &responseError) && responseError.HTTPStatusCode() == http.StatusTooManyRequests
shouldRetry := throttled || retry.NewStandard().IsErrorRetryable(err) || errors.As(err, &networkError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, context.DeadlineExceeded)
return nil, shouldRetry, fmt.Errorf("read archive range: %w", err)
}
defer object.Body.Close()
want := end - start + 1
+2
View File
@@ -340,6 +340,8 @@ func TestS3UploadRejectsUnconfirmedEncryption(t *testing.T) {
switch r.Method {
case "PUT":
_, _ = io.Copy(io.Discard, r.Body)
case "DELETE":
w.WriteHeader(204)
case "HEAD":
w.Header().Set("ETag", `"test-etag"`)
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
+231
View File
@@ -0,0 +1,231 @@
package archivestore
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/pem"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"testing"
)
func archiveSource(t *testing.T, data []byte) (string, string) {
t.Helper()
filename := filepath.Join(t.TempDir(), "source")
if err := os.WriteFile(filename, data, 0600); err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(data)
return filename, hex.EncodeToString(sum[:])
}
func TestS3VerificationFailureCleansUploadedVersion(t *testing.T) {
for _, failure := range []string{"wrong-kms", "checksum", "cancelled", "delete-denied"} {
t.Run(failure, func(t *testing.T) {
data := []byte("private archive")
source, checksum := archiveSource(t, data)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var present, deleted atomic.Bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "PUT":
_, _ = io.Copy(io.Discard, r.Body)
present.Store(true)
w.Header().Set("X-Amz-Version-Id", "uploaded-version")
case "HEAD":
w.Header().Set("ETag", `"test-etag"`)
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
w.Header().Set("X-Amz-Server-Side-Encryption", "aws:kms")
kms := "arn:aws:kms:us-east-1:123456789012:key/expected"
if failure == "wrong-kms" {
kms = "arn:aws:kms:us-east-1:123456789012:key/different"
}
w.Header().Set("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id", kms)
if failure == "cancelled" {
cancel()
}
case "GET":
corrupt := bytes.Clone(data)
corrupt[0] ^= 0xff
serveArchiveRange(t, w, r, corrupt)
case "DELETE":
if r.URL.Query().Get("versionId") != "uploaded-version" {
t.Error("cleanup did not target uploaded version")
}
deleted.Store(true)
if failure == "delete-denied" {
w.WriteHeader(403)
return
}
present.Store(false)
w.WriteHeader(204)
default:
t.Errorf("unexpected request %s", r.Method)
w.WriteHeader(400)
}
}))
defer srv.Close()
store, err := New(Config{Endpoint: srv.URL, Bucket: "test", AccessKey: "test", SecretKey: "test", PathStyle: true,
SSE: "aws:kms", KMSKeyID: "arn:aws:kms:us-east-1:123456789012:key/expected"})
if err != nil {
t.Fatal(err)
}
err = store.Upload(ctx, store.URI(1, 2, checksum), source, checksum)
if err == nil {
t.Fatal("accepted failed verification")
}
if failure == "wrong-kms" && !strings.Contains(err.Error(), "encryption") {
t.Fatal(err)
}
if failure == "cancelled" && !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
if failure == "delete-denied" && (!strings.Contains(err.Error(), "SHA-256") || !strings.Contains(err.Error(), "delete unverified archive")) {
t.Fatalf("verification and cleanup errors must both survive: %v", err)
}
if !deleted.Load() || present.Load() != (failure == "delete-denied") {
t.Fatal("unverified object cleanup was not attempted/completed")
}
if got, readErr := os.ReadFile(source); readErr != nil || !bytes.Equal(got, data) {
t.Fatal("source changed", readErr)
}
})
}
}
func TestS3CancelledMultipartAbortsWithFreshContext(t *testing.T) {
for _, denyAbort := range []bool{false, true} {
t.Run(strconv.FormatBool(denyAbort), func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var aborted atomic.Bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
switch {
case r.Method == "POST" && q.Has("uploads"):
fmt.Fprint(w, `<InitiateMultipartUploadResult><UploadId>cancelled-upload</UploadId></InitiateMultipartUploadResult>`)
case r.Method == "PUT" && q.Has("partNumber"):
cancel()
_, _ = io.Copy(io.Discard, r.Body)
w.WriteHeader(403)
case r.Method == "DELETE" && q.Get("uploadId") == "cancelled-upload":
aborted.Store(true)
if denyAbort {
w.WriteHeader(403)
} else {
w.WriteHeader(204)
}
default:
t.Errorf("unexpected request %s", r.Method)
w.WriteHeader(400)
}
}))
defer srv.Close()
data := bytes.Repeat([]byte("x"), 9<<20)
source, sum := archiveSource(t, data)
store := testRangeStore(t, srv.URL)
err := store.Upload(ctx, store.URI(1, 2, sum), source, sum)
if !errors.Is(err, context.Canceled) || !aborted.Load() {
t.Fatalf("cancelled upload not aborted: %v", err)
}
if denyAbort && !strings.Contains(err.Error(), "abort archive multipart upload") {
t.Fatalf("cleanup failure lost: %v", err)
}
if got, readErr := os.ReadFile(source); readErr != nil || !bytes.Equal(got, data) {
t.Fatal("source changed", readErr)
}
})
}
}
func TestS3SignedRequestsNeverFollowRedirects(t *testing.T) {
var reached atomic.Int32
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reached.Add(1); w.WriteHeader(204) }))
defer target.Close()
for _, status := range []int{301, 302, 303, 307, 308} {
t.Run(strconv.Itoa(status), func(t *testing.T) {
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") == "" || r.Header.Get("X-Amz-Security-Token") != "fixture-session" {
t.Error("fixture request must be signed with session credentials")
}
w.Header().Set("Location", target.URL+"/redirected")
w.WriteHeader(status)
}))
defer origin.Close()
store, err := New(Config{Endpoint: origin.URL, Bucket: "test", AccessKey: "test", SecretKey: "test", SessionToken: "fixture-session", PathStyle: true})
if err != nil {
t.Fatal(err)
}
if err := store.Delete(context.Background(), store.URI(1, 2, "test")); err == nil {
t.Fatal("redirect accepted")
}
if reached.Load() != 0 {
t.Fatal("signed request followed redirect")
}
})
}
}
func TestS3CustomCABundleForBothCredentialModes(t *testing.T) {
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) }))
defer srv.Close()
for _, explicit := range []bool{false, true} {
t.Run(strconv.FormatBool(explicit), func(t *testing.T) {
isolateAWSCredentials(t)
ca := filepath.Join(t.TempDir(), "ca.pem")
if err := os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw}), 0600); err != nil {
t.Fatal(err)
}
t.Setenv("AWS_CA_BUNDLE", ca)
c := Config{Endpoint: srv.URL, Bucket: "test", PathStyle: true}
if explicit {
c.AccessKey, c.SecretKey = "fixture", "fixture"
} else {
t.Setenv("AWS_ACCESS_KEY_ID", "fixture")
t.Setenv("AWS_SECRET_ACCESS_KEY", "fixture")
}
store, err := New(c)
if err != nil {
t.Fatal(err)
}
if err := store.Delete(context.Background(), store.URI(1, 2, "test")); err != nil {
t.Fatalf("configured CA not used: %v", err)
}
})
}
}
func TestS3RangeRetriesHaveOneWireAttemptEach(t *testing.T) {
for _, status := range []int{403, 429, 503} {
t.Run(strconv.Itoa(status), func(t *testing.T) {
var gets atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gets.Add(1)
w.WriteHeader(status)
}))
defer srv.Close()
store := testRangeStore(t, srv.URL)
if _, err := store.readArchiveChunk(context.Background(), "key", `"etag"`, 0, 9, 10); err == nil {
t.Fatal("failed request accepted")
}
want := int32(archiveReadAttempts)
if status == 403 {
want = 1
}
if got := gets.Load(); got != want {
t.Fatalf("wire requests=%d want=%d", got, want)
}
})
}
}